From 2cbea54929c419a841c9f728ce1244102e849d6d Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Tue, 17 Jun 2014 07:44:55 +0000
Subject: [PATCH 001/380] 4390 i/o errors when deleting filesystem/zvol can
 lead to space map corruption

Reviewed by: George Wilson <george.wilson@delphix.com>
Reviewed by: Christopher Siden <christopher.siden@delphix.com>
Reviewed by: Adam Leventhal <ahl@delphix.com>
Reviewed by: Dan McDonald <danmcd@omniti.com>
Reviewed by: Saso Kiselkov <saso.kiselkov@nexenta.com>
Approved by: Dan McDonald <danmcd@omniti.com>

illumos/illumos-dist@7fd05ac4dec0c343d2f68f310d3718b715ecfbaf
---
 common/zfs/zpool_prop.c           |   2 +
 uts/common/fs/zfs/bptree.c        | 101 +++++++++++++++++----
 uts/common/fs/zfs/dmu_traverse.c  | 126 +++++++++++---------------
 uts/common/fs/zfs/dsl_pool.c      |   9 ++
 uts/common/fs/zfs/dsl_scan.c      | 144 ++++++++++++++++++++----------
 uts/common/fs/zfs/spa.c           |  14 ++-
 uts/common/fs/zfs/spa_misc.c      |  33 ++++++-
 uts/common/fs/zfs/sys/bptree.h    |   3 +-
 uts/common/fs/zfs/sys/dmu.h       |   1 -
 uts/common/fs/zfs/sys/dsl_dir.h   |   1 +
 uts/common/fs/zfs/sys/dsl_pool.h  |   1 +
 uts/common/fs/zfs/sys/dsl_scan.h  |   1 +
 uts/common/fs/zfs/sys/zfs_debug.h |   3 +-
 uts/common/fs/zfs/zfs_debug.c     |   2 +-
 uts/common/fs/zfs/zio.c           |   7 --
 uts/common/sys/fs/zfs.h           |   1 +
 16 files changed, 293 insertions(+), 156 deletions(-)

diff --git a/common/zfs/zpool_prop.c b/common/zfs/zpool_prop.c
index 72db87937110..51041a8766cc 100644
--- a/common/zfs/zpool_prop.c
+++ b/common/zfs/zpool_prop.c
@@ -81,6 +81,8 @@ zpool_prop_init(void)
 	    ZFS_TYPE_POOL, "<size>", "FREE");
 	zprop_register_number(ZPOOL_PROP_FREEING, "freeing", 0, PROP_READONLY,
 	    ZFS_TYPE_POOL, "<size>", "FREEING");
+	zprop_register_number(ZPOOL_PROP_LEAKED, "leaked", 0, PROP_READONLY,
+	    ZFS_TYPE_POOL, "<size>", "LEAKED");
 	zprop_register_number(ZPOOL_PROP_ALLOCATED, "allocated", 0,
 	    PROP_READONLY, ZFS_TYPE_POOL, "<size>", "ALLOC");
 	zprop_register_number(ZPOOL_PROP_EXPANDSZ, "expandsize", 0,
diff --git a/uts/common/fs/zfs/bptree.c b/uts/common/fs/zfs/bptree.c
index 83f365864dec..77067d24e98a 100644
--- a/uts/common/fs/zfs/bptree.c
+++ b/uts/common/fs/zfs/bptree.c
@@ -102,13 +102,27 @@ bptree_free(objset_t *os, uint64_t obj, dmu_tx_t *tx)
 	return (dmu_object_free(os, obj, tx));
 }
 
+boolean_t
+bptree_is_empty(objset_t *os, uint64_t obj)
+{
+	dmu_buf_t *db;
+	bptree_phys_t *bt;
+	boolean_t rv;
+
+	VERIFY0(dmu_bonus_hold(os, obj, FTAG, &db));
+	bt = db->db_data;
+	rv = (bt->bt_begin == bt->bt_end);
+	dmu_buf_rele(db, FTAG);
+	return (rv);
+}
+
 void
 bptree_add(objset_t *os, uint64_t obj, blkptr_t *bp, uint64_t birth_txg,
     uint64_t bytes, uint64_t comp, uint64_t uncomp, dmu_tx_t *tx)
 {
 	dmu_buf_t *db;
 	bptree_phys_t *bt;
-	bptree_entry_phys_t bte;
+	bptree_entry_phys_t bte = { 0 };
 
 	/*
 	 * bptree objects are in the pool mos, therefore they can only be
@@ -122,7 +136,6 @@ bptree_add(objset_t *os, uint64_t obj, blkptr_t *bp, uint64_t birth_txg,
 
 	bte.be_birth_txg = birth_txg;
 	bte.be_bp = *bp;
-	bzero(&bte.be_zb, sizeof (bte.be_zb));
 	dmu_write(os, obj, bt->bt_end * sizeof (bte), sizeof (bte), &bte, tx);
 
 	dmu_buf_will_dirty(db, tx);
@@ -153,10 +166,27 @@ bptree_visit_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 	return (err);
 }
 
+/*
+ * If "free" is set:
+ *  - It is assumed that "func" will be freeing the block pointers.
+ *  - If "func" returns nonzero, the bookmark will be remembered and
+ *    iteration will be restarted from this point on next invocation.
+ *  - If an i/o error is encountered (e.g. "func" returns EIO or ECKSUM),
+ *    bptree_iterate will remember the bookmark, continue traversing
+ *    any additional entries, and return 0.
+ *
+ * If "free" is not set, traversal will stop and return an error if
+ * an i/o error is encountered.
+ *
+ * In either case, if zfs_free_leak_on_eio is set, i/o errors will be
+ * ignored and traversal will continue (i.e. TRAVERSE_HARD will be passed to
+ * traverse_dataset_destroyed()).
+ */
 int
 bptree_iterate(objset_t *os, uint64_t obj, boolean_t free, bptree_itor_t func,
     void *arg, dmu_tx_t *tx)
 {
+	boolean_t ioerr = B_FALSE;
 	int err;
 	uint64_t i;
 	dmu_buf_t *db;
@@ -182,49 +212,82 @@ bptree_iterate(objset_t *os, uint64_t obj, boolean_t free, bptree_itor_t func,
 		bptree_entry_phys_t bte;
 		int flags = TRAVERSE_PREFETCH_METADATA | TRAVERSE_POST;
 
-		ASSERT(!free || i == ba.ba_phys->bt_begin);
-
 		err = dmu_read(os, obj, i * sizeof (bte), sizeof (bte),
 		    &bte, DMU_READ_NO_PREFETCH);
 		if (err != 0)
 			break;
 
-		if (zfs_recover)
+		if (zfs_free_leak_on_eio)
 			flags |= TRAVERSE_HARD;
+		zfs_dbgmsg("bptree index %d: traversing from min_txg=%lld "
+		    "bookmark %lld/%lld/%lld/%lld",
+		    i, (longlong_t)bte.be_birth_txg,
+		    (longlong_t)bte.be_zb.zb_objset,
+		    (longlong_t)bte.be_zb.zb_object,
+		    (longlong_t)bte.be_zb.zb_level,
+		    (longlong_t)bte.be_zb.zb_blkid);
 		err = traverse_dataset_destroyed(os->os_spa, &bte.be_bp,
 		    bte.be_birth_txg, &bte.be_zb, flags,
 		    bptree_visit_cb, &ba);
 		if (free) {
-			if (err == ERESTART) {
+			/*
+			 * The callback has freed the visited block pointers.
+			 * Record our traversal progress on disk, either by
+			 * updating this record's bookmark, or by logically
+			 * removing this record by advancing bt_begin.
+			 */
+			if (err != 0) {
 				/* save bookmark for future resume */
 				ASSERT3U(bte.be_zb.zb_objset, ==,
 				    ZB_DESTROYED_OBJSET);
 				ASSERT0(bte.be_zb.zb_level);
 				dmu_write(os, obj, i * sizeof (bte),
 				    sizeof (bte), &bte, tx);
-				break;
-			}
-			if (err != 0) {
+				if (err == EIO || err == ECKSUM ||
+				    err == ENXIO) {
+					/*
+					 * Skip the rest of this tree and
+					 * continue on to the next entry.
+					 */
+					err = 0;
+					ioerr = B_TRUE;
+				} else {
+					break;
+				}
+			} else if (ioerr) {
 				/*
-				 * We can not properly handle an i/o
-				 * error, because the traversal code
-				 * does not know how to resume from an
-				 * arbitrary bookmark.
+				 * This entry is finished, but there were
+				 * i/o errors on previous entries, so we
+				 * can't adjust bt_begin.  Set this entry's
+				 * be_birth_txg such that it will be
+				 * treated as a no-op in future traversals.
 				 */
-				zfs_panic_recover("error %u from "
-				    "traverse_dataset_destroyed()", err);
+				bte.be_birth_txg = UINT64_MAX;
+				dmu_write(os, obj, i * sizeof (bte),
+				    sizeof (bte), &bte, tx);
 			}
 
-			ba.ba_phys->bt_begin++;
-			(void) dmu_free_range(os, obj,
-			    i * sizeof (bte), sizeof (bte), tx);
+			if (!ioerr) {
+				ba.ba_phys->bt_begin++;
+				(void) dmu_free_range(os, obj,
+				    i * sizeof (bte), sizeof (bte), tx);
+			}
+		} else if (err != 0) {
+			break;
 		}
 	}
 
-	ASSERT(!free || err != 0 || ba.ba_phys->bt_begin == ba.ba_phys->bt_end);
+	ASSERT(!free || err != 0 || ioerr ||
+	    ba.ba_phys->bt_begin == ba.ba_phys->bt_end);
 
 	/* if all blocks are free there should be no used space */
 	if (ba.ba_phys->bt_begin == ba.ba_phys->bt_end) {
+		if (zfs_free_leak_on_eio) {
+			ba.ba_phys->bt_bytes = 0;
+			ba.ba_phys->bt_comp = 0;
+			ba.ba_phys->bt_uncomp = 0;
+		}
+
 		ASSERT0(ba.ba_phys->bt_bytes);
 		ASSERT0(ba.ba_phys->bt_comp);
 		ASSERT0(ba.ba_phys->bt_uncomp);
diff --git a/uts/common/fs/zfs/dmu_traverse.c b/uts/common/fs/zfs/dmu_traverse.c
index db0869ca6f18..1c698b5ee03c 100644
--- a/uts/common/fs/zfs/dmu_traverse.c
+++ b/uts/common/fs/zfs/dmu_traverse.c
@@ -58,6 +58,7 @@ typedef struct traverse_data {
 	zbookmark_t *td_resume;
 	int td_flags;
 	prefetch_data_t *td_pfd;
+	boolean_t td_paused;
 	blkptr_cb_t *td_func;
 	void *td_arg;
 } traverse_data_t;
@@ -163,7 +164,6 @@ resume_skip_check(traverse_data_t *td, const dnode_phys_t *dnp,
 		 * If we found the block we're trying to resume from, zero
 		 * the bookmark out to indicate that we have resumed.
 		 */
-		ASSERT3U(zb->zb_object, <=, td->td_resume->zb_object);
 		if (bcmp(zb, td->td_resume, sizeof (*zb)) == 0) {
 			bzero(td->td_resume, sizeof (*zb));
 			if (td->td_flags & TRAVERSE_POST)
@@ -173,14 +173,6 @@ resume_skip_check(traverse_data_t *td, const dnode_phys_t *dnp,
 	return (RESUME_SKIP_NONE);
 }
 
-static void
-traverse_pause(traverse_data_t *td, const zbookmark_t *zb)
-{
-	ASSERT(td->td_resume != NULL);
-	ASSERT0(zb->zb_level);
-	bcopy(zb, td->td_resume, sizeof (*td->td_resume));
-}
-
 static void
 traverse_prefetch_metadata(traverse_data_t *td,
     const blkptr_t *bp, const zbookmark_t *zb)
@@ -210,11 +202,10 @@ traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
     const blkptr_t *bp, const zbookmark_t *zb)
 {
 	zbookmark_t czb;
-	int err = 0, lasterr = 0;
+	int err = 0;
 	arc_buf_t *buf = NULL;
 	prefetch_data_t *pd = td->td_pfd;
 	boolean_t hard = td->td_flags & TRAVERSE_HARD;
-	boolean_t pause = B_FALSE;
 
 	switch (resume_skip_check(td, dnp, zb)) {
 	case RESUME_SKIP_ALL:
@@ -253,7 +244,9 @@ traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
 
 	if (BP_IS_HOLE(bp)) {
 		err = td->td_func(td->td_spa, NULL, bp, zb, dnp, td->td_arg);
-		return (err);
+		if (err != 0)
+			goto post;
+		return (0);
 	}
 
 	if (pd && !pd->pd_exited &&
@@ -273,8 +266,6 @@ traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
 		    td->td_arg);
 		if (err == TRAVERSE_VISIT_NO_CHILDREN)
 			return (0);
-		if (err == ERESTART)
-			pause = B_TRUE; /* handle pausing at a common point */
 		if (err != 0)
 			goto post;
 	}
@@ -288,7 +279,7 @@ traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
 		err = arc_read(NULL, td->td_spa, bp, arc_getbuf_func, &buf,
 		    ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL, &flags, zb);
 		if (err != 0)
-			return (err);
+			goto post;
 		cbp = buf->b_data;
 
 		for (i = 0; i < epb; i++) {
@@ -304,11 +295,8 @@ traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
 			    zb->zb_level - 1,
 			    zb->zb_blkid * epb + i);
 			err = traverse_visitbp(td, dnp, &cbp[i], &czb);
-			if (err != 0) {
-				if (!hard)
-					break;
-				lasterr = err;
-			}
+			if (err != 0)
+				break;
 		}
 	} else if (BP_GET_TYPE(bp) == DMU_OT_DNODE) {
 		uint32_t flags = ARC_WAIT;
@@ -318,7 +306,7 @@ traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
 		err = arc_read(NULL, td->td_spa, bp, arc_getbuf_func, &buf,
 		    ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL, &flags, zb);
 		if (err != 0)
-			return (err);
+			goto post;
 		dnp = buf->b_data;
 
 		for (i = 0; i < epb; i++) {
@@ -330,11 +318,8 @@ traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
 		for (i = 0; i < epb; i++) {
 			err = traverse_dnode(td, &dnp[i], zb->zb_objset,
 			    zb->zb_blkid * epb + i);
-			if (err != 0) {
-				if (!hard)
-					break;
-				lasterr = err;
-			}
+			if (err != 0)
+				break;
 		}
 	} else if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) {
 		uint32_t flags = ARC_WAIT;
@@ -344,7 +329,7 @@ traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
 		err = arc_read(NULL, td->td_spa, bp, arc_getbuf_func, &buf,
 		    ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL, &flags, zb);
 		if (err != 0)
-			return (err);
+			goto post;
 
 		osp = buf->b_data;
 		dnp = &osp->os_meta_dnode;
@@ -359,19 +344,11 @@ traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
 
 		err = traverse_dnode(td, dnp, zb->zb_objset,
 		    DMU_META_DNODE_OBJECT);
-		if (err && hard) {
-			lasterr = err;
-			err = 0;
-		}
 		if (err == 0 && arc_buf_size(buf) >= sizeof (objset_phys_t)) {
 			dnp = &osp->os_groupused_dnode;
 			err = traverse_dnode(td, dnp, zb->zb_objset,
 			    DMU_GROUPUSED_OBJECT);
 		}
-		if (err && hard) {
-			lasterr = err;
-			err = 0;
-		}
 		if (err == 0 && arc_buf_size(buf) >= sizeof (objset_phys_t)) {
 			dnp = &osp->os_userused_dnode;
 			err = traverse_dnode(td, dnp, zb->zb_objset,
@@ -383,19 +360,37 @@ traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
 		(void) arc_buf_remove_ref(buf, &buf);
 
 post:
-	if (err == 0 && (td->td_flags & TRAVERSE_POST)) {
+	if (err == 0 && (td->td_flags & TRAVERSE_POST))
 		err = td->td_func(td->td_spa, NULL, bp, zb, dnp, td->td_arg);
-		if (err == ERESTART)
-			pause = B_TRUE;
+
+	if (hard && (err == EIO || err == ECKSUM)) {
+		/*
+		 * Ignore this disk error as requested by the HARD flag,
+		 * and continue traversal.
+		 */
+		err = 0;
 	}
 
-	if (pause && td->td_resume != NULL) {
-		ASSERT3U(err, ==, ERESTART);
-		ASSERT(!hard);
-		traverse_pause(td, zb);
+	/*
+	 * If we are stopping here, set td_resume.
+	 */
+	if (td->td_resume != NULL && err != 0 && !td->td_paused) {
+		td->td_resume->zb_objset = zb->zb_objset;
+		td->td_resume->zb_object = zb->zb_object;
+		td->td_resume->zb_level = 0;
+		/*
+		 * If we have stopped on an indirect block (e.g. due to
+		 * i/o error), we have not visited anything below it.
+		 * Set the bookmark to the first level-0 block that we need
+		 * to visit.  This way, the resuming code does not need to
+		 * deal with resuming from indirect blocks.
+		 */
+		td->td_resume->zb_blkid = zb->zb_blkid <<
+		    (zb->zb_level * (dnp->dn_indblkshift - SPA_BLKPTRSHIFT));
+		td->td_paused = B_TRUE;
 	}
 
-	return (err != 0 ? err : lasterr);
+	return (err);
 }
 
 static void
@@ -420,30 +415,21 @@ static int
 traverse_dnode(traverse_data_t *td, const dnode_phys_t *dnp,
     uint64_t objset, uint64_t object)
 {
-	int j, err = 0, lasterr = 0;
+	int j, err = 0;
 	zbookmark_t czb;
-	boolean_t hard = (td->td_flags & TRAVERSE_HARD);
 
 	for (j = 0; j < dnp->dn_nblkptr; j++) {
 		SET_BOOKMARK(&czb, objset, object, dnp->dn_nlevels - 1, j);
 		err = traverse_visitbp(td, dnp, &dnp->dn_blkptr[j], &czb);
-		if (err != 0) {
-			if (!hard)
-				break;
-			lasterr = err;
-		}
+		if (err != 0)
+			break;
 	}
 
 	if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
 		SET_BOOKMARK(&czb, objset, object, 0, DMU_SPILL_BLKID);
 		err = traverse_visitbp(td, dnp, &dnp->dn_spill, &czb);
-		if (err != 0) {
-			if (!hard)
-				return (err);
-			lasterr = err;
-		}
 	}
-	return (err != 0 ? err : lasterr);
+	return (err);
 }
 
 /* ARGSUSED */
@@ -530,6 +516,7 @@ traverse_impl(spa_t *spa, dsl_dataset_t *ds, uint64_t objset, blkptr_t *rootbp,
 	td.td_arg = arg;
 	td.td_pfd = &pd;
 	td.td_flags = flags;
+	td.td_paused = B_FALSE;
 
 	pd.pd_blks_max = zfs_pd_blks_max;
 	pd.pd_flags = flags;
@@ -603,7 +590,7 @@ int
 traverse_pool(spa_t *spa, uint64_t txg_start, int flags,
     blkptr_cb_t func, void *arg)
 {
-	int err, lasterr = 0;
+	int err;
 	uint64_t obj;
 	dsl_pool_t *dp = spa_get_dsl(spa);
 	objset_t *mos = dp->dp_meta_objset;
@@ -616,16 +603,15 @@ traverse_pool(spa_t *spa, uint64_t txg_start, int flags,
 		return (err);
 
 	/* visit each dataset */
-	for (obj = 1; err == 0 || (err != ESRCH && hard);
+	for (obj = 1; err == 0;
 	    err = dmu_object_next(mos, &obj, FALSE, txg_start)) {
 		dmu_object_info_t doi;
 
 		err = dmu_object_info(mos, obj, &doi);
 		if (err != 0) {
-			if (!hard)
-				return (err);
-			lasterr = err;
-			continue;
+			if (hard)
+				continue;
+			break;
 		}
 
 		if (doi.doi_bonus_type == DMU_OT_DSL_DATASET) {
@@ -636,23 +622,19 @@ traverse_pool(spa_t *spa, uint64_t txg_start, int flags,
 			err = dsl_dataset_hold_obj(dp, obj, FTAG, &ds);
 			dsl_pool_config_exit(dp, FTAG);
 			if (err != 0) {
-				if (!hard)
-					return (err);
-				lasterr = err;
-				continue;
+				if (hard)
+					continue;
+				break;
 			}
 			if (ds->ds_phys->ds_prev_snap_txg > txg)
 				txg = ds->ds_phys->ds_prev_snap_txg;
 			err = traverse_dataset(ds, txg, flags, func, arg);
 			dsl_dataset_rele(ds, FTAG);
-			if (err != 0) {
-				if (!hard)
-					return (err);
-				lasterr = err;
-			}
+			if (err != 0)
+				break;
 		}
 	}
 	if (err == ESRCH)
 		err = 0;
-	return (err != 0 ? err : lasterr);
+	return (err);
 }
diff --git a/uts/common/fs/zfs/dsl_pool.c b/uts/common/fs/zfs/dsl_pool.c
index b6bdc78f09c2..76ca96b1431a 100644
--- a/uts/common/fs/zfs/dsl_pool.c
+++ b/uts/common/fs/zfs/dsl_pool.c
@@ -250,6 +250,13 @@ dsl_pool_open(dsl_pool_t *dp)
 		    dp->dp_meta_objset, obj));
 	}
 
+	/*
+	 * Note: errors ignored, because the leak dir will not exist if we
+	 * have not encountered a leak yet.
+	 */
+	(void) dsl_pool_open_special_dir(dp, LEAK_DIR_NAME,
+	    &dp->dp_leak_dir);
+
 	if (spa_feature_is_active(dp->dp_spa, SPA_FEATURE_ASYNC_DESTROY)) {
 		err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
 		    DMU_POOL_BPTREE_OBJ, sizeof (uint64_t), 1,
@@ -297,6 +304,8 @@ dsl_pool_close(dsl_pool_t *dp)
 		dsl_dir_rele(dp->dp_mos_dir, dp);
 	if (dp->dp_free_dir)
 		dsl_dir_rele(dp->dp_free_dir, dp);
+	if (dp->dp_leak_dir)
+		dsl_dir_rele(dp->dp_leak_dir, dp);
 	if (dp->dp_root_dir)
 		dsl_dir_rele(dp->dp_root_dir, dp);
 
diff --git a/uts/common/fs/zfs/dsl_scan.c b/uts/common/fs/zfs/dsl_scan.c
index 484420ea3f17..860f9dfb4e86 100644
--- a/uts/common/fs/zfs/dsl_scan.c
+++ b/uts/common/fs/zfs/dsl_scan.c
@@ -52,9 +52,7 @@
 
 typedef int (scan_cb_t)(dsl_pool_t *, const blkptr_t *, const zbookmark_t *);
 
-static scan_cb_t dsl_scan_defrag_cb;
 static scan_cb_t dsl_scan_scrub_cb;
-static scan_cb_t dsl_scan_remove_cb;
 static void dsl_scan_cancel_sync(void *, dmu_tx_t *);
 static void dsl_scan_sync_state(dsl_scan_t *, dmu_tx_t *tx);
 
@@ -67,7 +65,7 @@ int zfs_scan_min_time_ms = 1000; /* min millisecs to scrub per txg */
 int zfs_free_min_time_ms = 1000; /* min millisecs to free per txg */
 int zfs_resilver_min_time_ms = 3000; /* min millisecs to resilver per txg */
 boolean_t zfs_no_scrub_io = B_FALSE; /* set to disable scrub i/o */
-boolean_t zfs_no_scrub_prefetch = B_FALSE; /* set to disable srub prefetching */
+boolean_t zfs_no_scrub_prefetch = B_FALSE; /* set to disable scrub prefetch */
 enum ddt_class zfs_scrub_ddt_class_max = DDT_CLASS_DUPLICATE;
 int dsl_scan_delay_completion = B_FALSE; /* set to delay scan completion */
 
@@ -1358,7 +1356,7 @@ dsl_scan_active(dsl_scan_t *scn)
 	if (spa_shutting_down(spa))
 		return (B_FALSE);
 	if (scn->scn_phys.scn_state == DSS_SCANNING ||
-	    scn->scn_async_destroying)
+	    (scn->scn_async_destroying && !scn->scn_async_stalled))
 		return (B_TRUE);
 
 	if (spa_version(scn->scn_dp->dp_spa) >= SPA_VERSION_DEADLISTS) {
@@ -1373,7 +1371,7 @@ dsl_scan_sync(dsl_pool_t *dp, dmu_tx_t *tx)
 {
 	dsl_scan_t *scn = dp->dp_scan;
 	spa_t *spa = dp->dp_spa;
-	int err;
+	int err = 0;
 
 	/*
 	 * Check for scn_restart_txg before checking spa_load_state, so
@@ -1391,7 +1389,10 @@ dsl_scan_sync(dsl_pool_t *dp, dmu_tx_t *tx)
 		dsl_scan_setup_sync(&func, tx);
 	}
 
-	if (!dsl_scan_active(scn) ||
+	/*
+	 * If the scan is inactive due to a stalled async destroy, try again.
+	 */
+	if ((!scn->scn_async_stalled && !dsl_scan_active(scn)) ||
 	    spa_sync_pass(dp->dp_spa) > 1)
 		return;
 
@@ -1401,10 +1402,11 @@ dsl_scan_sync(dsl_pool_t *dp, dmu_tx_t *tx)
 	spa->spa_scrub_active = B_TRUE;
 
 	/*
-	 * First process the free list.  If we pause the free, don't do
-	 * any scanning.  This ensures that there is no free list when
-	 * we are scanning, so the scan code doesn't have to worry about
-	 * traversing it.
+	 * First process the async destroys.  If we pause, don't do
+	 * any scrubbing or resilvering.  This ensures that there are no
+	 * async destroys while we are scanning, so the scan code doesn't
+	 * have to worry about traversing it.  It is also faster to free the
+	 * blocks than to scrub them.
 	 */
 	if (spa_version(dp->dp_spa) >= SPA_VERSION_DEADLISTS) {
 		scn->scn_is_bptree = B_FALSE;
@@ -1414,48 +1416,92 @@ dsl_scan_sync(dsl_pool_t *dp, dmu_tx_t *tx)
 		    dsl_scan_free_block_cb, scn, tx);
 		VERIFY3U(0, ==, zio_wait(scn->scn_zio_root));
 
-		if (err == 0 && spa_feature_is_active(spa,
-		    SPA_FEATURE_ASYNC_DESTROY)) {
-			ASSERT(scn->scn_async_destroying);
-			scn->scn_is_bptree = B_TRUE;
-			scn->scn_zio_root = zio_root(dp->dp_spa, NULL,
-			    NULL, ZIO_FLAG_MUSTSUCCEED);
-			err = bptree_iterate(dp->dp_meta_objset,
-			    dp->dp_bptree_obj, B_TRUE, dsl_scan_free_block_cb,
-			    scn, tx);
-			VERIFY0(zio_wait(scn->scn_zio_root));
+		if (err != 0 && err != ERESTART)
+			zfs_panic_recover("error %u from bpobj_iterate()", err);
+	}
 
-			if (err == 0) {
-				/* finished; deactivate async destroy feature */
-				spa_feature_decr(spa, SPA_FEATURE_ASYNC_DESTROY,
-				    tx);
-				ASSERT(!spa_feature_is_active(spa,
-				    SPA_FEATURE_ASYNC_DESTROY));
-				VERIFY0(zap_remove(dp->dp_meta_objset,
-				    DMU_POOL_DIRECTORY_OBJECT,
-				    DMU_POOL_BPTREE_OBJ, tx));
-				VERIFY0(bptree_free(dp->dp_meta_objset,
-				    dp->dp_bptree_obj, tx));
-				dp->dp_bptree_obj = 0;
-				scn->scn_async_destroying = B_FALSE;
-			}
+	if (err == 0 && spa_feature_is_active(spa, SPA_FEATURE_ASYNC_DESTROY)) {
+		ASSERT(scn->scn_async_destroying);
+		scn->scn_is_bptree = B_TRUE;
+		scn->scn_zio_root = zio_root(dp->dp_spa, NULL,
+		    NULL, ZIO_FLAG_MUSTSUCCEED);
+		err = bptree_iterate(dp->dp_meta_objset,
+		    dp->dp_bptree_obj, B_TRUE, dsl_scan_free_block_cb, scn, tx);
+		VERIFY0(zio_wait(scn->scn_zio_root));
+
+		if (err == EIO || err == ECKSUM) {
+			err = 0;
+		} else if (err != 0 && err != ERESTART) {
+			zfs_panic_recover("error %u from "
+			    "traverse_dataset_destroyed()", err);
 		}
-		if (scn->scn_visited_this_txg) {
-			zfs_dbgmsg("freed %llu blocks in %llums from "
-			    "free_bpobj/bptree txg %llu",
-			    (longlong_t)scn->scn_visited_this_txg,
-			    (longlong_t)
-			    NSEC2MSEC(gethrtime() - scn->scn_sync_start_time),
-			    (longlong_t)tx->tx_txg);
-			scn->scn_visited_this_txg = 0;
-			/*
-			 * Re-sync the ddt so that we can further modify
-			 * it when doing bprewrite.
-			 */
-			ddt_sync(spa, tx->tx_txg);
+
+		/*
+		 * If we didn't make progress, mark the async destroy as
+		 * stalled, so that we will not initiate a spa_sync() on
+		 * its behalf.
+		 */
+		scn->scn_async_stalled = (scn->scn_visited_this_txg == 0);
+
+		if (bptree_is_empty(dp->dp_meta_objset, dp->dp_bptree_obj)) {
+			/* finished; deactivate async destroy feature */
+			spa_feature_decr(spa, SPA_FEATURE_ASYNC_DESTROY, tx);
+			ASSERT(!spa_feature_is_active(spa,
+			    SPA_FEATURE_ASYNC_DESTROY));
+			VERIFY0(zap_remove(dp->dp_meta_objset,
+			    DMU_POOL_DIRECTORY_OBJECT,
+			    DMU_POOL_BPTREE_OBJ, tx));
+			VERIFY0(bptree_free(dp->dp_meta_objset,
+			    dp->dp_bptree_obj, tx));
+			dp->dp_bptree_obj = 0;
+			scn->scn_async_destroying = B_FALSE;
 		}
-		if (err == ERESTART)
-			return;
+	}
+	if (scn->scn_visited_this_txg) {
+		zfs_dbgmsg("freed %llu blocks in %llums from "
+		    "free_bpobj/bptree txg %llu; err=%u",
+		    (longlong_t)scn->scn_visited_this_txg,
+		    (longlong_t)
+		    NSEC2MSEC(gethrtime() - scn->scn_sync_start_time),
+		    (longlong_t)tx->tx_txg, err);
+		scn->scn_visited_this_txg = 0;
+
+		/*
+		 * Write out changes to the DDT that may be required as a
+		 * result of the blocks freed.  This ensures that the DDT
+		 * is clean when a scrub/resilver runs.
+		 */
+		ddt_sync(spa, tx->tx_txg);
+	}
+	if (err != 0)
+		return;
+	if (!scn->scn_async_destroying && zfs_free_leak_on_eio &&
+	    (dp->dp_free_dir->dd_phys->dd_used_bytes != 0 ||
+	    dp->dp_free_dir->dd_phys->dd_compressed_bytes != 0 ||
+	    dp->dp_free_dir->dd_phys->dd_uncompressed_bytes != 0)) {
+		/*
+		 * We have finished background destroying, but there is still
+		 * some space left in the dp_free_dir. Transfer this leaked
+		 * space to the dp_leak_dir.
+		 */
+		if (dp->dp_leak_dir == NULL) {
+			rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG);
+			(void) dsl_dir_create_sync(dp, dp->dp_root_dir,
+			    LEAK_DIR_NAME, tx);
+			VERIFY0(dsl_pool_open_special_dir(dp,
+			    LEAK_DIR_NAME, &dp->dp_leak_dir));
+			rrw_exit(&dp->dp_config_rwlock, FTAG);
+		}
+		dsl_dir_diduse_space(dp->dp_leak_dir, DD_USED_HEAD,
+		    dp->dp_free_dir->dd_phys->dd_used_bytes,
+		    dp->dp_free_dir->dd_phys->dd_compressed_bytes,
+		    dp->dp_free_dir->dd_phys->dd_uncompressed_bytes, tx);
+		dsl_dir_diduse_space(dp->dp_free_dir, DD_USED_HEAD,
+		    -dp->dp_free_dir->dd_phys->dd_used_bytes,
+		    -dp->dp_free_dir->dd_phys->dd_compressed_bytes,
+		    -dp->dp_free_dir->dd_phys->dd_uncompressed_bytes, tx);
+	}
+	if (!scn->scn_async_destroying) {
 		/* finished; verify that space accounting went to zero */
 		ASSERT0(dp->dp_free_dir->dd_phys->dd_used_bytes);
 		ASSERT0(dp->dp_free_dir->dd_phys->dd_compressed_bytes);
diff --git a/uts/common/fs/zfs/spa.c b/uts/common/fs/zfs/spa.c
index d4734d21d0f6..f708ce3ef22a 100644
--- a/uts/common/fs/zfs/spa.c
+++ b/uts/common/fs/zfs/spa.c
@@ -241,19 +241,25 @@ spa_prop_get_config(spa_t *spa, nvlist_t **nvp)
 	}
 
 	if (pool != NULL) {
-		dsl_dir_t *freedir = pool->dp_free_dir;
-
 		/*
 		 * The $FREE directory was introduced in SPA_VERSION_DEADLISTS,
 		 * when opening pools before this version freedir will be NULL.
 		 */
-		if (freedir != NULL) {
+		if (pool->dp_free_dir != NULL) {
 			spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING, NULL,
-			    freedir->dd_phys->dd_used_bytes, src);
+			    pool->dp_free_dir->dd_phys->dd_used_bytes, src);
 		} else {
 			spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING,
 			    NULL, 0, src);
 		}
+
+		if (pool->dp_leak_dir != NULL) {
+			spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED, NULL,
+			    pool->dp_leak_dir->dd_phys->dd_used_bytes, src);
+		} else {
+			spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED,
+			    NULL, 0, src);
+		}
 	}
 
 	spa_prop_add_list(*nvp, ZPOOL_PROP_GUID, NULL, spa_guid(spa), src);
diff --git a/uts/common/fs/zfs/spa_misc.c b/uts/common/fs/zfs/spa_misc.c
index 1c5743ac4ed0..ea386591b27f 100644
--- a/uts/common/fs/zfs/spa_misc.c
+++ b/uts/common/fs/zfs/spa_misc.c
@@ -250,7 +250,38 @@ int zfs_flags = 0;
  * This should only be used as a last resort, as it typically results
  * in leaked space, or worse.
  */
-int zfs_recover = 0;
+boolean_t zfs_recover = B_FALSE;
+
+/*
+ * If destroy encounters an EIO while reading metadata (e.g. indirect
+ * blocks), space referenced by the missing metadata can not be freed.
+ * Normally this causes the background destroy to become "stalled", as
+ * it is unable to make forward progress.  While in this stalled state,
+ * all remaining space to free from the error-encountering filesystem is
+ * "temporarily leaked".  Set this flag to cause it to ignore the EIO,
+ * permanently leak the space from indirect blocks that can not be read,
+ * and continue to free everything else that it can.
+ *
+ * The default, "stalling" behavior is useful if the storage partially
+ * fails (i.e. some but not all i/os fail), and then later recovers.  In
+ * this case, we will be able to continue pool operations while it is
+ * partially failed, and when it recovers, we can continue to free the
+ * space, with no leaks.  However, note that this case is actually
+ * fairly rare.
+ *
+ * Typically pools either (a) fail completely (but perhaps temporarily,
+ * e.g. a top-level vdev going offline), or (b) have localized,
+ * permanent errors (e.g. disk returns the wrong data due to bit flip or
+ * firmware bug).  In case (a), this setting does not matter because the
+ * pool will be suspended and the sync thread will not be able to make
+ * forward progress regardless.  In case (b), because the error is
+ * permanent, the best we can do is leak the minimum amount of space,
+ * which is what setting this flag will do.  Therefore, it is reasonable
+ * for this flag to normally be set, but we chose the more conservative
+ * approach of not setting it, so that there is no possibility of
+ * leaking space in the "partial temporary" failure case.
+ */
+boolean_t zfs_free_leak_on_eio = B_FALSE;
 
 /*
  * Expiration time in milliseconds. This value has two meanings. First it is
diff --git a/uts/common/fs/zfs/sys/bptree.h b/uts/common/fs/zfs/sys/bptree.h
index 971507211875..a533cb949021 100644
--- a/uts/common/fs/zfs/sys/bptree.h
+++ b/uts/common/fs/zfs/sys/bptree.h
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright (c) 2012 by Delphix. All rights reserved.
+ * Copyright (c) 2013 by Delphix. All rights reserved.
  */
 
 #ifndef	_SYS_BPTREE_H
@@ -50,6 +50,7 @@ typedef int bptree_itor_t(void *arg, const blkptr_t *bp, dmu_tx_t *tx);
 
 uint64_t bptree_alloc(objset_t *os, dmu_tx_t *tx);
 int bptree_free(objset_t *os, uint64_t obj, dmu_tx_t *tx);
+boolean_t bptree_is_empty(objset_t *os, uint64_t obj);
 
 void bptree_add(objset_t *os, uint64_t obj, blkptr_t *bp, uint64_t birth_txg,
     uint64_t bytes, uint64_t comp, uint64_t uncomp, dmu_tx_t *tx);
diff --git a/uts/common/fs/zfs/sys/dmu.h b/uts/common/fs/zfs/sys/dmu.h
index e9ab6d7534e2..9a6006ce6646 100644
--- a/uts/common/fs/zfs/sys/dmu.h
+++ b/uts/common/fs/zfs/sys/dmu.h
@@ -253,7 +253,6 @@ void zfs_znode_byteswap(void *buf, size_t size);
 
 #define	DMU_USERUSED_OBJECT	(-1ULL)
 #define	DMU_GROUPUSED_OBJECT	(-2ULL)
-#define	DMU_DEADLIST_OBJECT	(-3ULL)
 
 /*
  * artificial blkids for bonus buffer and spill blocks
diff --git a/uts/common/fs/zfs/sys/dsl_dir.h b/uts/common/fs/zfs/sys/dsl_dir.h
index 752ccadb729d..a9c4f67515a9 100644
--- a/uts/common/fs/zfs/sys/dsl_dir.h
+++ b/uts/common/fs/zfs/sys/dsl_dir.h
@@ -160,6 +160,7 @@ boolean_t dsl_dir_is_zapified(dsl_dir_t *dd);
 #define	ORIGIN_DIR_NAME "$ORIGIN"
 #define	XLATION_DIR_NAME "$XLATION"
 #define	FREE_DIR_NAME "$FREE"
+#define	LEAK_DIR_NAME "$LEAK"
 
 #ifdef ZFS_DEBUG
 #define	dprintf_dd(dd, fmt, ...) do { \
diff --git a/uts/common/fs/zfs/sys/dsl_pool.h b/uts/common/fs/zfs/sys/dsl_pool.h
index ea180c9a5b8d..a6fb201047bb 100644
--- a/uts/common/fs/zfs/sys/dsl_pool.h
+++ b/uts/common/fs/zfs/sys/dsl_pool.h
@@ -84,6 +84,7 @@ typedef struct dsl_pool {
 	struct dsl_dir *dp_root_dir;
 	struct dsl_dir *dp_mos_dir;
 	struct dsl_dir *dp_free_dir;
+	struct dsl_dir *dp_leak_dir;
 	struct dsl_dataset *dp_origin_snap;
 	uint64_t dp_root_dir_obj;
 	struct taskq *dp_vnrele_taskq;
diff --git a/uts/common/fs/zfs/sys/dsl_scan.h b/uts/common/fs/zfs/sys/dsl_scan.h
index bf8c5ac824a1..d9f396680792 100644
--- a/uts/common/fs/zfs/sys/dsl_scan.h
+++ b/uts/common/fs/zfs/sys/dsl_scan.h
@@ -114,6 +114,7 @@ typedef struct dsl_scan {
 	/* for freeing blocks */
 	boolean_t scn_is_bptree;
 	boolean_t scn_async_destroying;
+	boolean_t scn_async_stalled;
 
 	/* for debugging / information */
 	uint64_t scn_visited_this_txg;
diff --git a/uts/common/fs/zfs/sys/zfs_debug.h b/uts/common/fs/zfs/sys/zfs_debug.h
index e6a1fa282878..3c8e2177bccc 100644
--- a/uts/common/fs/zfs/sys/zfs_debug.h
+++ b/uts/common/fs/zfs/sys/zfs_debug.h
@@ -47,7 +47,8 @@ extern "C" {
 #endif
 
 extern int zfs_flags;
-extern int zfs_recover;
+extern boolean_t zfs_recover;
+extern boolean_t zfs_free_leak_on_eio;
 
 #define	ZFS_DEBUG_DPRINTF	(1<<0)
 #define	ZFS_DEBUG_DBUF_VERIFY	(1<<1)
diff --git a/uts/common/fs/zfs/zfs_debug.c b/uts/common/fs/zfs/zfs_debug.c
index 26ea561eb1d6..85fa7600d9b9 100644
--- a/uts/common/fs/zfs/zfs_debug.c
+++ b/uts/common/fs/zfs/zfs_debug.c
@@ -28,7 +28,7 @@
 list_t zfs_dbgmsgs;
 int zfs_dbgmsg_size;
 kmutex_t zfs_dbgmsgs_lock;
-int zfs_dbgmsg_maxsize = 1<<20; /* 1MB */
+int zfs_dbgmsg_maxsize = 4<<20; /* 4MB */
 
 void
 zfs_dbgmsg_init(void)
diff --git a/uts/common/fs/zfs/zio.c b/uts/common/fs/zfs/zio.c
index 8fdbbf843231..506897c02d41 100644
--- a/uts/common/fs/zfs/zio.c
+++ b/uts/common/fs/zfs/zio.c
@@ -3208,13 +3208,6 @@ zbookmark_is_before(const dnode_phys_t *dnp, const zbookmark_t *zb1,
 	ASSERT(zb1->zb_objset == zb2->zb_objset);
 	ASSERT(zb2->zb_level == 0);
 
-	/*
-	 * A bookmark in the deadlist is considered to be after
-	 * everything else.
-	 */
-	if (zb2->zb_object == DMU_DEADLIST_OBJECT)
-		return (B_TRUE);
-
 	/* The objset_phys_t isn't before anything. */
 	if (dnp == NULL)
 		return (B_FALSE);
diff --git a/uts/common/sys/fs/zfs.h b/uts/common/sys/fs/zfs.h
index caad33c567fe..01deecdf02da 100644
--- a/uts/common/sys/fs/zfs.h
+++ b/uts/common/sys/fs/zfs.h
@@ -189,6 +189,7 @@ typedef enum {
 	ZPOOL_PROP_COMMENT,
 	ZPOOL_PROP_EXPANDSZ,
 	ZPOOL_PROP_FREEING,
+	ZPOOL_PROP_LEAKED,
 	ZPOOL_NUM_PROPS
 } zpool_prop_t;
 

From bd47f3f79411ba6725c929367575f53e0e7e0c87 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Tue, 17 Jun 2014 07:46:55 +0000
Subject: [PATCH 002/380] 4881 zfs send performance degradation when embedded
 block pointers are encountered

Reviewed by: George Wilson <george.wilson@delphix.com>
Reviewed by: Christopher Siden <christopher.siden@delphix.com>
Approved by: Dan McDonald <danmcd@omniti.com>

illumos/illumos-gate@06315b795c0d54f0228e0b8af497a28752dd92da
---
 uts/common/fs/zfs/dmu_traverse.c | 35 ++++++++++++++++++--------------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/uts/common/fs/zfs/dmu_traverse.c b/uts/common/fs/zfs/dmu_traverse.c
index 1c698b5ee03c..ad2f09ed4a55 100644
--- a/uts/common/fs/zfs/dmu_traverse.c
+++ b/uts/common/fs/zfs/dmu_traverse.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -197,6 +197,16 @@ traverse_prefetch_metadata(traverse_data_t *td,
 	    ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL, &flags, zb);
 }
 
+static boolean_t
+prefetch_needed(prefetch_data_t *pfd, const blkptr_t *bp)
+{
+	ASSERT(pfd->pd_flags & TRAVERSE_PREFETCH_DATA);
+	if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp) ||
+	    BP_GET_TYPE(bp) == DMU_OT_INTENT_LOG)
+		return (B_FALSE);
+	return (B_TRUE);
+}
+
 static int
 traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
     const blkptr_t *bp, const zbookmark_t *zb)
@@ -242,16 +252,7 @@ traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
 		return (0);
 	}
 
-	if (BP_IS_HOLE(bp)) {
-		err = td->td_func(td->td_spa, NULL, bp, zb, dnp, td->td_arg);
-		if (err != 0)
-			goto post;
-		return (0);
-	}
-
-	if (pd && !pd->pd_exited &&
-	    ((pd->pd_flags & TRAVERSE_PREFETCH_DATA) ||
-	    BP_GET_TYPE(bp) == DMU_OT_DNODE || BP_GET_LEVEL(bp) > 0)) {
+	if (pd != NULL && !pd->pd_exited && prefetch_needed(pd, bp)) {
 		mutex_enter(&pd->pd_mtx);
 		ASSERT(pd->pd_blks_fetched >= 0);
 		while (pd->pd_blks_fetched == 0 && !pd->pd_exited)
@@ -261,6 +262,13 @@ traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
 		mutex_exit(&pd->pd_mtx);
 	}
 
+	if (BP_IS_HOLE(bp)) {
+		err = td->td_func(td->td_spa, NULL, bp, zb, dnp, td->td_arg);
+		if (err != 0)
+			goto post;
+		return (0);
+	}
+
 	if (td->td_flags & TRAVERSE_PRE) {
 		err = td->td_func(td->td_spa, NULL, bp, zb, dnp,
 		    td->td_arg);
@@ -444,10 +452,7 @@ traverse_prefetcher(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 	if (pfd->pd_cancel)
 		return (SET_ERROR(EINTR));
 
-	if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp) ||
-	    !((pfd->pd_flags & TRAVERSE_PREFETCH_DATA) ||
-	    BP_GET_TYPE(bp) == DMU_OT_DNODE || BP_GET_LEVEL(bp) > 0) ||
-	    BP_GET_TYPE(bp) == DMU_OT_INTENT_LOG)
+	if (!prefetch_needed(pfd, bp))
 		return (0);
 
 	mutex_enter(&pfd->pd_mtx);

From 93a52c22ac7c8de75dd8911a7902e35222c1dd36 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Tue, 17 Jun 2014 07:58:53 +0000
Subject: [PATCH 003/380] 4897 Space accounting mismatch in L2ARC/zpool

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Boris Protopopov <bprotopopov@hotmail.com>
Approved by: Dan McDonald <danmcd@omniti.com>

illumos/illumos-dist@3038a2b421b40dc5ac11cd88423696618584f85a
---
 uts/common/fs/zfs/arc.c | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index 256ec3b34cfb..811a2d1b3930 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -23,7 +23,7 @@
  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
  * Copyright (c) 2013 by Delphix. All rights reserved.
  * Copyright (c) 2014 by Saso Kiselkov. All rights reserved.
- * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
  */
 
 /*
@@ -1592,6 +1592,8 @@ arc_hdr_destroy(arc_buf_hdr_t *hdr)
 			list_remove(l2hdr->b_dev->l2ad_buflist, hdr);
 			ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
 			ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
+			vdev_space_update(l2hdr->b_dev->l2ad_vdev,
+			    -l2hdr->b_asize, 0, 0);
 			kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t));
 			if (hdr->b_state == arc_l2c_only)
 				l2arc_hdr_stat_remove();
@@ -3445,6 +3447,8 @@ arc_release(arc_buf_t *buf, void *tag)
 
 	if (l2hdr) {
 		ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
+		vdev_space_update(l2hdr->b_dev->l2ad_vdev,
+		    -l2hdr->b_asize, 0, 0);
 		kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t));
 		ARCSTAT_INCR(arcstat_l2_size, -buf_size);
 		mutex_exit(&l2arc_buflist_mtx);
@@ -4241,6 +4245,7 @@ l2arc_write_done(zio_t *zio)
 	arc_buf_hdr_t *head, *ab, *ab_prev;
 	l2arc_buf_hdr_t *abl2;
 	kmutex_t *hash_lock;
+	int64_t bytes_dropped = 0;
 
 	cb = zio->io_private;
 	ASSERT(cb != NULL);
@@ -4288,6 +4293,7 @@ l2arc_write_done(zio_t *zio)
 			 */
 			list_remove(buflist, ab);
 			ARCSTAT_INCR(arcstat_l2_asize, -abl2->b_asize);
+			bytes_dropped += abl2->b_asize;
 			ab->b_l2hdr = NULL;
 			kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
 			ARCSTAT_INCR(arcstat_l2_size, -ab->b_size);
@@ -4306,6 +4312,8 @@ l2arc_write_done(zio_t *zio)
 	kmem_cache_free(hdr_cache, head);
 	mutex_exit(&l2arc_buflist_mtx);
 
+	vdev_space_update(dev->l2ad_vdev, -bytes_dropped, 0, 0);
+
 	l2arc_do_free_on_write();
 
 	kmem_free(cb, sizeof (l2arc_write_callback_t));
@@ -4444,6 +4452,7 @@ l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
 	arc_buf_hdr_t *ab, *ab_prev;
 	kmutex_t *hash_lock;
 	uint64_t taddr;
+	int64_t bytes_evicted = 0;
 
 	buflist = dev->l2ad_buflist;
 
@@ -4542,6 +4551,7 @@ top:
 			if (ab->b_l2hdr != NULL) {
 				abl2 = ab->b_l2hdr;
 				ARCSTAT_INCR(arcstat_l2_asize, -abl2->b_asize);
+				bytes_evicted += abl2->b_asize;
 				ab->b_l2hdr = NULL;
 				kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
 				ARCSTAT_INCR(arcstat_l2_size, -ab->b_size);
@@ -4558,7 +4568,7 @@ top:
 	}
 	mutex_exit(&l2arc_buflist_mtx);
 
-	vdev_space_update(dev->l2ad_vdev, -(taddr - dev->l2ad_evict), 0, 0);
+	vdev_space_update(dev->l2ad_vdev, -bytes_evicted, 0, 0);
 	dev->l2ad_evict = taddr;
 }
 
@@ -4801,15 +4811,13 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 	ARCSTAT_INCR(arcstat_l2_write_bytes, write_asize);
 	ARCSTAT_INCR(arcstat_l2_size, write_sz);
 	ARCSTAT_INCR(arcstat_l2_asize, write_asize);
-	vdev_space_update(dev->l2ad_vdev, write_psize, 0, 0);
+	vdev_space_update(dev->l2ad_vdev, write_asize, 0, 0);
 
 	/*
 	 * Bump device hand to the device start if it is approaching the end.
 	 * l2arc_evict() will already have evicted ahead for this case.
 	 */
 	if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) {
-		vdev_space_update(dev->l2ad_vdev,
-		    dev->l2ad_end - dev->l2ad_hand, 0, 0);
 		dev->l2ad_hand = dev->l2ad_start;
 		dev->l2ad_evict = dev->l2ad_start;
 		dev->l2ad_first = B_FALSE;

From 929353c3a13aefafe6edb149e6e56421915d911a Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Tue, 17 Jun 2014 08:02:50 +0000
Subject: [PATCH 004/380] 4756 metaslab_group_preload() could deadlock

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Christopher Siden <christopher.siden@delphix.com>
Reviewed by: Dan McDonald <danmcd@omniti.com>
Reviewed by: Saso Kiselkov <saso.kiselkov@nexenta.com>
Approved by: Garrett D'Amore <garrett@damore.org>

illumos/illumos-gate@30beaff42d8240ebf5386e8b7a14e3d137a1631f
---
 uts/common/fs/zfs/metaslab.c | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/uts/common/fs/zfs/metaslab.c b/uts/common/fs/zfs/metaslab.c
index 4c854b710589..bba1dbf5148c 100644
--- a/uts/common/fs/zfs/metaslab.c
+++ b/uts/common/fs/zfs/metaslab.c
@@ -1207,6 +1207,8 @@ metaslab_preload(void *arg)
 	metaslab_t *msp = arg;
 	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
 
+	ASSERT(!MUTEX_HELD(&msp->ms_group->mg_lock));
+
 	mutex_enter(&msp->ms_lock);
 	metaslab_load_wait(msp);
 	if (!msp->ms_loaded)
@@ -1231,19 +1233,36 @@ metaslab_group_preload(metaslab_group_t *mg)
 		taskq_wait(mg->mg_taskq);
 		return;
 	}
-	mutex_enter(&mg->mg_lock);
 
+	mutex_enter(&mg->mg_lock);
 	/*
-	 * Prefetch the next potential metaslabs
+	 * Load the next potential metaslabs
 	 */
-	for (msp = avl_first(t); msp != NULL; msp = AVL_NEXT(t, msp)) {
+	msp = avl_first(t);
+	while (msp != NULL) {
+		metaslab_t *msp_next = AVL_NEXT(t, msp);
 
 		/* If we have reached our preload limit then we're done */
 		if (++m > metaslab_preload_limit)
 			break;
 
+		/*
+		 * We must drop the metaslab group lock here to preserve
+		 * lock ordering with the ms_lock (when grabbing both
+		 * the mg_lock and the ms_lock, the ms_lock must be taken
+		 * first).  As a result, it is possible that the ordering
+		 * of the metaslabs within the avl tree may change before
+		 * we reacquire the lock. The metaslab cannot be removed from
+		 * the tree while we're in syncing context so it is safe to
+		 * drop the mg_lock here. If the metaslabs are reordered
+		 * nothing will break -- we just may end up loading a
+		 * less than optimal one.
+		 */
+		mutex_exit(&mg->mg_lock);
 		VERIFY(taskq_dispatch(mg->mg_taskq, metaslab_preload,
 		    msp, TQ_SLEEP) != NULL);
+		mutex_enter(&mg->mg_lock);
+		msp = msp_next;
 	}
 	mutex_exit(&mg->mg_lock);
 }

From cfd11dccb95eac86d4ef3516628a99ee4fde89ab Mon Sep 17 00:00:00 2001
From: pfg <pfg@FreeBSD.org>
Date: Thu, 26 Jun 2014 19:45:35 +0000
Subject: [PATCH 005/380] Bring a new header from OpenSolaris/Illumos.

This is required for r266987 and probably other DTrace probes.
---
 common/util/strtolctype.h | 79 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 79 insertions(+)
 create mode 100644 common/util/strtolctype.h

diff --git a/common/util/strtolctype.h b/common/util/strtolctype.h
new file mode 100644
index 000000000000..5675e42be7a0
--- /dev/null
+++ b/common/util/strtolctype.h
@@ -0,0 +1,79 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*	Copyright (c) 1988 AT&T	*/
+/*	  All Rights Reserved  	*/
+
+#ifndef	_COMMON_UTIL_CTYPE_H
+#define	_COMMON_UTIL_CTYPE_H
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+/*
+ * This header file contains a collection of macros that the strtou?ll?
+ * functions in common/util use to test characters.  What we need is a kernel
+ * version of ctype.h.
+ *
+ * NOTE: These macros are used within several DTrace probe context functions.
+ * They must not be altered to make function calls or perform actions not
+ * safe in probe context.
+ */
+
+#if	defined(_KERNEL) && !defined(_BOOT)
+
+#define	isalnum(ch)	(isalpha(ch) || isdigit(ch))
+#define	isalpha(ch)	(isupper(ch) || islower(ch))
+#define	isdigit(ch)	((ch) >= '0' && (ch) <= '9')
+#define	islower(ch)	((ch) >= 'a' && (ch) <= 'z')
+#define	isspace(ch)	(((ch) == ' ') || ((ch) == '\r') || ((ch) == '\n') || \
+			((ch) == '\t') || ((ch) == '\f'))
+#define	isupper(ch)	((ch) >= 'A' && (ch) <= 'Z')
+#define	isxdigit(ch)	(isdigit(ch) || ((ch) >= 'a' && (ch) <= 'f') || \
+			((ch) >= 'A' && (ch) <= 'F'))
+
+#endif	/* _KERNEL && !_BOOT */
+
+#define	DIGIT(x)	\
+	(isdigit(x) ? (x) - '0' : islower(x) ? (x) + 10 - 'a' : (x) + 10 - 'A')
+
+#define	MBASE	('z' - 'a' + 1 + 10)
+
+/*
+ * The following macro is a version of isalnum() that limits alphabetic
+ * characters to the ranges a-z and A-Z; locale dependent characters will not
+ * return 1.  The members of a-z and A-Z are assumed to be in ascending order
+ * and contiguous.
+ */
+#define	lisalnum(x)	\
+	(isdigit(x) || ((x) >= 'a' && (x) <= 'z') || ((x) >= 'A' && (x) <= 'Z'))
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _COMMON_UTIL_CTYPE_H */

From b28d56edcac8773b148fa4452514d5532081ff01 Mon Sep 17 00:00:00 2001
From: pfg <pfg@FreeBSD.org>
Date: Thu, 26 Jun 2014 20:02:29 +0000
Subject: [PATCH 006/380] Revert r267930.

File looks better in vendor/ and was brought there in r267931.
---
 common/util/strtolctype.h | 79 ---------------------------------------
 1 file changed, 79 deletions(-)
 delete mode 100644 common/util/strtolctype.h

diff --git a/common/util/strtolctype.h b/common/util/strtolctype.h
deleted file mode 100644
index 5675e42be7a0..000000000000
--- a/common/util/strtolctype.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-
-/*
- * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
- */
-
-/*	Copyright (c) 1988 AT&T	*/
-/*	  All Rights Reserved  	*/
-
-#ifndef	_COMMON_UTIL_CTYPE_H
-#define	_COMMON_UTIL_CTYPE_H
-
-#ifdef	__cplusplus
-extern "C" {
-#endif
-
-/*
- * This header file contains a collection of macros that the strtou?ll?
- * functions in common/util use to test characters.  What we need is a kernel
- * version of ctype.h.
- *
- * NOTE: These macros are used within several DTrace probe context functions.
- * They must not be altered to make function calls or perform actions not
- * safe in probe context.
- */
-
-#if	defined(_KERNEL) && !defined(_BOOT)
-
-#define	isalnum(ch)	(isalpha(ch) || isdigit(ch))
-#define	isalpha(ch)	(isupper(ch) || islower(ch))
-#define	isdigit(ch)	((ch) >= '0' && (ch) <= '9')
-#define	islower(ch)	((ch) >= 'a' && (ch) <= 'z')
-#define	isspace(ch)	(((ch) == ' ') || ((ch) == '\r') || ((ch) == '\n') || \
-			((ch) == '\t') || ((ch) == '\f'))
-#define	isupper(ch)	((ch) >= 'A' && (ch) <= 'Z')
-#define	isxdigit(ch)	(isdigit(ch) || ((ch) >= 'a' && (ch) <= 'f') || \
-			((ch) >= 'A' && (ch) <= 'F'))
-
-#endif	/* _KERNEL && !_BOOT */
-
-#define	DIGIT(x)	\
-	(isdigit(x) ? (x) - '0' : islower(x) ? (x) + 10 - 'a' : (x) + 10 - 'A')
-
-#define	MBASE	('z' - 'a' + 1 + 10)
-
-/*
- * The following macro is a version of isalnum() that limits alphabetic
- * characters to the ranges a-z and A-Z; locale dependent characters will not
- * return 1.  The members of a-z and A-Z are assumed to be in ascending order
- * and contiguous.
- */
-#define	lisalnum(x)	\
-	(isdigit(x) || ((x) >= 'a' && (x) <= 'z') || ((x) >= 'A' && (x) <= 'Z'))
-
-#ifdef	__cplusplus
-}
-#endif
-
-#endif	/* _COMMON_UTIL_CTYPE_H */

From 229d137acfb5c3e2d13ab8a13fee0663c0ae78b9 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Tue, 1 Jul 2014 21:14:35 +0000
Subject: [PATCH 007/380] 4914 zfs on-disk bookmark structure should be named
 *_phys_t Reviewed by: George Wilson <george.wilson@delphix.com> Reviewed by:
 Christopher Siden <christopher.siden@delphix.com> Reviewed by: Richard Lowe
 <richlowe@richlowe.net> Reviewed by: Saso Kiselkov <skiselkov.ml@gmail.com>
 Approved by: Robert Mustacchi <rm@joyent.com>

illumos/illumos-gate@7802d7bf98dec568dadf72286893b1fe5abd8602
---
 uts/common/fs/zfs/arc.c              | 10 ++++----
 uts/common/fs/zfs/bptree.c           |  4 ++--
 uts/common/fs/zfs/dbuf.c             |  6 ++---
 uts/common/fs/zfs/dmu.c              |  6 ++---
 uts/common/fs/zfs/dmu_diff.c         |  4 ++--
 uts/common/fs/zfs/dmu_objset.c       |  4 ++--
 uts/common/fs/zfs/dmu_send.c         |  4 ++--
 uts/common/fs/zfs/dmu_traverse.c     | 28 +++++++++++-----------
 uts/common/fs/zfs/dsl_destroy.c      |  4 ++--
 uts/common/fs/zfs/dsl_scan.c         | 35 ++++++++++++++--------------
 uts/common/fs/zfs/spa.c              |  4 ++--
 uts/common/fs/zfs/spa_errlog.c       | 20 ++++++++--------
 uts/common/fs/zfs/sys/arc.h          |  6 ++---
 uts/common/fs/zfs/sys/bptree.h       |  4 ++--
 uts/common/fs/zfs/sys/dmu.h          |  2 +-
 uts/common/fs/zfs/sys/dmu_traverse.h |  6 ++---
 uts/common/fs/zfs/sys/dsl_scan.h     |  4 ++--
 uts/common/fs/zfs/sys/spa.h          |  4 ++--
 uts/common/fs/zfs/sys/spa_impl.h     |  8 +++----
 uts/common/fs/zfs/sys/zio.h          | 24 +++++++++----------
 uts/common/fs/zfs/zil.c              |  8 +++----
 uts/common/fs/zfs/zio.c              | 12 +++++-----
 uts/common/fs/zfs/zio_inject.c       |  4 ++--
 uts/common/fs/zfs/zvol.c             |  4 ++--
 24 files changed, 108 insertions(+), 107 deletions(-)

diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index 811a2d1b3930..752cacdb126c 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -21,7 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2014 by Saso Kiselkov. All rights reserved.
  * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
  */
@@ -664,7 +664,7 @@ typedef struct l2arc_read_callback {
 	arc_buf_t		*l2rcb_buf;		/* read buffer */
 	spa_t			*l2rcb_spa;		/* spa */
 	blkptr_t		l2rcb_bp;		/* original blkptr */
-	zbookmark_t		l2rcb_zb;		/* original bookmark */
+	zbookmark_phys_t	l2rcb_zb;		/* original bookmark */
 	int			l2rcb_flags;		/* original flags */
 	enum zio_compress	l2rcb_compress;		/* applied compress */
 } l2arc_read_callback_t;
@@ -2899,7 +2899,7 @@ arc_read_done(zio_t *zio)
 int
 arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_done_func_t *done,
     void *private, zio_priority_t priority, int zio_flags, uint32_t *arc_flags,
-    const zbookmark_t *zb)
+    const zbookmark_phys_t *zb)
 {
 	arc_buf_hdr_t *hdr = NULL;
 	arc_buf_t *buf = NULL;
@@ -3093,7 +3093,7 @@ top:
 		 */
 		ASSERT3U(hdr->b_size, ==, size);
 		DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp,
-		    uint64_t, size, zbookmark_t *, zb);
+		    uint64_t, size, zbookmark_phys_t *, zb);
 		ARCSTAT_BUMP(arcstat_misses);
 		ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
 		    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
@@ -3616,7 +3616,7 @@ arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
     blkptr_t *bp, arc_buf_t *buf, boolean_t l2arc, boolean_t l2arc_compress,
     const zio_prop_t *zp, arc_done_func_t *ready, arc_done_func_t *physdone,
     arc_done_func_t *done, void *private, zio_priority_t priority,
-    int zio_flags, const zbookmark_t *zb)
+    int zio_flags, const zbookmark_phys_t *zb)
 {
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 	arc_write_callback_t *callback;
diff --git a/uts/common/fs/zfs/bptree.c b/uts/common/fs/zfs/bptree.c
index 77067d24e98a..c724ed074103 100644
--- a/uts/common/fs/zfs/bptree.c
+++ b/uts/common/fs/zfs/bptree.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  */
 
 #include <sys/arc.h>
@@ -149,7 +149,7 @@ bptree_add(objset_t *os, uint64_t obj, blkptr_t *bp, uint64_t birth_txg,
 /* ARGSUSED */
 static int
 bptree_visit_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
-    const zbookmark_t *zb, const dnode_phys_t *dnp, void *arg)
+    const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg)
 {
 	int err;
 	struct bptree_args *ba = arg;
diff --git a/uts/common/fs/zfs/dbuf.c b/uts/common/fs/zfs/dbuf.c
index a7e12d5bcdf8..b7567d77df02 100644
--- a/uts/common/fs/zfs/dbuf.c
+++ b/uts/common/fs/zfs/dbuf.c
@@ -521,7 +521,7 @@ static void
 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags)
 {
 	dnode_t *dn;
-	zbookmark_t zb;
+	zbookmark_phys_t zb;
 	uint32_t aflags = ARC_NOWAIT;
 
 	DB_DNODE_ENTER(db);
@@ -1856,7 +1856,7 @@ dbuf_prefetch(dnode_t *dn, uint64_t blkid, zio_priority_t prio)
 		if (bp && !BP_IS_HOLE(bp) && !BP_IS_EMBEDDED(bp)) {
 			dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;
 			uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH;
-			zbookmark_t zb;
+			zbookmark_phys_t zb;
 
 			SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET,
 			    dn->dn_object, 0, blkid);
@@ -2721,7 +2721,7 @@ dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx)
 	objset_t *os;
 	dmu_buf_impl_t *parent = db->db_parent;
 	uint64_t txg = tx->tx_txg;
-	zbookmark_t zb;
+	zbookmark_phys_t zb;
 	zio_prop_t zp;
 	zio_t *zio;
 	int wp_flag = 0;
diff --git a/uts/common/fs/zfs/dmu.c b/uts/common/fs/zfs/dmu.c
index 4f621a2297af..1a62ddb48b15 100644
--- a/uts/common/fs/zfs/dmu.c
+++ b/uts/common/fs/zfs/dmu.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  */
 /* Copyright (c) 2013 by Saso Kiselkov. All rights reserved. */
 /* Copyright (c) 2013, Joyent, Inc. All rights reserved. */
@@ -1379,7 +1379,7 @@ dmu_sync_late_arrival_done(zio_t *zio)
 
 static int
 dmu_sync_late_arrival(zio_t *pio, objset_t *os, dmu_sync_cb_t *done, zgd_t *zgd,
-    zio_prop_t *zp, zbookmark_t *zb)
+    zio_prop_t *zp, zbookmark_phys_t *zb)
 {
 	dmu_sync_arg_t *dsa;
 	dmu_tx_t *tx;
@@ -1440,7 +1440,7 @@ dmu_sync(zio_t *pio, uint64_t txg, dmu_sync_cb_t *done, zgd_t *zgd)
 	dsl_dataset_t *ds = os->os_dsl_dataset;
 	dbuf_dirty_record_t *dr;
 	dmu_sync_arg_t *dsa;
-	zbookmark_t zb;
+	zbookmark_phys_t zb;
 	zio_prop_t zp;
 	dnode_t *dn;
 
diff --git a/uts/common/fs/zfs/dmu_diff.c b/uts/common/fs/zfs/dmu_diff.c
index a2130b1319b5..30fabbb07957 100644
--- a/uts/common/fs/zfs/dmu_diff.c
+++ b/uts/common/fs/zfs/dmu_diff.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #include <sys/dmu.h>
@@ -107,7 +107,7 @@ report_dnode(struct diffarg *da, uint64_t object, dnode_phys_t *dnp)
 /* ARGSUSED */
 static int
 diff_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
-    const zbookmark_t *zb, const dnode_phys_t *dnp, void *arg)
+    const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg)
 {
 	struct diffarg *da = arg;
 	int err = 0;
diff --git a/uts/common/fs/zfs/dmu_objset.c b/uts/common/fs/zfs/dmu_objset.c
index efed341d6f7f..9cfe42a32382 100644
--- a/uts/common/fs/zfs/dmu_objset.c
+++ b/uts/common/fs/zfs/dmu_objset.c
@@ -286,7 +286,7 @@ dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
 	os->os_rootbp = bp;
 	if (!BP_IS_HOLE(os->os_rootbp)) {
 		uint32_t aflags = ARC_WAIT;
-		zbookmark_t zb;
+		zbookmark_phys_t zb;
 		SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET,
 		    ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID);
 
@@ -1023,7 +1023,7 @@ void
 dmu_objset_sync(objset_t *os, zio_t *pio, dmu_tx_t *tx)
 {
 	int txgoff;
-	zbookmark_t zb;
+	zbookmark_phys_t zb;
 	zio_prop_t zp;
 	zio_t *zio;
 	list_t *list;
diff --git a/uts/common/fs/zfs/dmu_send.c b/uts/common/fs/zfs/dmu_send.c
index 5da3f8b0e75d..20d196a83303 100644
--- a/uts/common/fs/zfs/dmu_send.c
+++ b/uts/common/fs/zfs/dmu_send.c
@@ -21,7 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2014, Joyent, Inc. All rights reserved.
  */
 
@@ -420,7 +420,7 @@ backup_do_embed(dmu_sendarg_t *dsp, const blkptr_t *bp)
 /* ARGSUSED */
 static int
 backup_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
-    const zbookmark_t *zb, const dnode_phys_t *dnp, void *arg)
+    const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg)
 {
 	dmu_sendarg_t *dsp = arg;
 	dmu_object_type_t type = bp ? BP_GET_TYPE(bp) : DMU_OT_NONE;
diff --git a/uts/common/fs/zfs/dmu_traverse.c b/uts/common/fs/zfs/dmu_traverse.c
index ad2f09ed4a55..fb1ec0755cf1 100644
--- a/uts/common/fs/zfs/dmu_traverse.c
+++ b/uts/common/fs/zfs/dmu_traverse.c
@@ -55,7 +55,7 @@ typedef struct traverse_data {
 	uint64_t td_objset;
 	blkptr_t *td_rootbp;
 	uint64_t td_min_txg;
-	zbookmark_t *td_resume;
+	zbookmark_phys_t *td_resume;
 	int td_flags;
 	prefetch_data_t *td_pfd;
 	boolean_t td_paused;
@@ -72,7 +72,7 @@ static int
 traverse_zil_block(zilog_t *zilog, blkptr_t *bp, void *arg, uint64_t claim_txg)
 {
 	traverse_data_t *td = arg;
-	zbookmark_t zb;
+	zbookmark_phys_t zb;
 
 	if (BP_IS_HOLE(bp))
 		return (0);
@@ -96,7 +96,7 @@ traverse_zil_record(zilog_t *zilog, lr_t *lrc, void *arg, uint64_t claim_txg)
 	if (lrc->lrc_txtype == TX_WRITE) {
 		lr_write_t *lr = (lr_write_t *)lrc;
 		blkptr_t *bp = &lr->lr_blkptr;
-		zbookmark_t zb;
+		zbookmark_phys_t zb;
 
 		if (BP_IS_HOLE(bp))
 			return (0);
@@ -150,7 +150,7 @@ typedef enum resume_skip {
  */
 static resume_skip_t
 resume_skip_check(traverse_data_t *td, const dnode_phys_t *dnp,
-    const zbookmark_t *zb)
+    const zbookmark_phys_t *zb)
 {
 	if (td->td_resume != NULL && !ZB_IS_ZERO(td->td_resume)) {
 		/*
@@ -175,7 +175,7 @@ resume_skip_check(traverse_data_t *td, const dnode_phys_t *dnp,
 
 static void
 traverse_prefetch_metadata(traverse_data_t *td,
-    const blkptr_t *bp, const zbookmark_t *zb)
+    const blkptr_t *bp, const zbookmark_phys_t *zb)
 {
 	uint32_t flags = ARC_NOWAIT | ARC_PREFETCH;
 
@@ -209,9 +209,9 @@ prefetch_needed(prefetch_data_t *pfd, const blkptr_t *bp)
 
 static int
 traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
-    const blkptr_t *bp, const zbookmark_t *zb)
+    const blkptr_t *bp, const zbookmark_phys_t *zb)
 {
-	zbookmark_t czb;
+	zbookmark_phys_t czb;
 	int err = 0;
 	arc_buf_t *buf = NULL;
 	prefetch_data_t *pd = td->td_pfd;
@@ -406,7 +406,7 @@ prefetch_dnode_metadata(traverse_data_t *td, const dnode_phys_t *dnp,
     uint64_t objset, uint64_t object)
 {
 	int j;
-	zbookmark_t czb;
+	zbookmark_phys_t czb;
 
 	for (j = 0; j < dnp->dn_nblkptr; j++) {
 		SET_BOOKMARK(&czb, objset, object, dnp->dn_nlevels - 1, j);
@@ -424,7 +424,7 @@ traverse_dnode(traverse_data_t *td, const dnode_phys_t *dnp,
     uint64_t objset, uint64_t object)
 {
 	int j, err = 0;
-	zbookmark_t czb;
+	zbookmark_phys_t czb;
 
 	for (j = 0; j < dnp->dn_nblkptr; j++) {
 		SET_BOOKMARK(&czb, objset, object, dnp->dn_nlevels - 1, j);
@@ -443,7 +443,7 @@ traverse_dnode(traverse_data_t *td, const dnode_phys_t *dnp,
 /* ARGSUSED */
 static int
 traverse_prefetcher(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
-    const zbookmark_t *zb, const dnode_phys_t *dnp, void *arg)
+    const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg)
 {
 	prefetch_data_t *pfd = arg;
 	uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH;
@@ -473,7 +473,7 @@ traverse_prefetch_thread(void *arg)
 {
 	traverse_data_t *td_main = arg;
 	traverse_data_t td = *td_main;
-	zbookmark_t czb;
+	zbookmark_phys_t czb;
 
 	td.td_func = traverse_prefetcher;
 	td.td_arg = td_main->td_pfd;
@@ -495,12 +495,12 @@ traverse_prefetch_thread(void *arg)
  */
 static int
 traverse_impl(spa_t *spa, dsl_dataset_t *ds, uint64_t objset, blkptr_t *rootbp,
-    uint64_t txg_start, zbookmark_t *resume, int flags,
+    uint64_t txg_start, zbookmark_phys_t *resume, int flags,
     blkptr_cb_t func, void *arg)
 {
 	traverse_data_t td;
 	prefetch_data_t pd = { 0 };
-	zbookmark_t czb;
+	zbookmark_phys_t czb;
 	int err;
 
 	ASSERT(ds == NULL || objset == ds->ds_object);
@@ -581,7 +581,7 @@ traverse_dataset(dsl_dataset_t *ds, uint64_t txg_start, int flags,
 
 int
 traverse_dataset_destroyed(spa_t *spa, blkptr_t *blkptr,
-    uint64_t txg_start, zbookmark_t *resume, int flags,
+    uint64_t txg_start, zbookmark_phys_t *resume, int flags,
     blkptr_cb_t func, void *arg)
 {
 	return (traverse_impl(spa, NULL, ZB_DESTROYED_OBJSET,
diff --git a/uts/common/fs/zfs/dsl_destroy.c b/uts/common/fs/zfs/dsl_destroy.c
index 441036c25afa..0813f3288dde 100644
--- a/uts/common/fs/zfs/dsl_destroy.c
+++ b/uts/common/fs/zfs/dsl_destroy.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
  * Copyright (c) 2013 by Joyent, Inc. All rights reserved.
  */
@@ -534,7 +534,7 @@ struct killarg {
 /* ARGSUSED */
 static int
 kill_blkptr(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
-    const zbookmark_t *zb, const dnode_phys_t *dnp, void *arg)
+    const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg)
 {
 	struct killarg *ka = arg;
 	dmu_tx_t *tx = ka->tx;
diff --git a/uts/common/fs/zfs/dsl_scan.c b/uts/common/fs/zfs/dsl_scan.c
index 860f9dfb4e86..1f705580d0bb 100644
--- a/uts/common/fs/zfs/dsl_scan.c
+++ b/uts/common/fs/zfs/dsl_scan.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  */
 
 #include <sys/dsl_scan.h>
@@ -50,7 +50,8 @@
 #include <sys/zfs_vfsops.h>
 #endif
 
-typedef int (scan_cb_t)(dsl_pool_t *, const blkptr_t *, const zbookmark_t *);
+typedef int (scan_cb_t)(dsl_pool_t *, const blkptr_t *,
+    const zbookmark_phys_t *);
 
 static scan_cb_t dsl_scan_scrub_cb;
 static void dsl_scan_cancel_sync(void *, dmu_tx_t *);
@@ -349,7 +350,7 @@ dsl_scan_cancel(dsl_pool_t *dp)
 }
 
 static void dsl_scan_visitbp(blkptr_t *bp,
-    const zbookmark_t *zb, dnode_phys_t *dnp, arc_buf_t *pbuf,
+    const zbookmark_phys_t *zb, dnode_phys_t *dnp, arc_buf_t *pbuf,
     dsl_dataset_t *ds, dsl_scan_t *scn, dmu_objset_type_t ostype,
     dmu_tx_t *tx);
 static void dsl_scan_visitdnode(dsl_scan_t *, dsl_dataset_t *ds,
@@ -388,7 +389,7 @@ dsl_scan_sync_state(dsl_scan_t *scn, dmu_tx_t *tx)
 }
 
 static boolean_t
-dsl_scan_check_pause(dsl_scan_t *scn, const zbookmark_t *zb)
+dsl_scan_check_pause(dsl_scan_t *scn, const zbookmark_phys_t *zb)
 {
 	uint64_t elapsed_nanosecs;
 	int mintime;
@@ -446,7 +447,7 @@ dsl_scan_zil_block(zilog_t *zilog, blkptr_t *bp, void *arg, uint64_t claim_txg)
 	dsl_pool_t *dp = zsa->zsa_dp;
 	dsl_scan_t *scn = dp->dp_scan;
 	zil_header_t *zh = zsa->zsa_zh;
-	zbookmark_t zb;
+	zbookmark_phys_t zb;
 
 	if (BP_IS_HOLE(bp) || bp->blk_birth <= scn->scn_phys.scn_cur_min_txg)
 		return (0);
@@ -478,7 +479,7 @@ dsl_scan_zil_record(zilog_t *zilog, lr_t *lrc, void *arg, uint64_t claim_txg)
 		zil_header_t *zh = zsa->zsa_zh;
 		lr_write_t *lr = (lr_write_t *)lrc;
 		blkptr_t *bp = &lr->lr_blkptr;
-		zbookmark_t zb;
+		zbookmark_phys_t zb;
 
 		if (BP_IS_HOLE(bp) ||
 		    bp->blk_birth <= scn->scn_phys.scn_cur_min_txg)
@@ -528,7 +529,7 @@ static void
 dsl_scan_prefetch(dsl_scan_t *scn, arc_buf_t *buf, blkptr_t *bp,
     uint64_t objset, uint64_t object, uint64_t blkid)
 {
-	zbookmark_t czb;
+	zbookmark_phys_t czb;
 	uint32_t flags = ARC_NOWAIT | ARC_PREFETCH;
 
 	if (zfs_no_scrub_prefetch)
@@ -547,7 +548,7 @@ dsl_scan_prefetch(dsl_scan_t *scn, arc_buf_t *buf, blkptr_t *bp,
 
 static boolean_t
 dsl_scan_check_resume(dsl_scan_t *scn, const dnode_phys_t *dnp,
-    const zbookmark_t *zb)
+    const zbookmark_phys_t *zb)
 {
 	/*
 	 * We never skip over user/group accounting objects (obj<0)
@@ -587,7 +588,7 @@ dsl_scan_check_resume(dsl_scan_t *scn, const dnode_phys_t *dnp,
 static int
 dsl_scan_recurse(dsl_scan_t *scn, dsl_dataset_t *ds, dmu_objset_type_t ostype,
     dnode_phys_t *dnp, const blkptr_t *bp,
-    const zbookmark_t *zb, dmu_tx_t *tx, arc_buf_t **bufp)
+    const zbookmark_phys_t *zb, dmu_tx_t *tx, arc_buf_t **bufp)
 {
 	dsl_pool_t *dp = scn->scn_dp;
 	int zio_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_SCAN_THREAD;
@@ -610,7 +611,7 @@ dsl_scan_recurse(dsl_scan_t *scn, dsl_dataset_t *ds, dmu_objset_type_t ostype,
 			    zb->zb_object, zb->zb_blkid * epb + i);
 		}
 		for (i = 0, cbp = (*bufp)->b_data; i < epb; i++, cbp++) {
-			zbookmark_t czb;
+			zbookmark_phys_t czb;
 
 			SET_BOOKMARK(&czb, zb->zb_objset, zb->zb_object,
 			    zb->zb_level - 1,
@@ -694,7 +695,7 @@ dsl_scan_visitdnode(dsl_scan_t *scn, dsl_dataset_t *ds,
 	int j;
 
 	for (j = 0; j < dnp->dn_nblkptr; j++) {
-		zbookmark_t czb;
+		zbookmark_phys_t czb;
 
 		SET_BOOKMARK(&czb, ds ? ds->ds_object : 0, object,
 		    dnp->dn_nlevels - 1, j);
@@ -703,7 +704,7 @@ dsl_scan_visitdnode(dsl_scan_t *scn, dsl_dataset_t *ds,
 	}
 
 	if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
-		zbookmark_t czb;
+		zbookmark_phys_t czb;
 		SET_BOOKMARK(&czb, ds ? ds->ds_object : 0, object,
 		    0, DMU_SPILL_BLKID);
 		dsl_scan_visitbp(&dnp->dn_spill,
@@ -716,7 +717,7 @@ dsl_scan_visitdnode(dsl_scan_t *scn, dsl_dataset_t *ds,
  * first 5; we want them to be useful.
  */
 static void
-dsl_scan_visitbp(blkptr_t *bp, const zbookmark_t *zb,
+dsl_scan_visitbp(blkptr_t *bp, const zbookmark_phys_t *zb,
     dnode_phys_t *dnp, arc_buf_t *pbuf,
     dsl_dataset_t *ds, dsl_scan_t *scn, dmu_objset_type_t ostype,
     dmu_tx_t *tx)
@@ -780,7 +781,7 @@ static void
 dsl_scan_visit_rootbp(dsl_scan_t *scn, dsl_dataset_t *ds, blkptr_t *bp,
     dmu_tx_t *tx)
 {
-	zbookmark_t zb;
+	zbookmark_phys_t zb;
 
 	SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET,
 	    ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID);
@@ -1207,7 +1208,7 @@ dsl_scan_ddt_entry(dsl_scan_t *scn, enum zio_checksum checksum,
 	const ddt_key_t *ddk = &dde->dde_key;
 	ddt_phys_t *ddp = dde->dde_phys;
 	blkptr_t bp;
-	zbookmark_t zb = { 0 };
+	zbookmark_phys_t zb = { 0 };
 
 	if (scn->scn_phys.scn_state != DSS_SCANNING)
 		return;
@@ -1275,7 +1276,7 @@ dsl_scan_visit(dsl_scan_t *scn, dmu_tx_t *tx)
 	 * In case we were paused right at the end of the ds, zero the
 	 * bookmark so we don't think that we're still trying to resume.
 	 */
-	bzero(&scn->scn_phys.scn_bookmark, sizeof (zbookmark_t));
+	bzero(&scn->scn_phys.scn_bookmark, sizeof (zbookmark_phys_t));
 
 	/* keep pulling things out of the zap-object-as-queue */
 	while (zap_cursor_init(&zc, dp->dp_meta_objset,
@@ -1672,7 +1673,7 @@ dsl_scan_scrub_done(zio_t *zio)
 
 static int
 dsl_scan_scrub_cb(dsl_pool_t *dp,
-    const blkptr_t *bp, const zbookmark_t *zb)
+    const blkptr_t *bp, const zbookmark_phys_t *zb)
 {
 	dsl_scan_t *scn = dp->dp_scan;
 	size_t size = BP_GET_PSIZE(bp);
diff --git a/uts/common/fs/zfs/spa.c b/uts/common/fs/zfs/spa.c
index f708ce3ef22a..2aedc4279dba 100644
--- a/uts/common/fs/zfs/spa.c
+++ b/uts/common/fs/zfs/spa.c
@@ -793,7 +793,7 @@ spa_error_entry_compare(const void *a, const void *b)
 	int ret;
 
 	ret = bcmp(&sa->se_bookmark, &sb->se_bookmark,
-	    sizeof (zbookmark_t));
+	    sizeof (zbookmark_phys_t));
 
 	if (ret < 0)
 		return (-1);
@@ -1831,7 +1831,7 @@ spa_load_verify_done(zio_t *zio)
 /*ARGSUSED*/
 static int
 spa_load_verify_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
-    const zbookmark_t *zb, const dnode_phys_t *dnp, void *arg)
+    const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg)
 {
 	if (!BP_IS_HOLE(bp) && !BP_IS_EMBEDDED(bp)) {
 		zio_t *rio = arg;
diff --git a/uts/common/fs/zfs/spa_errlog.c b/uts/common/fs/zfs/spa_errlog.c
index 0dd6c7a489b7..168f3acc2e8e 100644
--- a/uts/common/fs/zfs/spa_errlog.c
+++ b/uts/common/fs/zfs/spa_errlog.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
  */
 
 /*
@@ -35,7 +35,7 @@
  * deleted from the log when the scrub completes.
  *
  * The log is stored using a ZAP object whose key is a string form of the
- * zbookmark tuple (objset, object, level, blkid), and whose contents is an
+ * zbookmark_phys tuple (objset, object, level, blkid), and whose contents is an
  * optional 'objset:object' human-readable string describing the data.  When an
  * error is first logged, this string will be empty, indicating that no name is
  * known.  This prevents us from having to issue a potentially large amount of
@@ -59,7 +59,7 @@
  * Convert a bookmark to a string.
  */
 static void
-bookmark_to_name(zbookmark_t *zb, char *buf, size_t len)
+bookmark_to_name(zbookmark_phys_t *zb, char *buf, size_t len)
 {
 	(void) snprintf(buf, len, "%llx:%llx:%llx:%llx",
 	    (u_longlong_t)zb->zb_objset, (u_longlong_t)zb->zb_object,
@@ -71,7 +71,7 @@ bookmark_to_name(zbookmark_t *zb, char *buf, size_t len)
  */
 #ifdef _KERNEL
 static void
-name_to_bookmark(char *buf, zbookmark_t *zb)
+name_to_bookmark(char *buf, zbookmark_phys_t *zb)
 {
 	zb->zb_objset = strtonum(buf, &buf);
 	ASSERT(*buf == ':');
@@ -92,7 +92,7 @@ name_to_bookmark(char *buf, zbookmark_t *zb)
 void
 spa_log_error(spa_t *spa, zio_t *zio)
 {
-	zbookmark_t *zb = &zio->io_logical->io_bookmark;
+	zbookmark_phys_t *zb = &zio->io_logical->io_bookmark;
 	spa_error_entry_t search;
 	spa_error_entry_t *new;
 	avl_tree_t *tree;
@@ -165,7 +165,7 @@ process_error_log(spa_t *spa, uint64_t obj, void *addr, size_t *count)
 {
 	zap_cursor_t zc;
 	zap_attribute_t za;
-	zbookmark_t zb;
+	zbookmark_phys_t zb;
 
 	if (obj == 0)
 		return (0);
@@ -182,8 +182,8 @@ process_error_log(spa_t *spa, uint64_t obj, void *addr, size_t *count)
 		name_to_bookmark(za.za_name, &zb);
 
 		if (copyout(&zb, (char *)addr +
-		    (*count - 1) * sizeof (zbookmark_t),
-		    sizeof (zbookmark_t)) != 0) {
+		    (*count - 1) * sizeof (zbookmark_phys_t),
+		    sizeof (zbookmark_phys_t)) != 0) {
 			zap_cursor_fini(&zc);
 			return (SET_ERROR(EFAULT));
 		}
@@ -207,8 +207,8 @@ process_error_list(avl_tree_t *list, void *addr, size_t *count)
 			return (SET_ERROR(ENOMEM));
 
 		if (copyout(&se->se_bookmark, (char *)addr +
-		    (*count - 1) * sizeof (zbookmark_t),
-		    sizeof (zbookmark_t)) != 0)
+		    (*count - 1) * sizeof (zbookmark_phys_t),
+		    sizeof (zbookmark_phys_t)) != 0)
 			return (SET_ERROR(EFAULT));
 
 		*count -= 1;
diff --git a/uts/common/fs/zfs/sys/arc.h b/uts/common/fs/zfs/sys/arc.h
index b1c60905721d..15ef0f58d02b 100644
--- a/uts/common/fs/zfs/sys/arc.h
+++ b/uts/common/fs/zfs/sys/arc.h
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  */
 
@@ -105,12 +105,12 @@ int arc_referenced(arc_buf_t *buf);
 
 int arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp,
     arc_done_func_t *done, void *private, zio_priority_t priority, int flags,
-    uint32_t *arc_flags, const zbookmark_t *zb);
+    uint32_t *arc_flags, const zbookmark_phys_t *zb);
 zio_t *arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
     blkptr_t *bp, arc_buf_t *buf, boolean_t l2arc, boolean_t l2arc_compress,
     const zio_prop_t *zp, arc_done_func_t *ready, arc_done_func_t *physdone,
     arc_done_func_t *done, void *private, zio_priority_t priority,
-    int zio_flags, const zbookmark_t *zb);
+    int zio_flags, const zbookmark_phys_t *zb);
 void arc_freed(spa_t *spa, const blkptr_t *bp);
 
 void arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private);
diff --git a/uts/common/fs/zfs/sys/bptree.h b/uts/common/fs/zfs/sys/bptree.h
index a533cb949021..327c128bf493 100644
--- a/uts/common/fs/zfs/sys/bptree.h
+++ b/uts/common/fs/zfs/sys/bptree.h
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #ifndef	_SYS_BPTREE_H
@@ -43,7 +43,7 @@ typedef struct bptree_phys {
 typedef struct bptree_entry_phys {
 	blkptr_t be_bp;
 	uint64_t be_birth_txg; /* only delete blocks born after this txg */
-	zbookmark_t be_zb; /* holds traversal resume point if needed */
+	zbookmark_phys_t be_zb; /* holds traversal resume point if needed */
 } bptree_entry_phys_t;
 
 typedef int bptree_itor_t(void *arg, const blkptr_t *bp, dmu_tx_t *tx);
diff --git a/uts/common/fs/zfs/sys/dmu.h b/uts/common/fs/zfs/sys/dmu.h
index 9a6006ce6646..f968f4bc1220 100644
--- a/uts/common/fs/zfs/sys/dmu.h
+++ b/uts/common/fs/zfs/sys/dmu.h
@@ -65,7 +65,7 @@ struct dsl_pool;
 struct dnode;
 struct drr_begin;
 struct drr_end;
-struct zbookmark;
+struct zbookmark_phys;
 struct spa;
 struct nvlist;
 struct arc_buf;
diff --git a/uts/common/fs/zfs/sys/dmu_traverse.h b/uts/common/fs/zfs/sys/dmu_traverse.h
index bc1590bc3aa8..544b721e4612 100644
--- a/uts/common/fs/zfs/sys/dmu_traverse.h
+++ b/uts/common/fs/zfs/sys/dmu_traverse.h
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #ifndef	_SYS_DMU_TRAVERSE_H
@@ -40,7 +40,7 @@ struct zilog;
 struct arc_buf;
 
 typedef int (blkptr_cb_t)(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
-    const zbookmark_t *zb, const struct dnode_phys *dnp, void *arg);
+    const zbookmark_phys_t *zb, const struct dnode_phys *dnp, void *arg);
 
 #define	TRAVERSE_PRE			(1<<0)
 #define	TRAVERSE_POST			(1<<1)
@@ -55,7 +55,7 @@ typedef int (blkptr_cb_t)(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 int traverse_dataset(struct dsl_dataset *ds,
     uint64_t txg_start, int flags, blkptr_cb_t func, void *arg);
 int traverse_dataset_destroyed(spa_t *spa, blkptr_t *blkptr,
-    uint64_t txg_start, zbookmark_t *resume, int flags,
+    uint64_t txg_start, zbookmark_phys_t *resume, int flags,
     blkptr_cb_t func, void *arg);
 int traverse_pool(spa_t *spa,
     uint64_t txg_start, int flags, blkptr_cb_t func, void *arg);
diff --git a/uts/common/fs/zfs/sys/dsl_scan.h b/uts/common/fs/zfs/sys/dsl_scan.h
index d9f396680792..ee8512c07dac 100644
--- a/uts/common/fs/zfs/sys/dsl_scan.h
+++ b/uts/common/fs/zfs/sys/dsl_scan.h
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #ifndef	_SYS_DSL_SCAN_H
@@ -62,7 +62,7 @@ typedef struct dsl_scan_phys {
 	uint64_t scn_errors;	/* scan I/O error count */
 	uint64_t scn_ddt_class_max;
 	ddt_bookmark_t scn_ddt_bookmark;
-	zbookmark_t scn_bookmark;
+	zbookmark_phys_t scn_bookmark;
 	uint64_t scn_flags; /* dsl_scan_flags_t */
 } dsl_scan_phys_t;
 
diff --git a/uts/common/fs/zfs/sys/spa.h b/uts/common/fs/zfs/sys/spa.h
index 4bb3d0a9adf7..a2f200e49353 100644
--- a/uts/common/fs/zfs/sys/spa.h
+++ b/uts/common/fs/zfs/sys/spa.h
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  */
 
@@ -799,7 +799,7 @@ extern void spa_history_log_internal_dd(dsl_dir_t *dd, const char *operation,
     dmu_tx_t *tx, const char *fmt, ...);
 
 /* error handling */
-struct zbookmark;
+struct zbookmark_phys;
 extern void spa_log_error(spa_t *spa, zio_t *zio);
 extern void zfs_ereport_post(const char *class, spa_t *spa, vdev_t *vd,
     zio_t *zio, uint64_t stateoroffset, uint64_t length);
diff --git a/uts/common/fs/zfs/sys/spa_impl.h b/uts/common/fs/zfs/sys/spa_impl.h
index 9d74efa1849b..739b95988c25 100644
--- a/uts/common/fs/zfs/sys/spa_impl.h
+++ b/uts/common/fs/zfs/sys/spa_impl.h
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  */
 
@@ -46,9 +46,9 @@ extern "C" {
 #endif
 
 typedef struct spa_error_entry {
-	zbookmark_t	se_bookmark;
-	char		*se_name;
-	avl_node_t	se_avl;
+	zbookmark_phys_t	se_bookmark;
+	char			*se_name;
+	avl_node_t		se_avl;
 } spa_error_entry_t;
 
 typedef struct spa_history_phys {
diff --git a/uts/common/fs/zfs/sys/zio.h b/uts/common/fs/zfs/sys/zio.h
index 88e58884cf47..4c241e9f2680 100644
--- a/uts/common/fs/zfs/sys/zio.h
+++ b/uts/common/fs/zfs/sys/zio.h
@@ -22,7 +22,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  */
@@ -260,16 +260,16 @@ extern const char *zio_type_name[ZIO_TYPES];
  * Note: this structure is called a bookmark because its original purpose
  * was to remember where to resume a pool-wide traverse.
  *
- * Note: this structure is passed between userland and the kernel.
- * Therefore it must not change size or alignment between 32/64 bit
- * compilation options.
+ * Note: this structure is passed between userland and the kernel, and is
+ * stored on disk (by virtue of being incorporated into other on-disk
+ * structures, e.g. dsl_scan_phys_t).
  */
-typedef struct zbookmark {
+typedef struct zbookmark_phys {
 	uint64_t	zb_objset;
 	uint64_t	zb_object;
 	int64_t		zb_level;
 	uint64_t	zb_blkid;
-} zbookmark_t;
+} zbookmark_phys_t;
 
 #define	SET_BOOKMARK(zb, objset, object, level, blkid)  \
 {                                                       \
@@ -380,7 +380,7 @@ typedef struct zio_link {
 
 struct zio {
 	/* Core information about this I/O */
-	zbookmark_t	io_bookmark;
+	zbookmark_phys_t	io_bookmark;
 	zio_prop_t	io_prop;
 	zio_type_t	io_type;
 	enum zio_child	io_child_type;
@@ -459,17 +459,17 @@ extern zio_t *zio_root(spa_t *spa,
 
 extern zio_t *zio_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, void *data,
     uint64_t size, zio_done_func_t *done, void *private,
-    zio_priority_t priority, enum zio_flag flags, const zbookmark_t *zb);
+    zio_priority_t priority, enum zio_flag flags, const zbookmark_phys_t *zb);
 
 extern zio_t *zio_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp,
     void *data, uint64_t size, const zio_prop_t *zp,
     zio_done_func_t *ready, zio_done_func_t *physdone, zio_done_func_t *done,
     void *private,
-    zio_priority_t priority, enum zio_flag flags, const zbookmark_t *zb);
+    zio_priority_t priority, enum zio_flag flags, const zbookmark_phys_t *zb);
 
 extern zio_t *zio_rewrite(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp,
     void *data, uint64_t size, zio_done_func_t *done, void *private,
-    zio_priority_t priority, enum zio_flag flags, zbookmark_t *zb);
+    zio_priority_t priority, enum zio_flag flags, zbookmark_phys_t *zb);
 
 extern void zio_write_override(zio_t *zio, blkptr_t *bp, int copies,
     boolean_t nopwrite);
@@ -588,9 +588,9 @@ extern void zfs_ereport_post_checksum(spa_t *spa, vdev_t *vd,
 /* Called from spa_sync(), but primarily an injection handler */
 extern void spa_handle_ignored_writes(spa_t *spa);
 
-/* zbookmark functions */
+/* zbookmark_phys functions */
 boolean_t zbookmark_is_before(const struct dnode_phys *dnp,
-    const zbookmark_t *zb1, const zbookmark_t *zb2);
+    const zbookmark_phys_t *zb1, const zbookmark_phys_t *zb2);
 
 #ifdef	__cplusplus
 }
diff --git a/uts/common/fs/zfs/zil.c b/uts/common/fs/zfs/zil.c
index b873b12bd76e..a5fe3fb00f82 100644
--- a/uts/common/fs/zfs/zil.c
+++ b/uts/common/fs/zfs/zil.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  */
 
 /* Portions Copyright 2010 Robert Milkowski */
@@ -183,7 +183,7 @@ zil_read_log_block(zilog_t *zilog, const blkptr_t *bp, blkptr_t *nbp, void *dst,
 	enum zio_flag zio_flags = ZIO_FLAG_CANFAIL;
 	uint32_t aflags = ARC_WAIT;
 	arc_buf_t *abuf = NULL;
-	zbookmark_t zb;
+	zbookmark_phys_t zb;
 	int error;
 
 	if (zilog->zl_header->zh_claim_txg == 0)
@@ -256,7 +256,7 @@ zil_read_log_data(zilog_t *zilog, const lr_write_t *lr, void *wbuf)
 	const blkptr_t *bp = &lr->lr_blkptr;
 	uint32_t aflags = ARC_WAIT;
 	arc_buf_t *abuf = NULL;
-	zbookmark_t zb;
+	zbookmark_phys_t zb;
 	int error;
 
 	if (BP_IS_HOLE(bp)) {
@@ -866,7 +866,7 @@ zil_lwb_write_done(zio_t *zio)
 static void
 zil_lwb_write_init(zilog_t *zilog, lwb_t *lwb)
 {
-	zbookmark_t zb;
+	zbookmark_phys_t zb;
 
 	SET_BOOKMARK(&zb, lwb->lwb_blk.blk_cksum.zc_word[ZIL_ZC_OBJSET],
 	    ZB_ZIL_OBJECT, ZB_ZIL_LEVEL,
diff --git a/uts/common/fs/zfs/zio.c b/uts/common/fs/zfs/zio.c
index 506897c02d41..e60f0eb614ac 100644
--- a/uts/common/fs/zfs/zio.c
+++ b/uts/common/fs/zfs/zio.c
@@ -489,7 +489,7 @@ static zio_t *
 zio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp,
     void *data, uint64_t size, zio_done_func_t *done, void *private,
     zio_type_t type, zio_priority_t priority, enum zio_flag flags,
-    vdev_t *vd, uint64_t offset, const zbookmark_t *zb,
+    vdev_t *vd, uint64_t offset, const zbookmark_phys_t *zb,
     enum zio_stage stage, enum zio_stage pipeline)
 {
 	zio_t *zio;
@@ -598,7 +598,7 @@ zio_root(spa_t *spa, zio_done_func_t *done, void *private, enum zio_flag flags)
 zio_t *
 zio_read(zio_t *pio, spa_t *spa, const blkptr_t *bp,
     void *data, uint64_t size, zio_done_func_t *done, void *private,
-    zio_priority_t priority, enum zio_flag flags, const zbookmark_t *zb)
+    zio_priority_t priority, enum zio_flag flags, const zbookmark_phys_t *zb)
 {
 	zio_t *zio;
 
@@ -616,7 +616,7 @@ zio_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp,
     void *data, uint64_t size, const zio_prop_t *zp,
     zio_done_func_t *ready, zio_done_func_t *physdone, zio_done_func_t *done,
     void *private,
-    zio_priority_t priority, enum zio_flag flags, const zbookmark_t *zb)
+    zio_priority_t priority, enum zio_flag flags, const zbookmark_phys_t *zb)
 {
 	zio_t *zio;
 
@@ -654,7 +654,7 @@ zio_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp,
 zio_t *
 zio_rewrite(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, void *data,
     uint64_t size, zio_done_func_t *done, void *private,
-    zio_priority_t priority, enum zio_flag flags, zbookmark_t *zb)
+    zio_priority_t priority, enum zio_flag flags, zbookmark_phys_t *zb)
 {
 	zio_t *zio;
 
@@ -3200,8 +3200,8 @@ static zio_pipe_stage_t *zio_pipeline[] = {
 
 /* dnp is the dnode for zb1->zb_object */
 boolean_t
-zbookmark_is_before(const dnode_phys_t *dnp, const zbookmark_t *zb1,
-    const zbookmark_t *zb2)
+zbookmark_is_before(const dnode_phys_t *dnp, const zbookmark_phys_t *zb1,
+    const zbookmark_phys_t *zb2)
 {
 	uint64_t zb1nextL0, zb2thisobj;
 
diff --git a/uts/common/fs/zfs/zio_inject.c b/uts/common/fs/zfs/zio_inject.c
index 9f3db704b2fa..b2976111d65e 100644
--- a/uts/common/fs/zfs/zio_inject.c
+++ b/uts/common/fs/zfs/zio_inject.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 /*
@@ -64,7 +64,7 @@ static int inject_next_id = 1;
  * Returns true if the given record matches the I/O in progress.
  */
 static boolean_t
-zio_match_handler(zbookmark_t *zb, uint64_t type,
+zio_match_handler(zbookmark_phys_t *zb, uint64_t type,
     zinject_record_t *record, int error)
 {
 	/*
diff --git a/uts/common/fs/zfs/zvol.c b/uts/common/fs/zfs/zvol.c
index 19f6bb97536c..2c225e4e9dcc 100644
--- a/uts/common/fs/zfs/zvol.c
+++ b/uts/common/fs/zfs/zvol.c
@@ -24,7 +24,7 @@
  * Portions Copyright 2010 Robert Milkowski
  *
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  */
 
@@ -244,7 +244,7 @@ struct maparg {
 /*ARGSUSED*/
 static int
 zvol_map_block(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
-    const zbookmark_t *zb, const dnode_phys_t *dnp, void *arg)
+    const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg)
 {
 	struct maparg *ma = arg;
 	zvol_extent_t *ze;

From bb7e6d84964734155f4de126fd46261349151b82 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Tue, 1 Jul 2014 21:16:27 +0000
Subject: [PATCH 008/380] 4936 lz4 could theoretically overflow a pointer with
 a certain input Reviewed by: Saso Kiselkov <skiselkov.ml@gmail.com> Reviewed
 by: Keith Wesolowski <keith.wesolowski@joyent.com> Approved by: Gordon Ross
 <gordon.ross@nexenta.com>

illumos/illumos-gate@58d0718061c87e3d647c891ec5281b93c08dba4e
---
 uts/common/fs/zfs/lz4.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/uts/common/fs/zfs/lz4.c b/uts/common/fs/zfs/lz4.c
index 40cb0711e08f..656360a6f2e0 100644
--- a/uts/common/fs/zfs/lz4.c
+++ b/uts/common/fs/zfs/lz4.c
@@ -960,6 +960,9 @@ real_LZ4_uncompress(const char *source, char *dest, int osize)
 		}
 		/* copy literals */
 		cpy = op + length;
+		/* CORNER-CASE: cpy might overflow. */
+		if (cpy < op)
+			goto _output_error;	/* cpy was overflowed, bail! */
 		if unlikely(cpy > oend - COPYLENGTH) {
 			if (cpy != oend)
 				/* Error: we must necessarily stand at EOF */
@@ -1075,6 +1078,9 @@ LZ4_uncompress_unknownOutputSize(const char *source, char *dest, int isize,
 		}
 		/* copy literals */
 		cpy = op + length;
+		/* CORNER-CASE: cpy might overflow. */
+		if (cpy < op)
+			goto _output_error;	/* cpy was overflowed, bail! */
 		if ((cpy > oend - COPYLENGTH) ||
 		    (ip + length > iend - COPYLENGTH)) {
 			if (cpy > oend)

From 6c37cbaf2596c77fb699e7fd869b87af872c621e Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Tue, 1 Jul 2014 21:19:10 +0000
Subject: [PATCH 009/380] 4924 LZ4 Compression for metadata Reviewed by Matthew
 Ahrens <mahrens@delphix.com> Reviewed by Saso Kiselkov
 <skiselkov.ml@gmail.com> Approved by: Christopher Siden
 <christopher.siden@delphix.com>

illumos/illumos-gate@b8289d24d866c1af02d7007348f7f057693c15d3
---
 common/zfs/zfeature_common.c  |  3 ++-
 uts/common/fs/zfs/dmu.c       | 14 ++++++++++++--
 uts/common/fs/zfs/spa.c       | 18 +++++++++++++++++-
 uts/common/fs/zfs/zfs_ioctl.c | 32 +-------------------------------
 4 files changed, 32 insertions(+), 35 deletions(-)

diff --git a/common/zfs/zfeature_common.c b/common/zfs/zfeature_common.c
index 447c64cc2007..9b046ab07db1 100644
--- a/common/zfs/zfeature_common.c
+++ b/common/zfs/zfeature_common.c
@@ -23,6 +23,7 @@
  * Copyright (c) 2013 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
+ * Copyright (c) 2014, Nexenta Systems, Inc. All rights reserved.
  */
 
 #ifdef _KERNEL
@@ -169,7 +170,7 @@ zpool_feature_init(void)
 	zfeature_register(SPA_FEATURE_LZ4_COMPRESS,
 	    "org.illumos:lz4_compress", "lz4_compress",
 	    "LZ4 compression algorithm support.", B_FALSE, B_FALSE,
-	    B_FALSE, NULL);
+	    B_TRUE, NULL);
 
 	zfeature_register(SPA_FEATURE_MULTI_VDEV_CRASH_DUMP,
 	    "com.joyent:multi_vdev_crash_dump", "multi_vdev_crash_dump",
diff --git a/uts/common/fs/zfs/dmu.c b/uts/common/fs/zfs/dmu.c
index 1a62ddb48b15..a1bd0d810301 100644
--- a/uts/common/fs/zfs/dmu.c
+++ b/uts/common/fs/zfs/dmu.c
@@ -24,6 +24,7 @@
  */
 /* Copyright (c) 2013 by Saso Kiselkov. All rights reserved. */
 /* Copyright (c) 2013, Joyent, Inc. All rights reserved. */
+/* Copyright (c) 2014, Nexenta Systems, Inc. All rights reserved. */
 
 #include <sys/dmu.h>
 #include <sys/dmu_impl.h>
@@ -44,6 +45,7 @@
 #include <sys/zio_checksum.h>
 #include <sys/zio_compress.h>
 #include <sys/sa.h>
+#include <sys/zfeature.h>
 #ifdef _KERNEL
 #include <sys/vmsystm.h>
 #include <sys/zfs_znode.h>
@@ -1635,8 +1637,16 @@ dmu_write_policy(objset_t *os, dnode_t *dn, int level, int wp, zio_prop_t *zp)
 		 * XXX -- we should design a compression algorithm
 		 * that specializes in arrays of bps.
 		 */
-		compress = zfs_mdcomp_disable ? ZIO_COMPRESS_EMPTY :
-		    ZIO_COMPRESS_LZJB;
+		boolean_t lz4_ac = spa_feature_is_active(os->os_spa,
+		    SPA_FEATURE_LZ4_COMPRESS);
+
+		if (zfs_mdcomp_disable) {
+			compress = ZIO_COMPRESS_EMPTY;
+		} else if (lz4_ac) {
+			compress = ZIO_COMPRESS_LZ4;
+		} else {
+			compress = ZIO_COMPRESS_LZJB;
+		}
 
 		/*
 		 * Metadata always gets checksummed.  If the data
diff --git a/uts/common/fs/zfs/spa.c b/uts/common/fs/zfs/spa.c
index 2aedc4279dba..c8cabb2c32e3 100644
--- a/uts/common/fs/zfs/spa.c
+++ b/uts/common/fs/zfs/spa.c
@@ -22,7 +22,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
- * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013, 2014, Nexenta Systems, Inc.  All rights reserved.
  */
 
 /*
@@ -6106,6 +6106,22 @@ spa_sync_upgrades(spa_t *spa, dmu_tx_t *tx)
 	    spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) {
 		spa_feature_create_zap_objects(spa, tx);
 	}
+
+	/*
+	 * LZ4_COMPRESS feature's behaviour was changed to activate_on_enable
+	 * when possibility to use lz4 compression for metadata was added
+	 * Old pools that have this feature enabled must be upgraded to have
+	 * this feature active
+	 */
+	if (spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) {
+		boolean_t lz4_en = spa_feature_is_enabled(spa,
+		    SPA_FEATURE_LZ4_COMPRESS);
+		boolean_t lz4_ac = spa_feature_is_active(spa,
+		    SPA_FEATURE_LZ4_COMPRESS);
+
+		if (lz4_en && !lz4_ac)
+			spa_feature_incr(spa, SPA_FEATURE_LZ4_COMPRESS, tx);
+	}
 	rrw_exit(&dp->dp_config_rwlock, FTAG);
 }
 
diff --git a/uts/common/fs/zfs/zfs_ioctl.c b/uts/common/fs/zfs/zfs_ioctl.c
index 01105be5c51c..83ec68854ed2 100644
--- a/uts/common/fs/zfs/zfs_ioctl.c
+++ b/uts/common/fs/zfs/zfs_ioctl.c
@@ -27,6 +27,7 @@
  * Copyright (c) 2013 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
+ * Copyright (c) 2014, Nexenta Systems, Inc. All rights reserved.
  */
 
 /*
@@ -2453,37 +2454,6 @@ zfs_prop_set_special(const char *dsname, zprop_source_t source,
 		}
 		break;
 	}
-	case ZFS_PROP_COMPRESSION:
-	{
-		if (intval == ZIO_COMPRESS_LZ4) {
-			spa_t *spa;
-
-			if ((err = spa_open(dsname, &spa, FTAG)) != 0)
-				return (err);
-
-			/*
-			 * Setting the LZ4 compression algorithm activates
-			 * the feature.
-			 */
-			if (!spa_feature_is_active(spa,
-			    SPA_FEATURE_LZ4_COMPRESS)) {
-				if ((err = zfs_prop_activate_feature(spa,
-				    SPA_FEATURE_LZ4_COMPRESS)) != 0) {
-					spa_close(spa, FTAG);
-					return (err);
-				}
-			}
-
-			spa_close(spa, FTAG);
-		}
-		/*
-		 * We still want the default set action to be performed in the
-		 * caller, we only performed zfeature settings here.
-		 */
-		err = -1;
-		break;
-	}
-
 	default:
 		err = -1;
 	}

From fcbf7cd0f6a14e541b4b43773b2880e28e41d1d0 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Tue, 1 Jul 2014 21:21:56 +0000
Subject: [PATCH 010/380] 4929 want prevsnap property Reviewed by: Adam
 Leventhal <adam.leventhal@delphix.com> Reviewed by: Matt Amdur
 <matt.amdur@delphix.com> Reviewed by: Saso Kiselkov <skiselkov.ml@gmail.com>
 Reviewed by: Boris Protopopov <bprotopopov@hotmail.com> Reviewed by: Richard
 Lowe <richlowe@richlowe.net> Approved by: Dan McDonald <danmcd@omniti.com>

illumos/illumos-gate@b461c7460e5e77cf65f00151162e654220c6e2fb
---
 common/zfs/zfs_prop.c           | 2 ++
 uts/common/fs/zfs/dsl_dataset.c | 8 +++++++-
 uts/common/sys/fs/zfs.h         | 1 +
 3 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/common/zfs/zfs_prop.c b/common/zfs/zfs_prop.c
index 7c12d6f3f471..0b4927243fb6 100644
--- a/common/zfs/zfs_prop.c
+++ b/common/zfs/zfs_prop.c
@@ -423,6 +423,8 @@ zfs_prop_init(void)
 	    PROP_READONLY, ZFS_TYPE_DATASET, "OBJSETID");
 	zprop_register_hidden(ZFS_PROP_INCONSISTENT, "inconsistent",
 	    PROP_TYPE_NUMBER, PROP_READONLY, ZFS_TYPE_DATASET, "INCONSISTENT");
+	zprop_register_hidden(ZFS_PROP_PREV_SNAP, "prevsnap", PROP_TYPE_STRING,
+	    PROP_READONLY, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME, "PREVSNAP");
 
 	/* oddball properties */
 	zprop_register_impl(ZFS_PROP_CREATION, "creation", PROP_TYPE_NUMBER, 0,
diff --git a/uts/common/fs/zfs/dsl_dataset.c b/uts/common/fs/zfs/dsl_dataset.c
index c5872379694c..425c0d45515c 100644
--- a/uts/common/fs/zfs/dsl_dataset.c
+++ b/uts/common/fs/zfs/dsl_dataset.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2014, Joyent, Inc. All rights reserved.
  * Copyright (c) 2014 RackTop Systems.
  */
@@ -1543,6 +1543,12 @@ dsl_dataset_stats(dsl_dataset_t *ds, nvlist_t *nv)
 		    ds->ds_phys->ds_unique_bytes);
 		get_clones_stat(ds, nv);
 	} else {
+		if (ds->ds_prev != NULL && ds->ds_prev != dp->dp_origin_snap) {
+			char buf[MAXNAMELEN];
+			dsl_dataset_name(ds->ds_prev, buf);
+			dsl_prop_nvlist_add_string(nv, ZFS_PROP_PREV_SNAP, buf);
+		}
+
 		dsl_dir_stats(ds->ds_dir, nv);
 	}
 
diff --git a/uts/common/sys/fs/zfs.h b/uts/common/sys/fs/zfs.h
index 01deecdf02da..f1615194b16a 100644
--- a/uts/common/sys/fs/zfs.h
+++ b/uts/common/sys/fs/zfs.h
@@ -147,6 +147,7 @@ typedef enum {
 	ZFS_PROP_FILESYSTEM_COUNT,
 	ZFS_PROP_SNAPSHOT_COUNT,
 	ZFS_PROP_REDUNDANT_METADATA,
+	ZFS_PROP_PREV_SNAP,
 	ZFS_NUM_PROPS
 } zfs_prop_t;
 

From 129f80f73e8f04be6e2819c3960869a5ec14f512 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Wed, 9 Jul 2014 08:14:13 +0000
Subject: [PATCH 011/380] 4950 files sometimes can't be removed from a full
 filesystem Reviewed by: Adam Leventhal <adam.leventhal@delphix.com> Reviewed
 by: George Wilson <george.wilson@delphix.com> Reviewed by: Sebastien Roy
 <sebastien.roy@delphix.com> Reviewed by: Boris Protopopov
 <bprotopopov@hotmail.com> Approved by: Dan McDonald <danmcd@omniti.com>

illumos/illumos-gate@4bb73804952172060c9efb163b89c17f56804fe8
---
 uts/common/fs/zfs/dmu.c       |  7 +++++++
 uts/common/fs/zfs/dmu_tx.c    | 28 +++++++++++++++++++++++++++-
 uts/common/fs/zfs/sys/dmu.h   |  1 +
 uts/common/fs/zfs/zfs_dir.c   |  3 ++-
 uts/common/fs/zfs/zfs_vnops.c | 35 +++++++++++++++++++++--------------
 uts/common/fs/zfs/zfs_znode.c | 11 ++++++-----
 uts/common/fs/zfs/zvol.c      |  2 ++
 7 files changed, 66 insertions(+), 21 deletions(-)

diff --git a/uts/common/fs/zfs/dmu.c b/uts/common/fs/zfs/dmu.c
index a1bd0d810301..ff67a894bc25 100644
--- a/uts/common/fs/zfs/dmu.c
+++ b/uts/common/fs/zfs/dmu.c
@@ -666,6 +666,12 @@ dmu_free_long_range_impl(objset_t *os, dnode_t *dn, uint64_t offset,
 		dmu_tx_t *tx = dmu_tx_create(os);
 		dmu_tx_hold_free(tx, dn->dn_object,
 		    chunk_begin, chunk_end - chunk_begin);
+
+		/*
+		 * Mark this transaction as typically resulting in a net
+		 * reduction in space used.
+		 */
+		dmu_tx_mark_netfree(tx);
 		err = dmu_tx_assign(tx, TXG_WAIT);
 		if (err) {
 			dmu_tx_abort(tx);
@@ -717,6 +723,7 @@ dmu_free_long_object(objset_t *os, uint64_t object)
 	tx = dmu_tx_create(os);
 	dmu_tx_hold_bonus(tx, object);
 	dmu_tx_hold_free(tx, object, 0, DMU_OBJECT_END);
+	dmu_tx_mark_netfree(tx);
 	err = dmu_tx_assign(tx, TXG_WAIT);
 	if (err == 0) {
 		err = dmu_object_free(os, object, tx);
diff --git a/uts/common/fs/zfs/dmu_tx.c b/uts/common/fs/zfs/dmu_tx.c
index 70c91aca8d0c..55ce31eda85c 100644
--- a/uts/common/fs/zfs/dmu_tx.c
+++ b/uts/common/fs/zfs/dmu_tx.c
@@ -21,7 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #include <sys/dmu.h>
@@ -583,6 +583,32 @@ dmu_tx_count_free(dmu_tx_hold_t *txh, uint64_t off, uint64_t len)
 	txh->txh_space_tounref += unref;
 }
 
+/*
+ * This function marks the transaction as being a "net free".  The end
+ * result is that refquotas will be disabled for this transaction, and
+ * this transaction will be able to use half of the pool space overhead
+ * (see dsl_pool_adjustedsize()).  Therefore this function should only
+ * be called for transactions that we expect will not cause a net increase
+ * in the amount of space used (but it's OK if that is occasionally not true).
+ */
+void
+dmu_tx_mark_netfree(dmu_tx_t *tx)
+{
+	dmu_tx_hold_t *txh;
+
+	txh = dmu_tx_hold_object_impl(tx, tx->tx_objset,
+	    DMU_NEW_OBJECT, THT_FREE, 0, 0);
+
+	/*
+	 * Pretend that this operation will free 1GB of space.  This
+	 * should be large enough to cancel out the largest write.
+	 * We don't want to use something like UINT64_MAX, because that would
+	 * cause overflows when doing math with these values (e.g. in
+	 * dmu_tx_try_assign()).
+	 */
+	txh->txh_space_tofree = txh->txh_space_tounref = 1024 * 1024 * 1024;
+}
+
 void
 dmu_tx_hold_free(dmu_tx_t *tx, uint64_t object, uint64_t off, uint64_t len)
 {
diff --git a/uts/common/fs/zfs/sys/dmu.h b/uts/common/fs/zfs/sys/dmu.h
index f968f4bc1220..b73de0649377 100644
--- a/uts/common/fs/zfs/sys/dmu.h
+++ b/uts/common/fs/zfs/sys/dmu.h
@@ -567,6 +567,7 @@ void dmu_tx_abort(dmu_tx_t *tx);
 int dmu_tx_assign(dmu_tx_t *tx, enum txg_how txg_how);
 void dmu_tx_wait(dmu_tx_t *tx);
 void dmu_tx_commit(dmu_tx_t *tx);
+void dmu_tx_mark_netfree(dmu_tx_t *tx);
 
 /*
  * To register a commit callback, dmu_tx_callback_register() must be called.
diff --git a/uts/common/fs/zfs/zfs_dir.c b/uts/common/fs/zfs/zfs_dir.c
index cb3b4aca15c1..bd7424b55b5e 100644
--- a/uts/common/fs/zfs/zfs_dir.c
+++ b/uts/common/fs/zfs/zfs_dir.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
  */
 
 #include <sys/types.h>
@@ -554,6 +554,7 @@ zfs_purgedir(znode_t *dzp)
 		dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL);
 		/* Is this really needed ? */
 		zfs_sa_upgrade_txholds(tx, xzp);
+		dmu_tx_mark_netfree(tx);
 		error = dmu_tx_assign(tx, TXG_WAIT);
 		if (error) {
 			dmu_tx_abort(tx);
diff --git a/uts/common/fs/zfs/zfs_vnops.c b/uts/common/fs/zfs/zfs_vnops.c
index 767d3f765d07..6fd0bfed0636 100644
--- a/uts/common/fs/zfs/zfs_vnops.c
+++ b/uts/common/fs/zfs/zfs_vnops.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
  * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
  */
 
@@ -1301,7 +1301,7 @@ zfs_lookup(vnode_t *dvp, char *nm, vnode_t **vpp, struct pathname *pnp,
  *		cr	- credentials of caller.
  *		flag	- large file flag [UNUSED].
  *		ct	- caller context
- *		vsecp 	- ACL to be set
+ *		vsecp	- ACL to be set
  *
  *	OUT:	vpp	- vnode of created or trunc'd entry.
  *
@@ -1577,7 +1577,7 @@ zfs_remove(vnode_t *dvp, char *name, cred_t *cr, caller_context_t *ct,
 	zfsvfs_t	*zfsvfs = dzp->z_zfsvfs;
 	zilog_t		*zilog;
 	uint64_t	acl_obj, xattr_obj;
-	uint64_t 	xattr_obj_unlinked = 0;
+	uint64_t	xattr_obj_unlinked = 0;
 	uint64_t	obj = 0;
 	zfs_dirlock_t	*dl;
 	dmu_tx_t	*tx;
@@ -1677,6 +1677,14 @@ top:
 	/* charge as an update -- would be nice not to charge at all */
 	dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL);
 
+	/*
+	 * Mark this transaction as typically resulting in a net free of
+	 * space, unless object removal will be delayed indefinitely
+	 * (due to active holds on the vnode due to the file being open).
+	 */
+	if (may_delete_now)
+		dmu_tx_mark_netfree(tx);
+
 	error = dmu_tx_assign(tx, waited ? TXG_WAITED : TXG_NOWAIT);
 	if (error) {
 		zfs_dirent_unlock(dl);
@@ -1707,7 +1715,6 @@ top:
 	}
 
 	if (unlinked) {
-
 		/*
 		 * Hold z_lock so that we can make sure that the ACL obj
 		 * hasn't changed.  Could have been deleted due to
@@ -4692,13 +4699,13 @@ zfs_addmap(vnode_t *vp, offset_t off, struct as *as, caddr_t addr,
  * last page is pushed.  The problem occurs when the msync() call is omitted,
  * which by far the most common case:
  *
- * 	open()
- * 	mmap()
- * 	<modify memory>
- * 	munmap()
- * 	close()
- * 	<time lapse>
- * 	putpage() via fsflush
+ *	open()
+ *	mmap()
+ *	<modify memory>
+ *	munmap()
+ *	close()
+ *	<time lapse>
+ *	putpage() via fsflush
  *
  * If we wait until fsflush to come along, we can have a modification time that
  * is some arbitrary point in the future.  In order to prevent this in the
@@ -5172,7 +5179,7 @@ const fs_operation_def_t zfs_dvnodeops_template[] = {
 	VOPNAME_PATHCONF,	{ .vop_pathconf = zfs_pathconf },
 	VOPNAME_GETSECATTR,	{ .vop_getsecattr = zfs_getsecattr },
 	VOPNAME_SETSECATTR,	{ .vop_setsecattr = zfs_setsecattr },
-	VOPNAME_VNEVENT, 	{ .vop_vnevent = fs_vnevent_support },
+	VOPNAME_VNEVENT,	{ .vop_vnevent = fs_vnevent_support },
 	NULL,			NULL
 };
 
@@ -5206,8 +5213,8 @@ const fs_operation_def_t zfs_fvnodeops_template[] = {
 	VOPNAME_GETSECATTR,	{ .vop_getsecattr = zfs_getsecattr },
 	VOPNAME_SETSECATTR,	{ .vop_setsecattr = zfs_setsecattr },
 	VOPNAME_VNEVENT,	{ .vop_vnevent = fs_vnevent_support },
-	VOPNAME_REQZCBUF, 	{ .vop_reqzcbuf = zfs_reqzcbuf },
-	VOPNAME_RETZCBUF, 	{ .vop_retzcbuf = zfs_retzcbuf },
+	VOPNAME_REQZCBUF,	{ .vop_reqzcbuf = zfs_reqzcbuf },
+	VOPNAME_RETZCBUF,	{ .vop_retzcbuf = zfs_retzcbuf },
 	NULL,			NULL
 };
 
diff --git a/uts/common/fs/zfs/zfs_znode.c b/uts/common/fs/zfs/zfs_znode.c
index d441faf96de6..7073e2fd0f6b 100644
--- a/uts/common/fs/zfs/zfs_znode.c
+++ b/uts/common/fs/zfs/zfs_znode.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 /* Portions Copyright 2007 Jeremy Teo */
@@ -1442,7 +1442,7 @@ zfs_no_putpage(vnode_t *vp, page_t *pp, u_offset_t *offp, size_t *lenp,
  *	IN:	zp	- znode of file to free data in.
  *		end	- new end-of-file
  *
- * 	RETURN:	0 on success, error code on failure
+ *	RETURN:	0 on success, error code on failure
  */
 static int
 zfs_extend(znode_t *zp, uint64_t end)
@@ -1513,7 +1513,7 @@ zfs_extend(znode_t *zp, uint64_t end)
  *		off	- start of section to free.
  *		len	- length of section to free.
  *
- * 	RETURN:	0 on success, error code on failure
+ *	RETURN:	0 on success, error code on failure
  */
 static int
 zfs_free_range(znode_t *zp, uint64_t off, uint64_t len)
@@ -1551,7 +1551,7 @@ zfs_free_range(znode_t *zp, uint64_t off, uint64_t len)
  *	IN:	zp	- znode of file to free data in.
  *		end	- new end-of-file.
  *
- * 	RETURN:	0 on success, error code on failure
+ *	RETURN:	0 on success, error code on failure
  */
 static int
 zfs_trunc(znode_t *zp, uint64_t end)
@@ -1585,6 +1585,7 @@ zfs_trunc(znode_t *zp, uint64_t end)
 	tx = dmu_tx_create(zfsvfs->z_os);
 	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
 	zfs_sa_upgrade_txholds(tx, zp);
+	dmu_tx_mark_netfree(tx);
 	error = dmu_tx_assign(tx, TXG_WAIT);
 	if (error) {
 		dmu_tx_abort(tx);
@@ -1643,7 +1644,7 @@ zfs_trunc(znode_t *zp, uint64_t end)
  *		flag	- current file open mode flags.
  *		log	- TRUE if this action should be logged
  *
- * 	RETURN:	0 on success, error code on failure
+ *	RETURN:	0 on success, error code on failure
  */
 int
 zfs_freesp(znode_t *zp, uint64_t off, uint64_t len, int flag, boolean_t log)
diff --git a/uts/common/fs/zfs/zvol.c b/uts/common/fs/zfs/zvol.c
index 2c225e4e9dcc..1564e0da684c 100644
--- a/uts/common/fs/zfs/zvol.c
+++ b/uts/common/fs/zfs/zvol.c
@@ -717,6 +717,7 @@ zvol_update_volsize(objset_t *os, uint64_t volsize)
 
 	tx = dmu_tx_create(os);
 	dmu_tx_hold_zap(tx, ZVOL_ZAP_OBJ, TRUE, NULL);
+	dmu_tx_mark_netfree(tx);
 	error = dmu_tx_assign(tx, TXG_WAIT);
 	if (error) {
 		dmu_tx_abort(tx);
@@ -1786,6 +1787,7 @@ zvol_ioctl(dev_t dev, int cmd, intptr_t arg, int flag, cred_t *cr, int *rvalp)
 		rl = zfs_range_lock(&zv->zv_znode, df.df_start, df.df_length,
 		    RL_WRITER);
 		tx = dmu_tx_create(zv->zv_objset);
+		dmu_tx_mark_netfree(tx);
 		error = dmu_tx_assign(tx, TXG_WAIT);
 		if (error != 0) {
 			dmu_tx_abort(tx);

From 4edc11018d364d7d998a4f032ca0eb23a2ab11b5 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Wed, 9 Jul 2014 08:23:22 +0000
Subject: [PATCH 012/380] 4951 ZFS administrative commands should use reserved
 space, not fail with ENOSPC Reviewed by: John Kennedy
 <john.kennedy@delphix.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Reviewed by: Christopher Siden
 <christopher.siden@delphix.com> Reviewed by: Dan McDonald <danmcd@omniti.com>
 Approved by: Garrett D'Amore <garrett@damore.org>

illumos/illumos-gate@7d46dc6ca63a6f3f0d51aa655bfcf10cf2405a9e
---
 uts/common/fs/zfs/dmu_objset.c       |  6 ++--
 uts/common/fs/zfs/dmu_send.c         |  6 ++--
 uts/common/fs/zfs/dsl_bookmark.c     |  8 +++--
 uts/common/fs/zfs/dsl_dataset.c      | 18 ++++++-----
 uts/common/fs/zfs/dsl_deleg.c        |  4 +--
 uts/common/fs/zfs/dsl_destroy.c      |  7 +++--
 uts/common/fs/zfs/dsl_dir.c          | 12 +++++---
 uts/common/fs/zfs/dsl_prop.c         |  4 +--
 uts/common/fs/zfs/dsl_scan.c         |  4 +--
 uts/common/fs/zfs/dsl_synctask.c     | 46 ++++++++++++++++++----------
 uts/common/fs/zfs/dsl_userhold.c     |  7 +++--
 uts/common/fs/zfs/spa.c              |  7 +++--
 uts/common/fs/zfs/spa_history.c      |  6 ++--
 uts/common/fs/zfs/spa_misc.c         | 38 +++++++++++++++++++++++
 uts/common/fs/zfs/sys/dsl_synctask.h | 42 +++++++++++++++++++++----
 uts/common/fs/zfs/zfs_ioctl.c        |  4 +--
 uts/common/fs/zfs/zvol.c             |  3 +-
 17 files changed, 158 insertions(+), 64 deletions(-)

diff --git a/uts/common/fs/zfs/dmu_objset.c b/uts/common/fs/zfs/dmu_objset.c
index 9cfe42a32382..73b8e056cc8c 100644
--- a/uts/common/fs/zfs/dmu_objset.c
+++ b/uts/common/fs/zfs/dmu_objset.c
@@ -832,7 +832,8 @@ dmu_objset_create(const char *name, dmu_objset_type_t type, uint64_t flags,
 	doca.doca_type = type;
 
 	return (dsl_sync_task(name,
-	    dmu_objset_create_check, dmu_objset_create_sync, &doca, 5));
+	    dmu_objset_create_check, dmu_objset_create_sync, &doca,
+	    5, ZFS_SPACE_CHECK_NORMAL));
 }
 
 typedef struct dmu_objset_clone_arg {
@@ -931,7 +932,8 @@ dmu_objset_clone(const char *clone, const char *origin)
 	doca.doca_cred = CRED();
 
 	return (dsl_sync_task(clone,
-	    dmu_objset_clone_check, dmu_objset_clone_sync, &doca, 5));
+	    dmu_objset_clone_check, dmu_objset_clone_sync, &doca,
+	    5, ZFS_SPACE_CHECK_NORMAL));
 }
 
 int
diff --git a/uts/common/fs/zfs/dmu_send.c b/uts/common/fs/zfs/dmu_send.c
index 20d196a83303..97c9926bd79b 100644
--- a/uts/common/fs/zfs/dmu_send.c
+++ b/uts/common/fs/zfs/dmu_send.c
@@ -1162,7 +1162,7 @@ dmu_recv_begin(char *tofs, char *tosnap, struct drr_begin *drrb,
 	drba.drba_cred = CRED();
 
 	return (dsl_sync_task(tofs, dmu_recv_begin_check, dmu_recv_begin_sync,
-	    &drba, 5));
+	    &drba, 5, ZFS_SPACE_CHECK_NORMAL));
 }
 
 struct restorearg {
@@ -2022,7 +2022,7 @@ dmu_recv_existing_end(dmu_recv_cookie_t *drc)
 
 	error = dsl_sync_task(drc->drc_tofs,
 	    dmu_recv_end_check, dmu_recv_end_sync, drc,
-	    dmu_recv_end_modified_blocks);
+	    dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL);
 
 	if (error != 0)
 		dmu_recv_cleanup_ds(drc);
@@ -2036,7 +2036,7 @@ dmu_recv_new_end(dmu_recv_cookie_t *drc)
 
 	error = dsl_sync_task(drc->drc_tofs,
 	    dmu_recv_end_check, dmu_recv_end_sync, drc,
-	    dmu_recv_end_modified_blocks);
+	    dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL);
 
 	if (error != 0) {
 		dmu_recv_cleanup_ds(drc);
diff --git a/uts/common/fs/zfs/dsl_bookmark.c b/uts/common/fs/zfs/dsl_bookmark.c
index 35e55ece4f4d..21784630da6b 100644
--- a/uts/common/fs/zfs/dsl_bookmark.c
+++ b/uts/common/fs/zfs/dsl_bookmark.c
@@ -13,7 +13,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -246,7 +246,8 @@ dsl_bookmark_create(nvlist_t *bmarks, nvlist_t *errors)
 	dbca.dbca_errors = errors;
 
 	return (dsl_sync_task(nvpair_name(pair), dsl_bookmark_create_check,
-	    dsl_bookmark_create_sync, &dbca, fnvlist_num_pairs(bmarks)));
+	    dsl_bookmark_create_sync, &dbca,
+	    fnvlist_num_pairs(bmarks), ZFS_SPACE_CHECK_NORMAL));
 }
 
 int
@@ -448,7 +449,8 @@ dsl_bookmark_destroy(nvlist_t *bmarks, nvlist_t *errors)
 	dbda.dbda_success = fnvlist_alloc();
 
 	rv = dsl_sync_task(nvpair_name(pair), dsl_bookmark_destroy_check,
-	    dsl_bookmark_destroy_sync, &dbda, fnvlist_num_pairs(bmarks));
+	    dsl_bookmark_destroy_sync, &dbda, fnvlist_num_pairs(bmarks),
+	    ZFS_SPACE_CHECK_RESERVED);
 	fnvlist_free(dbda.dbda_success);
 	return (rv);
 }
diff --git a/uts/common/fs/zfs/dsl_dataset.c b/uts/common/fs/zfs/dsl_dataset.c
index 425c0d45515c..937a0eb59e0c 100644
--- a/uts/common/fs/zfs/dsl_dataset.c
+++ b/uts/common/fs/zfs/dsl_dataset.c
@@ -1352,7 +1352,7 @@ dsl_dataset_snapshot(nvlist_t *snaps, nvlist_t *props, nvlist_t *errors)
 	if (error == 0) {
 		error = dsl_sync_task(firstname, dsl_dataset_snapshot_check,
 		    dsl_dataset_snapshot_sync, &ddsa,
-		    fnvlist_num_pairs(snaps) * 3);
+		    fnvlist_num_pairs(snaps) * 3, ZFS_SPACE_CHECK_NORMAL);
 	}
 
 	if (suspended != NULL) {
@@ -1454,7 +1454,7 @@ dsl_dataset_snapshot_tmp(const char *fsname, const char *snapname,
 	}
 
 	error = dsl_sync_task(fsname, dsl_dataset_snapshot_tmp_check,
-	    dsl_dataset_snapshot_tmp_sync, &ddsta, 3);
+	    dsl_dataset_snapshot_tmp_sync, &ddsta, 3, ZFS_SPACE_CHECK_RESERVED);
 
 	if (needsuspend)
 		zil_resume(cookie);
@@ -1805,7 +1805,8 @@ dsl_dataset_rename_snapshot(const char *fsname,
 	ddrsa.ddrsa_recursive = recursive;
 
 	return (dsl_sync_task(fsname, dsl_dataset_rename_snapshot_check,
-	    dsl_dataset_rename_snapshot_sync, &ddrsa, 1));
+	    dsl_dataset_rename_snapshot_sync, &ddrsa,
+	    1, ZFS_SPACE_CHECK_RESERVED));
 }
 
 /*
@@ -1980,7 +1981,8 @@ dsl_dataset_rollback(const char *fsname, void *owner, nvlist_t *result)
 	ddra.ddra_result = result;
 
 	return (dsl_sync_task(fsname, dsl_dataset_rollback_check,
-	    dsl_dataset_rollback_sync, &ddra, 1));
+	    dsl_dataset_rollback_sync, &ddra,
+	    1, ZFS_SPACE_CHECK_RESERVED));
 }
 
 struct promotenode {
@@ -2501,7 +2503,8 @@ dsl_dataset_promote(const char *name, char *conflsnap)
 	ddpa.cr = CRED();
 
 	return (dsl_sync_task(name, dsl_dataset_promote_check,
-	    dsl_dataset_promote_sync, &ddpa, 2 + numsnaps));
+	    dsl_dataset_promote_sync, &ddpa,
+	    2 + numsnaps, ZFS_SPACE_CHECK_RESERVED));
 }
 
 int
@@ -2846,7 +2849,7 @@ dsl_dataset_set_refquota(const char *dsname, zprop_source_t source,
 	ddsqra.ddsqra_value = refquota;
 
 	return (dsl_sync_task(dsname, dsl_dataset_set_refquota_check,
-	    dsl_dataset_set_refquota_sync, &ddsqra, 0));
+	    dsl_dataset_set_refquota_sync, &ddsqra, 0, ZFS_SPACE_CHECK_NONE));
 }
 
 static int
@@ -2961,7 +2964,8 @@ dsl_dataset_set_refreservation(const char *dsname, zprop_source_t source,
 	ddsqra.ddsqra_value = refreservation;
 
 	return (dsl_sync_task(dsname, dsl_dataset_set_refreservation_check,
-	    dsl_dataset_set_refreservation_sync, &ddsqra, 0));
+	    dsl_dataset_set_refreservation_sync, &ddsqra,
+	    0, ZFS_SPACE_CHECK_NONE));
 }
 
 /*
diff --git a/uts/common/fs/zfs/dsl_deleg.c b/uts/common/fs/zfs/dsl_deleg.c
index 4f98c843d3ce..9c6f38a6bdea 100644
--- a/uts/common/fs/zfs/dsl_deleg.c
+++ b/uts/common/fs/zfs/dsl_deleg.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  */
 
 /*
@@ -282,7 +282,7 @@ dsl_deleg_set(const char *ddname, nvlist_t *nvp, boolean_t unset)
 
 	return (dsl_sync_task(ddname, dsl_deleg_check,
 	    unset ? dsl_deleg_unset_sync : dsl_deleg_set_sync,
-	    &dda, fnvlist_num_pairs(nvp)));
+	    &dda, fnvlist_num_pairs(nvp), ZFS_SPACE_CHECK_RESERVED));
 }
 
 /*
diff --git a/uts/common/fs/zfs/dsl_destroy.c b/uts/common/fs/zfs/dsl_destroy.c
index 0813f3288dde..f8a4546535e4 100644
--- a/uts/common/fs/zfs/dsl_destroy.c
+++ b/uts/common/fs/zfs/dsl_destroy.c
@@ -506,7 +506,7 @@ dsl_destroy_snapshots_nvl(nvlist_t *snaps, boolean_t defer,
 
 	error = dsl_sync_task(nvpair_name(pair),
 	    dsl_destroy_snapshot_check, dsl_destroy_snapshot_sync,
-	    &dsda, 0);
+	    &dsda, 0, ZFS_SPACE_CHECK_NONE);
 	fnvlist_free(dsda.dsda_successful_snaps);
 
 	return (error);
@@ -899,7 +899,8 @@ dsl_destroy_head(const char *name)
 		objset_t *os;
 
 		error = dsl_sync_task(name, dsl_destroy_head_check,
-		    dsl_destroy_head_begin_sync, &ddha, 0);
+		    dsl_destroy_head_begin_sync, &ddha,
+		    0, ZFS_SPACE_CHECK_NONE);
 		if (error != 0)
 			return (error);
 
@@ -923,7 +924,7 @@ dsl_destroy_head(const char *name)
 	}
 
 	return (dsl_sync_task(name, dsl_destroy_head_check,
-	    dsl_destroy_head_sync, &ddha, 0));
+	    dsl_destroy_head_sync, &ddha, 0, ZFS_SPACE_CHECK_NONE));
 }
 
 /*
diff --git a/uts/common/fs/zfs/dsl_dir.c b/uts/common/fs/zfs/dsl_dir.c
index 285634fa880c..415bd91e9041 100644
--- a/uts/common/fs/zfs/dsl_dir.c
+++ b/uts/common/fs/zfs/dsl_dir.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 Martin Matuska. All rights reserved.
  * Copyright (c) 2014 Joyent, Inc. All rights reserved.
  */
@@ -637,7 +637,8 @@ dsl_dir_activate_fs_ss_limit(const char *ddname)
 	int error;
 
 	error = dsl_sync_task(ddname, dsl_dir_actv_fs_ss_limit_check,
-	    dsl_dir_actv_fs_ss_limit_sync, (void *)ddname, 0);
+	    dsl_dir_actv_fs_ss_limit_sync, (void *)ddname, 0,
+	    ZFS_SPACE_CHECK_RESERVED);
 
 	if (error == EALREADY)
 		error = 0;
@@ -1487,7 +1488,7 @@ dsl_dir_set_quota(const char *ddname, zprop_source_t source, uint64_t quota)
 	ddsqra.ddsqra_value = quota;
 
 	return (dsl_sync_task(ddname, dsl_dir_set_quota_check,
-	    dsl_dir_set_quota_sync, &ddsqra, 0));
+	    dsl_dir_set_quota_sync, &ddsqra, 0, ZFS_SPACE_CHECK_NONE));
 }
 
 int
@@ -1609,7 +1610,7 @@ dsl_dir_set_reservation(const char *ddname, zprop_source_t source,
 	ddsqra.ddsqra_value = reservation;
 
 	return (dsl_sync_task(ddname, dsl_dir_set_reservation_check,
-	    dsl_dir_set_reservation_sync, &ddsqra, 0));
+	    dsl_dir_set_reservation_sync, &ddsqra, 0, ZFS_SPACE_CHECK_NONE));
 }
 
 static dsl_dir_t *
@@ -1884,7 +1885,8 @@ dsl_dir_rename(const char *oldname, const char *newname)
 	ddra.ddra_cred = CRED();
 
 	return (dsl_sync_task(oldname,
-	    dsl_dir_rename_check, dsl_dir_rename_sync, &ddra, 3));
+	    dsl_dir_rename_check, dsl_dir_rename_sync, &ddra,
+	    3, ZFS_SPACE_CHECK_RESERVED));
 }
 
 int
diff --git a/uts/common/fs/zfs/dsl_prop.c b/uts/common/fs/zfs/dsl_prop.c
index 2eada5cd16dc..7e62635176df 100644
--- a/uts/common/fs/zfs/dsl_prop.c
+++ b/uts/common/fs/zfs/dsl_prop.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 Martin Matuska. All rights reserved.
  */
 
@@ -835,7 +835,7 @@ dsl_props_set(const char *dsname, zprop_source_t source, nvlist_t *props)
 		nblks = 2 * fnvlist_num_pairs(props);
 
 	return (dsl_sync_task(dsname, dsl_props_set_check, dsl_props_set_sync,
-	    &dpsa, nblks));
+	    &dpsa, nblks, ZFS_SPACE_CHECK_RESERVED));
 }
 
 typedef enum dsl_prop_getflags {
diff --git a/uts/common/fs/zfs/dsl_scan.c b/uts/common/fs/zfs/dsl_scan.c
index 1f705580d0bb..d4e234937a06 100644
--- a/uts/common/fs/zfs/dsl_scan.c
+++ b/uts/common/fs/zfs/dsl_scan.c
@@ -346,7 +346,7 @@ int
 dsl_scan_cancel(dsl_pool_t *dp)
 {
 	return (dsl_sync_task(spa_name(dp->dp_spa), dsl_scan_cancel_check,
-	    dsl_scan_cancel_sync, NULL, 3));
+	    dsl_scan_cancel_sync, NULL, 3, ZFS_SPACE_CHECK_RESERVED));
 }
 
 static void dsl_scan_visitbp(blkptr_t *bp,
@@ -1784,5 +1784,5 @@ dsl_scan(dsl_pool_t *dp, pool_scan_func_t func)
 	(void) spa_vdev_state_exit(spa, NULL, 0);
 
 	return (dsl_sync_task(spa_name(spa), dsl_scan_setup_check,
-	    dsl_scan_setup_sync, &func, 0));
+	    dsl_scan_setup_sync, &func, 0, ZFS_SPACE_CHECK_NONE));
 }
diff --git a/uts/common/fs/zfs/dsl_synctask.c b/uts/common/fs/zfs/dsl_synctask.c
index 58f7d3795282..8e09347678dd 100644
--- a/uts/common/fs/zfs/dsl_synctask.c
+++ b/uts/common/fs/zfs/dsl_synctask.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #include <sys/dmu.h>
@@ -64,7 +64,8 @@ dsl_null_checkfunc(void *arg, dmu_tx_t *tx)
  */
 int
 dsl_sync_task(const char *pool, dsl_checkfunc_t *checkfunc,
-    dsl_syncfunc_t *syncfunc, void *arg, int blocks_modified)
+    dsl_syncfunc_t *syncfunc, void *arg,
+    int blocks_modified, zfs_space_check_t space_check)
 {
 	spa_t *spa;
 	dmu_tx_t *tx;
@@ -84,6 +85,7 @@ top:
 	dst.dst_pool = dp;
 	dst.dst_txg = dmu_tx_get_txg(tx);
 	dst.dst_space = blocks_modified << DST_AVG_BLKSHIFT;
+	dst.dst_space_check = space_check;
 	dst.dst_checkfunc = checkfunc != NULL ? checkfunc : dsl_null_checkfunc;
 	dst.dst_syncfunc = syncfunc;
 	dst.dst_arg = arg;
@@ -117,13 +119,14 @@ top:
 
 void
 dsl_sync_task_nowait(dsl_pool_t *dp, dsl_syncfunc_t *syncfunc, void *arg,
-    int blocks_modified, dmu_tx_t *tx)
+    int blocks_modified, zfs_space_check_t space_check, dmu_tx_t *tx)
 {
 	dsl_sync_task_t *dst = kmem_zalloc(sizeof (*dst), KM_SLEEP);
 
 	dst->dst_pool = dp;
 	dst->dst_txg = dmu_tx_get_txg(tx);
 	dst->dst_space = blocks_modified << DST_AVG_BLKSHIFT;
+	dst->dst_space_check = space_check;
 	dst->dst_checkfunc = dsl_null_checkfunc;
 	dst->dst_syncfunc = syncfunc;
 	dst->dst_arg = arg;
@@ -140,25 +143,34 @@ void
 dsl_sync_task_sync(dsl_sync_task_t *dst, dmu_tx_t *tx)
 {
 	dsl_pool_t *dp = dst->dst_pool;
-	uint64_t quota, used;
 
 	ASSERT0(dst->dst_error);
 
 	/*
-	 * Check for sufficient space.  We just check against what's
-	 * on-disk; we don't want any in-flight accounting to get in our
-	 * way, because open context may have already used up various
-	 * in-core limits (arc_tempreserve, dsl_pool_tempreserve).
+	 * Check for sufficient space.
+	 *
+	 * When the sync task was created, the caller specified the
+	 * type of space checking required.  See the comment in
+	 * zfs_space_check_t for details on the semantics of each
+	 * type of space checking.
+	 *
+	 * We just check against what's on-disk; we don't want any
+	 * in-flight accounting to get in our way, because open context
+	 * may have already used up various in-core limits
+	 * (arc_tempreserve, dsl_pool_tempreserve).
 	 */
-	quota = dsl_pool_adjustedsize(dp, B_FALSE) -
-	    metaslab_class_get_deferred(spa_normal_class(dp->dp_spa));
-	used = dp->dp_root_dir->dd_phys->dd_used_bytes;
-	/* MOS space is triple-dittoed, so we multiply by 3. */
-	if (dst->dst_space > 0 && used + dst->dst_space * 3 > quota) {
-		dst->dst_error = SET_ERROR(ENOSPC);
-		if (dst->dst_nowaiter)
-			kmem_free(dst, sizeof (*dst));
-		return;
+	if (dst->dst_space_check != ZFS_SPACE_CHECK_NONE) {
+		uint64_t quota = dsl_pool_adjustedsize(dp,
+		    dst->dst_space_check == ZFS_SPACE_CHECK_RESERVED) -
+		    metaslab_class_get_deferred(spa_normal_class(dp->dp_spa));
+		uint64_t used = dp->dp_root_dir->dd_phys->dd_used_bytes;
+		/* MOS space is triple-dittoed, so we multiply by 3. */
+		if (dst->dst_space > 0 && used + dst->dst_space * 3 > quota) {
+			dst->dst_error = SET_ERROR(ENOSPC);
+			if (dst->dst_nowaiter)
+				kmem_free(dst, sizeof (*dst));
+			return;
+		}
 	}
 
 	/*
diff --git a/uts/common/fs/zfs/dsl_userhold.c b/uts/common/fs/zfs/dsl_userhold.c
index be5b7102cad4..b6878d7ecc44 100644
--- a/uts/common/fs/zfs/dsl_userhold.c
+++ b/uts/common/fs/zfs/dsl_userhold.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
  */
 
@@ -317,7 +317,8 @@ dsl_dataset_user_hold(nvlist_t *holds, minor_t cleanup_minor, nvlist_t *errlist)
 	dduha.dduha_minor = cleanup_minor;
 
 	ret = dsl_sync_task(nvpair_name(pair), dsl_dataset_user_hold_check,
-	    dsl_dataset_user_hold_sync, &dduha, fnvlist_num_pairs(holds));
+	    dsl_dataset_user_hold_sync, &dduha,
+	    fnvlist_num_pairs(holds), ZFS_SPACE_CHECK_RESERVED);
 	fnvlist_free(dduha.dduha_chkholds);
 
 	return (ret);
@@ -600,7 +601,7 @@ dsl_dataset_user_release_impl(nvlist_t *holds, nvlist_t *errlist,
 	ddura.ddura_chkholds = fnvlist_alloc();
 
 	error = dsl_sync_task(pool, dsl_dataset_user_release_check,
-	    dsl_dataset_user_release_sync, &ddura, 0);
+	    dsl_dataset_user_release_sync, &ddura, 0, ZFS_SPACE_CHECK_NONE);
 	fnvlist_free(ddura.ddura_todelete);
 	fnvlist_free(ddura.ddura_chkholds);
 
diff --git a/uts/common/fs/zfs/spa.c b/uts/common/fs/zfs/spa.c
index c8cabb2c32e3..1f8648272544 100644
--- a/uts/common/fs/zfs/spa.c
+++ b/uts/common/fs/zfs/spa.c
@@ -673,7 +673,8 @@ spa_prop_set(spa_t *spa, nvlist_t *nvp)
 			 * feature descriptions object.
 			 */
 			error = dsl_sync_task(spa->spa_name, NULL,
-			    spa_sync_version, &ver, 6);
+			    spa_sync_version, &ver,
+			    6, ZFS_SPACE_CHECK_RESERVED);
 			if (error)
 				return (error);
 			continue;
@@ -685,7 +686,7 @@ spa_prop_set(spa_t *spa, nvlist_t *nvp)
 
 	if (need_sync) {
 		return (dsl_sync_task(spa->spa_name, NULL, spa_sync_props,
-		    nvp, 6));
+		    nvp, 6, ZFS_SPACE_CHECK_RESERVED));
 	}
 
 	return (0);
@@ -766,7 +767,7 @@ spa_change_guid(spa_t *spa)
 	guid = spa_generate_guid(NULL);
 
 	error = dsl_sync_task(spa->spa_name, spa_change_guid_check,
-	    spa_change_guid_sync, &guid, 5);
+	    spa_change_guid_sync, &guid, 5, ZFS_SPACE_CHECK_RESERVED);
 
 	if (error == 0) {
 		spa_config_sync(spa, B_FALSE, B_TRUE);
diff --git a/uts/common/fs/zfs/spa_history.c b/uts/common/fs/zfs/spa_history.c
index 9e172482fe25..cf72c8ad8827 100644
--- a/uts/common/fs/zfs/spa_history.c
+++ b/uts/common/fs/zfs/spa_history.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  */
 
 #include <sys/spa.h>
@@ -322,7 +322,7 @@ spa_history_log_nvl(spa_t *spa, nvlist_t *nvl)
 
 	/* Kick this off asynchronously; errors are ignored. */
 	dsl_sync_task_nowait(spa_get_dsl(spa), spa_history_log_sync,
-	    nvarg, 0, tx);
+	    nvarg, 0, ZFS_SPACE_CHECK_NONE, tx);
 	dmu_tx_commit(tx);
 
 	/* spa_history_log_sync will free nvl */
@@ -458,7 +458,7 @@ log_internal(nvlist_t *nvl, const char *operation, spa_t *spa,
 		spa_history_log_sync(nvl, tx);
 	} else {
 		dsl_sync_task_nowait(spa_get_dsl(spa),
-		    spa_history_log_sync, nvl, 0, tx);
+		    spa_history_log_sync, nvl, 0, ZFS_SPACE_CHECK_NONE, tx);
 	}
 	/* spa_history_log_sync() will free nvl */
 }
diff --git a/uts/common/fs/zfs/spa_misc.c b/uts/common/fs/zfs/spa_misc.c
index ea386591b27f..14bd2a01762d 100644
--- a/uts/common/fs/zfs/spa_misc.c
+++ b/uts/common/fs/zfs/spa_misc.c
@@ -316,6 +316,32 @@ int zfs_deadman_enabled = -1;
  */
 int spa_asize_inflation = 24;
 
+/*
+ * Normally, we don't allow the last 3.2% (1/(2^spa_slop_shift)) of space in
+ * the pool to be consumed.  This ensures that we don't run the pool
+ * completely out of space, due to unaccounted changes (e.g. to the MOS).
+ * It also limits the worst-case time to allocate space.  If we have
+ * less than this amount of free space, most ZPL operations (e.g. write,
+ * create) will return ENOSPC.
+ *
+ * Certain operations (e.g. file removal, most administrative actions) can
+ * use half the slop space.  They will only return ENOSPC if less than half
+ * the slop space is free.  Typically, once the pool has less than the slop
+ * space free, the user will use these operations to free up space in the pool.
+ * These are the operations that call dsl_pool_adjustedsize() with the netfree
+ * argument set to TRUE.
+ *
+ * A very restricted set of operations are always permitted, regardless of
+ * the amount of free space.  These are the operations that call
+ * dsl_sync_task(ZFS_SPACE_CHECK_NONE), e.g. "zfs destroy".  If these
+ * operations result in a net increase in the amount of space used,
+ * it is possible to run the pool completely out of space, causing it to
+ * be permanently read-only.
+ *
+ * See also the comments in zfs_space_check_t.
+ */
+int spa_slop_shift = 5;
+
 /*
  * ==========================================================================
  * SPA config locking
@@ -1578,6 +1604,18 @@ spa_get_asize(spa_t *spa, uint64_t lsize)
 	return (lsize * spa_asize_inflation);
 }
 
+/*
+ * Return the amount of slop space in bytes.  It is 1/32 of the pool (3.2%),
+ * or at least 32MB.
+ *
+ * See the comment above spa_slop_shift for details.
+ */
+uint64_t
+spa_get_slop_space(spa_t *spa) {
+	uint64_t space = spa_get_dspace(spa);
+	return (MAX(space >> spa_slop_shift, SPA_MINDEVSIZE >> 1));
+}
+
 uint64_t
 spa_get_dspace(spa_t *spa)
 {
diff --git a/uts/common/fs/zfs/sys/dsl_synctask.h b/uts/common/fs/zfs/sys/dsl_synctask.h
index ef86fb64cf0c..6139303c1564 100644
--- a/uts/common/fs/zfs/sys/dsl_synctask.h
+++ b/uts/common/fs/zfs/sys/dsl_synctask.h
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #ifndef	_SYS_DSL_SYNCTASK_H
@@ -38,11 +38,41 @@ struct dsl_pool;
 typedef int (dsl_checkfunc_t)(void *, dmu_tx_t *);
 typedef void (dsl_syncfunc_t)(void *, dmu_tx_t *);
 
+typedef enum zfs_space_check {
+	/*
+	 * Normal space check: if there is less than 3.2% free space,
+	 * the operation will fail.  Operations which are logically
+	 * creating things should use this (e.g. "zfs create", "zfs snapshot").
+	 * User writes (via the ZPL / ZVOL) also fail at this point.
+	 */
+	ZFS_SPACE_CHECK_NORMAL,
+
+	/*
+	 * Space check allows use of half the slop space.  If there
+	 * is less than 1.6% free space, the operation will fail.  Most
+	 * operations should use this (e.g. "zfs set", "zfs rename"),
+	 * because we want them to succeed even after user writes are failing,
+	 * so that they can be used as part of the space recovery process.
+	 */
+	ZFS_SPACE_CHECK_RESERVED,
+
+	/*
+	 * No space check is performed.  Only operations which we expect to
+	 * result in a net reduction in space should use this
+	 * (e.g. "zfs destroy". Setting quotas & reservations also uses
+	 * this because it needs to circumvent the quota/reservation checks).
+	 *
+	 * See also the comments above spa_slop_shift.
+	 */
+	ZFS_SPACE_CHECK_NONE,
+} zfs_space_check_t;
+
 typedef struct dsl_sync_task {
 	txg_node_t dst_node;
 	struct dsl_pool *dst_pool;
 	uint64_t dst_txg;
 	int dst_space;
+	zfs_space_check_t dst_space_check;
 	dsl_checkfunc_t *dst_checkfunc;
 	dsl_syncfunc_t *dst_syncfunc;
 	void *dst_arg;
@@ -50,11 +80,11 @@ typedef struct dsl_sync_task {
 	boolean_t dst_nowaiter;
 } dsl_sync_task_t;
 
-void dsl_sync_task_sync(dsl_sync_task_t *dst, dmu_tx_t *tx);
-int dsl_sync_task(const char *pool, dsl_checkfunc_t *checkfunc,
-    dsl_syncfunc_t *syncfunc, void *arg, int blocks_modified);
-void dsl_sync_task_nowait(struct dsl_pool *dp, dsl_syncfunc_t *syncfunc,
-    void *arg, int blocks_modified, dmu_tx_t *tx);
+void dsl_sync_task_sync(dsl_sync_task_t *, dmu_tx_t *);
+int dsl_sync_task(const char *, dsl_checkfunc_t *,
+    dsl_syncfunc_t *, void *, int, zfs_space_check_t);
+void dsl_sync_task_nowait(struct dsl_pool *, dsl_syncfunc_t *,
+    void *, int, zfs_space_check_t, dmu_tx_t *);
 
 #ifdef	__cplusplus
 }
diff --git a/uts/common/fs/zfs/zfs_ioctl.c b/uts/common/fs/zfs/zfs_ioctl.c
index 83ec68854ed2..7f07ec61f615 100644
--- a/uts/common/fs/zfs/zfs_ioctl.c
+++ b/uts/common/fs/zfs/zfs_ioctl.c
@@ -24,7 +24,7 @@
  * Portions Copyright 2011 Martin Matuska
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2014, Joyent, Inc. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
  * Copyright (c) 2014, Nexenta Systems, Inc. All rights reserved.
@@ -3885,7 +3885,7 @@ zfs_prop_activate_feature(spa_t *spa, spa_feature_t feature)
 	/* EBUSY here indicates that the feature is already active */
 	err = dsl_sync_task(spa_name(spa),
 	    zfs_prop_activate_feature_check, zfs_prop_activate_feature_sync,
-	    &feature, 2);
+	    &feature, 2, ZFS_SPACE_CHECK_RESERVED);
 
 	if (err != 0 && err != EBUSY)
 		return (err);
diff --git a/uts/common/fs/zfs/zvol.c b/uts/common/fs/zfs/zvol.c
index 1564e0da684c..5b511ffa0224 100644
--- a/uts/common/fs/zfs/zvol.c
+++ b/uts/common/fs/zfs/zvol.c
@@ -1910,7 +1910,8 @@ zvol_dump_init(zvol_state_t *zv, boolean_t resize)
 			return (SET_ERROR(ENOTSUP));
 		(void) dsl_sync_task(spa_name(spa),
 		    zfs_mvdev_dump_feature_check,
-		    zfs_mvdev_dump_activate_feature_sync, NULL, 2);
+		    zfs_mvdev_dump_activate_feature_sync, NULL,
+		    2, ZFS_SPACE_CHECK_RESERVED);
 	}
 
 	tx = dmu_tx_create(os);

From 708a8883150012b3259ccc6d303512aae9f6f38e Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Mon, 14 Jul 2014 18:02:36 +0000
Subject: [PATCH 013/380] 4951 ZFS administrative commands should use reserved
 space, not fail with ENOSPC (fix call)

Approved by: Christopher Siden <christopher.siden@delphix.com>

illumos/illumos-gate@c39f2c8cacf297ec37acd421f4c4fec127991995
---
 uts/common/fs/zfs/dsl_pool.c | 7 +------
 uts/common/fs/zfs/sys/spa.h  | 1 +
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/uts/common/fs/zfs/dsl_pool.c b/uts/common/fs/zfs/dsl_pool.c
index 76ca96b1431a..058dc90e2500 100644
--- a/uts/common/fs/zfs/dsl_pool.c
+++ b/uts/common/fs/zfs/dsl_pool.c
@@ -612,17 +612,12 @@ dsl_pool_adjustedsize(dsl_pool_t *dp, boolean_t netfree)
 	uint64_t space, resv;
 
 	/*
-	 * Reserve about 1.6% (1/64), or at least 32MB, for allocation
-	 * efficiency.
-	 * XXX The intent log is not accounted for, so it must fit
-	 * within this slop.
-	 *
 	 * If we're trying to assess whether it's OK to do a free,
 	 * cut the reservation in half to allow forward progress
 	 * (e.g. make it possible to rm(1) files from a full pool).
 	 */
 	space = spa_get_dspace(dp->dp_spa);
-	resv = MAX(space >> 6, SPA_MINDEVSIZE >> 1);
+	resv = spa_get_slop_space(dp->dp_spa);
 	if (netfree)
 		resv >>= 1;
 
diff --git a/uts/common/fs/zfs/sys/spa.h b/uts/common/fs/zfs/sys/spa.h
index a2f200e49353..36b6c93bde8a 100644
--- a/uts/common/fs/zfs/sys/spa.h
+++ b/uts/common/fs/zfs/sys/spa.h
@@ -739,6 +739,7 @@ extern spa_load_state_t spa_load_state(spa_t *spa);
 extern uint64_t spa_freeze_txg(spa_t *spa);
 extern uint64_t spa_get_asize(spa_t *spa, uint64_t lsize);
 extern uint64_t spa_get_dspace(spa_t *spa);
+extern uint64_t spa_get_slop_space(spa_t *spa);
 extern void spa_update_dspace(spa_t *spa);
 extern uint64_t spa_version(spa_t *spa);
 extern boolean_t spa_deflate(spa_t *spa);

From c8707f3e7372d23c2dffe55ffa7b47bf1a1d32bd Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Tue, 15 Jul 2014 17:39:53 +0000
Subject: [PATCH 014/380] 4975 missing mutex_destroy() calls in zfs Reviewed
 by: Matthew Ahrens <matthew.ahrens@delphix.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Reviewed by: Rich Lowe <richlowe@richlowe.net>
 Reviewed by: Seth Nimbosa <darth.Serious@gmail.com> Reviewed by: Dan McDonald
 <danmcd@omniti.com> Reviewed by: Don Brady <dev.fs.zfs@gmail.com> Approved
 by: Dan McDonald <danmcd@omniti.com>

illumos/illumos-gate@d2b3cbbd7f3a37bc7c01b526d3eb312acd070423
---
 uts/common/fs/zfs/dbuf.c        | 6 ++++++
 uts/common/fs/zfs/dnode.c       | 5 +++++
 uts/common/fs/zfs/dnode_sync.c  | 3 +++
 uts/common/fs/zfs/dsl_dataset.c | 3 +++
 uts/common/fs/zfs/sa.c          | 4 ++++
 5 files changed, 21 insertions(+)

diff --git a/uts/common/fs/zfs/dbuf.c b/uts/common/fs/zfs/dbuf.c
index b7567d77df02..a635f245131f 100644
--- a/uts/common/fs/zfs/dbuf.c
+++ b/uts/common/fs/zfs/dbuf.c
@@ -1353,6 +1353,12 @@ dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
 		if (dr->dt.dl.dr_data != db->db_buf)
 			VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db));
 	}
+
+	if (db->db_level != 0) {
+		mutex_destroy(&dr->dt.di.dr_mtx);
+		list_destroy(&dr->dt.di.dr_children);
+	}
+
 	kmem_free(dr, sizeof (dbuf_dirty_record_t));
 
 	ASSERT(db->db_dirtycnt > 0);
diff --git a/uts/common/fs/zfs/dnode.c b/uts/common/fs/zfs/dnode.c
index 7261314ad57d..d7bfacad8631 100644
--- a/uts/common/fs/zfs/dnode.c
+++ b/uts/common/fs/zfs/dnode.c
@@ -1078,6 +1078,11 @@ dnode_hold_impl(objset_t *os, uint64_t object, int flag,
 		}
 		if (winner = dmu_buf_set_user(&db->db, children_dnodes, NULL,
 		    dnode_buf_pageout)) {
+
+			for (i = 0; i < epb; i++) {
+				zrl_destroy(&dnh[i].dnh_zrlock);
+			}
+
 			kmem_free(children_dnodes, sizeof (dnode_children_t) +
 			    (epb - 1) * sizeof (dnode_handle_t));
 			children_dnodes = winner;
diff --git a/uts/common/fs/zfs/dnode_sync.c b/uts/common/fs/zfs/dnode_sync.c
index 04fbd37b6e60..aca4203180a6 100644
--- a/uts/common/fs/zfs/dnode_sync.c
+++ b/uts/common/fs/zfs/dnode_sync.c
@@ -475,6 +475,9 @@ dnode_undirty_dbufs(list_t *list)
 			ASSERT(db->db_blkid == DMU_BONUS_BLKID ||
 			    dr->dt.dl.dr_data == db->db_buf);
 			dbuf_unoverride(dr);
+		} else {
+			mutex_destroy(&dr->dt.di.dr_mtx);
+			list_destroy(&dr->dt.di.dr_children);
 		}
 		kmem_free(dr, sizeof (dbuf_dirty_record_t));
 		dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg);
diff --git a/uts/common/fs/zfs/dsl_dataset.c b/uts/common/fs/zfs/dsl_dataset.c
index 937a0eb59e0c..c075c6ac3dd5 100644
--- a/uts/common/fs/zfs/dsl_dataset.c
+++ b/uts/common/fs/zfs/dsl_dataset.c
@@ -265,6 +265,7 @@ dsl_dataset_evict(dmu_buf_t *db, void *dsv)
 
 	mutex_destroy(&ds->ds_lock);
 	mutex_destroy(&ds->ds_opening_lock);
+	mutex_destroy(&ds->ds_sendstream_lock);
 	refcount_destroy(&ds->ds_longholds);
 
 	kmem_free(ds, sizeof (dsl_dataset_t));
@@ -393,6 +394,7 @@ dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
 		if (err != 0) {
 			mutex_destroy(&ds->ds_lock);
 			mutex_destroy(&ds->ds_opening_lock);
+			mutex_destroy(&ds->ds_sendstream_lock);
 			refcount_destroy(&ds->ds_longholds);
 			bplist_destroy(&ds->ds_pending_deadlist);
 			dsl_deadlist_close(&ds->ds_deadlist);
@@ -449,6 +451,7 @@ dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
 			dsl_dir_rele(ds->ds_dir, ds);
 			mutex_destroy(&ds->ds_lock);
 			mutex_destroy(&ds->ds_opening_lock);
+			mutex_destroy(&ds->ds_sendstream_lock);
 			refcount_destroy(&ds->ds_longholds);
 			kmem_free(ds, sizeof (dsl_dataset_t));
 			if (err != 0) {
diff --git a/uts/common/fs/zfs/sa.c b/uts/common/fs/zfs/sa.c
index 6a87231c7b8a..bd2ebce5df34 100644
--- a/uts/common/fs/zfs/sa.c
+++ b/uts/common/fs/zfs/sa.c
@@ -1108,6 +1108,9 @@ fail:
 	if (sa->sa_user_table)
 		kmem_free(sa->sa_user_table, sa->sa_user_table_sz);
 	mutex_exit(&sa->sa_lock);
+	avl_destroy(&sa->sa_layout_hash_tree);
+	avl_destroy(&sa->sa_layout_num_tree);
+	mutex_destroy(&sa->sa_lock);
 	kmem_free(sa, sizeof (sa_os_t));
 	return ((error == ECKSUM) ? EIO : error);
 }
@@ -1143,6 +1146,7 @@ sa_tear_down(objset_t *os)
 
 	avl_destroy(&sa->sa_layout_hash_tree);
 	avl_destroy(&sa->sa_layout_num_tree);
+	mutex_destroy(&sa->sa_lock);
 
 	kmem_free(sa, sizeof (sa_os_t));
 	os->os_sa = NULL;

From 4848d9863884c7dd37190cb3dcb98bf281298cef Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Tue, 15 Jul 2014 20:35:56 +0000
Subject: [PATCH 015/380] 4970 need controls on i/o issued by zpool import -XF
 4971 zpool import -T should accept hex values 4972 zpool import -T implies
 extreme rewind, and thus a scrub 4973 spa_load_retry retries the same txg
 4974 spa_load_verify() reads all data twice Reviewed by: Christopher Siden
 <christopher.siden@delphix.com> Reviewed by: Dan McDonald <danmcd@omniti.com>
 Reviewed by: George Wilson <george.wilson@delphix.com> Approved by: Robert
 Mustacchi <rm@joyent.com>

illumos/illumos-gate@e42d205944d245bf5d1c4fc45261cbe09e28a7b9
---
 uts/common/fs/zfs/spa.c | 61 ++++++++++++++++++++++++++++++++---------
 1 file changed, 48 insertions(+), 13 deletions(-)

diff --git a/uts/common/fs/zfs/spa.c b/uts/common/fs/zfs/spa.c
index 1f8648272544..dc87100e6b81 100644
--- a/uts/common/fs/zfs/spa.c
+++ b/uts/common/fs/zfs/spa.c
@@ -1818,6 +1818,7 @@ spa_load_verify_done(zio_t *zio)
 	spa_load_error_t *sle = zio->io_private;
 	dmu_object_type_t type = BP_GET_TYPE(bp);
 	int error = zio->io_error;
+	spa_t *spa = zio->io_spa;
 
 	if (error) {
 		if ((BP_GET_LEVEL(bp) != 0 || DMU_OT_IS_METADATA(type)) &&
@@ -1827,23 +1828,52 @@ spa_load_verify_done(zio_t *zio)
 			atomic_add_64(&sle->sle_data_count, 1);
 	}
 	zio_data_buf_free(zio->io_data, zio->io_size);
+
+	mutex_enter(&spa->spa_scrub_lock);
+	spa->spa_scrub_inflight--;
+	cv_broadcast(&spa->spa_scrub_io_cv);
+	mutex_exit(&spa->spa_scrub_lock);
 }
 
+/*
+ * Maximum number of concurrent scrub i/os to create while verifying
+ * a pool while importing it.
+ */
+int spa_load_verify_maxinflight = 10000;
+boolean_t spa_load_verify_metadata = B_TRUE;
+boolean_t spa_load_verify_data = B_TRUE;
+
 /*ARGSUSED*/
 static int
 spa_load_verify_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
     const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg)
 {
-	if (!BP_IS_HOLE(bp) && !BP_IS_EMBEDDED(bp)) {
-		zio_t *rio = arg;
-		size_t size = BP_GET_PSIZE(bp);
-		void *data = zio_data_buf_alloc(size);
+	if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp))
+		return (0);
+	/*
+	 * Note: normally this routine will not be called if
+	 * spa_load_verify_metadata is not set.  However, it may be useful
+	 * to manually set the flag after the traversal has begun.
+	 */
+	if (!spa_load_verify_metadata)
+		return (0);
+	if (BP_GET_BUFC_TYPE(bp) == ARC_BUFC_DATA && !spa_load_verify_data)
+		return (0);
 
-		zio_nowait(zio_read(rio, spa, bp, data, size,
-		    spa_load_verify_done, rio->io_private, ZIO_PRIORITY_SCRUB,
-		    ZIO_FLAG_SPECULATIVE | ZIO_FLAG_CANFAIL |
-		    ZIO_FLAG_SCRUB | ZIO_FLAG_RAW, zb));
-	}
+	zio_t *rio = arg;
+	size_t size = BP_GET_PSIZE(bp);
+	void *data = zio_data_buf_alloc(size);
+
+	mutex_enter(&spa->spa_scrub_lock);
+	while (spa->spa_scrub_inflight >= spa_load_verify_maxinflight)
+		cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock);
+	spa->spa_scrub_inflight++;
+	mutex_exit(&spa->spa_scrub_lock);
+
+	zio_nowait(zio_read(rio, spa, bp, data, size,
+	    spa_load_verify_done, rio->io_private, ZIO_PRIORITY_SCRUB,
+	    ZIO_FLAG_SPECULATIVE | ZIO_FLAG_CANFAIL |
+	    ZIO_FLAG_SCRUB | ZIO_FLAG_RAW, zb));
 	return (0);
 }
 
@@ -1854,7 +1884,7 @@ spa_load_verify(spa_t *spa)
 	spa_load_error_t sle = { 0 };
 	zpool_rewind_policy_t policy;
 	boolean_t verify_ok = B_FALSE;
-	int error;
+	int error = 0;
 
 	zpool_get_rewind_policy(spa->spa_config, &policy);
 
@@ -1864,8 +1894,11 @@ spa_load_verify(spa_t *spa)
 	rio = zio_root(spa, NULL, &sle,
 	    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE);
 
-	error = traverse_pool(spa, spa->spa_verify_min_txg,
-	    TRAVERSE_PRE | TRAVERSE_PREFETCH, spa_load_verify_cb, rio);
+	if (spa_load_verify_metadata) {
+		error = traverse_pool(spa, spa->spa_verify_min_txg,
+		    TRAVERSE_PRE | TRAVERSE_PREFETCH_METADATA,
+		    spa_load_verify_cb, rio);
+	}
 
 	(void) zio_wait(rio);
 
@@ -2740,7 +2773,7 @@ spa_load_retry(spa_t *spa, spa_load_state_t state, int mosconfig)
 	spa_unload(spa);
 	spa_deactivate(spa);
 
-	spa->spa_load_max_txg--;
+	spa->spa_load_max_txg = spa->spa_uberblock.ub_txg - 1;
 
 	spa_activate(spa, mode);
 	spa_async_suspend(spa);
@@ -2770,6 +2803,8 @@ spa_load_best(spa_t *spa, spa_load_state_t state, int mosconfig,
 		spa_set_log_state(spa, SPA_LOG_CLEAR);
 	} else {
 		spa->spa_load_max_txg = max_request;
+		if (max_request != UINT64_MAX)
+			spa->spa_extreme_rewind = B_TRUE;
 	}
 
 	load_error = rewind_error = spa_load(spa, state, SPA_IMPORT_EXISTING,

From edc0259371054251bf21054b88b2f5ec685074bf Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Fri, 18 Jul 2014 17:55:12 +0000
Subject: [PATCH 016/380] 4958 zdb trips assert on pools with ashift >= 0xe
 Reviewed by: Matthew Ahrens <mahrens@delphix.com> Reviewed by: Max Grossman
 <max.grossman@delphix.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Reviewed by: Christopher Siden
 <christopher.siden@delphix.com> Approved by: Garrett D'Amore
 <garrett@damore.org>

illumos/illumos-gate@2a104a5236475eb73aa41eaaf3ed9f3ccbe0ca55
---
 uts/common/fs/zfs/dsl_pool.c      |  2 +-
 uts/common/fs/zfs/metaslab.c      | 39 ++++++++++++++++++++++++---
 uts/common/fs/zfs/spa_misc.c      |  2 +-
 uts/common/fs/zfs/sys/vdev_impl.h |  7 +++--
 uts/common/fs/zfs/sys/zio.h       | 45 ++++++++++++++++---------------
 uts/common/fs/zfs/zfs_debug.c     |  7 +++--
 uts/common/fs/zfs/zio.c           | 30 ++++++++++++++++-----
 7 files changed, 93 insertions(+), 39 deletions(-)

diff --git a/uts/common/fs/zfs/dsl_pool.c b/uts/common/fs/zfs/dsl_pool.c
index 058dc90e2500..9435e74c0e7f 100644
--- a/uts/common/fs/zfs/dsl_pool.c
+++ b/uts/common/fs/zfs/dsl_pool.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
  */
 
diff --git a/uts/common/fs/zfs/metaslab.c b/uts/common/fs/zfs/metaslab.c
index bba1dbf5148c..2831fb2f7bee 100644
--- a/uts/common/fs/zfs/metaslab.c
+++ b/uts/common/fs/zfs/metaslab.c
@@ -61,6 +61,21 @@ uint64_t metaslab_gang_bang = SPA_MAXBLOCKSIZE + 1;	/* force gang blocks */
  */
 int zfs_condense_pct = 200;
 
+/*
+ * Condensing a metaslab is not guaranteed to actually reduce the amount of
+ * space used on disk. In particular, a space map uses data in increments of
+ * MAX(1 << ashift, SPACE_MAP_INITIAL_BLOCKSIZE), so a metaslab might use the
+ * same number of blocks after condensing. Since the goal of condensing is to
+ * reduce the number of IOPs required to read the space map, we only want to
+ * condense when we can be sure we will reduce the number of blocks used by the
+ * space map. Unfortunately, we cannot precisely compute whether or not this is
+ * the case in metaslab_should_condense since we are holding ms_lock. Instead,
+ * we apply the following heuristic: do not condense a spacemap unless the
+ * uncondensed size consumes greater than zfs_metaslab_condense_block_threshold
+ * blocks.
+ */
+int zfs_metaslab_condense_block_threshold = 4;
+
 /*
  * The zfs_mg_noalloc_threshold defines which metaslab groups should
  * be eligible for allocation. The value is defined as a percentage of
@@ -1279,6 +1294,8 @@ metaslab_group_preload(metaslab_group_t *mg)
  * times the size than the free space range tree representation
  * (i.e. zfs_condense_pct = 110 and in-core = 1MB, minimal = 1.1.MB).
  *
+ * 3. The on-disk size of the space map should actually decrease.
+ *
  * Checking the first condition is tricky since we don't want to walk
  * the entire AVL tree calculating the estimated on-disk size. Instead we
  * use the size-ordered range tree in the metaslab and calculate the
@@ -1289,13 +1306,21 @@ metaslab_group_preload(metaslab_group_t *mg)
  * To determine the second criterion we use a best-case estimate and assume
  * each segment can be represented on-disk as a single 64-bit entry. We refer
  * to this best-case estimate as the space map's minimal form.
+ *
+ * Unfortunately, we cannot compute the on-disk size of the space map in this
+ * context because we cannot accurately compute the effects of compression, etc.
+ * Instead, we apply the heuristic described in the block comment for
+ * zfs_metaslab_condense_block_threshold - we only condense if the space used
+ * is greater than a threshold number of blocks.
  */
 static boolean_t
 metaslab_should_condense(metaslab_t *msp)
 {
 	space_map_t *sm = msp->ms_sm;
 	range_seg_t *rs;
-	uint64_t size, entries, segsz;
+	uint64_t size, entries, segsz, object_size, optimal_size, record_size;
+	dmu_object_info_t doi;
+	uint64_t vdev_blocksize = 1 << msp->ms_group->mg_vd->vdev_ashift;
 
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 	ASSERT(msp->ms_loaded);
@@ -1319,9 +1344,15 @@ metaslab_should_condense(metaslab_t *msp)
 	entries = size / (MIN(size, SM_RUN_MAX));
 	segsz = entries * sizeof (uint64_t);
 
-	return (segsz <= space_map_length(msp->ms_sm) &&
-	    space_map_length(msp->ms_sm) >= (zfs_condense_pct *
-	    sizeof (uint64_t) * avl_numnodes(&msp->ms_tree->rt_root)) / 100);
+	optimal_size = sizeof (uint64_t) * avl_numnodes(&msp->ms_tree->rt_root);
+	object_size = space_map_length(msp->ms_sm);
+
+	dmu_object_info_from_db(sm->sm_dbuf, &doi);
+	record_size = MAX(doi.doi_data_block_size, vdev_blocksize);
+
+	return (segsz <= object_size &&
+	    object_size >= (optimal_size * zfs_condense_pct / 100) &&
+	    object_size > zfs_metaslab_condense_block_threshold * record_size);
 }
 
 /*
diff --git a/uts/common/fs/zfs/spa_misc.c b/uts/common/fs/zfs/spa_misc.c
index 14bd2a01762d..ff8739e8dd85 100644
--- a/uts/common/fs/zfs/spa_misc.c
+++ b/uts/common/fs/zfs/spa_misc.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  */
 
diff --git a/uts/common/fs/zfs/sys/vdev_impl.h b/uts/common/fs/zfs/sys/vdev_impl.h
index 6d2ea2fc4228..9c22384f733d 100644
--- a/uts/common/fs/zfs/sys/vdev_impl.h
+++ b/uts/common/fs/zfs/sys/vdev_impl.h
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #ifndef _SYS_VDEV_IMPL_H
@@ -229,8 +229,11 @@ struct vdev {
 #define	VDEV_PHYS_SIZE		(112 << 10)
 #define	VDEV_UBERBLOCK_RING	(128 << 10)
 
+/* The largest uberblock we support is 8k. */
+#define	MAX_UBERBLOCK_SHIFT (13)
 #define	VDEV_UBERBLOCK_SHIFT(vd)	\
-	MAX((vd)->vdev_top->vdev_ashift, UBERBLOCK_SHIFT)
+	MIN(MAX((vd)->vdev_top->vdev_ashift, UBERBLOCK_SHIFT), \
+	    MAX_UBERBLOCK_SHIFT)
 #define	VDEV_UBERBLOCK_COUNT(vd)	\
 	(VDEV_UBERBLOCK_RING >> VDEV_UBERBLOCK_SHIFT(vd))
 #define	VDEV_UBERBLOCK_OFFSET(vd, n)	\
diff --git a/uts/common/fs/zfs/sys/zio.h b/uts/common/fs/zfs/sys/zio.h
index 4c241e9f2680..f158042eb40c 100644
--- a/uts/common/fs/zfs/sys/zio.h
+++ b/uts/common/fs/zfs/sys/zio.h
@@ -165,19 +165,20 @@ enum zio_flag {
 	ZIO_FLAG_RESILVER	= 1 << 3,
 	ZIO_FLAG_SCRUB		= 1 << 4,
 	ZIO_FLAG_SCAN_THREAD	= 1 << 5,
+	ZIO_FLAG_PHYSICAL	= 1 << 6,
 
 #define	ZIO_FLAG_AGG_INHERIT	(ZIO_FLAG_CANFAIL - 1)
 
 	/*
 	 * Flags inherited by ddt, gang, and vdev children.
 	 */
-	ZIO_FLAG_CANFAIL	= 1 << 6,	/* must be first for INHERIT */
-	ZIO_FLAG_SPECULATIVE	= 1 << 7,
-	ZIO_FLAG_CONFIG_WRITER	= 1 << 8,
-	ZIO_FLAG_DONT_RETRY	= 1 << 9,
-	ZIO_FLAG_DONT_CACHE	= 1 << 10,
-	ZIO_FLAG_NODATA		= 1 << 11,
-	ZIO_FLAG_INDUCE_DAMAGE	= 1 << 12,
+	ZIO_FLAG_CANFAIL	= 1 << 7,	/* must be first for INHERIT */
+	ZIO_FLAG_SPECULATIVE	= 1 << 8,
+	ZIO_FLAG_CONFIG_WRITER	= 1 << 9,
+	ZIO_FLAG_DONT_RETRY	= 1 << 10,
+	ZIO_FLAG_DONT_CACHE	= 1 << 11,
+	ZIO_FLAG_NODATA		= 1 << 12,
+	ZIO_FLAG_INDUCE_DAMAGE	= 1 << 13,
 
 #define	ZIO_FLAG_DDT_INHERIT	(ZIO_FLAG_IO_RETRY - 1)
 #define	ZIO_FLAG_GANG_INHERIT	(ZIO_FLAG_IO_RETRY - 1)
@@ -185,27 +186,27 @@ enum zio_flag {
 	/*
 	 * Flags inherited by vdev children.
 	 */
-	ZIO_FLAG_IO_RETRY	= 1 << 13,	/* must be first for INHERIT */
-	ZIO_FLAG_PROBE		= 1 << 14,
-	ZIO_FLAG_TRYHARD	= 1 << 15,
-	ZIO_FLAG_OPTIONAL	= 1 << 16,
+	ZIO_FLAG_IO_RETRY	= 1 << 14,	/* must be first for INHERIT */
+	ZIO_FLAG_PROBE		= 1 << 15,
+	ZIO_FLAG_TRYHARD	= 1 << 16,
+	ZIO_FLAG_OPTIONAL	= 1 << 17,
 
 #define	ZIO_FLAG_VDEV_INHERIT	(ZIO_FLAG_DONT_QUEUE - 1)
 
 	/*
 	 * Flags not inherited by any children.
 	 */
-	ZIO_FLAG_DONT_QUEUE	= 1 << 17,	/* must be first for INHERIT */
-	ZIO_FLAG_DONT_PROPAGATE	= 1 << 18,
-	ZIO_FLAG_IO_BYPASS	= 1 << 19,
-	ZIO_FLAG_IO_REWRITE	= 1 << 20,
-	ZIO_FLAG_RAW		= 1 << 21,
-	ZIO_FLAG_GANG_CHILD	= 1 << 22,
-	ZIO_FLAG_DDT_CHILD	= 1 << 23,
-	ZIO_FLAG_GODFATHER	= 1 << 24,
-	ZIO_FLAG_NOPWRITE	= 1 << 25,
-	ZIO_FLAG_REEXECUTED	= 1 << 26,
-	ZIO_FLAG_DELEGATED	= 1 << 27,
+	ZIO_FLAG_DONT_QUEUE	= 1 << 18,	/* must be first for INHERIT */
+	ZIO_FLAG_DONT_PROPAGATE	= 1 << 19,
+	ZIO_FLAG_IO_BYPASS	= 1 << 20,
+	ZIO_FLAG_IO_REWRITE	= 1 << 21,
+	ZIO_FLAG_RAW		= 1 << 22,
+	ZIO_FLAG_GANG_CHILD	= 1 << 23,
+	ZIO_FLAG_DDT_CHILD	= 1 << 24,
+	ZIO_FLAG_GODFATHER	= 1 << 25,
+	ZIO_FLAG_NOPWRITE	= 1 << 26,
+	ZIO_FLAG_REEXECUTED	= 1 << 27,
+	ZIO_FLAG_DELEGATED	= 1 << 28,
 };
 
 #define	ZIO_FLAG_MUSTSUCCEED		0
diff --git a/uts/common/fs/zfs/zfs_debug.c b/uts/common/fs/zfs/zfs_debug.c
index 85fa7600d9b9..a9cbe4dfe392 100644
--- a/uts/common/fs/zfs/zfs_debug.c
+++ b/uts/common/fs/zfs/zfs_debug.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -57,7 +57,10 @@ zfs_dbgmsg_fini(void)
  * echo ::zfs_dbgmsg | mdb -k
  *
  * Monitor these messages by running:
- * 	dtrace -q -n 'zfs-dbgmsg{printf("%s\n", stringof(arg0))}'
+ * dtrace -qn 'zfs-dbgmsg{printf("%s\n", stringof(arg0))}'
+ *
+ * When used with libzpool, monitor with:
+ * dtrace -qn 'zfs$pid::zfs_dbgmsg:probe1{printf("%s\n", copyinstr(arg1))}'
  */
 void
 zfs_dbgmsg(const char *fmt, ...)
diff --git a/uts/common/fs/zfs/zio.c b/uts/common/fs/zfs/zio.c
index e60f0eb614ac..d8833f67079d 100644
--- a/uts/common/fs/zfs/zio.c
+++ b/uts/common/fs/zfs/zio.c
@@ -815,8 +815,8 @@ zio_read_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size,
 	ASSERT3U(offset + size, <=, vd->vdev_psize);
 
 	zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, done, private,
-	    ZIO_TYPE_READ, priority, flags, vd, offset, NULL,
-	    ZIO_STAGE_OPEN, ZIO_READ_PHYS_PIPELINE);
+	    ZIO_TYPE_READ, priority, flags | ZIO_FLAG_PHYSICAL, vd, offset,
+	    NULL, ZIO_STAGE_OPEN, ZIO_READ_PHYS_PIPELINE);
 
 	zio->io_prop.zp_checksum = checksum;
 
@@ -836,8 +836,8 @@ zio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size,
 	ASSERT3U(offset + size, <=, vd->vdev_psize);
 
 	zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, done, private,
-	    ZIO_TYPE_WRITE, priority, flags, vd, offset, NULL,
-	    ZIO_STAGE_OPEN, ZIO_WRITE_PHYS_PIPELINE);
+	    ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_PHYSICAL, vd, offset,
+	    NULL, ZIO_STAGE_OPEN, ZIO_WRITE_PHYS_PIPELINE);
 
 	zio->io_prop.zp_checksum = checksum;
 
@@ -2508,7 +2508,9 @@ zio_vdev_io_start(zio_t *zio)
 
 	align = 1ULL << vd->vdev_top->vdev_ashift;
 
-	if (P2PHASE(zio->io_size, align) != 0) {
+	if (!(zio->io_flags & ZIO_FLAG_PHYSICAL) &&
+	    P2PHASE(zio->io_size, align) != 0) {
+		/* Transform logical writes to be a full physical block size. */
 		uint64_t asize = P2ROUNDUP(zio->io_size, align);
 		char *abuf = zio_buf_alloc(asize);
 		ASSERT(vd == vd->vdev_top);
@@ -2519,8 +2521,22 @@ zio_vdev_io_start(zio_t *zio)
 		zio_push_transform(zio, abuf, asize, asize, zio_subblock);
 	}
 
-	ASSERT(P2PHASE(zio->io_offset, align) == 0);
-	ASSERT(P2PHASE(zio->io_size, align) == 0);
+	/*
+	 * If this is not a physical io, make sure that it is properly aligned
+	 * before proceeding.
+	 */
+	if (!(zio->io_flags & ZIO_FLAG_PHYSICAL)) {
+		ASSERT0(P2PHASE(zio->io_offset, align));
+		ASSERT0(P2PHASE(zio->io_size, align));
+	} else {
+		/*
+		 * For physical writes, we allow 512b aligned writes and assume
+		 * the device will perform a read-modify-write as necessary.
+		 */
+		ASSERT0(P2PHASE(zio->io_offset, SPA_MINBLOCKSIZE));
+		ASSERT0(P2PHASE(zio->io_size, SPA_MINBLOCKSIZE));
+	}
+
 	VERIFY(zio->io_type != ZIO_TYPE_WRITE || spa_writeable(spa));
 
 	/*

From 0bebdc44704febcd7f6321268b81fa398f7a4600 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Fri, 18 Jul 2014 18:02:34 +0000
Subject: [PATCH 017/380] 4631 zvol_get_stats triggering too many reads
 Reviewed by: Adam Leventhal <ahl@delphix.com> Reviewed by: Sebastien Roy
 <sebastien.roy@delphix.com> Reviewed by: Matt Ahrens <mahrens@delphix.com>
 Approved by: Dan McDonald <danmcd@omniti.com>

illumos/illumos-gate@bbfa8ea8bb4168c969ba27d632dfe0aeec3fc0da
---
 uts/common/fs/zfs/arc.c     | 104 ++++++++++++++----------------------
 uts/common/fs/zfs/dbuf.c    |  38 ++++++++-----
 uts/common/fs/zfs/sys/arc.h |   3 +-
 3 files changed, 65 insertions(+), 80 deletions(-)

diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index 752cacdb126c..dd5b3accc445 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -105,7 +105,7 @@
  * with the buffer may be evicted prior to the callback.  The callback
  * must be made with *no locks held* (to prevent deadlock).  Additionally,
  * the users of callbacks must ensure that their private data is
- * protected from simultaneous callbacks from arc_buf_evict()
+ * protected from simultaneous callbacks from arc_clear_callback()
  * and arc_do_user_evicts().
  *
  * Note that the majority of the performance stats are manipulated
@@ -1495,8 +1495,12 @@ arc_buf_data_free(arc_buf_t *buf, void (*free_func)(void *, size_t))
 	}
 }
 
+/*
+ * Free up buf->b_data and if 'remove' is set, then pull the
+ * arc_buf_t off of the the arc_buf_hdr_t's list and free it.
+ */
 static void
-arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all)
+arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t remove)
 {
 	arc_buf_t **bufp;
 
@@ -1547,7 +1551,7 @@ arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all)
 	}
 
 	/* only remove the buf if requested */
-	if (!all)
+	if (!remove)
 		return;
 
 	/* remove the buf from the hdr list */
@@ -2121,7 +2125,7 @@ arc_do_user_evicts(void)
 		mutex_exit(&arc_eviction_mtx);
 
 		if (buf->b_efunc != NULL)
-			VERIFY(buf->b_efunc(buf) == 0);
+			VERIFY0(buf->b_efunc(buf->b_private));
 
 		buf->b_efunc = NULL;
 		buf->b_private = NULL;
@@ -3240,16 +3244,25 @@ arc_freed(spa_t *spa, const blkptr_t *bp)
 }
 
 /*
- * This is used by the DMU to let the ARC know that a buffer is
- * being evicted, so the ARC should clean up.  If this arc buf
- * is not yet in the evicted state, it will be put there.
+ * Clear the user eviction callback set by arc_set_callback(), first calling
+ * it if it exists.  Because the presence of a callback keeps an arc_buf cached
+ * clearing the callback may result in the arc_buf being destroyed.  However,
+ * it will not result in the *last* arc_buf being destroyed, hence the data
+ * will remain cached in the ARC. We make a copy of the arc buffer here so
+ * that we can process the callback without holding any locks.
+ *
+ * It's possible that the callback is already in the process of being cleared
+ * by another thread.  In this case we can not clear the callback.
+ *
+ * Returns B_TRUE if the callback was successfully called and cleared.
  */
-int
-arc_buf_evict(arc_buf_t *buf)
+boolean_t
+arc_clear_callback(arc_buf_t *buf)
 {
 	arc_buf_hdr_t *hdr;
 	kmutex_t *hash_lock;
-	arc_buf_t **bufp;
+	arc_evict_func_t *efunc = buf->b_efunc;
+	void *private = buf->b_private;
 
 	mutex_enter(&buf->b_evict_lock);
 	hdr = buf->b_hdr;
@@ -3259,17 +3272,16 @@ arc_buf_evict(arc_buf_t *buf)
 		 */
 		ASSERT(buf->b_data == NULL);
 		mutex_exit(&buf->b_evict_lock);
-		return (0);
+		return (B_FALSE);
 	} else if (buf->b_data == NULL) {
-		arc_buf_t copy = *buf; /* structure assignment */
 		/*
 		 * We are on the eviction list; process this buffer now
 		 * but let arc_do_user_evicts() do the reaping.
 		 */
 		buf->b_efunc = NULL;
 		mutex_exit(&buf->b_evict_lock);
-		VERIFY(copy.b_efunc(&copy) == 0);
-		return (1);
+		VERIFY0(efunc(private));
+		return (B_TRUE);
 	}
 	hash_lock = HDR_LOCK(hdr);
 	mutex_enter(hash_lock);
@@ -3279,48 +3291,21 @@ arc_buf_evict(arc_buf_t *buf)
 	ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt);
 	ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
 
-	/*
-	 * Pull this buffer off of the hdr
-	 */
-	bufp = &hdr->b_buf;
-	while (*bufp != buf)
-		bufp = &(*bufp)->b_next;
-	*bufp = buf->b_next;
-
-	ASSERT(buf->b_data != NULL);
-	arc_buf_destroy(buf, FALSE, FALSE);
-
-	if (hdr->b_datacnt == 0) {
-		arc_state_t *old_state = hdr->b_state;
-		arc_state_t *evicted_state;
-
-		ASSERT(hdr->b_buf == NULL);
-		ASSERT(refcount_is_zero(&hdr->b_refcnt));
-
-		evicted_state =
-		    (old_state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
-
-		mutex_enter(&old_state->arcs_mtx);
-		mutex_enter(&evicted_state->arcs_mtx);
-
-		arc_change_state(evicted_state, hdr, hash_lock);
-		ASSERT(HDR_IN_HASH_TABLE(hdr));
-		hdr->b_flags |= ARC_IN_HASH_TABLE;
-		hdr->b_flags &= ~ARC_BUF_AVAILABLE;
-
-		mutex_exit(&evicted_state->arcs_mtx);
-		mutex_exit(&old_state->arcs_mtx);
-	}
-	mutex_exit(hash_lock);
-	mutex_exit(&buf->b_evict_lock);
-
-	VERIFY(buf->b_efunc(buf) == 0);
 	buf->b_efunc = NULL;
 	buf->b_private = NULL;
-	buf->b_hdr = NULL;
-	buf->b_next = NULL;
-	kmem_cache_free(buf_cache, buf);
-	return (1);
+
+	if (hdr->b_datacnt > 1) {
+		mutex_exit(&buf->b_evict_lock);
+		arc_buf_destroy(buf, FALSE, TRUE);
+	} else {
+		ASSERT(buf == hdr->b_buf);
+		hdr->b_flags |= ARC_BUF_AVAILABLE;
+		mutex_exit(&buf->b_evict_lock);
+	}
+
+	mutex_exit(hash_lock);
+	VERIFY0(efunc(private));
+	return (B_TRUE);
 }
 
 /*
@@ -3466,17 +3451,6 @@ arc_released(arc_buf_t *buf)
 	return (released);
 }
 
-int
-arc_has_callback(arc_buf_t *buf)
-{
-	int callback;
-
-	mutex_enter(&buf->b_evict_lock);
-	callback = (buf->b_efunc != NULL);
-	mutex_exit(&buf->b_evict_lock);
-	return (callback);
-}
-
 #ifdef ZFS_DEBUG
 int
 arc_referenced(arc_buf_t *buf)
diff --git a/uts/common/fs/zfs/dbuf.c b/uts/common/fs/zfs/dbuf.c
index a635f245131f..759cfcfb8e3a 100644
--- a/uts/common/fs/zfs/dbuf.c
+++ b/uts/common/fs/zfs/dbuf.c
@@ -181,8 +181,7 @@ dbuf_hash_insert(dmu_buf_impl_t *db)
 }
 
 /*
- * Remove an entry from the hash table.  This operation will
- * fail if there are any existing holds on the db.
+ * Remove an entry from the hash table.  It must be in the EVICTING state.
  */
 static void
 dbuf_hash_remove(dmu_buf_impl_t *db)
@@ -194,7 +193,7 @@ dbuf_hash_remove(dmu_buf_impl_t *db)
 	dmu_buf_impl_t *dbf, **dbp;
 
 	/*
-	 * We musn't hold db_mtx to maintin lock ordering:
+	 * We musn't hold db_mtx to maintain lock ordering:
 	 * DBUF_HASH_MUTEX > db_mtx.
 	 */
 	ASSERT(refcount_is_zero(&db->db_holds));
@@ -431,7 +430,6 @@ static void
 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf)
 {
 	ASSERT(MUTEX_HELD(&db->db_mtx));
-	ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf));
 	db->db_buf = buf;
 	if (buf != NULL) {
 		ASSERT(buf->b_data != NULL);
@@ -1544,12 +1542,15 @@ dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx)
  * when we are not holding the dn_dbufs_mtx, we can't clear the
  * entry in the dn_dbufs list.  We have to wait until dbuf_destroy()
  * in this case.  For callers from the DMU we will usually see:
- *	dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy()
+ *	dbuf_clear()->arc_clear_callback()->dbuf_do_evict()->dbuf_destroy()
  * For the arc callback, we will usually see:
  *	dbuf_do_evict()->dbuf_clear();dbuf_destroy()
  * Sometimes, though, we will get a mix of these two:
- *	DMU: dbuf_clear()->arc_buf_evict()
+ *	DMU: dbuf_clear()->arc_clear_callback()
  *	ARC: dbuf_do_evict()->dbuf_destroy()
+ *
+ * This routine will dissociate the dbuf from the arc, by calling
+ * arc_clear_callback(), but will not evict the data from the ARC.
  */
 void
 dbuf_clear(dmu_buf_impl_t *db)
@@ -1557,7 +1558,7 @@ dbuf_clear(dmu_buf_impl_t *db)
 	dnode_t *dn;
 	dmu_buf_impl_t *parent = db->db_parent;
 	dmu_buf_impl_t *dndb;
-	int dbuf_gone = FALSE;
+	boolean_t dbuf_gone = B_FALSE;
 
 	ASSERT(MUTEX_HELD(&db->db_mtx));
 	ASSERT(refcount_is_zero(&db->db_holds));
@@ -1603,7 +1604,7 @@ dbuf_clear(dmu_buf_impl_t *db)
 	}
 
 	if (db->db_buf)
-		dbuf_gone = arc_buf_evict(db->db_buf);
+		dbuf_gone = arc_clear_callback(db->db_buf);
 
 	if (!dbuf_gone)
 		mutex_exit(&db->db_mtx);
@@ -1771,8 +1772,7 @@ dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid,
 static int
 dbuf_do_evict(void *private)
 {
-	arc_buf_t *buf = private;
-	dmu_buf_impl_t *db = buf->b_private;
+	dmu_buf_impl_t *db = private;
 
 	if (!MUTEX_HELD(&db->db_mtx))
 		mutex_enter(&db->db_mtx);
@@ -2135,11 +2135,23 @@ dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag)
 			 * block on-disk. If so, then we simply evict
 			 * ourselves.
 			 */
-			if (!DBUF_IS_CACHEABLE(db) ||
-			    arc_buf_eviction_needed(db->db_buf))
+			if (!DBUF_IS_CACHEABLE(db)) {
+				if (db->db_blkptr != NULL &&
+				    !BP_IS_HOLE(db->db_blkptr) &&
+				    !BP_IS_EMBEDDED(db->db_blkptr)) {
+					spa_t *spa =
+					    dmu_objset_spa(db->db_objset);
+					blkptr_t bp = *db->db_blkptr;
+					dbuf_clear(db);
+					arc_freed(spa, &bp);
+				} else {
+					dbuf_clear(db);
+				}
+			} else if (arc_buf_eviction_needed(db->db_buf)) {
 				dbuf_clear(db);
-			else
+			} else {
 				mutex_exit(&db->db_mtx);
+			}
 		}
 	} else {
 		mutex_exit(&db->db_mtx);
diff --git a/uts/common/fs/zfs/sys/arc.h b/uts/common/fs/zfs/sys/arc.h
index 15ef0f58d02b..04a20c13eeef 100644
--- a/uts/common/fs/zfs/sys/arc.h
+++ b/uts/common/fs/zfs/sys/arc.h
@@ -95,7 +95,6 @@ boolean_t arc_buf_remove_ref(arc_buf_t *buf, void *tag);
 int arc_buf_size(arc_buf_t *buf);
 void arc_release(arc_buf_t *buf, void *tag);
 int arc_released(arc_buf_t *buf);
-int arc_has_callback(arc_buf_t *buf);
 void arc_buf_freeze(arc_buf_t *buf);
 void arc_buf_thaw(arc_buf_t *buf);
 boolean_t arc_buf_eviction_needed(arc_buf_t *buf);
@@ -114,7 +113,7 @@ zio_t *arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
 void arc_freed(spa_t *spa, const blkptr_t *bp);
 
 void arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private);
-int arc_buf_evict(arc_buf_t *buf);
+boolean_t arc_clear_callback(arc_buf_t *buf);
 
 void arc_flush(spa_t *spa);
 void arc_tempreserve_clear(uint64_t reserve);

From eee619b6068db9b5bba4c3c0a49fd48923216448 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Fri, 18 Jul 2014 18:05:09 +0000
Subject: [PATCH 018/380] 4753 increase number of outstanding async writes when
 sync task is waiting Reviewed by: Matthew Ahrens <mahrens@delphix.com>
 Reviewed by: George Wilson <george.wilson@delphix.com> Reviewed by: Adam
 Leventhal <ahl@delphix.com> Reviewed by: Christopher Siden
 <christopher.siden@delphix.com> Reviewed by: Dan McDonald <danmcd@omniti.com>
 Approved by: Garrett D'Amore <garrett@damore.org>

illumos/illumos-gate@73527f441cbbd953fa42cc5a30a413bad75f24a9
---
 uts/common/fs/zfs/spa_misc.c   | 10 ++++++++++
 uts/common/fs/zfs/sys/spa.h    |  1 +
 uts/common/fs/zfs/sys/txg.h    |  3 ++-
 uts/common/fs/zfs/txg.c        | 20 +++++++++++++++++++-
 uts/common/fs/zfs/vdev_queue.c | 16 ++++++++++++----
 5 files changed, 44 insertions(+), 6 deletions(-)

diff --git a/uts/common/fs/zfs/spa_misc.c b/uts/common/fs/zfs/spa_misc.c
index ff8739e8dd85..050436272696 100644
--- a/uts/common/fs/zfs/spa_misc.c
+++ b/uts/common/fs/zfs/spa_misc.c
@@ -1876,6 +1876,16 @@ spa_writeable(spa_t *spa)
 	return (!!(spa->spa_mode & FWRITE));
 }
 
+/*
+ * Returns true if there is a pending sync task in any of the current
+ * syncing txg, the current quiescing txg, or the current open txg.
+ */
+boolean_t
+spa_has_pending_synctask(spa_t *spa)
+{
+	return (!txg_all_lists_empty(&spa->spa_dsl_pool->dp_sync_tasks));
+}
+
 int
 spa_mode(spa_t *spa)
 {
diff --git a/uts/common/fs/zfs/sys/spa.h b/uts/common/fs/zfs/sys/spa.h
index 36b6c93bde8a..6c7d34cb8235 100644
--- a/uts/common/fs/zfs/sys/spa.h
+++ b/uts/common/fs/zfs/sys/spa.h
@@ -780,6 +780,7 @@ extern uint64_t bp_get_dsize(spa_t *spa, const blkptr_t *bp);
 extern boolean_t spa_has_slogs(spa_t *spa);
 extern boolean_t spa_is_root(spa_t *spa);
 extern boolean_t spa_writeable(spa_t *spa);
+extern boolean_t spa_has_pending_synctask(spa_t *spa);
 
 extern int spa_mode(spa_t *spa);
 extern uint64_t strtonum(const char *str, char **nptr);
diff --git a/uts/common/fs/zfs/sys/txg.h b/uts/common/fs/zfs/sys/txg.h
index e96c1fa8aac1..77e2dba251cd 100644
--- a/uts/common/fs/zfs/sys/txg.h
+++ b/uts/common/fs/zfs/sys/txg.h
@@ -23,7 +23,7 @@
  * Use is subject to license terms.
  */
 /*
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #ifndef _SYS_TXG_H
@@ -112,6 +112,7 @@ extern boolean_t txg_sync_waiting(struct dsl_pool *dp);
 extern void txg_list_create(txg_list_t *tl, size_t offset);
 extern void txg_list_destroy(txg_list_t *tl);
 extern boolean_t txg_list_empty(txg_list_t *tl, uint64_t txg);
+extern boolean_t txg_all_lists_empty(txg_list_t *tl);
 extern boolean_t txg_list_add(txg_list_t *tl, void *p, uint64_t txg);
 extern boolean_t txg_list_add_tail(txg_list_t *tl, void *p, uint64_t txg);
 extern void *txg_list_remove(txg_list_t *tl, uint64_t txg);
diff --git a/uts/common/fs/zfs/txg.c b/uts/common/fs/zfs/txg.c
index 2bee4294b444..191259e75b92 100644
--- a/uts/common/fs/zfs/txg.c
+++ b/uts/common/fs/zfs/txg.c
@@ -21,7 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Portions Copyright 2011 Martin Matuska
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -717,6 +717,24 @@ txg_list_empty(txg_list_t *tl, uint64_t txg)
 	return (tl->tl_head[txg & TXG_MASK] == NULL);
 }
 
+/*
+ * Returns true if all txg lists are empty.
+ *
+ * Warning: this is inherently racy (an item could be added immediately after this
+ * function returns). We don't bother with the lock because it wouldn't change the
+ * semantics.
+ */
+boolean_t
+txg_all_lists_empty(txg_list_t *tl)
+{
+	for (int i = 0; i < TXG_SIZE; i++) {
+		if (!txg_list_empty(tl, i)) {
+			return (B_FALSE);
+		}
+	}
+	return (B_TRUE);
+}
+
 /*
  * Add an entry to the list (unless it's already on the list).
  * Returns B_TRUE if it was actually added.
diff --git a/uts/common/fs/zfs/vdev_queue.c b/uts/common/fs/zfs/vdev_queue.c
index a394b80c67f4..561e6a4482eb 100644
--- a/uts/common/fs/zfs/vdev_queue.c
+++ b/uts/common/fs/zfs/vdev_queue.c
@@ -24,7 +24,7 @@
  */
 
 /*
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -350,14 +350,23 @@ vdev_queue_class_min_active(zio_priority_t p)
 }
 
 static int
-vdev_queue_max_async_writes(uint64_t dirty)
+vdev_queue_max_async_writes(spa_t *spa)
 {
 	int writes;
+	uint64_t dirty = spa->spa_dsl_pool->dp_dirty_total;
 	uint64_t min_bytes = zfs_dirty_data_max *
 	    zfs_vdev_async_write_active_min_dirty_percent / 100;
 	uint64_t max_bytes = zfs_dirty_data_max *
 	    zfs_vdev_async_write_active_max_dirty_percent / 100;
 
+	/*
+	 * Sync tasks correspond to interactive user actions. To reduce the
+	 * execution time of those actions we push data out as fast as possible.
+	 */
+	if (spa_has_pending_synctask(spa)) {
+		return (zfs_vdev_async_write_max_active);
+	}
+
 	if (dirty < min_bytes)
 		return (zfs_vdev_async_write_min_active);
 	if (dirty > max_bytes)
@@ -390,8 +399,7 @@ vdev_queue_class_max_active(spa_t *spa, zio_priority_t p)
 	case ZIO_PRIORITY_ASYNC_READ:
 		return (zfs_vdev_async_read_max_active);
 	case ZIO_PRIORITY_ASYNC_WRITE:
-		return (vdev_queue_max_async_writes(
-		    spa->spa_dsl_pool->dp_dirty_total));
+		return (vdev_queue_max_async_writes(spa));
 	case ZIO_PRIORITY_SCRUB:
 		return (zfs_vdev_scrub_max_active);
 	default:

From 1c202a05f319a72d8f2c534c4a881511bc56c95d Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Fri, 18 Jul 2014 18:09:20 +0000
Subject: [PATCH 019/380] 5008 lock contention (rrw_exit) while running a read
 only load Reviewed by: Matthew Ahrens <matthew.ahrens@delphix.com> Reviewed
 by: George Wilson <george.wilson@delphix.com> Reviewed by: Alex Reece
 <alex.reece@delphix.com> Reviewed by: Christopher Siden
 <christopher.siden@delphix.com> Reviewed by: Richard Yao <ryao@gentoo.org>
 Reviewed by: Saso Kiselkov <skiselkov.ml@gmail.com> Approved by: Garrett
 D'Amore <garrett@damore.org>

illumos/illumos-gate@c9030f6c93613fe30ee0c16f92b96da7816ac052
---
 uts/common/fs/zfs/rrwlock.c        | 88 ++++++++++++++++++++++++++++++
 uts/common/fs/zfs/sys/rrwlock.h    | 25 +++++++++
 uts/common/fs/zfs/sys/zfs_vfsops.h |  2 +-
 uts/common/fs/zfs/sys/zfs_znode.h  |  4 +-
 uts/common/fs/zfs/zfs_ioctl.c      |  6 +-
 uts/common/fs/zfs/zfs_vfsops.c     | 14 ++---
 uts/common/fs/zfs/zfs_znode.c      |  2 +-
 7 files changed, 127 insertions(+), 14 deletions(-)

diff --git a/uts/common/fs/zfs/rrwlock.c b/uts/common/fs/zfs/rrwlock.c
index 8e80166c7d14..e3aa01a31c77 100644
--- a/uts/common/fs/zfs/rrwlock.c
+++ b/uts/common/fs/zfs/rrwlock.c
@@ -286,3 +286,91 @@ rrw_tsd_destroy(void *arg)
 		    (void *)curthread, (void *)rn->rn_rrl);
 	}
 }
+
+/*
+ * A reader-mostly lock implementation, tuning above reader-writer locks
+ * for hightly parallel read acquisitions, while pessimizing writes.
+ *
+ * The idea is to split single busy lock into array of locks, so that
+ * each reader can lock only one of them for read, depending on result
+ * of simple hash function.  That proportionally reduces lock congestion.
+ * Writer same time has to sequentially aquire write on all the locks.
+ * That makes write aquisition proportionally slower, but in places where
+ * it is used (filesystem unmount) performance is not critical.
+ *
+ * All the functions below are direct wrappers around functions above.
+ */
+void
+rrm_init(rrmlock_t *rrl, boolean_t track_all)
+{
+	int i;
+
+	for (i = 0; i < RRM_NUM_LOCKS; i++)
+		rrw_init(&rrl->locks[i], track_all);
+}
+
+void
+rrm_destroy(rrmlock_t *rrl)
+{
+	int i;
+
+	for (i = 0; i < RRM_NUM_LOCKS; i++)
+		rrw_destroy(&rrl->locks[i]);
+}
+
+void
+rrm_enter(rrmlock_t *rrl, krw_t rw, void *tag)
+{
+	if (rw == RW_READER)
+		rrm_enter_read(rrl, tag);
+	else
+		rrm_enter_write(rrl);
+}
+
+/*
+ * This maps the current thread to a specific lock.  Note that the lock
+ * must be released by the same thread that acquired it.  We do this
+ * mapping by taking the thread pointer mod a prime number.  We examine
+ * only the low 32 bits of the thread pointer, because 32-bit division
+ * is faster than 64-bit division, and the high 32 bits have little
+ * entropy anyway.
+ */
+#define	RRM_TD_LOCK()	(((uint32_t)(uintptr_t)(curthread)) % RRM_NUM_LOCKS)
+
+void
+rrm_enter_read(rrmlock_t *rrl, void *tag)
+{
+	rrw_enter_read(&rrl->locks[RRM_TD_LOCK()], tag);
+}
+
+void
+rrm_enter_write(rrmlock_t *rrl)
+{
+	int i;
+
+	for (i = 0; i < RRM_NUM_LOCKS; i++)
+		rrw_enter_write(&rrl->locks[i]);
+}
+
+void
+rrm_exit(rrmlock_t *rrl, void *tag)
+{
+	int i;
+
+	if (rrl->locks[0].rr_writer == curthread) {
+		for (i = 0; i < RRM_NUM_LOCKS; i++)
+			rrw_exit(&rrl->locks[i], tag);
+	} else {
+		rrw_exit(&rrl->locks[RRM_TD_LOCK()], tag);
+	}
+}
+
+boolean_t
+rrm_held(rrmlock_t *rrl, krw_t rw)
+{
+	if (rw == RW_WRITER) {
+		return (rrw_held(&rrl->locks[0], rw));
+	} else {
+		return (rrw_held(&rrl->locks[RRM_TD_LOCK()], rw));
+	}
+}
diff --git a/uts/common/fs/zfs/sys/rrwlock.h b/uts/common/fs/zfs/sys/rrwlock.h
index e1e6d3122373..bddf0768ae07 100644
--- a/uts/common/fs/zfs/sys/rrwlock.h
+++ b/uts/common/fs/zfs/sys/rrwlock.h
@@ -80,6 +80,31 @@ void rrw_tsd_destroy(void *arg);
 #define	RRW_LOCK_HELD(x) \
 	(rrw_held(x, RW_WRITER) || rrw_held(x, RW_READER))
 
+/*
+ * A reader-mostly lock implementation, tuning above reader-writer locks
+ * for hightly parallel read acquisitions, pessimizing write acquisitions.
+ *
+ * This should be a prime number.  See comment in rrwlock.c near
+ * RRM_TD_LOCK() for details.
+ */
+#define	RRM_NUM_LOCKS		17
+typedef struct rrmlock {
+	rrwlock_t	locks[RRM_NUM_LOCKS];
+} rrmlock_t;
+
+void rrm_init(rrmlock_t *rrl, boolean_t track_all);
+void rrm_destroy(rrmlock_t *rrl);
+void rrm_enter(rrmlock_t *rrl, krw_t rw, void *tag);
+void rrm_enter_read(rrmlock_t *rrl, void *tag);
+void rrm_enter_write(rrmlock_t *rrl);
+void rrm_exit(rrmlock_t *rrl, void *tag);
+boolean_t rrm_held(rrmlock_t *rrl, krw_t rw);
+
+#define	RRM_READ_HELD(x)	rrm_held(x, RW_READER)
+#define	RRM_WRITE_HELD(x)	rrm_held(x, RW_WRITER)
+#define	RRM_LOCK_HELD(x) \
+	(rrm_held(x, RW_WRITER) || rrm_held(x, RW_READER))
+
 #ifdef	__cplusplus
 }
 #endif
diff --git a/uts/common/fs/zfs/sys/zfs_vfsops.h b/uts/common/fs/zfs/sys/zfs_vfsops.h
index 9af5cef05863..b382d27bdfd8 100644
--- a/uts/common/fs/zfs/sys/zfs_vfsops.h
+++ b/uts/common/fs/zfs/sys/zfs_vfsops.h
@@ -64,7 +64,7 @@ struct zfsvfs {
 	int		z_norm;		/* normalization flags */
 	boolean_t	z_atime;	/* enable atimes mount option */
 	boolean_t	z_unmounted;	/* unmounted */
-	rrwlock_t	z_teardown_lock;
+	rrmlock_t	z_teardown_lock;
 	krwlock_t	z_teardown_inactive_lock;
 	list_t		z_all_znodes;	/* all vnodes in the fs */
 	kmutex_t	z_znodes_lock;	/* lock for z_all_znodes */
diff --git a/uts/common/fs/zfs/sys/zfs_znode.h b/uts/common/fs/zfs/sys/zfs_znode.h
index 43986afda237..bc389b4a4366 100644
--- a/uts/common/fs/zfs/sys/zfs_znode.h
+++ b/uts/common/fs/zfs/sys/zfs_znode.h
@@ -238,7 +238,7 @@ typedef struct znode {
 /* Called on entry to each ZFS vnode and vfs operation  */
 #define	ZFS_ENTER(zfsvfs) \
 	{ \
-		rrw_enter_read(&(zfsvfs)->z_teardown_lock, FTAG); \
+		rrm_enter_read(&(zfsvfs)->z_teardown_lock, FTAG); \
 		if ((zfsvfs)->z_unmounted) { \
 			ZFS_EXIT(zfsvfs); \
 			return (EIO); \
@@ -246,7 +246,7 @@ typedef struct znode {
 	}
 
 /* Must be called before exiting the vop */
-#define	ZFS_EXIT(zfsvfs) rrw_exit(&(zfsvfs)->z_teardown_lock, FTAG)
+#define	ZFS_EXIT(zfsvfs) rrm_exit(&(zfsvfs)->z_teardown_lock, FTAG)
 
 /* Verifies the znode is valid */
 #define	ZFS_VERIFY_ZP(zp) \
diff --git a/uts/common/fs/zfs/zfs_ioctl.c b/uts/common/fs/zfs/zfs_ioctl.c
index 7f07ec61f615..0cec1c2adad3 100644
--- a/uts/common/fs/zfs/zfs_ioctl.c
+++ b/uts/common/fs/zfs/zfs_ioctl.c
@@ -1420,7 +1420,7 @@ zfsvfs_hold(const char *name, void *tag, zfsvfs_t **zfvp, boolean_t writer)
 	if (getzfsvfs(name, zfvp) != 0)
 		error = zfsvfs_create(name, zfvp);
 	if (error == 0) {
-		rrw_enter(&(*zfvp)->z_teardown_lock, (writer) ? RW_WRITER :
+		rrm_enter(&(*zfvp)->z_teardown_lock, (writer) ? RW_WRITER :
 		    RW_READER, tag);
 		if ((*zfvp)->z_unmounted) {
 			/*
@@ -1428,7 +1428,7 @@ zfsvfs_hold(const char *name, void *tag, zfsvfs_t **zfvp, boolean_t writer)
 			 * thread should be just about to disassociate the
 			 * objset from the zfsvfs.
 			 */
-			rrw_exit(&(*zfvp)->z_teardown_lock, tag);
+			rrm_exit(&(*zfvp)->z_teardown_lock, tag);
 			return (SET_ERROR(EBUSY));
 		}
 	}
@@ -1438,7 +1438,7 @@ zfsvfs_hold(const char *name, void *tag, zfsvfs_t **zfvp, boolean_t writer)
 static void
 zfsvfs_rele(zfsvfs_t *zfsvfs, void *tag)
 {
-	rrw_exit(&zfsvfs->z_teardown_lock, tag);
+	rrm_exit(&zfsvfs->z_teardown_lock, tag);
 
 	if (zfsvfs->z_vfs) {
 		VFS_RELE(zfsvfs->z_vfs);
diff --git a/uts/common/fs/zfs/zfs_vfsops.c b/uts/common/fs/zfs/zfs_vfsops.c
index b7aa2e076bb2..8e3bf691672a 100644
--- a/uts/common/fs/zfs/zfs_vfsops.c
+++ b/uts/common/fs/zfs/zfs_vfsops.c
@@ -1004,7 +1004,7 @@ zfsvfs_create(const char *osname, zfsvfs_t **zfvp)
 	mutex_init(&zfsvfs->z_lock, NULL, MUTEX_DEFAULT, NULL);
 	list_create(&zfsvfs->z_all_znodes, sizeof (znode_t),
 	    offsetof(znode_t, z_link_node));
-	rrw_init(&zfsvfs->z_teardown_lock, B_FALSE);
+	rrm_init(&zfsvfs->z_teardown_lock, B_FALSE);
 	rw_init(&zfsvfs->z_teardown_inactive_lock, NULL, RW_DEFAULT, NULL);
 	rw_init(&zfsvfs->z_fuid_lock, NULL, RW_DEFAULT, NULL);
 	for (i = 0; i != ZFS_OBJ_MTX_SZ; i++)
@@ -1119,7 +1119,7 @@ zfsvfs_free(zfsvfs_t *zfsvfs)
 	mutex_destroy(&zfsvfs->z_znodes_lock);
 	mutex_destroy(&zfsvfs->z_lock);
 	list_destroy(&zfsvfs->z_all_znodes);
-	rrw_destroy(&zfsvfs->z_teardown_lock);
+	rrm_destroy(&zfsvfs->z_teardown_lock);
 	rw_destroy(&zfsvfs->z_teardown_inactive_lock);
 	rw_destroy(&zfsvfs->z_fuid_lock);
 	for (i = 0; i != ZFS_OBJ_MTX_SZ; i++)
@@ -1784,7 +1784,7 @@ zfsvfs_teardown(zfsvfs_t *zfsvfs, boolean_t unmounting)
 {
 	znode_t	*zp;
 
-	rrw_enter(&zfsvfs->z_teardown_lock, RW_WRITER, FTAG);
+	rrm_enter(&zfsvfs->z_teardown_lock, RW_WRITER, FTAG);
 
 	if (!unmounting) {
 		/*
@@ -1814,7 +1814,7 @@ zfsvfs_teardown(zfsvfs_t *zfsvfs, boolean_t unmounting)
 	 */
 	if (!unmounting && (zfsvfs->z_unmounted || zfsvfs->z_os == NULL)) {
 		rw_exit(&zfsvfs->z_teardown_inactive_lock);
-		rrw_exit(&zfsvfs->z_teardown_lock, FTAG);
+		rrm_exit(&zfsvfs->z_teardown_lock, FTAG);
 		return (SET_ERROR(EIO));
 	}
 
@@ -1841,7 +1841,7 @@ zfsvfs_teardown(zfsvfs_t *zfsvfs, boolean_t unmounting)
 	 */
 	if (unmounting) {
 		zfsvfs->z_unmounted = B_TRUE;
-		rrw_exit(&zfsvfs->z_teardown_lock, FTAG);
+		rrm_exit(&zfsvfs->z_teardown_lock, FTAG);
 		rw_exit(&zfsvfs->z_teardown_inactive_lock);
 	}
 
@@ -2073,7 +2073,7 @@ zfs_resume_fs(zfsvfs_t *zfsvfs, const char *osname)
 	znode_t *zp;
 	uint64_t sa_obj = 0;
 
-	ASSERT(RRW_WRITE_HELD(&zfsvfs->z_teardown_lock));
+	ASSERT(RRM_WRITE_HELD(&zfsvfs->z_teardown_lock));
 	ASSERT(RW_WRITE_HELD(&zfsvfs->z_teardown_inactive_lock));
 
 	/*
@@ -2129,7 +2129,7 @@ zfs_resume_fs(zfsvfs_t *zfsvfs, const char *osname)
 bail:
 	/* release the VOPs */
 	rw_exit(&zfsvfs->z_teardown_inactive_lock);
-	rrw_exit(&zfsvfs->z_teardown_lock, FTAG);
+	rrm_exit(&zfsvfs->z_teardown_lock, FTAG);
 
 	if (err) {
 		/*
diff --git a/uts/common/fs/zfs/zfs_znode.c b/uts/common/fs/zfs/zfs_znode.c
index 7073e2fd0f6b..7577250408df 100644
--- a/uts/common/fs/zfs/zfs_znode.c
+++ b/uts/common/fs/zfs/zfs_znode.c
@@ -276,7 +276,7 @@ zfs_znode_move(void *buf, void *newbuf, size_t size, void *arg)
 	 * can safely ensure that the filesystem is not and will not be
 	 * unmounted. The next statement is equivalent to ZFS_ENTER().
 	 */
-	rrw_enter(&zfsvfs->z_teardown_lock, RW_READER, FTAG);
+	rrm_enter(&zfsvfs->z_teardown_lock, RW_READER, FTAG);
 	if (zfsvfs->z_unmounted) {
 		ZFS_EXIT(zfsvfs);
 		rw_exit(&zfsvfs_lock);

From 79f6f7c83288fe0ce092d71943383659814984c3 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Wed, 23 Jul 2014 08:00:34 +0000
Subject: [PATCH 020/380] 4976 zfs should only avoid writing to a failing
 non-redundant top-level vdev 4977 mdb error in ::spa_space from space_cb() if
 a metaslab's ms_sm is NULL 4978 ztest fails in get_metaslab_refcount() 4979
 extend free space histogram to device and pool 4980 metaslabs should have a
 fragmentation metric 4981 remove fragmented ops vector from block allocator
 4982 space_map object should proactively upgrade when feature is enabled 4983
 need to collect metaslab information via mdb 4984 device selection should use
 fragmentation metric Reviewed by: Matthew Ahrens <mahrens@delphix.com>
 Reviewed by: Adam Leventhal <adam.leventhal@delphix.com> Reviewed by:
 Christopher Siden <christopher.siden@delphix.com> Approved by: Garrett
 D'Amore <garrett@damore.org>

illumos/illumos-gate@2e4c998613148111f2fc5371085331ffb39122ff
---
 common/zfs/zpool_prop.c               |   4 +-
 uts/common/fs/zfs/metaslab.c          | 608 ++++++++++++++++++++------
 uts/common/fs/zfs/range_tree.c        |   2 +
 uts/common/fs/zfs/spa.c               |  18 +-
 uts/common/fs/zfs/space_map.c         |   4 +-
 uts/common/fs/zfs/sys/metaslab.h      |  67 +--
 uts/common/fs/zfs/sys/metaslab_impl.h |  36 +-
 uts/common/fs/zfs/sys/space_map.h     |   8 +-
 uts/common/fs/zfs/sys/zfs_debug.h     |  17 +-
 uts/common/fs/zfs/vdev.c              |  31 +-
 uts/common/sys/fs/zfs.h               |   9 +
 11 files changed, 602 insertions(+), 202 deletions(-)

diff --git a/common/zfs/zpool_prop.c b/common/zfs/zpool_prop.c
index 51041a8766cc..a400f821e2e2 100644
--- a/common/zfs/zpool_prop.c
+++ b/common/zfs/zpool_prop.c
@@ -21,7 +21,7 @@
 /*
  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
- * Copyright (c) 2012 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #include <sys/zio.h>
@@ -87,6 +87,8 @@ zpool_prop_init(void)
 	    PROP_READONLY, ZFS_TYPE_POOL, "<size>", "ALLOC");
 	zprop_register_number(ZPOOL_PROP_EXPANDSZ, "expandsize", 0,
 	    PROP_READONLY, ZFS_TYPE_POOL, "<size>", "EXPANDSZ");
+	zprop_register_number(ZPOOL_PROP_FRAGMENTATION, "fragmentation", 0,
+	    PROP_READONLY, ZFS_TYPE_POOL, "<percent>", "FRAG");
 	zprop_register_number(ZPOOL_PROP_CAPACITY, "capacity", 0, PROP_READONLY,
 	    ZFS_TYPE_POOL, "<size>", "CAP");
 	zprop_register_number(ZPOOL_PROP_GUID, "guid", 0, PROP_READONLY,
diff --git a/uts/common/fs/zfs/metaslab.c b/uts/common/fs/zfs/metaslab.c
index 2831fb2f7bee..fcb290be317d 100644
--- a/uts/common/fs/zfs/metaslab.c
+++ b/uts/common/fs/zfs/metaslab.c
@@ -32,6 +32,7 @@
 #include <sys/vdev_impl.h>
 #include <sys/zio.h>
 #include <sys/spa_impl.h>
+#include <sys/zfeature.h>
 
 /*
  * Allow allocations to switch to gang blocks quickly. We do this to
@@ -79,7 +80,7 @@ int zfs_metaslab_condense_block_threshold = 4;
 /*
  * The zfs_mg_noalloc_threshold defines which metaslab groups should
  * be eligible for allocation. The value is defined as a percentage of
- * a free space. Metaslab groups that have more free space than
+ * free space. Metaslab groups that have more free space than
  * zfs_mg_noalloc_threshold are always eligible for allocations. Once
  * a metaslab group's free space is less than or equal to the
  * zfs_mg_noalloc_threshold the allocator will avoid allocating to that
@@ -91,6 +92,23 @@ int zfs_metaslab_condense_block_threshold = 4;
  */
 int zfs_mg_noalloc_threshold = 0;
 
+/*
+ * Metaslab groups are considered eligible for allocations if their
+ * fragmenation metric (measured as a percentage) is less than or equal to
+ * zfs_mg_fragmentation_threshold. If a metaslab group exceeds this threshold
+ * then it will be skipped unless all metaslab groups within the metaslab
+ * class have also crossed this threshold.
+ */
+int zfs_mg_fragmentation_threshold = 85;
+
+/*
+ * Allow metaslabs to keep their active state as long as their fragmentation
+ * percentage is less than or equal to zfs_metaslab_fragmentation_threshold. An
+ * active metaslab that exceeds this threshold will no longer keep its active
+ * status allowing better metaslabs to be selected.
+ */
+int zfs_metaslab_fragmentation_threshold = 70;
+
 /*
  * When set will load all metaslabs when pool is first opened.
  */
@@ -135,11 +153,6 @@ int metaslab_load_pct = 50;
  */
 int metaslab_unload_delay = TXG_SIZE * 2;
 
-/*
- * Should we be willing to write data to degraded vdevs?
- */
-boolean_t zfs_write_to_degraded = B_FALSE;
-
 /*
  * Max number of metaslabs per group to preload.
  */
@@ -151,10 +164,21 @@ int metaslab_preload_limit = SPA_DVAS_PER_BP;
 boolean_t metaslab_preload_enabled = B_TRUE;
 
 /*
- * Enable/disable additional weight factor for each metaslab.
+ * Enable/disable fragmentation weighting on metaslabs.
  */
-boolean_t metaslab_weight_factor_enable = B_FALSE;
+boolean_t metaslab_fragmentation_factor_enabled = B_TRUE;
 
+/*
+ * Enable/disable lba weighting (i.e. outer tracks are given preference).
+ */
+boolean_t metaslab_lba_weighting_enabled = B_TRUE;
+
+/*
+ * Enable/disable metaslab group biasing.
+ */
+boolean_t metaslab_bias_enabled = B_TRUE;
+
+static uint64_t metaslab_fragmentation(metaslab_t *);
 
 /*
  * ==========================================================================
@@ -247,6 +271,121 @@ metaslab_class_get_dspace(metaslab_class_t *mc)
 	return (spa_deflate(mc->mc_spa) ? mc->mc_dspace : mc->mc_space);
 }
 
+void
+metaslab_class_histogram_verify(metaslab_class_t *mc)
+{
+	vdev_t *rvd = mc->mc_spa->spa_root_vdev;
+	uint64_t *mc_hist;
+	int i;
+
+	if ((zfs_flags & ZFS_DEBUG_HISTOGRAM_VERIFY) == 0)
+		return;
+
+	mc_hist = kmem_zalloc(sizeof (uint64_t) * RANGE_TREE_HISTOGRAM_SIZE,
+	    KM_SLEEP);
+
+	for (int c = 0; c < rvd->vdev_children; c++) {
+		vdev_t *tvd = rvd->vdev_child[c];
+		metaslab_group_t *mg = tvd->vdev_mg;
+
+		/*
+		 * Skip any holes, uninitialized top-levels, or
+		 * vdevs that are not in this metalab class.
+		 */
+		if (tvd->vdev_ishole || tvd->vdev_ms_shift == 0 ||
+		    mg->mg_class != mc) {
+			continue;
+		}
+
+		for (i = 0; i < RANGE_TREE_HISTOGRAM_SIZE; i++)
+			mc_hist[i] += mg->mg_histogram[i];
+	}
+
+	for (i = 0; i < RANGE_TREE_HISTOGRAM_SIZE; i++)
+		VERIFY3U(mc_hist[i], ==, mc->mc_histogram[i]);
+
+	kmem_free(mc_hist, sizeof (uint64_t) * RANGE_TREE_HISTOGRAM_SIZE);
+}
+
+/*
+ * Calculate the metaslab class's fragmentation metric. The metric
+ * is weighted based on the space contribution of each metaslab group.
+ * The return value will be a number between 0 and 100 (inclusive), or
+ * ZFS_FRAG_INVALID if the metric has not been set. See comment above the
+ * zfs_frag_table for more information about the metric.
+ */
+uint64_t
+metaslab_class_fragmentation(metaslab_class_t *mc)
+{
+	vdev_t *rvd = mc->mc_spa->spa_root_vdev;
+	uint64_t fragmentation = 0;
+
+	spa_config_enter(mc->mc_spa, SCL_VDEV, FTAG, RW_READER);
+
+	for (int c = 0; c < rvd->vdev_children; c++) {
+		vdev_t *tvd = rvd->vdev_child[c];
+		metaslab_group_t *mg = tvd->vdev_mg;
+
+		/*
+		 * Skip any holes, uninitialized top-levels, or
+		 * vdevs that are not in this metalab class.
+		 */
+		if (tvd->vdev_ishole || tvd->vdev_ms_shift == 0 ||
+		    mg->mg_class != mc) {
+			continue;
+		}
+
+		/*
+		 * If a metaslab group does not contain a fragmentation
+		 * metric then just bail out.
+		 */
+		if (mg->mg_fragmentation == ZFS_FRAG_INVALID) {
+			spa_config_exit(mc->mc_spa, SCL_VDEV, FTAG);
+			return (ZFS_FRAG_INVALID);
+		}
+
+		/*
+		 * Determine how much this metaslab_group is contributing
+		 * to the overall pool fragmentation metric.
+		 */
+		fragmentation += mg->mg_fragmentation *
+		    metaslab_group_get_space(mg);
+	}
+	fragmentation /= metaslab_class_get_space(mc);
+
+	ASSERT3U(fragmentation, <=, 100);
+	spa_config_exit(mc->mc_spa, SCL_VDEV, FTAG);
+	return (fragmentation);
+}
+
+/*
+ * Calculate the amount of expandable space that is available in
+ * this metaslab class. If a device is expanded then its expandable
+ * space will be the amount of allocatable space that is currently not
+ * part of this metaslab class.
+ */
+uint64_t
+metaslab_class_expandable_space(metaslab_class_t *mc)
+{
+	vdev_t *rvd = mc->mc_spa->spa_root_vdev;
+	uint64_t space = 0;
+
+	spa_config_enter(mc->mc_spa, SCL_VDEV, FTAG, RW_READER);
+	for (int c = 0; c < rvd->vdev_children; c++) {
+		vdev_t *tvd = rvd->vdev_child[c];
+		metaslab_group_t *mg = tvd->vdev_mg;
+
+		if (tvd->vdev_ishole || tvd->vdev_ms_shift == 0 ||
+		    mg->mg_class != mc) {
+			continue;
+		}
+
+		space += tvd->vdev_max_asize - tvd->vdev_asize;
+	}
+	spa_config_exit(mc->mc_spa, SCL_VDEV, FTAG);
+	return (space);
+}
+
 /*
  * ==========================================================================
  * Metaslab groups
@@ -299,7 +438,15 @@ metaslab_group_alloc_update(metaslab_group_t *mg)
 	mg->mg_free_capacity = ((vs->vs_space - vs->vs_alloc) * 100) /
 	    (vs->vs_space + 1);
 
-	mg->mg_allocatable = (mg->mg_free_capacity > zfs_mg_noalloc_threshold);
+	/*
+	 * A metaslab group is considered allocatable if it has plenty
+	 * of free space or is not heavily fragmented. We only take
+	 * fragmentation into account if the metaslab group has a valid
+	 * fragmentation metric (i.e. a value between 0 and 100).
+	 */
+	mg->mg_allocatable = (mg->mg_free_capacity > zfs_mg_noalloc_threshold &&
+	    (mg->mg_fragmentation == ZFS_FRAG_INVALID ||
+	    mg->mg_fragmentation <= zfs_mg_fragmentation_threshold));
 
 	/*
 	 * The mc_alloc_groups maintains a count of the number of
@@ -320,6 +467,7 @@ metaslab_group_alloc_update(metaslab_group_t *mg)
 		mc->mc_alloc_groups--;
 	else if (!was_allocatable && mg->mg_allocatable)
 		mc->mc_alloc_groups++;
+
 	mutex_exit(&mg->mg_lock);
 }
 
@@ -409,6 +557,7 @@ metaslab_group_passivate(metaslab_group_t *mg)
 	}
 
 	taskq_wait(mg->mg_taskq);
+	metaslab_group_alloc_update(mg);
 
 	mgprev = mg->mg_prev;
 	mgnext = mg->mg_next;
@@ -425,20 +574,113 @@ metaslab_group_passivate(metaslab_group_t *mg)
 	mg->mg_next = NULL;
 }
 
+uint64_t
+metaslab_group_get_space(metaslab_group_t *mg)
+{
+	return ((1ULL << mg->mg_vd->vdev_ms_shift) * mg->mg_vd->vdev_ms_count);
+}
+
+void
+metaslab_group_histogram_verify(metaslab_group_t *mg)
+{
+	uint64_t *mg_hist;
+	vdev_t *vd = mg->mg_vd;
+	uint64_t ashift = vd->vdev_ashift;
+	int i;
+
+	if ((zfs_flags & ZFS_DEBUG_HISTOGRAM_VERIFY) == 0)
+		return;
+
+	mg_hist = kmem_zalloc(sizeof (uint64_t) * RANGE_TREE_HISTOGRAM_SIZE,
+	    KM_SLEEP);
+
+	ASSERT3U(RANGE_TREE_HISTOGRAM_SIZE, >=,
+	    SPACE_MAP_HISTOGRAM_SIZE + ashift);
+
+	for (int m = 0; m < vd->vdev_ms_count; m++) {
+		metaslab_t *msp = vd->vdev_ms[m];
+
+		if (msp->ms_sm == NULL)
+			continue;
+
+		for (i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++)
+			mg_hist[i + ashift] +=
+			    msp->ms_sm->sm_phys->smp_histogram[i];
+	}
+
+	for (i = 0; i < RANGE_TREE_HISTOGRAM_SIZE; i ++)
+		VERIFY3U(mg_hist[i], ==, mg->mg_histogram[i]);
+
+	kmem_free(mg_hist, sizeof (uint64_t) * RANGE_TREE_HISTOGRAM_SIZE);
+}
+
+static void
+metaslab_group_histogram_add(metaslab_group_t *mg, metaslab_t *msp)
+{
+	metaslab_class_t *mc = mg->mg_class;
+	uint64_t ashift = mg->mg_vd->vdev_ashift;
+
+	ASSERT(MUTEX_HELD(&msp->ms_lock));
+	if (msp->ms_sm == NULL)
+		return;
+
+	mutex_enter(&mg->mg_lock);
+	for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) {
+		mg->mg_histogram[i + ashift] +=
+		    msp->ms_sm->sm_phys->smp_histogram[i];
+		mc->mc_histogram[i + ashift] +=
+		    msp->ms_sm->sm_phys->smp_histogram[i];
+	}
+	mutex_exit(&mg->mg_lock);
+}
+
+void
+metaslab_group_histogram_remove(metaslab_group_t *mg, metaslab_t *msp)
+{
+	metaslab_class_t *mc = mg->mg_class;
+	uint64_t ashift = mg->mg_vd->vdev_ashift;
+
+	ASSERT(MUTEX_HELD(&msp->ms_lock));
+	if (msp->ms_sm == NULL)
+		return;
+
+	mutex_enter(&mg->mg_lock);
+	for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) {
+		ASSERT3U(mg->mg_histogram[i + ashift], >=,
+		    msp->ms_sm->sm_phys->smp_histogram[i]);
+		ASSERT3U(mc->mc_histogram[i + ashift], >=,
+		    msp->ms_sm->sm_phys->smp_histogram[i]);
+
+		mg->mg_histogram[i + ashift] -=
+		    msp->ms_sm->sm_phys->smp_histogram[i];
+		mc->mc_histogram[i + ashift] -=
+		    msp->ms_sm->sm_phys->smp_histogram[i];
+	}
+	mutex_exit(&mg->mg_lock);
+}
+
 static void
 metaslab_group_add(metaslab_group_t *mg, metaslab_t *msp)
 {
-	mutex_enter(&mg->mg_lock);
 	ASSERT(msp->ms_group == NULL);
+	mutex_enter(&mg->mg_lock);
 	msp->ms_group = mg;
 	msp->ms_weight = 0;
 	avl_add(&mg->mg_metaslab_tree, msp);
 	mutex_exit(&mg->mg_lock);
+
+	mutex_enter(&msp->ms_lock);
+	metaslab_group_histogram_add(mg, msp);
+	mutex_exit(&msp->ms_lock);
 }
 
 static void
 metaslab_group_remove(metaslab_group_t *mg, metaslab_t *msp)
 {
+	mutex_enter(&msp->ms_lock);
+	metaslab_group_histogram_remove(mg, msp);
+	mutex_exit(&msp->ms_lock);
+
 	mutex_enter(&mg->mg_lock);
 	ASSERT(msp->ms_group == mg);
 	avl_remove(&mg->mg_metaslab_tree, msp);
@@ -451,9 +693,9 @@ metaslab_group_sort(metaslab_group_t *mg, metaslab_t *msp, uint64_t weight)
 {
 	/*
 	 * Although in principle the weight can be any value, in
-	 * practice we do not use values in the range [1, 510].
+	 * practice we do not use values in the range [1, 511].
 	 */
-	ASSERT(weight >= SPA_MINBLOCKSIZE-1 || weight == 0);
+	ASSERT(weight >= SPA_MINBLOCKSIZE || weight == 0);
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 
 	mutex_enter(&mg->mg_lock);
@@ -464,10 +706,43 @@ metaslab_group_sort(metaslab_group_t *mg, metaslab_t *msp, uint64_t weight)
 	mutex_exit(&mg->mg_lock);
 }
 
+/*
+ * Calculate the fragmentation for a given metaslab group. We can use
+ * a simple average here since all metaslabs within the group must have
+ * the same size. The return value will be a value between 0 and 100
+ * (inclusive), or ZFS_FRAG_INVALID if less than half of the metaslab in this
+ * group have a fragmentation metric.
+ */
+uint64_t
+metaslab_group_fragmentation(metaslab_group_t *mg)
+{
+	vdev_t *vd = mg->mg_vd;
+	uint64_t fragmentation = 0;
+	uint64_t valid_ms = 0;
+
+	for (int m = 0; m < vd->vdev_ms_count; m++) {
+		metaslab_t *msp = vd->vdev_ms[m];
+
+		if (msp->ms_fragmentation == ZFS_FRAG_INVALID)
+			continue;
+
+		valid_ms++;
+		fragmentation += msp->ms_fragmentation;
+	}
+
+	if (valid_ms <= vd->vdev_ms_count / 2)
+		return (ZFS_FRAG_INVALID);
+
+	fragmentation /= valid_ms;
+	ASSERT3U(fragmentation, <=, 100);
+	return (fragmentation);
+}
+
 /*
  * Determine if a given metaslab group should skip allocations. A metaslab
- * group should avoid allocations if its used capacity has crossed the
- * zfs_mg_noalloc_threshold and there is at least one metaslab group
+ * group should avoid allocations if its free capacity is less than the
+ * zfs_mg_noalloc_threshold or its fragmentation metric is greater than
+ * zfs_mg_fragmentation_threshold and there is at least one metaslab group
  * that can still handle allocations.
  */
 static boolean_t
@@ -478,12 +753,19 @@ metaslab_group_allocatable(metaslab_group_t *mg)
 	metaslab_class_t *mc = mg->mg_class;
 
 	/*
-	 * A metaslab group is considered allocatable if its free capacity
-	 * is greater than the set value of zfs_mg_noalloc_threshold, it's
-	 * associated with a slog, or there are no other metaslab groups
-	 * with free capacity greater than zfs_mg_noalloc_threshold.
+	 * We use two key metrics to determine if a metaslab group is
+	 * considered allocatable -- free space and fragmentation. If
+	 * the free space is greater than the free space threshold and
+	 * the fragmentation is less than the fragmentation threshold then
+	 * consider the group allocatable. There are two case when we will
+	 * not consider these key metrics. The first is if the group is
+	 * associated with a slog device and the second is if all groups
+	 * in this metaslab class have already been consider ineligible
+	 * for allocations.
 	 */
-	return (mg->mg_free_capacity > zfs_mg_noalloc_threshold ||
+	return ((mg->mg_free_capacity > zfs_mg_noalloc_threshold &&
+	    (mg->mg_fragmentation == ZFS_FRAG_INVALID ||
+	    mg->mg_fragmentation <= zfs_mg_fragmentation_threshold)) ||
 	    mc != spa_normal_class(spa) || mc->mc_alloc_groups == 0);
 }
 
@@ -707,16 +989,8 @@ metaslab_ff_alloc(metaslab_t *msp, uint64_t size)
 	return (metaslab_block_picker(t, cursor, size, align));
 }
 
-/* ARGSUSED */
-static boolean_t
-metaslab_ff_fragmented(metaslab_t *msp)
-{
-	return (B_TRUE);
-}
-
 static metaslab_ops_t metaslab_ff_ops = {
-	metaslab_ff_alloc,
-	metaslab_ff_fragmented
+	metaslab_ff_alloc
 };
 
 /*
@@ -763,23 +1037,8 @@ metaslab_df_alloc(metaslab_t *msp, uint64_t size)
 	return (metaslab_block_picker(t, cursor, size, 1ULL));
 }
 
-static boolean_t
-metaslab_df_fragmented(metaslab_t *msp)
-{
-	range_tree_t *rt = msp->ms_tree;
-	uint64_t max_size = metaslab_block_maxsize(msp);
-	int free_pct = range_tree_space(rt) * 100 / msp->ms_size;
-
-	if (max_size >= metaslab_df_alloc_threshold &&
-	    free_pct >= metaslab_df_free_pct)
-		return (B_FALSE);
-
-	return (B_TRUE);
-}
-
 static metaslab_ops_t metaslab_df_ops = {
-	metaslab_df_alloc,
-	metaslab_df_fragmented
+	metaslab_df_alloc
 };
 
 /*
@@ -822,15 +1081,8 @@ metaslab_cf_alloc(metaslab_t *msp, uint64_t size)
 	return (offset);
 }
 
-static boolean_t
-metaslab_cf_fragmented(metaslab_t *msp)
-{
-	return (metaslab_block_maxsize(msp) < metaslab_min_alloc_size);
-}
-
 static metaslab_ops_t metaslab_cf_ops = {
-	metaslab_cf_alloc,
-	metaslab_cf_fragmented
+	metaslab_cf_alloc
 };
 
 /*
@@ -887,16 +1139,8 @@ metaslab_ndf_alloc(metaslab_t *msp, uint64_t size)
 	return (-1ULL);
 }
 
-static boolean_t
-metaslab_ndf_fragmented(metaslab_t *msp)
-{
-	return (metaslab_block_maxsize(msp) <=
-	    (metaslab_min_alloc_size << metaslab_ndf_clump_shift));
-}
-
 static metaslab_ops_t metaslab_ndf_ops = {
-	metaslab_ndf_alloc,
-	metaslab_ndf_fragmented
+	metaslab_ndf_alloc
 };
 
 metaslab_ops_t *zfs_metaslab_ops = &metaslab_df_ops;
@@ -998,6 +1242,7 @@ metaslab_init(metaslab_group_t *mg, uint64_t id, uint64_t object, uint64_t txg)
 	msp->ms_tree = range_tree_create(&metaslab_rt_ops, msp, &msp->ms_lock);
 	metaslab_group_add(mg, msp);
 
+	msp->ms_fragmentation = metaslab_fragmentation(msp);
 	msp->ms_ops = mg->mg_class->mc_ops;
 
 	/*
@@ -1063,69 +1308,113 @@ metaslab_fini(metaslab_t *msp)
 	kmem_free(msp, sizeof (metaslab_t));
 }
 
+#define	FRAGMENTATION_TABLE_SIZE	17
+
 /*
- * Apply a weighting factor based on the histogram information for this
- * metaslab. The current weighting factor is somewhat arbitrary and requires
- * additional investigation. The implementation provides a measure of
- * "weighted" free space and gives a higher weighting for larger contiguous
- * regions. The weighting factor is determined by counting the number of
- * sm_shift sectors that exist in each region represented by the histogram.
- * That value is then multiplied by the power of 2 exponent and the sm_shift
- * value.
+ * This table defines a segment size based fragmentation metric that will
+ * allow each metaslab to derive its own fragmentation value. This is done
+ * by calculating the space in each bucket of the spacemap histogram and
+ * multiplying that by the fragmetation metric in this table. Doing
+ * this for all buckets and dividing it by the total amount of free
+ * space in this metaslab (i.e. the total free space in all buckets) gives
+ * us the fragmentation metric. This means that a high fragmentation metric
+ * equates to most of the free space being comprised of small segments.
+ * Conversely, if the metric is low, then most of the free space is in
+ * large segments. A 10% change in fragmentation equates to approximately
+ * double the number of segments.
  *
- * For example, assume the 2^21 histogram bucket has 4 2MB regions and the
- * metaslab has an sm_shift value of 9 (512B):
- *
- * 1) calculate the number of sm_shift sectors in the region:
- *	2^21 / 2^9 = 2^12 = 4096 * 4 (number of regions) = 16384
- * 2) multiply by the power of 2 exponent and the sm_shift value:
- *	16384 * 21 * 9 = 3096576
- * This value will be added to the weighting of the metaslab.
+ * This table defines 0% fragmented space using 16MB segments. Testing has
+ * shown that segments that are greater than or equal to 16MB do not suffer
+ * from drastic performance problems. Using this value, we derive the rest
+ * of the table. Since the fragmentation value is never stored on disk, it
+ * is possible to change these calculations in the future.
+ */
+int zfs_frag_table[FRAGMENTATION_TABLE_SIZE] = {
+	100,	/* 512B	*/
+	100,	/* 1K	*/
+	98,	/* 2K	*/
+	95,	/* 4K	*/
+	90,	/* 8K	*/
+	80,	/* 16K	*/
+	70,	/* 32K	*/
+	60,	/* 64K	*/
+	50,	/* 128K	*/
+	40,	/* 256K	*/
+	30,	/* 512K	*/
+	20,	/* 1M	*/
+	15,	/* 2M	*/
+	10,	/* 4M	*/
+	5,	/* 8M	*/
+	0	/* 16M	*/
+};
+
+/*
+ * Calclate the metaslab's fragmentation metric. A return value
+ * of ZFS_FRAG_INVALID means that the metaslab has not been upgraded and does
+ * not support this metric. Otherwise, the return value should be in the
+ * range [0, 100].
  */
 static uint64_t
-metaslab_weight_factor(metaslab_t *msp)
+metaslab_fragmentation(metaslab_t *msp)
 {
-	uint64_t factor = 0;
-	uint64_t sectors;
-	int i;
+	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
+	uint64_t fragmentation = 0;
+	uint64_t total = 0;
+	boolean_t feature_enabled = spa_feature_is_enabled(spa,
+	    SPA_FEATURE_SPACEMAP_HISTOGRAM);
+
+	if (!feature_enabled)
+		return (ZFS_FRAG_INVALID);
 
 	/*
-	 * A null space map means that the entire metaslab is free,
-	 * calculate a weight factor that spans the entire size of the
-	 * metaslab.
+	 * A null space map means that the entire metaslab is free
+	 * and thus is not fragmented.
 	 */
-	if (msp->ms_sm == NULL) {
-		vdev_t *vd = msp->ms_group->mg_vd;
-
-		i = highbit64(msp->ms_size) - 1;
-		sectors = msp->ms_size >> vd->vdev_ashift;
-		return (sectors * i * vd->vdev_ashift);
-	}
-
-	if (msp->ms_sm->sm_dbuf->db_size != sizeof (space_map_phys_t))
+	if (msp->ms_sm == NULL)
 		return (0);
 
-	for (i = 0; i < SPACE_MAP_HISTOGRAM_SIZE(msp->ms_sm); i++) {
+	/*
+	 * If this metaslab's space_map has not been upgraded, flag it
+	 * so that we upgrade next time we encounter it.
+	 */
+	if (msp->ms_sm->sm_dbuf->db_size != sizeof (space_map_phys_t)) {
+		uint64_t txg = spa_syncing_txg(spa);
+		vdev_t *vd = msp->ms_group->mg_vd;
+
+		msp->ms_condense_wanted = B_TRUE;
+		vdev_dirty(vd, VDD_METASLAB, msp, txg + 1);
+		spa_dbgmsg(spa, "txg %llu, requesting force condense: "
+		    "msp %p, vd %p", txg, msp, vd);
+		return (ZFS_FRAG_INVALID);
+	}
+
+	for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) {
+		uint64_t space = 0;
+		uint8_t shift = msp->ms_sm->sm_shift;
+		int idx = MIN(shift - SPA_MINBLOCKSHIFT + i,
+		    FRAGMENTATION_TABLE_SIZE - 1);
+
 		if (msp->ms_sm->sm_phys->smp_histogram[i] == 0)
 			continue;
 
-		/*
-		 * Determine the number of sm_shift sectors in the region
-		 * indicated by the histogram. For example, given an
-		 * sm_shift value of 9 (512 bytes) and i = 4 then we know
-		 * that we're looking at an 8K region in the histogram
-		 * (i.e. 9 + 4 = 13, 2^13 = 8192). To figure out the
-		 * number of sm_shift sectors (512 bytes in this example),
-		 * we would take 8192 / 512 = 16. Since the histogram
-		 * is offset by sm_shift we can simply use the value of
-		 * of i to calculate this (i.e. 2^i = 16 where i = 4).
-		 */
-		sectors = msp->ms_sm->sm_phys->smp_histogram[i] << i;
-		factor += (i + msp->ms_sm->sm_shift) * sectors;
+		space = msp->ms_sm->sm_phys->smp_histogram[i] << (i + shift);
+		total += space;
+
+		ASSERT3U(idx, <, FRAGMENTATION_TABLE_SIZE);
+		fragmentation += space * zfs_frag_table[idx];
 	}
-	return (factor * msp->ms_sm->sm_shift);
+
+	if (total > 0)
+		fragmentation /= total;
+	ASSERT3U(fragmentation, <=, 100);
+	return (fragmentation);
 }
 
+/*
+ * Compute a weight -- a selection preference value -- for the given metaslab.
+ * This is based on the amount of free space, the level of fragmentation,
+ * the LBA range, and whether the metaslab is loaded.
+ */
 static uint64_t
 metaslab_weight(metaslab_t *msp)
 {
@@ -1149,6 +1438,29 @@ metaslab_weight(metaslab_t *msp)
 	 * The baseline weight is the metaslab's free space.
 	 */
 	space = msp->ms_size - space_map_allocated(msp->ms_sm);
+
+	msp->ms_fragmentation = metaslab_fragmentation(msp);
+	if (metaslab_fragmentation_factor_enabled &&
+	    msp->ms_fragmentation != ZFS_FRAG_INVALID) {
+		/*
+		 * Use the fragmentation information to inversely scale
+		 * down the baseline weight. We need to ensure that we
+		 * don't exclude this metaslab completely when it's 100%
+		 * fragmented. To avoid this we reduce the fragmented value
+		 * by 1.
+		 */
+		space = (space * (100 - (msp->ms_fragmentation - 1))) / 100;
+
+		/*
+		 * If space < SPA_MINBLOCKSIZE, then we will not allocate from
+		 * this metaslab again. The fragmentation metric may have
+		 * decreased the space to something smaller than
+		 * SPA_MINBLOCKSIZE, so reset the space to SPA_MINBLOCKSIZE
+		 * so that we can consume any remaining space.
+		 */
+		if (space > 0 && space < SPA_MINBLOCKSIZE)
+			space = SPA_MINBLOCKSIZE;
+	}
 	weight = space;
 
 	/*
@@ -1160,19 +1472,19 @@ metaslab_weight(metaslab_t *msp)
 	 * In effect, this means that we'll select the metaslab with the most
 	 * free bandwidth rather than simply the one with the most free space.
 	 */
-	weight = 2 * weight - (msp->ms_id * weight) / vd->vdev_ms_count;
-	ASSERT(weight >= space && weight <= 2 * space);
+	if (metaslab_lba_weighting_enabled) {
+		weight = 2 * weight - (msp->ms_id * weight) / vd->vdev_ms_count;
+		ASSERT(weight >= space && weight <= 2 * space);
+	}
 
-	msp->ms_factor = metaslab_weight_factor(msp);
-	if (metaslab_weight_factor_enable)
-		weight += msp->ms_factor;
-
-	if (msp->ms_loaded && !msp->ms_ops->msop_fragmented(msp)) {
-		/*
-		 * If this metaslab is one we're actively using, adjust its
-		 * weight to make it preferable to any inactive metaslab so
-		 * we'll polish it off.
-		 */
+	/*
+	 * If this metaslab is one we're actively using, adjust its
+	 * weight to make it preferable to any inactive metaslab so
+	 * we'll polish it off. If the fragmentation on this metaslab
+	 * has exceed our threshold, then don't mark it active.
+	 */
+	if (msp->ms_loaded && msp->ms_fragmentation != ZFS_FRAG_INVALID &&
+	    msp->ms_fragmentation <= zfs_metaslab_fragmentation_threshold) {
 		weight |= (msp->ms_weight & METASLAB_ACTIVE_MASK);
 	}
 
@@ -1257,9 +1569,16 @@ metaslab_group_preload(metaslab_group_t *mg)
 	while (msp != NULL) {
 		metaslab_t *msp_next = AVL_NEXT(t, msp);
 
-		/* If we have reached our preload limit then we're done */
-		if (++m > metaslab_preload_limit)
-			break;
+		/*
+		 * We preload only the maximum number of metaslabs specified
+		 * by metaslab_preload_limit. If a metaslab is being forced
+		 * to condense then we preload it too. This will ensure
+		 * that force condensing happens in the next txg.
+		 */
+		if (++m > metaslab_preload_limit && !msp->ms_condense_wanted) {
+			msp = msp_next;
+			continue;
+		}
 
 		/*
 		 * We must drop the metaslab group lock here to preserve
@@ -1327,11 +1646,12 @@ metaslab_should_condense(metaslab_t *msp)
 
 	/*
 	 * Use the ms_size_tree range tree, which is ordered by size, to
-	 * obtain the largest segment in the free tree. If the tree is empty
-	 * then we should condense the map.
+	 * obtain the largest segment in the free tree. We always condense
+	 * metaslabs that are empty and metaslabs for which a condense
+	 * request has been made.
 	 */
 	rs = avl_last(&msp->ms_size_tree);
-	if (rs == NULL)
+	if (rs == NULL || msp->ms_condense_wanted)
 		return (B_TRUE);
 
 	/*
@@ -1372,9 +1692,14 @@ metaslab_condense(metaslab_t *msp, uint64_t txg, dmu_tx_t *tx)
 	ASSERT3U(spa_sync_pass(spa), ==, 1);
 	ASSERT(msp->ms_loaded);
 
+
 	spa_dbgmsg(spa, "condensing: txg %llu, msp[%llu] %p, "
-	    "smp size %llu, segments %lu", txg, msp->ms_id, msp,
-	    space_map_length(msp->ms_sm), avl_numnodes(&msp->ms_tree->rt_root));
+	    "smp size %llu, segments %lu, forcing condense=%s", txg,
+	    msp->ms_id, msp, space_map_length(msp->ms_sm),
+	    avl_numnodes(&msp->ms_tree->rt_root),
+	    msp->ms_condense_wanted ? "TRUE" : "FALSE");
+
+	msp->ms_condense_wanted = B_FALSE;
 
 	/*
 	 * Create an range tree that is 100% allocated. We remove segments
@@ -1467,8 +1792,14 @@ metaslab_sync(metaslab_t *msp, uint64_t txg)
 	ASSERT3P(*freetree, !=, NULL);
 	ASSERT3P(*freed_tree, !=, NULL);
 
+	/*
+	 * Normally, we don't want to process a metaslab if there
+	 * are no allocations or frees to perform. However, if the metaslab
+	 * is being forced to condense we need to let it through.
+	 */
 	if (range_tree_space(alloctree) == 0 &&
-	    range_tree_space(*freetree) == 0)
+	    range_tree_space(*freetree) == 0 &&
+	    !msp->ms_condense_wanted)
 		return;
 
 	/*
@@ -1505,8 +1836,9 @@ metaslab_sync(metaslab_t *msp, uint64_t txg)
 		space_map_write(msp->ms_sm, *freetree, SM_FREE, tx);
 	}
 
-	range_tree_vacate(alloctree, NULL, NULL);
-
+	metaslab_group_histogram_verify(mg);
+	metaslab_class_histogram_verify(mg->mg_class);
+	metaslab_group_histogram_remove(mg, msp);
 	if (msp->ms_loaded) {
 		/*
 		 * When the space map is loaded, we have an accruate
@@ -1526,6 +1858,9 @@ metaslab_sync(metaslab_t *msp, uint64_t txg)
 		 */
 		space_map_histogram_add(msp->ms_sm, *freetree, tx);
 	}
+	metaslab_group_histogram_add(mg, msp);
+	metaslab_group_histogram_verify(mg);
+	metaslab_class_histogram_verify(mg->mg_class);
 
 	/*
 	 * For sync pass 1, we avoid traversing this txg's free range tree
@@ -1538,6 +1873,7 @@ metaslab_sync(metaslab_t *msp, uint64_t txg)
 	} else {
 		range_tree_vacate(*freetree, range_tree_add, *freed_tree);
 	}
+	range_tree_vacate(alloctree, NULL, NULL);
 
 	ASSERT0(range_tree_space(msp->ms_alloctree[txg & TXG_MASK]));
 	ASSERT0(range_tree_space(msp->ms_freetree[txg & TXG_MASK]));
@@ -1648,13 +1984,13 @@ metaslab_sync_done(metaslab_t *msp, uint64_t txg)
 
 	metaslab_group_sort(mg, msp, metaslab_weight(msp));
 	mutex_exit(&msp->ms_lock);
-
 }
 
 void
 metaslab_sync_reassess(metaslab_group_t *mg)
 {
 	metaslab_group_alloc_update(mg);
+	mg->mg_fragmentation = metaslab_group_fragmentation(mg);
 
 	/*
 	 * Preload the next potential metaslabs
@@ -1916,9 +2252,7 @@ top:
 		 */
 		if ((vd->vdev_stat.vs_write_errors > 0 ||
 		    vd->vdev_state < VDEV_STATE_HEALTHY) &&
-		    d == 0 && dshift == 3 &&
-		    !(zfs_write_to_degraded && vd->vdev_state ==
-		    VDEV_STATE_DEGRADED)) {
+		    d == 0 && dshift == 3 && vd->vdev_children == 0) {
 			all_zero = B_FALSE;
 			goto next;
 		}
@@ -1943,7 +2277,7 @@ top:
 			 * over- or under-used relative to the pool,
 			 * and set an allocation bias to even it out.
 			 */
-			if (mc->mc_aliquot == 0) {
+			if (mc->mc_aliquot == 0 && metaslab_bias_enabled) {
 				vdev_stat_t *vs = &vd->vdev_stat;
 				int64_t vu, cu;
 
@@ -1965,6 +2299,8 @@ top:
 				 */
 				mg->mg_bias = ((cu - vu) *
 				    (int64_t)mg->mg_aliquot) / 100;
+			} else if (!metaslab_bias_enabled) {
+				mg->mg_bias = 0;
 			}
 
 			if (atomic_add_64_nv(&mc->mc_aliquot, asize) >=
diff --git a/uts/common/fs/zfs/range_tree.c b/uts/common/fs/zfs/range_tree.c
index 15e123ef5fbe..22175e06ab30 100644
--- a/uts/common/fs/zfs/range_tree.c
+++ b/uts/common/fs/zfs/range_tree.c
@@ -81,6 +81,7 @@ range_tree_stat_incr(range_tree_t *rt, range_seg_t *rs)
 	uint64_t size = rs->rs_end - rs->rs_start;
 	int idx = highbit64(size) - 1;
 
+	ASSERT(size != 0);
 	ASSERT3U(idx, <,
 	    sizeof (rt->rt_histogram) / sizeof (*rt->rt_histogram));
 
@@ -95,6 +96,7 @@ range_tree_stat_decr(range_tree_t *rt, range_seg_t *rs)
 	uint64_t size = rs->rs_end - rs->rs_start;
 	int idx = highbit64(size) - 1;
 
+	ASSERT(size != 0);
 	ASSERT3U(idx, <,
 	    sizeof (rt->rt_histogram) / sizeof (*rt->rt_histogram));
 
diff --git a/uts/common/fs/zfs/spa.c b/uts/common/fs/zfs/spa.c
index dc87100e6b81..c10cac1b2009 100644
--- a/uts/common/fs/zfs/spa.c
+++ b/uts/common/fs/zfs/spa.c
@@ -194,12 +194,10 @@ spa_prop_get_config(spa_t *spa, nvlist_t **nvp)
 {
 	vdev_t *rvd = spa->spa_root_vdev;
 	dsl_pool_t *pool = spa->spa_dsl_pool;
-	uint64_t size;
-	uint64_t alloc;
-	uint64_t space;
-	uint64_t cap, version;
+	uint64_t size, alloc, cap, version;
 	zprop_source_t src = ZPROP_SRC_NONE;
 	spa_config_dirent_t *dp;
+	metaslab_class_t *mc = spa_normal_class(spa);
 
 	ASSERT(MUTEX_HELD(&spa->spa_props_lock));
 
@@ -212,14 +210,10 @@ spa_prop_get_config(spa_t *spa, nvlist_t **nvp)
 		spa_prop_add_list(*nvp, ZPOOL_PROP_FREE, NULL,
 		    size - alloc, src);
 
-		space = 0;
-		for (int c = 0; c < rvd->vdev_children; c++) {
-			vdev_t *tvd = rvd->vdev_child[c];
-			space += tvd->vdev_max_asize - tvd->vdev_asize;
-		}
-		spa_prop_add_list(*nvp, ZPOOL_PROP_EXPANDSZ, NULL, space,
-		    src);
-
+		spa_prop_add_list(*nvp, ZPOOL_PROP_FRAGMENTATION, NULL,
+		    metaslab_class_fragmentation(mc), src);
+		spa_prop_add_list(*nvp, ZPOOL_PROP_EXPANDSZ, NULL,
+		    metaslab_class_expandable_space(mc), src);
 		spa_prop_add_list(*nvp, ZPOOL_PROP_READONLY, NULL,
 		    (spa_mode(spa) == FREAD), src);
 
diff --git a/uts/common/fs/zfs/space_map.c b/uts/common/fs/zfs/space_map.c
index 629870b6784d..d158b246d8c2 100644
--- a/uts/common/fs/zfs/space_map.c
+++ b/uts/common/fs/zfs/space_map.c
@@ -202,10 +202,10 @@ space_map_histogram_add(space_map_t *sm, range_tree_t *rt, dmu_tx_t *tx)
 		 * reached the maximum bucket size. Accumulate all ranges
 		 * larger than the max bucket size into the last bucket.
 		 */
-		if (idx < SPACE_MAP_HISTOGRAM_SIZE(sm) - 1) {
+		if (idx < SPACE_MAP_HISTOGRAM_SIZE - 1) {
 			ASSERT3U(idx + sm->sm_shift, ==, i);
 			idx++;
-			ASSERT3U(idx, <, SPACE_MAP_HISTOGRAM_SIZE(sm));
+			ASSERT3U(idx, <, SPACE_MAP_HISTOGRAM_SIZE);
 		}
 	}
 }
diff --git a/uts/common/fs/zfs/sys/metaslab.h b/uts/common/fs/zfs/sys/metaslab.h
index 4ace5987e0d1..7fd47c08c9df 100644
--- a/uts/common/fs/zfs/sys/metaslab.h
+++ b/uts/common/fs/zfs/sys/metaslab.h
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  */
 
 #ifndef _SYS_METASLAB_H
@@ -38,23 +38,22 @@ extern "C" {
 
 typedef struct metaslab_ops {
 	uint64_t (*msop_alloc)(metaslab_t *msp, uint64_t size);
-	boolean_t (*msop_fragmented)(metaslab_t *msp);
 } metaslab_ops_t;
 
 extern metaslab_ops_t *zfs_metaslab_ops;
 
-metaslab_t *metaslab_init(metaslab_group_t *mg, uint64_t id,
-    uint64_t object, uint64_t txg);
-void metaslab_fini(metaslab_t *msp);
+metaslab_t *metaslab_init(metaslab_group_t *, uint64_t,
+    uint64_t, uint64_t);
+void metaslab_fini(metaslab_t *);
 
-void metaslab_load_wait(metaslab_t *msp);
-int metaslab_load(metaslab_t *msp);
-void metaslab_unload(metaslab_t *msp);
+void metaslab_load_wait(metaslab_t *);
+int metaslab_load(metaslab_t *);
+void metaslab_unload(metaslab_t *);
 
-void metaslab_sync(metaslab_t *msp, uint64_t txg);
-void metaslab_sync_done(metaslab_t *msp, uint64_t txg);
-void metaslab_sync_reassess(metaslab_group_t *mg);
-uint64_t metaslab_block_maxsize(metaslab_t *msp);
+void metaslab_sync(metaslab_t *, uint64_t);
+void metaslab_sync_done(metaslab_t *, uint64_t);
+void metaslab_sync_reassess(metaslab_group_t *);
+uint64_t metaslab_block_maxsize(metaslab_t *);
 
 #define	METASLAB_HINTBP_FAVOR	0x0
 #define	METASLAB_HINTBP_AVOID	0x1
@@ -62,28 +61,34 @@ uint64_t metaslab_block_maxsize(metaslab_t *msp);
 #define	METASLAB_GANG_CHILD	0x4
 #define	METASLAB_GANG_AVOID	0x8
 
-int metaslab_alloc(spa_t *spa, metaslab_class_t *mc, uint64_t psize,
-    blkptr_t *bp, int ncopies, uint64_t txg, blkptr_t *hintbp, int flags);
-void metaslab_free(spa_t *spa, const blkptr_t *bp, uint64_t txg, boolean_t now);
-int metaslab_claim(spa_t *spa, const blkptr_t *bp, uint64_t txg);
-void metaslab_check_free(spa_t *spa, const blkptr_t *bp);
+int metaslab_alloc(spa_t *, metaslab_class_t *, uint64_t,
+    blkptr_t *, int, uint64_t, blkptr_t *, int);
+void metaslab_free(spa_t *, const blkptr_t *, uint64_t, boolean_t);
+int metaslab_claim(spa_t *, const blkptr_t *, uint64_t);
+void metaslab_check_free(spa_t *, const blkptr_t *);
 
-metaslab_class_t *metaslab_class_create(spa_t *spa, metaslab_ops_t *ops);
-void metaslab_class_destroy(metaslab_class_t *mc);
-int metaslab_class_validate(metaslab_class_t *mc);
+metaslab_class_t *metaslab_class_create(spa_t *, metaslab_ops_t *);
+void metaslab_class_destroy(metaslab_class_t *);
+int metaslab_class_validate(metaslab_class_t *);
+void metaslab_class_histogram_verify(metaslab_class_t *);
+uint64_t metaslab_class_fragmentation(metaslab_class_t *);
+uint64_t metaslab_class_expandable_space(metaslab_class_t *);
 
-void metaslab_class_space_update(metaslab_class_t *mc,
-    int64_t alloc_delta, int64_t defer_delta,
-    int64_t space_delta, int64_t dspace_delta);
-uint64_t metaslab_class_get_alloc(metaslab_class_t *mc);
-uint64_t metaslab_class_get_space(metaslab_class_t *mc);
-uint64_t metaslab_class_get_dspace(metaslab_class_t *mc);
-uint64_t metaslab_class_get_deferred(metaslab_class_t *mc);
+void metaslab_class_space_update(metaslab_class_t *, int64_t, int64_t,
+    int64_t, int64_t);
+uint64_t metaslab_class_get_alloc(metaslab_class_t *);
+uint64_t metaslab_class_get_space(metaslab_class_t *);
+uint64_t metaslab_class_get_dspace(metaslab_class_t *);
+uint64_t metaslab_class_get_deferred(metaslab_class_t *);
 
-metaslab_group_t *metaslab_group_create(metaslab_class_t *mc, vdev_t *vd);
-void metaslab_group_destroy(metaslab_group_t *mg);
-void metaslab_group_activate(metaslab_group_t *mg);
-void metaslab_group_passivate(metaslab_group_t *mg);
+metaslab_group_t *metaslab_group_create(metaslab_class_t *, vdev_t *);
+void metaslab_group_destroy(metaslab_group_t *);
+void metaslab_group_activate(metaslab_group_t *);
+void metaslab_group_passivate(metaslab_group_t *);
+uint64_t metaslab_group_get_space(metaslab_group_t *);
+void metaslab_group_histogram_verify(metaslab_group_t *);
+uint64_t metaslab_group_fragmentation(metaslab_group_t *);
+void metaslab_group_histogram_remove(metaslab_group_t *, metaslab_t *);
 
 #ifdef	__cplusplus
 }
diff --git a/uts/common/fs/zfs/sys/metaslab_impl.h b/uts/common/fs/zfs/sys/metaslab_impl.h
index 08e4de47100c..27a53b515fbc 100644
--- a/uts/common/fs/zfs/sys/metaslab_impl.h
+++ b/uts/common/fs/zfs/sys/metaslab_impl.h
@@ -41,6 +41,23 @@
 extern "C" {
 #endif
 
+/*
+ * A metaslab class encompasses a category of allocatable top-level vdevs.
+ * Each top-level vdev is associated with a metaslab group which defines
+ * the allocatable region for that vdev. Examples of these categories include
+ * "normal" for data block allocations (i.e. main pool allocations) or "log"
+ * for allocations designated for intent log devices (i.e. slog devices).
+ * When a block allocation is requested from the SPA it is associated with a
+ * metaslab_class_t, and only top-level vdevs (i.e. metaslab groups) belonging
+ * to the class can be used to satisfy that request. Allocations are done
+ * by traversing the metaslab groups that are linked off of the mc_rotor field.
+ * This rotor points to the next metaslab group where allocations will be
+ * attempted. Allocating a block is a 3 step process -- select the metaslab
+ * group, select the metaslab, and then allocate the block. The metaslab
+ * class defines the low-level block allocator that will be used as the
+ * final step in allocation. These allocators are pluggable allowing each class
+ * to use a block allocator that best suits that class.
+ */
 struct metaslab_class {
 	spa_t			*mc_spa;
 	metaslab_group_t	*mc_rotor;
@@ -51,8 +68,18 @@ struct metaslab_class {
 	uint64_t		mc_deferred;	/* total deferred frees */
 	uint64_t		mc_space;	/* total space (alloc + free) */
 	uint64_t		mc_dspace;	/* total deflated space */
+	uint64_t		mc_histogram[RANGE_TREE_HISTOGRAM_SIZE];
 };
 
+/*
+ * Metaslab groups encapsulate all the allocatable regions (i.e. metaslabs)
+ * of a top-level vdev. They are linked togther to form a circular linked
+ * list and can belong to only one metaslab class. Metaslab groups may become
+ * ineligible for allocations for a number of reasons such as limited free
+ * space, fragmentation, or going offline. When this happens the allocator will
+ * simply find the next metaslab group in the linked list and attempt
+ * to allocate from that group instead.
+ */
 struct metaslab_group {
 	kmutex_t		mg_lock;
 	avl_tree_t		mg_metaslab_tree;
@@ -66,12 +93,14 @@ struct metaslab_group {
 	taskq_t			*mg_taskq;
 	metaslab_group_t	*mg_prev;
 	metaslab_group_t	*mg_next;
+	uint64_t		mg_fragmentation;
+	uint64_t		mg_histogram[RANGE_TREE_HISTOGRAM_SIZE];
 };
 
 /*
  * This value defines the number of elements in the ms_lbas array. The value
- * of 64 was chosen as it covers to cover all power of 2 buckets up to
- * UINT64_MAX. This is the equivalent of highbit(UINT64_MAX).
+ * of 64 was chosen as it covers all power of 2 buckets up to UINT64_MAX.
+ * This is the equivalent of highbit(UINT64_MAX).
  */
 #define	MAX_LBAS	64
 
@@ -134,6 +163,7 @@ struct metaslab {
 	uint64_t	ms_id;
 	uint64_t	ms_start;
 	uint64_t	ms_size;
+	uint64_t	ms_fragmentation;
 
 	range_tree_t	*ms_alloctree[TXG_SIZE];
 	range_tree_t	*ms_freetree[TXG_SIZE];
@@ -141,12 +171,12 @@ struct metaslab {
 	range_tree_t	*ms_tree;
 
 	boolean_t	ms_condensing;	/* condensing? */
+	boolean_t	ms_condense_wanted;
 	boolean_t	ms_loaded;
 	boolean_t	ms_loading;
 
 	int64_t		ms_deferspace;	/* sum of ms_defermap[] space	*/
 	uint64_t	ms_weight;	/* weight vs. others in group	*/
-	uint64_t	ms_factor;
 	uint64_t	ms_access_txg;
 
 	/*
diff --git a/uts/common/fs/zfs/sys/space_map.h b/uts/common/fs/zfs/sys/space_map.h
index 3691803306d8..67fa2767b1f8 100644
--- a/uts/common/fs/zfs/sys/space_map.h
+++ b/uts/common/fs/zfs/sys/space_map.h
@@ -24,7 +24,7 @@
  */
 
 /*
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #ifndef _SYS_SPACE_MAP_H
@@ -44,9 +44,7 @@ extern "C" {
  * maintain backward compatibility.
  */
 #define	SPACE_MAP_SIZE_V0	(3 * sizeof (uint64_t))
-#define	SPACE_MAP_HISTOGRAM_SIZE(sm)			\
-	(sizeof ((sm)->sm_phys->smp_histogram) /	\
-	sizeof ((sm)->sm_phys->smp_histogram[0]))
+#define	SPACE_MAP_HISTOGRAM_SIZE	32
 
 /*
  * The space_map_phys is the on-disk representation of the space map.
@@ -68,7 +66,7 @@ typedef struct space_map_phys {
 	 * whose size is:
 	 * 2^(i+sm_shift) <= size of free region in bytes < 2^(i+sm_shift+1)
 	 */
-	uint64_t	smp_histogram[32]; /* histogram of free space */
+	uint64_t	smp_histogram[SPACE_MAP_HISTOGRAM_SIZE];
 } space_map_phys_t;
 
 /*
diff --git a/uts/common/fs/zfs/sys/zfs_debug.h b/uts/common/fs/zfs/sys/zfs_debug.h
index 3c8e2177bccc..3dd992bd2559 100644
--- a/uts/common/fs/zfs/sys/zfs_debug.h
+++ b/uts/common/fs/zfs/sys/zfs_debug.h
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #ifndef _SYS_ZFS_DEBUG_H
@@ -50,13 +50,14 @@ extern int zfs_flags;
 extern boolean_t zfs_recover;
 extern boolean_t zfs_free_leak_on_eio;
 
-#define	ZFS_DEBUG_DPRINTF	(1<<0)
-#define	ZFS_DEBUG_DBUF_VERIFY	(1<<1)
-#define	ZFS_DEBUG_DNODE_VERIFY	(1<<2)
-#define	ZFS_DEBUG_SNAPNAMES	(1<<3)
-#define	ZFS_DEBUG_MODIFY	(1<<4)
-#define	ZFS_DEBUG_SPA		(1<<5)
-#define	ZFS_DEBUG_ZIO_FREE	(1<<6)
+#define	ZFS_DEBUG_DPRINTF		(1<<0)
+#define	ZFS_DEBUG_DBUF_VERIFY		(1<<1)
+#define	ZFS_DEBUG_DNODE_VERIFY		(1<<2)
+#define	ZFS_DEBUG_SNAPNAMES		(1<<3)
+#define	ZFS_DEBUG_MODIFY		(1<<4)
+#define	ZFS_DEBUG_SPA			(1<<5)
+#define	ZFS_DEBUG_ZIO_FREE		(1<<6)
+#define	ZFS_DEBUG_HISTOGRAM_VERIFY	(1<<7)
 
 #ifdef ZFS_DEBUG
 extern void __dprintf(const char *file, const char *func,
diff --git a/uts/common/fs/zfs/vdev.c b/uts/common/fs/zfs/vdev.c
index 8c647f47d430..e1a0fbfe4c59 100644
--- a/uts/common/fs/zfs/vdev.c
+++ b/uts/common/fs/zfs/vdev.c
@@ -2116,6 +2116,11 @@ vdev_remove(vdev_t *vd, uint64_t txg)
 	tx = dmu_tx_create_assigned(spa_get_dsl(spa), txg);
 
 	if (vd->vdev_ms != NULL) {
+		metaslab_group_t *mg = vd->vdev_mg;
+
+		metaslab_group_histogram_verify(mg);
+		metaslab_class_histogram_verify(mg->mg_class);
+
 		for (int m = 0; m < vd->vdev_ms_count; m++) {
 			metaslab_t *msp = vd->vdev_ms[m];
 
@@ -2123,12 +2128,27 @@ vdev_remove(vdev_t *vd, uint64_t txg)
 				continue;
 
 			mutex_enter(&msp->ms_lock);
+			/*
+			 * If the metaslab was not loaded when the vdev
+			 * was removed then the histogram accounting may
+			 * not be accurate. Update the histogram information
+			 * here so that we ensure that the metaslab group
+			 * and metaslab class are up-to-date.
+			 */
+			metaslab_group_histogram_remove(mg, msp);
+
 			VERIFY0(space_map_allocated(msp->ms_sm));
 			space_map_free(msp->ms_sm, tx);
 			space_map_close(msp->ms_sm);
 			msp->ms_sm = NULL;
 			mutex_exit(&msp->ms_lock);
 		}
+
+		metaslab_group_histogram_verify(mg);
+		metaslab_class_histogram_verify(mg->mg_class);
+		for (int i = 0; i < RANGE_TREE_HISTOGRAM_SIZE; i++)
+			ASSERT0(mg->mg_histogram[i]);
+
 	}
 
 	if (vd->vdev_ms_array) {
@@ -2580,7 +2600,10 @@ vdev_accessible(vdev_t *vd, zio_t *zio)
 void
 vdev_get_stats(vdev_t *vd, vdev_stat_t *vs)
 {
-	vdev_t *rvd = vd->vdev_spa->spa_root_vdev;
+	spa_t *spa = vd->vdev_spa;
+	vdev_t *rvd = spa->spa_root_vdev;
+
+	ASSERT(spa_config_held(spa, SCL_ALL, RW_READER) != 0);
 
 	mutex_enter(&vd->vdev_stat_lock);
 	bcopy(&vd->vdev_stat, vs, sizeof (*vs));
@@ -2590,7 +2613,8 @@ vdev_get_stats(vdev_t *vd, vdev_stat_t *vs)
 	if (vd->vdev_ops->vdev_op_leaf)
 		vs->vs_rsize += VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE;
 	vs->vs_esize = vd->vdev_max_asize - vd->vdev_asize;
-	mutex_exit(&vd->vdev_stat_lock);
+	if (vd->vdev_aux == NULL && vd == vd->vdev_top)
+		vs->vs_fragmentation = vd->vdev_mg->mg_fragmentation;
 
 	/*
 	 * If we're getting stats on the root vdev, aggregate the I/O counts
@@ -2601,15 +2625,14 @@ vdev_get_stats(vdev_t *vd, vdev_stat_t *vs)
 			vdev_t *cvd = rvd->vdev_child[c];
 			vdev_stat_t *cvs = &cvd->vdev_stat;
 
-			mutex_enter(&vd->vdev_stat_lock);
 			for (int t = 0; t < ZIO_TYPES; t++) {
 				vs->vs_ops[t] += cvs->vs_ops[t];
 				vs->vs_bytes[t] += cvs->vs_bytes[t];
 			}
 			cvs->vs_scan_removing = cvd->vdev_removing;
-			mutex_exit(&vd->vdev_stat_lock);
 		}
 	}
+	mutex_exit(&vd->vdev_stat_lock);
 }
 
 void
diff --git a/uts/common/sys/fs/zfs.h b/uts/common/sys/fs/zfs.h
index f1615194b16a..7155d9702bb2 100644
--- a/uts/common/sys/fs/zfs.h
+++ b/uts/common/sys/fs/zfs.h
@@ -190,6 +190,7 @@ typedef enum {
 	ZPOOL_PROP_COMMENT,
 	ZPOOL_PROP_EXPANDSZ,
 	ZPOOL_PROP_FREEING,
+	ZPOOL_PROP_FRAGMENTATION,
 	ZPOOL_PROP_LEAKED,
 	ZPOOL_NUM_PROPS
 } zpool_prop_t;
@@ -588,6 +589,13 @@ typedef struct zpool_rewind_policy {
  */
 #define	SPA_MINDEVSIZE		(64ULL << 20)
 
+/*
+ * Set if the fragmentation has not yet been calculated. This can happen
+ * because the space maps have not been upgraded or the histogram feature
+ * is not enabled.
+ */
+#define	ZFS_FRAG_INVALID	UINT64_MAX
+
 /*
  * The location of the pool configuration repository, shared between kernel and
  * userland.
@@ -725,6 +733,7 @@ typedef struct vdev_stat {
 	uint64_t	vs_self_healed;		/* self-healed bytes	*/
 	uint64_t	vs_scan_removing;	/* removing?	*/
 	uint64_t	vs_scan_processed;	/* scan processed bytes	*/
+	uint64_t	vs_fragmentation;	/* device fragmentation */
 } vdev_stat_t;
 
 /*

From e104f78ad5767b6a25803afb54eb5a37c6568f77 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Tue, 29 Jul 2014 07:08:32 +0000
Subject: [PATCH 021/380] 4873 zvol unmap calls can take a very long time for
 larger datasets Reviewed by: George Wilson <george@delphix.com> Reviewed by:
 Matthew Ahrens <mahrens@delphix.com> Reviewed by: Paul Dagnelie
 <paul.dagnelie@delphix.com> Reviewed by: Basil Crow <basil.crow@delphix.com>
 Reviewed by: Dan McDonald <danmcd@omniti.com> Approved by: Robert Mustacchi
 <rm@joyent.com>

illumos/illumos-gate@0f6d88aded0d165f5954688a9b13bac76c38da84
---
 common/avl/avl.c               | 28 ++++++++++++++++
 uts/common/fs/zfs/dbuf.c       | 52 ++++++++++++++++++++----------
 uts/common/fs/zfs/dnode.c      | 59 +++++++++++++++++++++++++++-------
 uts/common/fs/zfs/dnode_sync.c | 12 +++----
 uts/common/fs/zfs/sys/dbuf.h   |  7 ++--
 uts/common/fs/zfs/sys/dnode.h  |  2 +-
 uts/common/sys/avl.h           |  9 ++++++
 7 files changed, 130 insertions(+), 39 deletions(-)

diff --git a/common/avl/avl.c b/common/avl/avl.c
index 4c0c213a10e1..4223da49915c 100644
--- a/common/avl/avl.c
+++ b/common/avl/avl.c
@@ -23,6 +23,10 @@
  * Use is subject to license terms.
  */
 
+/*
+ * Copyright (c) 2014 by Delphix. All rights reserved.
+ */
+
 /*
  * AVL - generic AVL tree implementation for kernel use
  *
@@ -85,6 +89,12 @@
  *	  is a modified "avl_node_t *".  The bottom bit (normally 0 for a
  *	  pointer) is set to indicate if that the new node has a value greater
  *	  than the value of the indicated "avl_node_t *".
+ *
+ * Note - in addition to userland (e.g. libavl and libutil) and the kernel
+ * (e.g. genunix), avl.c is compiled into ld.so and kmdb's genunix module,
+ * which each have their own compilation environments and subsequent
+ * requirements. Each of these environments must be considered when adding
+ * dependencies from avl.c.
  */
 
 #include <sys/types.h>
@@ -864,6 +874,24 @@ avl_update(avl_tree_t *t, void *obj)
 	return (B_FALSE);
 }
 
+void
+avl_swap(avl_tree_t *tree1, avl_tree_t *tree2)
+{
+	avl_node_t *temp_node;
+	ulong_t temp_numnodes;
+
+	ASSERT3P(tree1->avl_compar, ==, tree2->avl_compar);
+	ASSERT3U(tree1->avl_offset, ==, tree2->avl_offset);
+	ASSERT3U(tree1->avl_size, ==, tree2->avl_size);
+
+	temp_node = tree1->avl_root;
+	temp_numnodes = tree1->avl_numnodes;
+	tree1->avl_root = tree2->avl_root;
+	tree1->avl_numnodes = tree2->avl_numnodes;
+	tree2->avl_root = temp_node;
+	tree2->avl_numnodes = temp_numnodes;
+}
+
 /*
  * initialize a new AVL tree
  */
diff --git a/uts/common/fs/zfs/dbuf.c b/uts/common/fs/zfs/dbuf.c
index 759cfcfb8e3a..8db626fadc3b 100644
--- a/uts/common/fs/zfs/dbuf.c
+++ b/uts/common/fs/zfs/dbuf.c
@@ -69,6 +69,9 @@ dbuf_cons(void *vdb, void *unused, int kmflag)
 	mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL);
 	cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL);
 	refcount_create(&db->db_holds);
+
+	db->db_creation = gethrtime();
+
 	return (0);
 }
 
@@ -330,7 +333,7 @@ dbuf_verify(dmu_buf_impl_t *db)
 		ASSERT3U(db->db_level, <, dn->dn_nlevels);
 		ASSERT(db->db_blkid == DMU_BONUS_BLKID ||
 		    db->db_blkid == DMU_SPILL_BLKID ||
-		    !list_is_empty(&dn->dn_dbufs));
+		    !avl_is_empty(&dn->dn_dbufs));
 	}
 	if (db->db_blkid == DMU_BONUS_BLKID) {
 		ASSERT(dn != NULL);
@@ -803,18 +806,30 @@ dbuf_unoverride(dbuf_dirty_record_t *dr)
  * receive; see comment below for details.
  */
 void
-dbuf_free_range(dnode_t *dn, uint64_t start, uint64_t end, dmu_tx_t *tx)
+dbuf_free_range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid,
+    dmu_tx_t *tx)
 {
-	dmu_buf_impl_t *db, *db_next;
+	dmu_buf_impl_t *db, *db_next, db_search;
 	uint64_t txg = tx->tx_txg;
+	avl_index_t where;
 
-	if (end > dn->dn_maxblkid && (end != DMU_SPILL_BLKID))
-		end = dn->dn_maxblkid;
-	dprintf_dnode(dn, "start=%llu end=%llu\n", start, end);
+	if (end_blkid > dn->dn_maxblkid && (end_blkid != DMU_SPILL_BLKID))
+		end_blkid = dn->dn_maxblkid;
+	dprintf_dnode(dn, "start=%llu end=%llu\n", start_blkid, end_blkid);
+
+	db_search.db_level = 0;
+	db_search.db_blkid = start_blkid;
+	db_search.db_creation = 0;
 
 	mutex_enter(&dn->dn_dbufs_mtx);
-	if (start >= dn->dn_unlisted_l0_blkid * dn->dn_datablksz) {
+	if (start_blkid >= dn->dn_unlisted_l0_blkid) {
 		/* There can't be any dbufs in this range; no need to search. */
+#ifdef DEBUG
+		db = avl_find(&dn->dn_dbufs, &db_search, &where);
+		ASSERT3P(db, ==, NULL);
+		db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER);
+		ASSERT(db == NULL || db->db_level > 0);
+#endif
 		mutex_exit(&dn->dn_dbufs_mtx);
 		return;
 	} else if (dmu_objset_is_receiving(dn->dn_objset)) {
@@ -828,14 +843,18 @@ dbuf_free_range(dnode_t *dn, uint64_t start, uint64_t end, dmu_tx_t *tx)
 		atomic_inc_64(&zfs_free_range_recv_miss);
 	}
 
-	for (db = list_head(&dn->dn_dbufs); db != NULL; db = db_next) {
-		db_next = list_next(&dn->dn_dbufs, db);
+	db = avl_find(&dn->dn_dbufs, &db_search, &where);
+	ASSERT3P(db, ==, NULL);
+	db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER);
+
+	for (; db != NULL; db = db_next) {
+		db_next = AVL_NEXT(&dn->dn_dbufs, db);
 		ASSERT(db->db_blkid != DMU_BONUS_BLKID);
 
-		if (db->db_level != 0)
-			continue;
-		if (db->db_blkid < start || db->db_blkid > end)
-			continue;
+		if (db->db_level != 0 || db->db_blkid > end_blkid) {
+			break;
+		}
+		ASSERT3U(db->db_blkid, >=, start_blkid);
 
 		/* found a level 0 buffer in the range */
 		mutex_enter(&db->db_mtx);
@@ -1585,7 +1604,7 @@ dbuf_clear(dmu_buf_impl_t *db)
 	dn = DB_DNODE(db);
 	dndb = dn->dn_dbuf;
 	if (db->db_blkid != DMU_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) {
-		list_remove(&dn->dn_dbufs, db);
+		avl_remove(&dn->dn_dbufs, db);
 		(void) atomic_dec_32_nv(&dn->dn_dbufs_count);
 		membar_producer();
 		DB_DNODE_EXIT(db);
@@ -1748,7 +1767,7 @@ dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid,
 		mutex_exit(&dn->dn_dbufs_mtx);
 		return (odb);
 	}
-	list_insert_head(&dn->dn_dbufs, db);
+	avl_add(&dn->dn_dbufs, db);
 	if (db->db_level == 0 && db->db_blkid >=
 	    dn->dn_unlisted_l0_blkid)
 		dn->dn_unlisted_l0_blkid = db->db_blkid + 1;
@@ -1807,7 +1826,7 @@ dbuf_destroy(dmu_buf_impl_t *db)
 			DB_DNODE_ENTER(db);
 			dn = DB_DNODE(db);
 			mutex_enter(&dn->dn_dbufs_mtx);
-			list_remove(&dn->dn_dbufs, db);
+			avl_remove(&dn->dn_dbufs, db);
 			(void) atomic_dec_32_nv(&dn->dn_dbufs_count);
 			mutex_exit(&dn->dn_dbufs_mtx);
 			DB_DNODE_EXIT(db);
@@ -1825,7 +1844,6 @@ dbuf_destroy(dmu_buf_impl_t *db)
 	db->db_parent = NULL;
 	db->db_buf = NULL;
 
-	ASSERT(!list_link_active(&db->db_link));
 	ASSERT(db->db.db_data == NULL);
 	ASSERT(db->db_hash_next == NULL);
 	ASSERT(db->db_blkptr == NULL);
diff --git a/uts/common/fs/zfs/dnode.c b/uts/common/fs/zfs/dnode.c
index d7bfacad8631..1eadb4f6954d 100644
--- a/uts/common/fs/zfs/dnode.c
+++ b/uts/common/fs/zfs/dnode.c
@@ -59,6 +59,43 @@ int zfs_default_ibs = DN_MAX_INDBLKSHIFT;
 
 static kmem_cbrc_t dnode_move(void *, void *, size_t, void *);
 
+static int
+dbuf_compare(const void *x1, const void *x2)
+{
+	const dmu_buf_impl_t *d1 = x1;
+	const dmu_buf_impl_t *d2 = x2;
+
+	if (d1->db_level < d2->db_level) {
+		return (-1);
+	} else if (d1->db_level > d2->db_level) {
+		return (1);
+	}
+
+	if (d1->db_blkid < d2->db_blkid) {
+		return (-1);
+	} else if (d1->db_blkid > d2->db_blkid) {
+		return (1);
+	}
+
+	/*
+	 * If a dbuf is being evicted while dn_dbufs_mutex is not held, we set
+	 * the db_state to DB_EVICTING but do not remove it from dn_dbufs. If
+	 * another thread creates a dbuf of the same blkid before the dbuf is
+	 * removed from dn_dbufs, we can reach a state where there are two
+	 * dbufs of the same blkid and level in db_dbufs. To maintain the avl
+	 * invariant that there cannot be duplicate items, we distinguish
+	 * between these two dbufs based on the time they were created.
+	 */
+	if (d1->db_creation < d2->db_creation) {
+		return (-1);
+	} else if (d1->db_creation > d2->db_creation) {
+		return (1);
+	} else {
+		ASSERT3P(d1, ==, d2);
+		return (0);
+	}
+}
+
 /* ARGSUSED */
 static int
 dnode_cons(void *arg, void *unused, int kmflag)
@@ -113,7 +150,7 @@ dnode_cons(void *arg, void *unused, int kmflag)
 
 	dn->dn_dbufs_count = 0;
 	dn->dn_unlisted_l0_blkid = 0;
-	list_create(&dn->dn_dbufs, sizeof (dmu_buf_impl_t),
+	avl_create(&dn->dn_dbufs, dbuf_compare, sizeof (dmu_buf_impl_t),
 	    offsetof(dmu_buf_impl_t, db_link));
 
 	dn->dn_moved = 0;
@@ -166,7 +203,7 @@ dnode_dest(void *arg, void *unused)
 
 	ASSERT0(dn->dn_dbufs_count);
 	ASSERT0(dn->dn_unlisted_l0_blkid);
-	list_destroy(&dn->dn_dbufs);
+	avl_destroy(&dn->dn_dbufs);
 }
 
 void
@@ -502,7 +539,7 @@ dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs,
 	ASSERT0(dn->dn_assigned_txg);
 	ASSERT(refcount_is_zero(&dn->dn_tx_holds));
 	ASSERT3U(refcount_count(&dn->dn_holds), <=, 1);
-	ASSERT3P(list_head(&dn->dn_dbufs), ==, NULL);
+	ASSERT(avl_is_empty(&dn->dn_dbufs));
 
 	for (i = 0; i < TXG_SIZE; i++) {
 		ASSERT0(dn->dn_next_nblkptr[i]);
@@ -687,8 +724,8 @@ dnode_move_impl(dnode_t *odn, dnode_t *ndn)
 	ndn->dn_dirtyctx_firstset = odn->dn_dirtyctx_firstset;
 	ASSERT(refcount_count(&odn->dn_tx_holds) == 0);
 	refcount_transfer(&ndn->dn_holds, &odn->dn_holds);
-	ASSERT(list_is_empty(&ndn->dn_dbufs));
-	list_move_tail(&ndn->dn_dbufs, &odn->dn_dbufs);
+	ASSERT(avl_is_empty(&ndn->dn_dbufs));
+	avl_swap(&ndn->dn_dbufs, &odn->dn_dbufs);
 	ndn->dn_dbufs_count = odn->dn_dbufs_count;
 	ndn->dn_unlisted_l0_blkid = odn->dn_unlisted_l0_blkid;
 	ndn->dn_bonus = odn->dn_bonus;
@@ -722,7 +759,7 @@ dnode_move_impl(dnode_t *odn, dnode_t *ndn)
 	 */
 	odn->dn_dbuf = NULL;
 	odn->dn_handle = NULL;
-	list_create(&odn->dn_dbufs, sizeof (dmu_buf_impl_t),
+	avl_create(&odn->dn_dbufs, dbuf_compare, sizeof (dmu_buf_impl_t),
 	    offsetof(dmu_buf_impl_t, db_link));
 	odn->dn_dbufs_count = 0;
 	odn->dn_unlisted_l0_blkid = 0;
@@ -1231,7 +1268,8 @@ dnode_setdirty(dnode_t *dn, dmu_tx_t *tx)
 		return;
 	}
 
-	ASSERT(!refcount_is_zero(&dn->dn_holds) || list_head(&dn->dn_dbufs));
+	ASSERT(!refcount_is_zero(&dn->dn_holds) ||
+	    !avl_is_empty(&dn->dn_dbufs));
 	ASSERT(dn->dn_datablksz != 0);
 	ASSERT0(dn->dn_next_bonuslen[txg&TXG_MASK]);
 	ASSERT0(dn->dn_next_blksz[txg&TXG_MASK]);
@@ -1304,7 +1342,7 @@ dnode_free(dnode_t *dn, dmu_tx_t *tx)
 int
 dnode_set_blksz(dnode_t *dn, uint64_t size, int ibs, dmu_tx_t *tx)
 {
-	dmu_buf_impl_t *db, *db_next;
+	dmu_buf_impl_t *db;
 	int err;
 
 	if (size == 0)
@@ -1327,9 +1365,8 @@ dnode_set_blksz(dnode_t *dn, uint64_t size, int ibs, dmu_tx_t *tx)
 		goto fail;
 
 	mutex_enter(&dn->dn_dbufs_mtx);
-	for (db = list_head(&dn->dn_dbufs); db; db = db_next) {
-		db_next = list_next(&dn->dn_dbufs, db);
-
+	for (db = avl_first(&dn->dn_dbufs); db != NULL;
+	    db = AVL_NEXT(&dn->dn_dbufs, db)) {
 		if (db->db_blkid != 0 && db->db_blkid != DMU_BONUS_BLKID &&
 		    db->db_blkid != DMU_SPILL_BLKID) {
 			mutex_exit(&dn->dn_dbufs_mtx);
diff --git a/uts/common/fs/zfs/dnode_sync.c b/uts/common/fs/zfs/dnode_sync.c
index aca4203180a6..ff666b212d5e 100644
--- a/uts/common/fs/zfs/dnode_sync.c
+++ b/uts/common/fs/zfs/dnode_sync.c
@@ -400,16 +400,13 @@ dnode_evict_dbufs(dnode_t *dn)
 	int pass = 0;
 
 	do {
-		dmu_buf_impl_t *db, marker;
+		dmu_buf_impl_t *db, *db_next;
 		int evicting = FALSE;
 
 		progress = FALSE;
 		mutex_enter(&dn->dn_dbufs_mtx);
-		list_insert_tail(&dn->dn_dbufs, &marker);
-		db = list_head(&dn->dn_dbufs);
-		for (; db != &marker; db = list_head(&dn->dn_dbufs)) {
-			list_remove(&dn->dn_dbufs, db);
-			list_insert_tail(&dn->dn_dbufs, db);
+		for (db = avl_first(&dn->dn_dbufs); db != NULL; db = db_next) {
+			db_next = AVL_NEXT(&dn->dn_dbufs, db);
 #ifdef	DEBUG
 			DB_DNODE_ENTER(db);
 			ASSERT3P(DB_DNODE(db), ==, dn);
@@ -429,7 +426,6 @@ dnode_evict_dbufs(dnode_t *dn)
 			}
 
 		}
-		list_remove(&dn->dn_dbufs, &marker);
 		/*
 		 * NB: we need to drop dn_dbufs_mtx between passes so
 		 * that any DB_EVICTING dbufs can make progress.
@@ -500,7 +496,7 @@ dnode_sync_free(dnode_t *dn, dmu_tx_t *tx)
 
 	dnode_undirty_dbufs(&dn->dn_dirty_records[txgoff]);
 	dnode_evict_dbufs(dn);
-	ASSERT3P(list_head(&dn->dn_dbufs), ==, NULL);
+	ASSERT(avl_is_empty(&dn->dn_dbufs));
 	ASSERT3P(dn->dn_bonus, ==, NULL);
 
 	/*
diff --git a/uts/common/fs/zfs/sys/dbuf.h b/uts/common/fs/zfs/sys/dbuf.h
index c80b7ffba34f..643d9685b61c 100644
--- a/uts/common/fs/zfs/sys/dbuf.h
+++ b/uts/common/fs/zfs/sys/dbuf.h
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  */
 
@@ -213,11 +213,14 @@ typedef struct dmu_buf_impl {
 	/* pointer to most recent dirty record for this buffer */
 	dbuf_dirty_record_t *db_last_dirty;
 
+	/* Creation time of dbuf (see comment in dbuf_compare). */
+	hrtime_t db_creation;
+
 	/*
 	 * Our link on the owner dnodes's dn_dbufs list.
 	 * Protected by its dn_dbufs_mtx.
 	 */
-	list_node_t db_link;
+	avl_node_t db_link;
 
 	/* Data which is unique to data (leaf) blocks: */
 
diff --git a/uts/common/fs/zfs/sys/dnode.h b/uts/common/fs/zfs/sys/dnode.h
index 35501ce022cf..7923a34a82bb 100644
--- a/uts/common/fs/zfs/sys/dnode.h
+++ b/uts/common/fs/zfs/sys/dnode.h
@@ -211,7 +211,7 @@ typedef struct dnode {
 	refcount_t dn_holds;
 
 	kmutex_t dn_dbufs_mtx;
-	list_t dn_dbufs;		/* descendent dbufs */
+	avl_tree_t dn_dbufs;		/* descendent dbufs */
 
 	/* protected by dn_struct_rwlock */
 	struct dmu_buf_impl *dn_bonus;	/* bonus buffer dbuf */
diff --git a/uts/common/sys/avl.h b/uts/common/sys/avl.h
index eea9c5558b1e..10e0ddaeef88 100644
--- a/uts/common/sys/avl.h
+++ b/uts/common/sys/avl.h
@@ -23,6 +23,10 @@
  * Use is subject to license terms.
  */
 
+/*
+ * Copyright (c) 2014 by Delphix. All rights reserved.
+ */
+
 #ifndef	_AVL_H
 #define	_AVL_H
 
@@ -259,6 +263,11 @@ extern boolean_t avl_update(avl_tree_t *, void *);
 extern boolean_t avl_update_lt(avl_tree_t *, void *);
 extern boolean_t avl_update_gt(avl_tree_t *, void *);
 
+/*
+ * Swaps the contents of the two trees.
+ */
+extern void avl_swap(avl_tree_t *tree1, avl_tree_t *tree2);
+
 /*
  * Return the number of nodes in the tree
  */

From d9258b1417e40f51bdf71d320f4afa2dbe907c1d Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Tue, 29 Jul 2014 07:13:03 +0000
Subject: [PATCH 022/380] 5034 ARC's buf_hash_table is too small Reviewed by:
 Christopher Siden <christopher.siden@delphix.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Reviewed by: Saso Kiselkov
 <skiselkov.ml@gmail.com> Reviewed by: Richard Elling
 <richard.elling@gmail.com> Approved by: Gordon Ross <gwr@nexenta.com>

illumos/illumos-gate@63e911b6fce0acc8e2a1d31ebdaf0c4c12580a14
---
 uts/common/fs/zfs/arc.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index dd5b3accc445..2353ac67ce49 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -200,6 +200,7 @@ int zfs_arc_grow_retry = 0;
 int zfs_arc_shrink_shift = 0;
 int zfs_arc_p_min_shift = 0;
 int zfs_disable_dup_eviction = 0;
+int zfs_arc_average_blocksize = 8 * 1024; /* 8KB */
 
 /*
  * Note that buffers can be in one of 6 states:
@@ -938,10 +939,11 @@ buf_init(void)
 
 	/*
 	 * The hash table is big enough to fill all of physical memory
-	 * with an average 64K block size.  The table will take up
-	 * totalmem*sizeof(void*)/64K (eg. 128KB/GB with 8-byte pointers).
+	 * with an average block size of zfs_arc_average_blocksize (default 8K).
+	 * By default, the table will take up
+	 * totalmem * sizeof(void*) / 8K (1MB per GB with 8-byte pointers).
 	 */
-	while (hsize * 65536 < physmem * PAGESIZE)
+	while (hsize * zfs_arc_average_blocksize < physmem * PAGESIZE)
 		hsize <<= 1;
 retry:
 	buf_hash_table.ht_mask = hsize - 1;

From 5950e31be7d8da2687b5bfa927fcb71d2547e936 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 2 Aug 2014 06:49:16 +0000
Subject: [PATCH 023/380] Author: Justin T. Gibbs <justing@spectralogic.com>
 Date:   Fri Aug 1 17:55:38 2014 -0800

    5038 Remove "old-style" flexible array usage in ZFS.
    Reviewed by: Matthew Ahrens <mahrens@delphix.com>
    Reviewed by: Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
    Approved by: Richard Lowe <richlowe@richlowe.net>

illumos/illumos-gate@7f18da4c54210b682e105b0e7be5195c60f98d20
---
 uts/common/fs/zfs/dnode.c     | 6 +++---
 uts/common/fs/zfs/sys/dnode.h | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/uts/common/fs/zfs/dnode.c b/uts/common/fs/zfs/dnode.c
index 1eadb4f6954d..f5f5c4016313 100644
--- a/uts/common/fs/zfs/dnode.c
+++ b/uts/common/fs/zfs/dnode.c
@@ -1021,7 +1021,7 @@ dnode_buf_pageout(dmu_buf_t *db, void *arg)
 		dnh->dnh_dnode = NULL;
 	}
 	kmem_free(children_dnodes, sizeof (dnode_children_t) +
-	    (epb - 1) * sizeof (dnode_handle_t));
+	    epb * sizeof (dnode_handle_t));
 }
 
 /*
@@ -1106,7 +1106,7 @@ dnode_hold_impl(objset_t *os, uint64_t object, int flag,
 		int i;
 		dnode_children_t *winner;
 		children_dnodes = kmem_alloc(sizeof (dnode_children_t) +
-		    (epb - 1) * sizeof (dnode_handle_t), KM_SLEEP);
+		    epb * sizeof (dnode_handle_t), KM_SLEEP);
 		children_dnodes->dnc_count = epb;
 		dnh = &children_dnodes->dnc_children[0];
 		for (i = 0; i < epb; i++) {
@@ -1121,7 +1121,7 @@ dnode_hold_impl(objset_t *os, uint64_t object, int flag,
 			}
 
 			kmem_free(children_dnodes, sizeof (dnode_children_t) +
-			    (epb - 1) * sizeof (dnode_handle_t));
+			    epb * sizeof (dnode_handle_t));
 			children_dnodes = winner;
 		}
 	}
diff --git a/uts/common/fs/zfs/sys/dnode.h b/uts/common/fs/zfs/sys/dnode.h
index 7923a34a82bb..6e4a845a36a1 100644
--- a/uts/common/fs/zfs/sys/dnode.h
+++ b/uts/common/fs/zfs/sys/dnode.h
@@ -245,7 +245,7 @@ typedef struct dnode_handle {
 
 typedef struct dnode_children {
 	size_t dnc_count;		/* number of children */
-	dnode_handle_t dnc_children[1];	/* sized dynamically */
+	dnode_handle_t dnc_children[];	/* sized dynamically */
 } dnode_children_t;
 
 typedef struct free_range {

From 16184c507086871e355d00410bfe1f4128e0908d Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Mon, 4 Aug 2014 23:57:46 +0000
Subject: [PATCH 024/380] 5049 panic when removing log device Reviewed by:
 George Wilson <george.wilson@delphix.com> Reviewed by: Mattew Ahrens
 <mahrens@delphix.com> Reviewed by: Paul Dagnelie <paul.dagnelie@delphix.com>
 Reviewed by: Dan Kimmel <dan.kimmel@delphix.com> Reviewed by: Saso Kiselkov
 <skiselkov@gmail.com> Approved by: Rich Lowe <richlowe@richlowe.net> Author:
 Alex Reece <alex@delphix.com>

illumos/illumos-gate@2986efa8094ce00f75df27fb74a184c65c38614a
---
 uts/common/fs/zfs/vdev.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/uts/common/fs/zfs/vdev.c b/uts/common/fs/zfs/vdev.c
index e1a0fbfe4c59..28ef0414a280 100644
--- a/uts/common/fs/zfs/vdev.c
+++ b/uts/common/fs/zfs/vdev.c
@@ -2613,8 +2613,9 @@ vdev_get_stats(vdev_t *vd, vdev_stat_t *vs)
 	if (vd->vdev_ops->vdev_op_leaf)
 		vs->vs_rsize += VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE;
 	vs->vs_esize = vd->vdev_max_asize - vd->vdev_asize;
-	if (vd->vdev_aux == NULL && vd == vd->vdev_top)
+	if (vd->vdev_aux == NULL && vd == vd->vdev_top && !vd->vdev_ishole) {
 		vs->vs_fragmentation = vd->vdev_mg->mg_fragmentation;
+	}
 
 	/*
 	 * If we're getting stats on the root vdev, aggregate the I/O counts

From ae12ec375ef8ffaad4cb09b6005c3ae3a53a3032 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Wed, 20 Aug 2014 06:15:53 +0000
Subject: [PATCH 025/380] 5042 stop using deprecated atomic functions Reviewed
 by: Dan McDonald <danmcd@omniti.com> Approved by: Robert Mustacchi
 <rm@joyent.com> Author:	Josef 'Jeff' Sipek <josef.sipek@nexenta.com>

illumos/illumos-gate@75d94465dbafa487b716482dc36d5150a4ec9853
---
 uts/common/dtrace/systrace.c | 8 ++++----
 uts/common/fs/vnode.c        | 7 ++++---
 uts/common/os/fm.c           | 2 +-
 uts/common/sys/bitmap.h      | 4 ++--
 uts/common/sys/cpuvar.h      | 4 ++--
 5 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/uts/common/dtrace/systrace.c b/uts/common/dtrace/systrace.c
index b864041c450d..3d36ce954774 100644
--- a/uts/common/dtrace/systrace.c
+++ b/uts/common/dtrace/systrace.c
@@ -164,11 +164,11 @@ systrace_enable(void *arg, dtrace_id_t id, void *parg)
 		return (0);
 	}
 
-	(void) casptr(&sysent[sysnum].sy_callc,
+	(void) atomic_cas_ptr(&sysent[sysnum].sy_callc,
 	    (void *)systrace_sysent[sysnum].stsy_underlying,
 	    (void *)dtrace_systrace_syscall);
 #ifdef _SYSCALL32_IMPL
-	(void) casptr(&sysent32[sysnum].sy_callc,
+	(void) atomic_cas_ptr(&sysent32[sysnum].sy_callc,
 	    (void *)systrace_sysent32[sysnum].stsy_underlying,
 	    (void *)dtrace_systrace_syscall32);
 #endif
@@ -184,12 +184,12 @@ systrace_disable(void *arg, dtrace_id_t id, void *parg)
 	    systrace_sysent[sysnum].stsy_return == DTRACE_IDNONE);
 
 	if (disable) {
-		(void) casptr(&sysent[sysnum].sy_callc,
+		(void) atomic_cas_ptr(&sysent[sysnum].sy_callc,
 		    (void *)dtrace_systrace_syscall,
 		    (void *)systrace_sysent[sysnum].stsy_underlying);
 
 #ifdef _SYSCALL32_IMPL
-		(void) casptr(&sysent32[sysnum].sy_callc,
+		(void) atomic_cas_ptr(&sysent32[sysnum].sy_callc,
 		    (void *)dtrace_systrace_syscall32,
 		    (void *)systrace_sysent32[sysnum].stsy_underlying);
 #endif
diff --git a/uts/common/fs/vnode.c b/uts/common/fs/vnode.c
index d67d1642868e..9cd8f0562bd9 100644
--- a/uts/common/fs/vnode.c
+++ b/uts/common/fs/vnode.c
@@ -2837,11 +2837,12 @@ vn_setops(vnode_t *vp, vnodeops_t *vnodeops)
 	op = vp->v_op;
 	membar_consumer();
 	/*
-	 * If vp->v_femhead == NULL, then we'll call casptr() to do the
-	 * compare-and-swap on vp->v_op.  If either fails, then FEM is
+	 * If vp->v_femhead == NULL, then we'll call atomic_cas_ptr() to do
+	 * the compare-and-swap on vp->v_op.  If either fails, then FEM is
 	 * in effect on the vnode and we need to have FEM deal with it.
 	 */
-	if (vp->v_femhead != NULL || casptr(&vp->v_op, op, vnodeops) != op) {
+	if (vp->v_femhead != NULL || atomic_cas_ptr(&vp->v_op, op, vnodeops) !=
+	    op) {
 		fem_setvnops(vp, vnodeops);
 	}
 }
diff --git a/uts/common/os/fm.c b/uts/common/os/fm.c
index eff91aee5e64..fbe45e833ef9 100644
--- a/uts/common/os/fm.c
+++ b/uts/common/os/fm.c
@@ -374,7 +374,7 @@ fm_panic(const char *format, ...)
 {
 	va_list ap;
 
-	(void) casptr((void *)&fm_panicstr, NULL, (void *)format);
+	(void) atomic_cas_ptr((void *)&fm_panicstr, NULL, (void *)format);
 #if defined(__i386) || defined(__amd64)
 	fastreboot_disable_highpil();
 #endif /* __i386 || __amd64 */
diff --git a/uts/common/sys/bitmap.h b/uts/common/sys/bitmap.h
index d2592820b6d2..8d7054954d06 100644
--- a/uts/common/sys/bitmap.h
+++ b/uts/common/sys/bitmap.h
@@ -171,9 +171,9 @@ extern int	odd_parity(ulong_t);
  * to 0 otherwise.
  */
 #define	BT_ATOMIC_SET(bitmap, bitindex) \
-	{ atomic_or_long(&(BT_WIM(bitmap, bitindex)), BT_BIW(bitindex)); }
+	{ atomic_or_ulong(&(BT_WIM(bitmap, bitindex)), BT_BIW(bitindex)); }
 #define	BT_ATOMIC_CLEAR(bitmap, bitindex) \
-	{ atomic_and_long(&(BT_WIM(bitmap, bitindex)), ~BT_BIW(bitindex)); }
+	{ atomic_and_ulong(&(BT_WIM(bitmap, bitindex)), ~BT_BIW(bitindex)); }
 
 #define	BT_ATOMIC_SET_EXCL(bitmap, bitindex, result) \
 	{ result = atomic_set_long_excl(&(BT_WIM(bitmap, bitindex)),	\
diff --git a/uts/common/sys/cpuvar.h b/uts/common/sys/cpuvar.h
index 6c07bcbc8a8c..bdefd08e3aeb 100644
--- a/uts/common/sys/cpuvar.h
+++ b/uts/common/sys/cpuvar.h
@@ -528,8 +528,8 @@ typedef	ulong_t	cpuset_t;	/* a set of CPUs */
 	largest = (uint_t)(highbit(set) - 1);		\
 }
 
-#define	CPUSET_ATOMIC_DEL(set, cpu)	atomic_and_long(&(set), ~CPUSET(cpu))
-#define	CPUSET_ATOMIC_ADD(set, cpu)	atomic_or_long(&(set), CPUSET(cpu))
+#define	CPUSET_ATOMIC_DEL(set, cpu)	atomic_and_ulong(&(set), ~CPUSET(cpu))
+#define	CPUSET_ATOMIC_ADD(set, cpu)	atomic_or_ulong(&(set), CPUSET(cpu))
 
 #define	CPUSET_ATOMIC_XADD(set, cpu, result) \
 	{ result = atomic_set_long_excl(&(set), (cpu)); }

From 37edb9eef7944effe6b4caf9fa552cadbae030d5 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Wed, 20 Aug 2014 06:20:06 +0000
Subject: [PATCH 026/380] 5043 remove deprecated atomic functions' prototypes
 Reviewed by: Garrett D'Amore <garrett@damore.org> Approved by: Robert
 Mustacchi <rm@joyent.com> Author: Josef 'Jeff' Sipek
 <josef.sipek@nexenta.com>

illumos/illumos-gate@6ed9368a130d7c9a82e574da808d34034da33748
---
 common/atomic/amd64/atomic.s | 3 ++-
 common/atomic/i386/atomic.s  | 3 ++-
 common/atomic/sparc/atomic.s | 3 ++-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/common/atomic/amd64/atomic.s b/common/atomic/amd64/atomic.s
index 4b0d66e4db20..1c382546d1c9 100644
--- a/common/atomic/amd64/atomic.s
+++ b/common/atomic/amd64/atomic.s
@@ -29,7 +29,8 @@
 
 #if defined(_KERNEL)
 	/*
-	 * Legacy kernel interfaces; they will go away (eventually).
+	 * Legacy kernel interfaces; they will go away the moment our closed
+	 * bins no longer require them.
 	 */
 	ANSI_PRAGMA_WEAK2(cas8,atomic_cas_8,function)
 	ANSI_PRAGMA_WEAK2(cas32,atomic_cas_32,function)
diff --git a/common/atomic/i386/atomic.s b/common/atomic/i386/atomic.s
index 4fa525ba20af..ce6ccbbcf767 100644
--- a/common/atomic/i386/atomic.s
+++ b/common/atomic/i386/atomic.s
@@ -30,7 +30,8 @@
 
 #if defined(_KERNEL)
 	/*
-	 * Legacy kernel interfaces; they will go away (eventually).
+	 * Legacy kernel interfaces; they will go away the moment our closed
+	 * bins no longer require them.
 	 */
 	ANSI_PRAGMA_WEAK2(cas8,atomic_cas_8,function)
 	ANSI_PRAGMA_WEAK2(cas32,atomic_cas_32,function)
diff --git a/common/atomic/sparc/atomic.s b/common/atomic/sparc/atomic.s
index 8aa240efa297..c7f108179547 100644
--- a/common/atomic/sparc/atomic.s
+++ b/common/atomic/sparc/atomic.s
@@ -30,7 +30,8 @@
 
 #if defined(_KERNEL)
 	/*
-	 * Legacy kernel interfaces; they will go away (eventually).
+	 * Legacy kernel interfaces; they will go away the moment our closed
+	 * bins no longer require them.
 	 */
 	ANSI_PRAGMA_WEAK2(cas8,atomic_cas_8,function)
 	ANSI_PRAGMA_WEAK2(cas32,atomic_cas_32,function)

From ce3fd300215550e4539bb501754e5c8ca917568c Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Wed, 20 Aug 2014 06:22:59 +0000
Subject: [PATCH 027/380] 5045 use atomic_{inc,dec}_* instead of atomic_add_*
 Reviewed by: Matthew Ahrens <mahrens@delphix.com> Reviewed by: Garrett
 D'Amore <garrett@damore.org> Approved by: Robert Mustacchi <rm@joyent.com>
 Author: Josef 'Jeff' Sipek <josef.sipek@nexenta.com>

illumos/illumos-gate@1a5e258f5471356ca102c7176637cdce45bac147
---
 uts/common/dtrace/fasttrap.c     |  12 ++--
 uts/common/dtrace/profile.c      |   6 +-
 uts/common/fs/vnode.c            |  30 ++++----
 uts/common/fs/zfs/dbuf.c         |   4 +-
 uts/common/fs/zfs/spa.c          |   4 +-
 uts/common/fs/zfs/sys/refcount.h |   4 +-
 uts/common/fs/zfs/vdev_cache.c   |   2 +-
 uts/common/fs/zfs/vdev_label.c   |   4 +-
 uts/common/fs/zfs/zfs_vfsops.c   |   4 +-
 uts/common/fs/zfs/zio_inject.c   |   4 +-
 uts/common/os/fm.c               | 117 +++++++++++++++----------------
 uts/sparc/dtrace/fasttrap_isa.c  |  22 +++---
 12 files changed, 104 insertions(+), 109 deletions(-)

diff --git a/uts/common/dtrace/fasttrap.c b/uts/common/dtrace/fasttrap.c
index f486c06b2cef..ce7b1c40e61c 100644
--- a/uts/common/dtrace/fasttrap.c
+++ b/uts/common/dtrace/fasttrap.c
@@ -1182,7 +1182,7 @@ fasttrap_proc_lookup(pid_t pid)
 			mutex_enter(&fprc->ftpc_mtx);
 			mutex_exit(&bucket->ftb_mtx);
 			fprc->ftpc_rcount++;
-			atomic_add_64(&fprc->ftpc_acount, 1);
+			atomic_inc_64(&fprc->ftpc_acount);
 			ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
 			mutex_exit(&fprc->ftpc_mtx);
 
@@ -1212,7 +1212,7 @@ fasttrap_proc_lookup(pid_t pid)
 			mutex_enter(&fprc->ftpc_mtx);
 			mutex_exit(&bucket->ftb_mtx);
 			fprc->ftpc_rcount++;
-			atomic_add_64(&fprc->ftpc_acount, 1);
+			atomic_inc_64(&fprc->ftpc_acount);
 			ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
 			mutex_exit(&fprc->ftpc_mtx);
 
@@ -1424,7 +1424,7 @@ fasttrap_provider_free(fasttrap_provider_t *provider)
 	 * count of active providers on the associated process structure.
 	 */
 	if (!provider->ftp_retired) {
-		atomic_add_64(&provider->ftp_proc->ftpc_acount, -1);
+		atomic_dec_64(&provider->ftp_proc->ftpc_acount);
 		ASSERT(provider->ftp_proc->ftpc_acount <
 		    provider->ftp_proc->ftpc_rcount);
 	}
@@ -1499,7 +1499,7 @@ fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
 	 * bucket lock therefore protects the integrity of the provider hash
 	 * table.
 	 */
-	atomic_add_64(&fp->ftp_proc->ftpc_acount, -1);
+	atomic_dec_64(&fp->ftp_proc->ftpc_acount);
 	ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount);
 
 	fp->ftp_retired = 1;
@@ -1595,10 +1595,10 @@ fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
 			    pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
 				continue;
 
-			atomic_add_32(&fasttrap_total, 1);
+			atomic_inc_32(&fasttrap_total);
 
 			if (fasttrap_total > fasttrap_max) {
-				atomic_add_32(&fasttrap_total, -1);
+				atomic_dec_32(&fasttrap_total);
 				goto no_mem;
 			}
 
diff --git a/uts/common/dtrace/profile.c b/uts/common/dtrace/profile.c
index fc809d3579a5..106fb735df63 100644
--- a/uts/common/dtrace/profile.c
+++ b/uts/common/dtrace/profile.c
@@ -171,9 +171,9 @@ profile_create(hrtime_t interval, const char *name, int kind)
 	if (dtrace_probe_lookup(profile_id, NULL, NULL, name) != 0)
 		return;
 
-	atomic_add_32(&profile_total, 1);
+	atomic_inc_32(&profile_total);
 	if (profile_total > profile_max) {
-		atomic_add_32(&profile_total, -1);
+		atomic_dec_32(&profile_total);
 		return;
 	}
 
@@ -328,7 +328,7 @@ profile_destroy(void *arg, dtrace_id_t id, void *parg)
 	kmem_free(prof, sizeof (profile_probe_t));
 
 	ASSERT(profile_total >= 1);
-	atomic_add_32(&profile_total, -1);
+	atomic_dec_32(&profile_total);
 }
 
 /*ARGSUSED*/
diff --git a/uts/common/fs/vnode.c b/uts/common/fs/vnode.c
index 9cd8f0562bd9..4abb040de0f0 100644
--- a/uts/common/fs/vnode.c
+++ b/uts/common/fs/vnode.c
@@ -1244,9 +1244,9 @@ vn_open_upgrade(
 	ASSERT(vp->v_type == VREG);
 
 	if (filemode & FREAD)
-		atomic_add_32(&(vp->v_rdcnt), 1);
+		atomic_inc_32(&vp->v_rdcnt);
 	if (filemode & FWRITE)
-		atomic_add_32(&(vp->v_wrcnt), 1);
+		atomic_inc_32(&vp->v_wrcnt);
 
 }
 
@@ -1259,11 +1259,11 @@ vn_open_downgrade(
 
 	if (filemode & FREAD) {
 		ASSERT(vp->v_rdcnt > 0);
-		atomic_add_32(&(vp->v_rdcnt), -1);
+		atomic_dec_32(&vp->v_rdcnt);
 	}
 	if (filemode & FWRITE) {
 		ASSERT(vp->v_wrcnt > 0);
-		atomic_add_32(&(vp->v_wrcnt), -1);
+		atomic_dec_32(&vp->v_wrcnt);
 	}
 
 }
@@ -2918,7 +2918,7 @@ fs_new_caller_id()
 {
 	static uint64_t next_caller_id = 0LL; /* First call returns 1 */
 
-	return ((u_longlong_t)atomic_add_64_nv(&next_caller_id, 1));
+	return ((u_longlong_t)atomic_inc_64_nv(&next_caller_id));
 }
 
 /*
@@ -3146,9 +3146,9 @@ fop_open(
 	 */
 	if ((*vpp)->v_type == VREG) {
 		if (mode & FREAD)
-			atomic_add_32(&((*vpp)->v_rdcnt), 1);
+			atomic_inc_32(&(*vpp)->v_rdcnt);
 		if (mode & FWRITE)
-			atomic_add_32(&((*vpp)->v_wrcnt), 1);
+			atomic_inc_32(&(*vpp)->v_wrcnt);
 	}
 
 	VOPXID_MAP_CR(vp, cr);
@@ -3162,9 +3162,9 @@ fop_open(
 		 */
 		VOPSTATS_UPDATE(vp, open);
 		if ((vp->v_type == VREG) && (mode & FREAD))
-			atomic_add_32(&(vp->v_rdcnt), -1);
+			atomic_dec_32(&vp->v_rdcnt);
 		if ((vp->v_type == VREG) && (mode & FWRITE))
-			atomic_add_32(&(vp->v_wrcnt), -1);
+			atomic_dec_32(&vp->v_wrcnt);
 	} else {
 		/*
 		 * Some filesystems will return a different vnode,
@@ -3178,13 +3178,13 @@ fop_open(
 		if (*vpp != vp && *vpp != NULL) {
 			vn_copypath(vp, *vpp);
 			if (((*vpp)->v_type == VREG) && (mode & FREAD))
-				atomic_add_32(&((*vpp)->v_rdcnt), 1);
+				atomic_inc_32(&(*vpp)->v_rdcnt);
 			if ((vp->v_type == VREG) && (mode & FREAD))
-				atomic_add_32(&(vp->v_rdcnt), -1);
+				atomic_dec_32(&vp->v_rdcnt);
 			if (((*vpp)->v_type == VREG) && (mode & FWRITE))
-				atomic_add_32(&((*vpp)->v_wrcnt), 1);
+				atomic_inc_32(&(*vpp)->v_wrcnt);
 			if ((vp->v_type == VREG) && (mode & FWRITE))
-				atomic_add_32(&(vp->v_wrcnt), -1);
+				atomic_dec_32(&vp->v_wrcnt);
 		}
 	}
 	VN_RELE(vp);
@@ -3213,11 +3213,11 @@ fop_close(
 	if ((vp->v_type == VREG) && (count == 1))  {
 		if (flag & FREAD) {
 			ASSERT(vp->v_rdcnt > 0);
-			atomic_add_32(&(vp->v_rdcnt), -1);
+			atomic_dec_32(&vp->v_rdcnt);
 		}
 		if (flag & FWRITE) {
 			ASSERT(vp->v_wrcnt > 0);
-			atomic_add_32(&(vp->v_wrcnt), -1);
+			atomic_dec_32(&vp->v_wrcnt);
 		}
 	}
 	return (err);
diff --git a/uts/common/fs/zfs/dbuf.c b/uts/common/fs/zfs/dbuf.c
index 8db626fadc3b..a56372197cdf 100644
--- a/uts/common/fs/zfs/dbuf.c
+++ b/uts/common/fs/zfs/dbuf.c
@@ -178,7 +178,7 @@ dbuf_hash_insert(dmu_buf_impl_t *db)
 	db->db_hash_next = h->hash_table[idx];
 	h->hash_table[idx] = db;
 	mutex_exit(DBUF_HASH_MUTEX(h, idx));
-	atomic_add_64(&dbuf_hash_count, 1);
+	atomic_inc_64(&dbuf_hash_count);
 
 	return (NULL);
 }
@@ -212,7 +212,7 @@ dbuf_hash_remove(dmu_buf_impl_t *db)
 	*dbp = db->db_hash_next;
 	db->db_hash_next = NULL;
 	mutex_exit(DBUF_HASH_MUTEX(h, idx));
-	atomic_add_64(&dbuf_hash_count, -1);
+	atomic_dec_64(&dbuf_hash_count);
 }
 
 static arc_evict_func_t dbuf_do_evict;
diff --git a/uts/common/fs/zfs/spa.c b/uts/common/fs/zfs/spa.c
index c10cac1b2009..72870b589891 100644
--- a/uts/common/fs/zfs/spa.c
+++ b/uts/common/fs/zfs/spa.c
@@ -1817,9 +1817,9 @@ spa_load_verify_done(zio_t *zio)
 	if (error) {
 		if ((BP_GET_LEVEL(bp) != 0 || DMU_OT_IS_METADATA(type)) &&
 		    type != DMU_OT_INTENT_LOG)
-			atomic_add_64(&sle->sle_meta_count, 1);
+			atomic_inc_64(&sle->sle_meta_count);
 		else
-			atomic_add_64(&sle->sle_data_count, 1);
+			atomic_inc_64(&sle->sle_data_count);
 	}
 	zio_data_buf_free(zio->io_data, zio->io_size);
 
diff --git a/uts/common/fs/zfs/sys/refcount.h b/uts/common/fs/zfs/sys/refcount.h
index 9efc5f1e445b..27c39135e056 100644
--- a/uts/common/fs/zfs/sys/refcount.h
+++ b/uts/common/fs/zfs/sys/refcount.h
@@ -87,8 +87,8 @@ typedef struct refcount {
 #define	refcount_destroy_many(rc, number) ((rc)->rc_count = 0)
 #define	refcount_is_zero(rc) ((rc)->rc_count == 0)
 #define	refcount_count(rc) ((rc)->rc_count)
-#define	refcount_add(rc, holder) atomic_add_64_nv(&(rc)->rc_count, 1)
-#define	refcount_remove(rc, holder) atomic_add_64_nv(&(rc)->rc_count, -1)
+#define	refcount_add(rc, holder) atomic_inc_64_nv(&(rc)->rc_count)
+#define	refcount_remove(rc, holder) atomic_dec_64_nv(&(rc)->rc_count)
 #define	refcount_add_many(rc, number, holder) \
 	atomic_add_64_nv(&(rc)->rc_count, number)
 #define	refcount_remove_many(rc, number, holder) \
diff --git a/uts/common/fs/zfs/vdev_cache.c b/uts/common/fs/zfs/vdev_cache.c
index 678c4e40ea6c..0b188dbc1668 100644
--- a/uts/common/fs/zfs/vdev_cache.c
+++ b/uts/common/fs/zfs/vdev_cache.c
@@ -102,7 +102,7 @@ static vdc_stats_t vdc_stats = {
 	{ "misses",		KSTAT_DATA_UINT64 }
 };
 
-#define	VDCSTAT_BUMP(stat)	atomic_add_64(&vdc_stats.stat.value.ui64, 1);
+#define	VDCSTAT_BUMP(stat)	atomic_inc_64(&vdc_stats.stat.value.ui64);
 
 static int
 vdev_cache_offset_compare(const void *a1, const void *a2)
diff --git a/uts/common/fs/zfs/vdev_label.c b/uts/common/fs/zfs/vdev_label.c
index c7ae60c68f0c..7bbd7f2bde69 100644
--- a/uts/common/fs/zfs/vdev_label.c
+++ b/uts/common/fs/zfs/vdev_label.c
@@ -976,7 +976,7 @@ vdev_uberblock_sync_done(zio_t *zio)
 	uint64_t *good_writes = zio->io_private;
 
 	if (zio->io_error == 0 && zio->io_vd->vdev_top->vdev_ms_array != 0)
-		atomic_add_64(good_writes, 1);
+		atomic_inc_64(good_writes);
 }
 
 /*
@@ -1051,7 +1051,7 @@ vdev_label_sync_done(zio_t *zio)
 	uint64_t *good_writes = zio->io_private;
 
 	if (zio->io_error == 0)
-		atomic_add_64(good_writes, 1);
+		atomic_inc_64(good_writes);
 }
 
 /*
diff --git a/uts/common/fs/zfs/zfs_vfsops.c b/uts/common/fs/zfs/zfs_vfsops.c
index 8e3bf691672a..0b0b0a99cc6e 100644
--- a/uts/common/fs/zfs/zfs_vfsops.c
+++ b/uts/common/fs/zfs/zfs_vfsops.c
@@ -1240,7 +1240,7 @@ out:
 		dmu_objset_disown(zfsvfs->z_os, zfsvfs);
 		zfsvfs_free(zfsvfs);
 	} else {
-		atomic_add_32(&zfs_active_fs_count, 1);
+		atomic_inc_32(&zfs_active_fs_count);
 	}
 
 	return (error);
@@ -2158,7 +2158,7 @@ zfs_freevfs(vfs_t *vfsp)
 
 	zfsvfs_free(zfsvfs);
 
-	atomic_add_32(&zfs_active_fs_count, -1);
+	atomic_dec_32(&zfs_active_fs_count);
 }
 
 /*
diff --git a/uts/common/fs/zfs/zio_inject.c b/uts/common/fs/zfs/zio_inject.c
index b2976111d65e..991a0a34ff6a 100644
--- a/uts/common/fs/zfs/zio_inject.c
+++ b/uts/common/fs/zfs/zio_inject.c
@@ -426,7 +426,7 @@ zio_inject_fault(char *name, int flags, int *id, zinject_record_t *record)
 		handler->zi_spa = spa;
 		handler->zi_record = *record;
 		list_insert_tail(&inject_handlers, handler);
-		atomic_add_32(&zio_injection_enabled, 1);
+		atomic_inc_32(&zio_injection_enabled);
 
 		rw_exit(&inject_lock);
 	}
@@ -503,7 +503,7 @@ zio_clear_fault(int id)
 
 	spa_inject_delref(handler->zi_spa);
 	kmem_free(handler, sizeof (inject_handler_t));
-	atomic_add_32(&zio_injection_enabled, -1);
+	atomic_dec_32(&zio_injection_enabled);
 
 	return (0);
 }
diff --git a/uts/common/os/fm.c b/uts/common/os/fm.c
index fbe45e833ef9..66fe699366a1 100644
--- a/uts/common/os/fm.c
+++ b/uts/common/os/fm.c
@@ -517,19 +517,19 @@ fm_ereport_post(nvlist_t *ereport, int evc_flag)
 
 	(void) nvlist_size(ereport, &nvl_size, NV_ENCODE_NATIVE);
 	if (nvl_size > ERPT_DATA_SZ || nvl_size == 0) {
-		atomic_add_64(&erpt_kstat_data.erpt_dropped.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.erpt_dropped.value.ui64);
 		return;
 	}
 
 	if (sysevent_evc_bind(FM_ERROR_CHAN, &error_chan,
 	    EVCH_CREAT|EVCH_HOLD_PEND) != 0) {
-		atomic_add_64(&erpt_kstat_data.erpt_dropped.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.erpt_dropped.value.ui64);
 		return;
 	}
 
 	if (sysevent_evc_publish(error_chan, EC_FM, ESC_FM_ERROR,
 	    SUNW_VENDOR, FM_PUB, ereport, evc_flag) != 0) {
-		atomic_add_64(&erpt_kstat_data.erpt_dropped.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.erpt_dropped.value.ui64);
 		(void) sysevent_evc_unbind(error_chan);
 		return;
 	}
@@ -791,8 +791,7 @@ fm_payload_set(nvlist_t *payload, ...)
 	va_end(ap);
 
 	if (ret)
-		atomic_add_64(
-		    &erpt_kstat_data.payload_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.payload_set_failed.value.ui64);
 }
 
 /*
@@ -825,24 +824,24 @@ fm_ereport_set(nvlist_t *ereport, int version, const char *erpt_class,
 	int ret;
 
 	if (version != FM_EREPORT_VERS0) {
-		atomic_add_64(&erpt_kstat_data.erpt_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.erpt_set_failed.value.ui64);
 		return;
 	}
 
 	(void) snprintf(ereport_class, FM_MAX_CLASS, "%s.%s",
 	    FM_EREPORT_CLASS, erpt_class);
 	if (nvlist_add_string(ereport, FM_CLASS, ereport_class) != 0) {
-		atomic_add_64(&erpt_kstat_data.erpt_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.erpt_set_failed.value.ui64);
 		return;
 	}
 
 	if (nvlist_add_uint64(ereport, FM_EREPORT_ENA, ena)) {
-		atomic_add_64(&erpt_kstat_data.erpt_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.erpt_set_failed.value.ui64);
 	}
 
 	if (nvlist_add_nvlist(ereport, FM_EREPORT_DETECTOR,
 	    (nvlist_t *)detector) != 0) {
-		atomic_add_64(&erpt_kstat_data.erpt_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.erpt_set_failed.value.ui64);
 	}
 
 	va_start(ap, detector);
@@ -851,7 +850,7 @@ fm_ereport_set(nvlist_t *ereport, int version, const char *erpt_class,
 	va_end(ap);
 
 	if (ret)
-		atomic_add_64(&erpt_kstat_data.erpt_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.erpt_set_failed.value.ui64);
 }
 
 /*
@@ -874,19 +873,19 @@ static int
 fm_fmri_hc_set_common(nvlist_t *fmri, int version, const nvlist_t *auth)
 {
 	if (version != FM_HC_SCHEME_VERSION) {
-		atomic_add_64(&erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.fmri_set_failed.value.ui64);
 		return (0);
 	}
 
 	if (nvlist_add_uint8(fmri, FM_VERSION, version) != 0 ||
 	    nvlist_add_string(fmri, FM_FMRI_SCHEME, FM_FMRI_SCHEME_HC) != 0) {
-		atomic_add_64(&erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.fmri_set_failed.value.ui64);
 		return (0);
 	}
 
 	if (auth != NULL && nvlist_add_nvlist(fmri, FM_FMRI_AUTHORITY,
 	    (nvlist_t *)auth) != 0) {
-		atomic_add_64(&erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.fmri_set_failed.value.ui64);
 		return (0);
 	}
 
@@ -918,22 +917,22 @@ fm_fmri_hc_set(nvlist_t *fmri, int version, const nvlist_t *auth,
 		pairs[i] = fm_nvlist_create(nva);
 		if (nvlist_add_string(pairs[i], FM_FMRI_HC_NAME, name) != 0 ||
 		    nvlist_add_string(pairs[i], FM_FMRI_HC_ID, idstr) != 0) {
-			atomic_add_64(
-			    &erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+			atomic_inc_64(
+			    &erpt_kstat_data.fmri_set_failed.value.ui64);
 		}
 	}
 	va_end(ap);
 
 	if (nvlist_add_nvlist_array(fmri, FM_FMRI_HC_LIST, pairs, npairs) != 0)
-		atomic_add_64(&erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.fmri_set_failed.value.ui64);
 
 	for (i = 0; i < npairs; i++)
 		fm_nvlist_destroy(pairs[i], FM_NVA_RETAIN);
 
 	if (snvl != NULL) {
 		if (nvlist_add_nvlist(fmri, FM_FMRI_HC_SPECIFIC, snvl) != 0) {
-			atomic_add_64(
-			    &erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+			atomic_inc_64(
+			    &erpt_kstat_data.fmri_set_failed.value.ui64);
 		}
 	}
 }
@@ -958,7 +957,7 @@ fm_fmri_dev_set(nvlist_t *fmri_dev, int version, const nvlist_t *auth,
 	int err = 0;
 
 	if (version != DEV_SCHEME_VERSION0) {
-		atomic_add_64(&erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.fmri_set_failed.value.ui64);
 		return;
 	}
 
@@ -979,7 +978,7 @@ fm_fmri_dev_set(nvlist_t *fmri_dev, int version, const nvlist_t *auth,
 		err |= nvlist_add_string(fmri_dev, FM_FMRI_DEV_TGTPTLUN0, tpl0);
 
 	if (err)
-		atomic_add_64(&erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.fmri_set_failed.value.ui64);
 
 }
 
@@ -1004,35 +1003,35 @@ fm_fmri_cpu_set(nvlist_t *fmri_cpu, int version, const nvlist_t *auth,
 	uint64_t *failedp = &erpt_kstat_data.fmri_set_failed.value.ui64;
 
 	if (version < CPU_SCHEME_VERSION1) {
-		atomic_add_64(failedp, 1);
+		atomic_inc_64(failedp);
 		return;
 	}
 
 	if (nvlist_add_uint8(fmri_cpu, FM_VERSION, version) != 0) {
-		atomic_add_64(failedp, 1);
+		atomic_inc_64(failedp);
 		return;
 	}
 
 	if (nvlist_add_string(fmri_cpu, FM_FMRI_SCHEME,
 	    FM_FMRI_SCHEME_CPU) != 0) {
-		atomic_add_64(failedp, 1);
+		atomic_inc_64(failedp);
 		return;
 	}
 
 	if (auth != NULL && nvlist_add_nvlist(fmri_cpu, FM_FMRI_AUTHORITY,
 	    (nvlist_t *)auth) != 0)
-		atomic_add_64(failedp, 1);
+		atomic_inc_64(failedp);
 
 	if (nvlist_add_uint32(fmri_cpu, FM_FMRI_CPU_ID, cpu_id) != 0)
-		atomic_add_64(failedp, 1);
+		atomic_inc_64(failedp);
 
 	if (cpu_maskp != NULL && nvlist_add_uint8(fmri_cpu, FM_FMRI_CPU_MASK,
 	    *cpu_maskp) != 0)
-		atomic_add_64(failedp, 1);
+		atomic_inc_64(failedp);
 
 	if (serial_idp == NULL || nvlist_add_string(fmri_cpu,
 	    FM_FMRI_CPU_SERIAL_ID, (char *)serial_idp) != 0)
-			atomic_add_64(failedp, 1);
+			atomic_inc_64(failedp);
 }
 
 /*
@@ -1053,49 +1052,47 @@ fm_fmri_mem_set(nvlist_t *fmri, int version, const nvlist_t *auth,
     const char *unum, const char *serial, uint64_t offset)
 {
 	if (version != MEM_SCHEME_VERSION0) {
-		atomic_add_64(&erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.fmri_set_failed.value.ui64);
 		return;
 	}
 
 	if (!serial && (offset != (uint64_t)-1)) {
-		atomic_add_64(&erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.fmri_set_failed.value.ui64);
 		return;
 	}
 
 	if (nvlist_add_uint8(fmri, FM_VERSION, version) != 0) {
-		atomic_add_64(&erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.fmri_set_failed.value.ui64);
 		return;
 	}
 
 	if (nvlist_add_string(fmri, FM_FMRI_SCHEME, FM_FMRI_SCHEME_MEM) != 0) {
-		atomic_add_64(&erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.fmri_set_failed.value.ui64);
 		return;
 	}
 
 	if (auth != NULL) {
 		if (nvlist_add_nvlist(fmri, FM_FMRI_AUTHORITY,
 		    (nvlist_t *)auth) != 0) {
-			atomic_add_64(
-			    &erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+			atomic_inc_64(
+			    &erpt_kstat_data.fmri_set_failed.value.ui64);
 		}
 	}
 
 	if (nvlist_add_string(fmri, FM_FMRI_MEM_UNUM, unum) != 0) {
-		atomic_add_64(&erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.fmri_set_failed.value.ui64);
 	}
 
 	if (serial != NULL) {
 		if (nvlist_add_string_array(fmri, FM_FMRI_MEM_SERIAL_ID,
 		    (char **)&serial, 1) != 0) {
-			atomic_add_64(
-			    &erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+			atomic_inc_64(
+			    &erpt_kstat_data.fmri_set_failed.value.ui64);
 		}
-		if (offset != (uint64_t)-1) {
-			if (nvlist_add_uint64(fmri, FM_FMRI_MEM_OFFSET,
-			    offset) != 0) {
-				atomic_add_64(&erpt_kstat_data.
-				    fmri_set_failed.value.ui64, 1);
-			}
+		if (offset != (uint64_t)-1 && nvlist_add_uint64(fmri,
+		    FM_FMRI_MEM_OFFSET, offset) != 0) {
+			atomic_inc_64(
+			    &erpt_kstat_data.fmri_set_failed.value.ui64);
 		}
 	}
 }
@@ -1105,28 +1102,28 @@ fm_fmri_zfs_set(nvlist_t *fmri, int version, uint64_t pool_guid,
     uint64_t vdev_guid)
 {
 	if (version != ZFS_SCHEME_VERSION0) {
-		atomic_add_64(&erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.fmri_set_failed.value.ui64);
 		return;
 	}
 
 	if (nvlist_add_uint8(fmri, FM_VERSION, version) != 0) {
-		atomic_add_64(&erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.fmri_set_failed.value.ui64);
 		return;
 	}
 
 	if (nvlist_add_string(fmri, FM_FMRI_SCHEME, FM_FMRI_SCHEME_ZFS) != 0) {
-		atomic_add_64(&erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.fmri_set_failed.value.ui64);
 		return;
 	}
 
 	if (nvlist_add_uint64(fmri, FM_FMRI_ZFS_POOL, pool_guid) != 0) {
-		atomic_add_64(&erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.fmri_set_failed.value.ui64);
 	}
 
 	if (vdev_guid != 0) {
 		if (nvlist_add_uint64(fmri, FM_FMRI_ZFS_VDEV, vdev_guid) != 0) {
-			atomic_add_64(
-			    &erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+			atomic_inc_64(
+			    &erpt_kstat_data.fmri_set_failed.value.ui64);
 		}
 	}
 }
@@ -1306,20 +1303,20 @@ fm_fmri_hc_create(nvlist_t *fmri, int version, const nvlist_t *auth,
 	 */
 	if (nvlist_lookup_nvlist_array(bboard, FM_FMRI_HC_LIST, &hcl, &n)
 	    != 0) {
-		atomic_add_64(&erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.fmri_set_failed.value.ui64);
 		return;
 	}
 
 	for (i = 0; i < n; i++) {
 		if (nvlist_lookup_string(hcl[i], FM_FMRI_HC_NAME,
 		    &hcname) != 0) {
-			atomic_add_64(
-			    &erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+			atomic_inc_64(
+			    &erpt_kstat_data.fmri_set_failed.value.ui64);
 			return;
 		}
 		if (nvlist_lookup_string(hcl[i], FM_FMRI_HC_ID, &hcid) != 0) {
-			atomic_add_64(
-			    &erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+			atomic_inc_64(
+			    &erpt_kstat_data.fmri_set_failed.value.ui64);
 			return;
 		}
 
@@ -1331,8 +1328,8 @@ fm_fmri_hc_create(nvlist_t *fmri, int version, const nvlist_t *auth,
 					fm_nvlist_destroy(pairs[j],
 					    FM_NVA_RETAIN);
 			}
-			atomic_add_64(
-			    &erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+			atomic_inc_64(
+			    &erpt_kstat_data.fmri_set_failed.value.ui64);
 			return;
 		}
 	}
@@ -1356,8 +1353,8 @@ fm_fmri_hc_create(nvlist_t *fmri, int version, const nvlist_t *auth,
 					fm_nvlist_destroy(pairs[j],
 					    FM_NVA_RETAIN);
 			}
-			atomic_add_64(
-			    &erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+			atomic_inc_64(
+			    &erpt_kstat_data.fmri_set_failed.value.ui64);
 			return;
 		}
 	}
@@ -1368,7 +1365,7 @@ fm_fmri_hc_create(nvlist_t *fmri, int version, const nvlist_t *auth,
 	 */
 	if (nvlist_add_nvlist_array(fmri, FM_FMRI_HC_LIST, pairs,
 	    npairs + n) != 0) {
-		atomic_add_64(&erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+		atomic_inc_64(&erpt_kstat_data.fmri_set_failed.value.ui64);
 		return;
 	}
 
@@ -1378,8 +1375,8 @@ fm_fmri_hc_create(nvlist_t *fmri, int version, const nvlist_t *auth,
 
 	if (snvl != NULL) {
 		if (nvlist_add_nvlist(fmri, FM_FMRI_HC_SPECIFIC, snvl) != 0) {
-			atomic_add_64(
-			    &erpt_kstat_data.fmri_set_failed.value.ui64, 1);
+			atomic_inc_64(
+			    &erpt_kstat_data.fmri_set_failed.value.ui64);
 			return;
 		}
 	}
diff --git a/uts/sparc/dtrace/fasttrap_isa.c b/uts/sparc/dtrace/fasttrap_isa.c
index 45d87478d6a2..735bb9d0ebd9 100644
--- a/uts/sparc/dtrace/fasttrap_isa.c
+++ b/uts/sparc/dtrace/fasttrap_isa.c
@@ -24,8 +24,6 @@
  * Use is subject to license terms.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <sys/fasttrap_isa.h>
 #include <sys/fasttrap_impl.h>
 #include <sys/dtrace.h>
@@ -1410,7 +1408,7 @@ fasttrap_getreg(struct regs *rp, uint_t reg)
 		value = dtrace_getreg_win(reg, 1);
 		dtrace_interrupt_enable(cookie);
 
-		atomic_add_64(&fasttrap_getreg_fast_cnt, 1);
+		atomic_inc_64(&fasttrap_getreg_fast_cnt);
 
 		return (value);
 	}
@@ -1435,7 +1433,7 @@ fasttrap_getreg(struct regs *rp, uint_t reg)
 				if ((long)mpcb->mpcb_spbuf[i] != rp->r_sp)
 					continue;
 
-				atomic_add_64(&fasttrap_getreg_mpcb_cnt, 1);
+				atomic_inc_64(&fasttrap_getreg_mpcb_cnt);
 				return (rwin[i].rw_local[reg - 16]);
 			} while (i > 0);
 		}
@@ -1455,7 +1453,7 @@ fasttrap_getreg(struct regs *rp, uint_t reg)
 				if ((long)mpcb->mpcb_spbuf[i] != rp->r_sp)
 					continue;
 
-				atomic_add_64(&fasttrap_getreg_mpcb_cnt, 1);
+				atomic_inc_64(&fasttrap_getreg_mpcb_cnt);
 				return (rwin[i].rw_local[reg - 16]);
 			} while (i > 0);
 		}
@@ -1466,7 +1464,7 @@ fasttrap_getreg(struct regs *rp, uint_t reg)
 		v32[0] = 0;
 	}
 
-	atomic_add_64(&fasttrap_getreg_slow_cnt, 1);
+	atomic_inc_64(&fasttrap_getreg_slow_cnt);
 	return (value);
 
 err:
@@ -1505,7 +1503,7 @@ fasttrap_putreg(struct regs *rp, uint_t reg, ulong_t value)
 	if (dtrace_getotherwin() > 0) {
 		dtrace_putreg_win(reg, value);
 		dtrace_interrupt_enable(cookie);
-		atomic_add_64(&fasttrap_putreg_fast_cnt, 1);
+		atomic_inc_64(&fasttrap_putreg_fast_cnt);
 		return;
 	}
 	dtrace_interrupt_enable(cookie);
@@ -1536,7 +1534,7 @@ fasttrap_putreg(struct regs *rp, uint_t reg, ulong_t value)
 					continue;
 
 				rwin[i].rw_local[reg - 16] = value;
-				atomic_add_64(&fasttrap_putreg_mpcb_cnt, 1);
+				atomic_inc_64(&fasttrap_putreg_mpcb_cnt);
 				return;
 			} while (i > 0);
 		}
@@ -1549,7 +1547,7 @@ fasttrap_putreg(struct regs *rp, uint_t reg, ulong_t value)
 			rwin[mpcb->mpcb_wbcnt].rw_local[reg - 16] = value;
 			mpcb->mpcb_spbuf[mpcb->mpcb_wbcnt] = (caddr_t)rp->r_sp;
 			mpcb->mpcb_wbcnt++;
-			atomic_add_64(&fasttrap_putreg_mpcb_cnt, 1);
+			atomic_inc_64(&fasttrap_putreg_mpcb_cnt);
 			return;
 		}
 	} else {
@@ -1567,7 +1565,7 @@ fasttrap_putreg(struct regs *rp, uint_t reg, ulong_t value)
 					continue;
 
 				rwin[i].rw_local[reg - 16] = v32;
-				atomic_add_64(&fasttrap_putreg_mpcb_cnt, 1);
+				atomic_inc_64(&fasttrap_putreg_mpcb_cnt);
 				return;
 			} while (i > 0);
 		}
@@ -1580,12 +1578,12 @@ fasttrap_putreg(struct regs *rp, uint_t reg, ulong_t value)
 			rwin[mpcb->mpcb_wbcnt].rw_local[reg - 16] = v32;
 			mpcb->mpcb_spbuf[mpcb->mpcb_wbcnt] = (caddr_t)rp->r_sp;
 			mpcb->mpcb_wbcnt++;
-			atomic_add_64(&fasttrap_putreg_mpcb_cnt, 1);
+			atomic_inc_64(&fasttrap_putreg_mpcb_cnt);
 			return;
 		}
 	}
 
-	atomic_add_64(&fasttrap_putreg_slow_cnt, 1);
+	atomic_inc_64(&fasttrap_putreg_slow_cnt);
 	return;
 
 err:

From 44d933c5eec8fcb22d67218f5800d1fb181b705c Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Wed, 20 Aug 2014 06:25:43 +0000
Subject: [PATCH 028/380] 5047 don't use atomic_*_nv if you discard the return
 value Reviewed by: Garrett D'Amore <garrett@damore.org> Reviewed by: Jason
 King <jason.brian.king@gmail.com> Reviewed by: Matthew Ahrens
 <mahrens@delphix.com> Approved by: Robert Mustacchi <rm@joyent.com> Author:
 Josef 'Jeff' Sipek <josef.sipek@nexenta.com>

illumos/illumos-gate@640c1670a105457bb0040e8e11037b53ab6ebcfa
---
 uts/common/fs/zfs/dbuf.c | 8 ++++----
 uts/common/fs/zfs/dmu.c  | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/uts/common/fs/zfs/dbuf.c b/uts/common/fs/zfs/dbuf.c
index a56372197cdf..f793192fd3c8 100644
--- a/uts/common/fs/zfs/dbuf.c
+++ b/uts/common/fs/zfs/dbuf.c
@@ -1605,7 +1605,7 @@ dbuf_clear(dmu_buf_impl_t *db)
 	dndb = dn->dn_dbuf;
 	if (db->db_blkid != DMU_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) {
 		avl_remove(&dn->dn_dbufs, db);
-		(void) atomic_dec_32_nv(&dn->dn_dbufs_count);
+		atomic_dec_32(&dn->dn_dbufs_count);
 		membar_producer();
 		DB_DNODE_EXIT(db);
 		/*
@@ -1781,7 +1781,7 @@ dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid,
 	ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT ||
 	    refcount_count(&dn->dn_holds) > 0);
 	(void) refcount_add(&dn->dn_holds, db);
-	(void) atomic_inc_32_nv(&dn->dn_dbufs_count);
+	atomic_inc_32(&dn->dn_dbufs_count);
 
 	dprintf_dbuf(db, "db=%p\n", db);
 
@@ -1827,7 +1827,7 @@ dbuf_destroy(dmu_buf_impl_t *db)
 			dn = DB_DNODE(db);
 			mutex_enter(&dn->dn_dbufs_mtx);
 			avl_remove(&dn->dn_dbufs, db);
-			(void) atomic_dec_32_nv(&dn->dn_dbufs_count);
+			atomic_dec_32(&dn->dn_dbufs_count);
 			mutex_exit(&dn->dn_dbufs_mtx);
 			DB_DNODE_EXIT(db);
 			/*
@@ -2111,7 +2111,7 @@ dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag)
 			 * until the move completes.
 			 */
 			DB_DNODE_ENTER(db);
-			(void) atomic_dec_32_nv(&DB_DNODE(db)->dn_dbufs_count);
+			atomic_dec_32(&DB_DNODE(db)->dn_dbufs_count);
 			DB_DNODE_EXIT(db);
 			/*
 			 * The bonus buffer's dnode hold is no longer discounted
diff --git a/uts/common/fs/zfs/dmu.c b/uts/common/fs/zfs/dmu.c
index ff67a894bc25..619b3b4aa458 100644
--- a/uts/common/fs/zfs/dmu.c
+++ b/uts/common/fs/zfs/dmu.c
@@ -284,7 +284,7 @@ dmu_bonus_hold(objset_t *os, uint64_t object, void *tag, dmu_buf_t **dbp)
 	/* as long as the bonus buf is held, the dnode will be held */
 	if (refcount_add(&db->db_holds, tag) == 1) {
 		VERIFY(dnode_add_ref(dn, db));
-		(void) atomic_inc_32_nv(&dn->dn_dbufs_count);
+		atomic_inc_32(&dn->dn_dbufs_count);
 	}
 
 	/*

From 49caed48e1d533d6faed3c9e77bae9f92c141f5e Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Wed, 20 Aug 2014 06:29:42 +0000
Subject: [PATCH 029/380] 5066 remove support for non-ANSI compilation 5068
 Remove SCCSID() macro from <macros.h> Reviewed by: Keith Wesolowski
 <keith.wesolowski@joyent.com> Reviewed by: Josef 'Jeff' Sipek
 <jeffpc@josefsipek.net> Approved by: Robert Mustacchi <rm@joyent.com> Author:
 Garrett D'Amore <garrett@damore.org>

illumos/illumos-gate@ba3594ba9b5dd4c846c472a8d657edcb7c8109ac
---
 uts/common/sys/acl.h           |  7 ++----
 uts/common/sys/debug.h         | 12 ++-------
 uts/common/sys/feature_tests.h | 46 ++++++++++++++++++++++++++++++----
 uts/common/sys/processor.h     | 13 ++--------
 4 files changed, 47 insertions(+), 31 deletions(-)

diff --git a/uts/common/sys/acl.h b/uts/common/sys/acl.h
index 35c9772b8e9b..2a923194081c 100644
--- a/uts/common/sys/acl.h
+++ b/uts/common/sys/acl.h
@@ -19,6 +19,8 @@
  * CDDL HEADER END
  */
 /*
+ * Copyright 2014 Garrett D'Amore <garrett@damore.org>
+ *
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
@@ -287,13 +289,8 @@ extern int cmp2acls(void *, void *);
 
 #endif	/* !defined(_KERNEL) */
 
-#if defined(__STDC__)
 extern int acl(const char *path, int cmd, int cnt, void *buf);
 extern int facl(int fd, int cmd, int cnt, void *buf);
-#else	/* !__STDC__ */
-extern int acl();
-extern int facl();
-#endif	/* defined(__STDC__) */
 
 #ifdef	__cplusplus
 }
diff --git a/uts/common/sys/debug.h b/uts/common/sys/debug.h
index 85b8ab5f90eb..27b84beb8807 100644
--- a/uts/common/sys/debug.h
+++ b/uts/common/sys/debug.h
@@ -19,6 +19,8 @@
  * CDDL HEADER END
  */
 /*
+ * Copyright 2014 Garrett D'Amore <garrett@damore.org>
+ *
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
@@ -48,7 +50,6 @@ extern "C" {
  * ASSERT and is evaluated on both debug and non-debug kernels.
  */
 
-#if defined(__STDC__)
 extern int assfail(const char *, const char *, int);
 #define	VERIFY(EX) ((void)((EX) || assfail(#EX, __FILE__, __LINE__)))
 #if DEBUG
@@ -56,15 +57,6 @@ extern int assfail(const char *, const char *, int);
 #else
 #define	ASSERT(x)  ((void)0)
 #endif
-#else	/* defined(__STDC__) */
-extern int assfail();
-#define	VERIFY(EX) ((void)((EX) || assfail("EX", __FILE__, __LINE__)))
-#if DEBUG
-#define	ASSERT(EX) ((void)((EX) || assfail("EX", __FILE__, __LINE__)))
-#else
-#define	ASSERT(x)  ((void)0)
-#endif
-#endif	/* defined(__STDC__) */
 
 /*
  * Assertion variants sensitive to the compilation data model
diff --git a/uts/common/sys/feature_tests.h b/uts/common/sys/feature_tests.h
index e6ababd3d409..863b976706bb 100644
--- a/uts/common/sys/feature_tests.h
+++ b/uts/common/sys/feature_tests.h
@@ -20,6 +20,8 @@
  */
 
 /*
+ * Copyright 2013 Garrett D'Amore <garrett@damore.org>
+ *
  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
@@ -43,15 +45,16 @@ extern "C" {
  *		199309L	    POSIX.1b-1993 compilation (Real Time)
  *		199506L	    POSIX.1c-1995 compilation (POSIX Threads)
  *		200112L	    POSIX.1-2001 compilation (Austin Group Revision)
+ *		200809L     POSIX.1-2008 compilation
  */
 #if defined(_POSIX_SOURCE) && !defined(_POSIX_C_SOURCE)
 #define	_POSIX_C_SOURCE 1
 #endif
 
 /*
- * The feature test macros __XOPEN_OR_POSIX, _STRICT_STDC, and _STDC_C99
- * are Sun implementation specific macros created in order to compress
- * common standards specified feature test macros for easier reading.
+ * The feature test macros __XOPEN_OR_POSIX, _STRICT_STDC, _STRICT_SYMBOLS,
+ * and _STDC_C99 are Sun implementation specific macros created in order to
+ * compress common standards specified feature test macros for easier reading.
  * These macros should not be used by the application developer as
  * unexpected results may occur. Instead, the user should reference
  * standards(5) for correct usage of the standards feature test macros.
@@ -77,6 +80,10 @@ extern "C" {
  *                      the C standard. A value of 199901L indicates a
  *                      compiler that complies with ISO/IEC 9899:1999, other-
  *                      wise known as the C99 standard.
+ *
+ * _STRICT_SYMBOLS	Used in cases where symbol visibility is restricted
+ *                      by the standards, and the user has not explicitly
+ *                      relaxed the strictness via __EXTENSIONS__.
  */
 
 #if defined(_XOPEN_SOURCE) || defined(_POSIX_C_SOURCE)
@@ -143,6 +150,14 @@ extern "C" {
 #define	_STDC_C99
 #endif
 
+/*
+ * Use strict symbol visibility.
+ */
+#if (defined(_STRICT_STDC) || defined(__XOPEN_OR_POSIX)) && \
+	!defined(__EXTENSIONS__)
+#define	_STRICT_SYMBOLS
+#endif
+
 /*
  * Large file interfaces:
  *
@@ -222,6 +237,8 @@ extern "C" {
  * X/Open CAE Specification, Issue 5 (XPG5)
  * Open Group Technical Standard, Issue 6 (XPG6), also referred to as
  *    IEEE Std. 1003.1-2001 and ISO/IEC 9945:2002.
+ * Open Group Technical Standard, Issue 7 (XPG7), also referred to as
+ *    IEEE Std. 1003.1-2008 and ISO/IEC 9945:2009.
  *
  * XPG4v2 is also referred to as UNIX 95 (SUS or SUSv1).
  * XPG5 is also referred to as UNIX 98 or the Single Unix Specification,
@@ -229,6 +246,7 @@ extern "C" {
  * XPG6 is the result of a merge of the X/Open and POSIX specifications
  *     and as such is also referred to as IEEE Std. 1003.1-2001 in
  *     addition to UNIX 03 and SUSv3.
+ * XPG7 is also referred to as UNIX 08 and SUSv4.
  *
  * When writing a conforming X/Open application, as per the specification
  * requirements, the appropriate feature test macros must be defined at
@@ -241,6 +259,7 @@ extern "C" {
  * _XOPEN_SOURCE && _XOPEN_SOURCE_EXTENDED = 1           XPG4v2
  * _XOPEN_SOURCE = 500                                   XPG5
  * _XOPEN_SOURCE = 600  (or POSIX_C_SOURCE=200112L)      XPG6
+ * _XOPEN_SOURCE = 700  (or POSIX_C_SOURCE=200809L)      XPG7
  *
  * In order to simplify the guards within the headers, the following
  * implementation private test macros have been created. Applications
@@ -260,6 +279,7 @@ extern "C" {
  * _XPG4_2  X/Open CAE Specification, Issue 4, Version 2 (XPG4v2/UNIX 95/SUS)
  * _XPG5    X/Open CAE Specification, Issue 5 (XPG5/UNIX 98/SUSv2)
  * _XPG6    Open Group Technical Standard, Issue 6 (XPG6/UNIX 03/SUSv3)
+ * _XPG7    Open Group Technical Standard, Issue 7 (XPG7/UNIX 08/SUSv4)
  */
 
 /* X/Open Portability Guide, Issue 3 */
@@ -294,6 +314,19 @@ extern "C" {
 #define	_POSIX_C_SOURCE			200112L
 #undef	_XOPEN_SOURCE
 #define	_XOPEN_SOURCE			600
+
+/* Open Group Technical Standard, Issue 7 */
+#elif	(_XOPEN_SOURCE - 0 == 700) || (_POSIX_C_SOURCE - 0 == 200809L)
+#define	_XPG7
+#define	_XPG6
+#define	_XPG5
+#define	_XPG4_2
+#define	_XPG4
+#define	_XPG3
+#undef	_POSIX_C_SOURCE
+#define	_POSIX_C_SOURCE			200809L
+#undef	_XOPEN_SOURCE
+#define	_XOPEN_SOURCE			700
 #endif
 
 /*
@@ -304,12 +337,15 @@ extern "C" {
  * with the value of 4 indicates an XPG4 or XPG4v2 (UNIX 95) application.
  * _XOPEN_VERSION  defined with a value of 500 indicates an XPG5 (UNIX 98)
  * application and with a value of 600 indicates an XPG6 (UNIX 03)
- * application.  The appropriate version is determined by the use of the
+ * application and with a value of 700 indicates an XPG7 (UNIX 08).
+ * The appropriate version is determined by the use of the
  * feature test macros described earlier.  The value of _XOPEN_VERSION
  * defaults to 3 otherwise indicating support for XPG3 applications.
  */
 #ifndef _XOPEN_VERSION
-#ifdef	_XPG6
+#if	defined(_XPG7)
+#define	_XOPEN_VERSION 700
+#elif	defined(_XPG6)
 #define	_XOPEN_VERSION 600
 #elif defined(_XPG5)
 #define	_XOPEN_VERSION 500
diff --git a/uts/common/sys/processor.h b/uts/common/sys/processor.h
index c0fe6e21b85f..ec4b7471e50c 100644
--- a/uts/common/sys/processor.h
+++ b/uts/common/sys/processor.h
@@ -25,6 +25,8 @@
  */
 
 /*
+ * Copyright 2014 Garrett D'Amore <garrett@damore.org>
+ *
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
@@ -112,7 +114,6 @@ typedef struct {
  * User-level system call interface prototypes
  */
 #ifndef _KERNEL
-#ifdef __STDC__
 
 extern int	p_online(processorid_t processorid, int flag);
 extern int	processor_info(processorid_t processorid,
@@ -122,16 +123,6 @@ extern int	processor_bind(idtype_t idtype, id_t id,
 extern processorid_t getcpuid(void);
 extern lgrpid_t gethomelgroup(void);
 
-#else
-
-extern int	p_online();
-extern int	processor_info();
-extern int	processor_bind();
-extern processorid_t getcpuid();
-extern lgrpid_t gethomelgroup();
-
-#endif /* __STDC__ */
-
 #else   /* _KERNEL */
 
 /*

From b07579009c6bc5f5bc7dee888e416a3a9704f854 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Wed, 20 Aug 2014 06:34:24 +0000
Subject: [PATCH 030/380] 5095 panic when adding a duplicate dbuf to dn_dbufs
 Reviewed by: Adam Leventhal <adam.leventhal@delphix.com> Reviewed by: George
 Wilson <george.wilson@delphix.com> Reviewed by: Mattew Ahrens
 <mahrens@delphix.com> Reviewed by: Dan Kimmel <dan.kimmel@delphix.com>
 Reviewed by: Dan McDonald <danmcd@omniti.com> Reviewed by: Josef Sipek
 <jeffpc@josefsipek.net> Approved by: Robert Mustacchi <rm@joyent.com> Author:
 Alex Reece <alex@delphix.com>

illumos/illumos-gate@86bb58aec7165f8a0303564575c65e5a2ad58bf1
---
 uts/common/fs/zfs/dbuf.c      |  4 +---
 uts/common/fs/zfs/dnode.c     | 36 ++++++++++++++++++-----------------
 uts/common/fs/zfs/sys/dbuf.h  |  8 +++++---
 uts/common/fs/zfs/sys/dnode.h | 13 ++++++++++++-
 4 files changed, 37 insertions(+), 24 deletions(-)

diff --git a/uts/common/fs/zfs/dbuf.c b/uts/common/fs/zfs/dbuf.c
index f793192fd3c8..4b0ec01a02cb 100644
--- a/uts/common/fs/zfs/dbuf.c
+++ b/uts/common/fs/zfs/dbuf.c
@@ -70,8 +70,6 @@ dbuf_cons(void *vdb, void *unused, int kmflag)
 	cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL);
 	refcount_create(&db->db_holds);
 
-	db->db_creation = gethrtime();
-
 	return (0);
 }
 
@@ -819,7 +817,7 @@ dbuf_free_range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid,
 
 	db_search.db_level = 0;
 	db_search.db_blkid = start_blkid;
-	db_search.db_creation = 0;
+	db_search.db_state = DB_SEARCH;
 
 	mutex_enter(&dn->dn_dbufs_mtx);
 	if (start_blkid >= dn->dn_unlisted_l0_blkid) {
diff --git a/uts/common/fs/zfs/dnode.c b/uts/common/fs/zfs/dnode.c
index f5f5c4016313..6f47b440e179 100644
--- a/uts/common/fs/zfs/dnode.c
+++ b/uts/common/fs/zfs/dnode.c
@@ -67,33 +67,35 @@ dbuf_compare(const void *x1, const void *x2)
 
 	if (d1->db_level < d2->db_level) {
 		return (-1);
-	} else if (d1->db_level > d2->db_level) {
+	}
+	if (d1->db_level > d2->db_level) {
 		return (1);
 	}
 
 	if (d1->db_blkid < d2->db_blkid) {
 		return (-1);
-	} else if (d1->db_blkid > d2->db_blkid) {
+	}
+	if (d1->db_blkid > d2->db_blkid) {
 		return (1);
 	}
 
-	/*
-	 * If a dbuf is being evicted while dn_dbufs_mutex is not held, we set
-	 * the db_state to DB_EVICTING but do not remove it from dn_dbufs. If
-	 * another thread creates a dbuf of the same blkid before the dbuf is
-	 * removed from dn_dbufs, we can reach a state where there are two
-	 * dbufs of the same blkid and level in db_dbufs. To maintain the avl
-	 * invariant that there cannot be duplicate items, we distinguish
-	 * between these two dbufs based on the time they were created.
-	 */
-	if (d1->db_creation < d2->db_creation) {
+	if (d1->db_state < d2->db_state) {
 		return (-1);
-	} else if (d1->db_creation > d2->db_creation) {
-		return (1);
-	} else {
-		ASSERT3P(d1, ==, d2);
-		return (0);
 	}
+	if (d1->db_state > d2->db_state) {
+		return (1);
+	}
+
+	ASSERT3S(d1->db_state, !=, DB_SEARCH);
+	ASSERT3S(d2->db_state, !=, DB_SEARCH);
+
+	if ((uintptr_t)d1 < (uintptr_t)d2) {
+		return (-1);
+	}
+	if ((uintptr_t)d1 > (uintptr_t)d2) {
+		return (1);
+	}
+	return (0);
 }
 
 /* ARGSUSED */
diff --git a/uts/common/fs/zfs/sys/dbuf.h b/uts/common/fs/zfs/sys/dbuf.h
index 643d9685b61c..8ca8753e5e6d 100644
--- a/uts/common/fs/zfs/sys/dbuf.h
+++ b/uts/common/fs/zfs/sys/dbuf.h
@@ -66,8 +66,13 @@ extern "C" {
  *		|			 |
  *		|			 |
  *		+--------> NOFILL -------+
+ *
+ * DB_SEARCH is an invalid state for a dbuf. It is used by dbuf_free_range
+ * to find all dbufs in a range of a dnode and must be less than any other
+ * dbuf_states_t (see comment on dn_dbufs in dnode.h).
  */
 typedef enum dbuf_states {
+	DB_SEARCH = -1,
 	DB_UNCACHED,
 	DB_FILL,
 	DB_NOFILL,
@@ -213,9 +218,6 @@ typedef struct dmu_buf_impl {
 	/* pointer to most recent dirty record for this buffer */
 	dbuf_dirty_record_t *db_last_dirty;
 
-	/* Creation time of dbuf (see comment in dbuf_compare). */
-	hrtime_t db_creation;
-
 	/*
 	 * Our link on the owner dnodes's dn_dbufs list.
 	 * Protected by its dn_dbufs_mtx.
diff --git a/uts/common/fs/zfs/sys/dnode.h b/uts/common/fs/zfs/sys/dnode.h
index 6e4a845a36a1..8a4f3f6185e3 100644
--- a/uts/common/fs/zfs/sys/dnode.h
+++ b/uts/common/fs/zfs/sys/dnode.h
@@ -211,7 +211,18 @@ typedef struct dnode {
 	refcount_t dn_holds;
 
 	kmutex_t dn_dbufs_mtx;
-	avl_tree_t dn_dbufs;		/* descendent dbufs */
+	/*
+	 * Descendent dbufs, ordered by dbuf_compare. Note that dn_dbufs
+	 * can contain multiple dbufs of the same (level, blkid) when a
+	 * dbuf is marked DB_EVICTING without being removed from
+	 * dn_dbufs. To maintain the avl invariant that there cannot be
+	 * duplicate entries, we order the dbufs by an arbitrary value -
+	 * their address in memory. This means that dn_dbufs cannot be used to
+	 * directly look up a dbuf. Instead, callers must use avl_walk, have
+	 * a reference to the dbuf, or look up a non-existant node with
+	 * db_state = DB_SEARCH (see dbuf_free_range for an example).
+	 */
+	avl_tree_t dn_dbufs;
 
 	/* protected by dn_struct_rwlock */
 	struct dmu_buf_impl *dn_bonus;	/* bonus buffer dbuf */

From 4923d576e31083a85ff9a36d3919d9121a4ec122 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sun, 7 Sep 2014 12:07:26 +0000
Subject: [PATCH 031/380] 5117 space map reallocation can cause corruption
 Reviewed by: Matthew Ahrens <mahrens@delphix.com> Reviewed by: Sebastien Roy
 <sebastien.roy@delphix.com> Reviewed by: Richard Elling
 <richard.elling@gmail.com> Approved by: Richard Lowe <richlowe@richlowe.net>
 Author: George Wilson <george.wilson@delphix.com>

illumos/illumos-gate@e503a685964805f048b35c2a4e70e0638344f2b7
---
 uts/common/fs/zfs/dnode_sync.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/uts/common/fs/zfs/dnode_sync.c b/uts/common/fs/zfs/dnode_sync.c
index ff666b212d5e..2a57aee904fd 100644
--- a/uts/common/fs/zfs/dnode_sync.c
+++ b/uts/common/fs/zfs/dnode_sync.c
@@ -684,6 +684,11 @@ dnode_sync(dnode_t *dn, dmu_tx_t *tx)
 		return;
 	}
 
+	if (dn->dn_next_nlevels[txgoff]) {
+		dnode_increase_indirection(dn, tx);
+		dn->dn_next_nlevels[txgoff] = 0;
+	}
+
 	if (dn->dn_next_nblkptr[txgoff]) {
 		/* this should only happen on a realloc */
 		ASSERT(dn->dn_allocated_txg == tx->tx_txg);
@@ -708,11 +713,6 @@ dnode_sync(dnode_t *dn, dmu_tx_t *tx)
 		mutex_exit(&dn->dn_mtx);
 	}
 
-	if (dn->dn_next_nlevels[txgoff]) {
-		dnode_increase_indirection(dn, tx);
-		dn->dn_next_nlevels[txgoff] = 0;
-	}
-
 	dbuf_sync_list(list, tx);
 
 	if (!DMU_OBJECT_IS_SPECIAL(dn->dn_object)) {

From 666b96024e831e872791c74910cc828ffa7db588 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 13 Sep 2014 14:23:51 +0000
Subject: [PATCH 032/380] 5151 blkdev should support reporting of physical
 block size Reviewed by: Josef 'Jeff' Sipek <jeffpc@josefsipek.net> Reviewed
 by: Garrett D'Amore <garrett@damore.org> Approved by: Dan McDonald
 <danmcd@omniti.com> Author: Hans Rosenfeld <hans.rosenfeld@nexenta.com>

illumos/illumos-gate@32ce6b819524e8719d4ce58db40a00e9f17843e2
---
 uts/common/fs/zfs/sys/dnode.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/uts/common/fs/zfs/sys/dnode.h b/uts/common/fs/zfs/sys/dnode.h
index 8a4f3f6185e3..5668af1fef08 100644
--- a/uts/common/fs/zfs/sys/dnode.h
+++ b/uts/common/fs/zfs/sys/dnode.h
@@ -56,7 +56,7 @@ extern "C" {
  * Fixed constants.
  */
 #define	DNODE_SHIFT		9	/* 512 bytes */
-#define	DN_MIN_INDBLKSHIFT	10	/* 1k */
+#define	DN_MIN_INDBLKSHIFT	12	/* 4k */
 #define	DN_MAX_INDBLKSHIFT	14	/* 16k */
 #define	DNODE_BLOCK_SHIFT	14	/* 16k */
 #define	DNODE_CORE_SIZE		64	/* 64 bytes for dnode sans blkptrs */

From 2baaad1981c6233d6c82f7e95a7dce534c4cd745 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 13 Sep 2014 15:05:10 +0000
Subject: [PATCH 033/380] 5136 fix write throttle comment in dsl_pool.c
 Reviewed by: Adam Leventhal <ahl@delphix.com> Reviewed by: Christopher Siden
 <christopher.siden@delphix.com> Reviewed by: Max Grossman
 <max.grossman@delphix.com> Reviewed by: Josef 'Jeff' Sipek
 <jeffpc@josefsipek.net> Reviewed by: Richard Elling
 <richard.elling@gmail.com> Approved by: Dan McDonald <danmcd@omniti.com>
 Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@d85a1e969ce08b3bc4a7cb3626a6e33a28b5caeb
---
 uts/common/fs/zfs/dsl_pool.c | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/uts/common/fs/zfs/dsl_pool.c b/uts/common/fs/zfs/dsl_pool.c
index 9435e74c0e7f..ad26a9070b68 100644
--- a/uts/common/fs/zfs/dsl_pool.c
+++ b/uts/common/fs/zfs/dsl_pool.c
@@ -111,8 +111,8 @@ int zfs_delay_min_dirty_percent = 60;
 
 /*
  * This controls how quickly the delay approaches infinity.
- * Larger values cause it to delay less for a given amount of dirty data.
- * Therefore larger values will cause there to be more dirty data for a
+ * Larger values cause it to delay more for a given amount of dirty data.
+ * Therefore larger values will cause there to be less dirty data for a
  * given throughput.
  *
  * For the smoothest delay, this value should be about 1 billion divided
@@ -125,12 +125,6 @@ int zfs_delay_min_dirty_percent = 60;
 uint64_t zfs_delay_scale = 1000 * 1000 * 1000 / 2000;
 
 
-/*
- * XXX someday maybe turn these into #defines, and you have to tune it on a
- * per-pool basis using zfs.conf.
- */
-
-
 hrtime_t zfs_throttle_delay = MSEC2NSEC(10);
 hrtime_t zfs_throttle_resolution = MSEC2NSEC(10);
 

From b326266194e317067040398dd645adf2d694f95f Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 13 Sep 2014 15:24:35 +0000
Subject: [PATCH 034/380] 5138 add tunable for maximum number of blocks freed
 in one txg Reviewed by: Adam Leventhal <adam.leventhal@delphix.com> Reviewed
 by: Mattew Ahrens <mahrens@delphix.com> Reviewed by: Josef 'Jeff' Sipek
 <jeffpc@josefsipek.net> Reviewed by: Richard Elling
 <richard.elling@gmail.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Approved by: Dan McDonald <danmcd@omniti.com>
 Author: Max Grossman <max.grossman@delphix.com>

illumos/illumos-gate@af3465da8fa420c4ec701e3e57704d537a6f755b
---
 uts/common/fs/zfs/dsl_scan.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/uts/common/fs/zfs/dsl_scan.c b/uts/common/fs/zfs/dsl_scan.c
index d4e234937a06..7638ee183a6f 100644
--- a/uts/common/fs/zfs/dsl_scan.c
+++ b/uts/common/fs/zfs/dsl_scan.c
@@ -69,6 +69,8 @@ boolean_t zfs_no_scrub_io = B_FALSE; /* set to disable scrub i/o */
 boolean_t zfs_no_scrub_prefetch = B_FALSE; /* set to disable scrub prefetch */
 enum ddt_class zfs_scrub_ddt_class_max = DDT_CLASS_DUPLICATE;
 int dsl_scan_delay_completion = B_FALSE; /* set to delay scan completion */
+/* max number of blocks to free in a single TXG */
+uint64_t zfs_free_max_blocks = UINT64_MAX;
 
 #define	DSL_SCAN_IS_SCRUB_RESILVER(scn) \
 	((scn)->scn_phys.scn_func == POOL_SCAN_SCRUB || \
@@ -1319,6 +1321,9 @@ dsl_scan_free_should_pause(dsl_scan_t *scn)
 	if (zfs_recover)
 		return (B_FALSE);
 
+	if (scn->scn_visited_this_txg >= zfs_free_max_blocks)
+		return (B_TRUE);
+
 	elapsed_nanosecs = gethrtime() - scn->scn_sync_start_time;
 	return (elapsed_nanosecs / NANOSEC > zfs_txg_timeout ||
 	    (NSEC2MSEC(elapsed_nanosecs) > zfs_free_min_time_ms &&

From 8c3fed76c83dd9b507752f5536340c5dbeab2286 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 13 Sep 2014 15:31:37 +0000
Subject: [PATCH 035/380] 5140 message about "%recv could not be opened" is
 printed when booting after crash Reviewed by: Christopher Siden
 <christopher.siden@delphix.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Reviewed by: Max Grossman
 <max.grossman@delphix.com> Reviewed by: Richard Elling
 <richard.elling@gmail.com> Approved by: Dan McDonald <danmcd@omniti.com>
 Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@22438533bcc131ecebd4c16c699c35e91d321b76
---
 uts/common/fs/zfs/zil.c | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/uts/common/fs/zfs/zil.c b/uts/common/fs/zfs/zil.c
index a5fe3fb00f82..6377285fe2a9 100644
--- a/uts/common/fs/zfs/zil.c
+++ b/uts/common/fs/zfs/zil.c
@@ -635,7 +635,14 @@ zil_claim(const char *osname, void *txarg)
 
 	error = dmu_objset_own(osname, DMU_OST_ANY, B_FALSE, FTAG, &os);
 	if (error != 0) {
-		cmn_err(CE_WARN, "can't open objset for %s", osname);
+		/*
+		 * EBUSY indicates that the objset is inconsistent, in which
+		 * case it can not have a ZIL.
+		 */
+		if (error != EBUSY) {
+			cmn_err(CE_WARN, "can't open objset for %s, error %u",
+			    osname, error);
+		}
 		return (0);
 	}
 

From b596a737fe8333830435a843a65098bc21f59747 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 13 Sep 2014 15:34:12 +0000
Subject: [PATCH 036/380] 5139 SEEK_HOLE failed to report a hole at end of file
 Reviewed by: Adam Leventhal <adam.leventhal@delphix.com> Reviewed by: Alex
 Reece <alex.reece@delphix.com> Reviewed by: Christopher Siden
 <christopher.siden@delphix.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Reviewed by: Max Grossman
 <max.grossman@delphix.com> Reviewed by: Peng Dai <peng.dai@delphix.com>
 Reviewed by: Richard Elling <richard.elling@gmail.com> Approved by: Dan
 McDonald <danmcd@omniti.com> Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@0fbc0cd0e52a11f6c4397a1714f94412cbf98b60
---
 uts/common/fs/zfs/dnode.c     |  9 +++++++++
 uts/common/fs/zfs/zfs_vnops.c | 23 +++++++++++++----------
 2 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/uts/common/fs/zfs/dnode.c b/uts/common/fs/zfs/dnode.c
index 6f47b440e179..175157714d06 100644
--- a/uts/common/fs/zfs/dnode.c
+++ b/uts/common/fs/zfs/dnode.c
@@ -1943,6 +1943,15 @@ dnode_next_offset(dnode_t *dn, int flags, uint64_t *offset,
 		    flags, offset, lvl, blkfill, txg);
 	}
 
+	/*
+	 * There's always a "virtual hole" at the end of the object, even
+	 * if all BP's which physically exist are non-holes.
+	 */
+	if ((flags & DNODE_FIND_HOLE) && error == ESRCH && txg == 0 &&
+	    minlvl == 1 && blkfill == 1 && !(flags & DNODE_FIND_BACKWARDS)) {
+		error = 0;
+	}
+
 	if (error == 0 && (flags & DNODE_FIND_BACKWARDS ?
 	    initial_offset < *offset : initial_offset > *offset))
 		error = SET_ERROR(ESRCH);
diff --git a/uts/common/fs/zfs/zfs_vnops.c b/uts/common/fs/zfs/zfs_vnops.c
index 6fd0bfed0636..e87c0027370e 100644
--- a/uts/common/fs/zfs/zfs_vnops.c
+++ b/uts/common/fs/zfs/zfs_vnops.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
  */
 
@@ -271,16 +271,19 @@ zfs_holey(vnode_t *vp, int cmd, offset_t *off)
 
 	error = dmu_offset_next(zp->z_zfsvfs->z_os, zp->z_id, hole, &noff);
 
-	/* end of file? */
-	if ((error == ESRCH) || (noff > file_sz)) {
-		/*
-		 * Handle the virtual hole at the end of file.
-		 */
-		if (hole) {
-			*off = file_sz;
-			return (0);
-		}
+	if (error == ESRCH)
 		return (SET_ERROR(ENXIO));
+
+	/*
+	 * We could find a hole that begins after the logical end-of-file,
+	 * because dmu_offset_next() only works on whole blocks.  If the
+	 * EOF falls mid-block, then indicate that the "virtual hole"
+	 * at the end of the file begins at the logical EOF, rather than
+	 * at the end of the last block.
+	 */
+	if (noff > file_sz) {
+		ASSERT(hole);
+		noff = file_sz;
 	}
 
 	if (noff < *off)

From 2e967c7403bdccf0a14be1d2582f695496094bd5 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 4 Oct 2014 07:26:42 +0000
Subject: [PATCH 037/380] 5164 space_map_max_blksz causes panic, does not work
 5165 zdb fails assertion when run on pool with recently-enabled
 spacemap_histogram feature Reviewed by: Christopher Siden
 <christopher.siden@delphix.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Reviewed by: Saso Kiselkov
 <skiselkov.ml@gmail.com> Approved by: Dan McDonald <danmcd@omniti.com>
 Author:	Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@b1be2892dd07cf9a97d47ad06334cdc879196aaf
---
 uts/common/fs/zfs/metaslab.c      |  24 ++++--
 uts/common/fs/zfs/space_map.c     | 128 ++++++++----------------------
 uts/common/fs/zfs/sys/space_map.h |  11 ---
 3 files changed, 50 insertions(+), 113 deletions(-)

diff --git a/uts/common/fs/zfs/metaslab.c b/uts/common/fs/zfs/metaslab.c
index fcb290be317d..a33ec7f62810 100644
--- a/uts/common/fs/zfs/metaslab.c
+++ b/uts/common/fs/zfs/metaslab.c
@@ -65,7 +65,7 @@ int zfs_condense_pct = 200;
 /*
  * Condensing a metaslab is not guaranteed to actually reduce the amount of
  * space used on disk. In particular, a space map uses data in increments of
- * MAX(1 << ashift, SPACE_MAP_INITIAL_BLOCKSIZE), so a metaslab might use the
+ * MAX(1 << ashift, space_map_blksize), so a metaslab might use the
  * same number of blocks after condensing. Since the goal of condensing is to
  * reduce the number of IOPs required to read the space map, we only want to
  * condense when we can be sure we will reduce the number of blocks used by the
@@ -1381,10 +1381,12 @@ metaslab_fragmentation(metaslab_t *msp)
 		uint64_t txg = spa_syncing_txg(spa);
 		vdev_t *vd = msp->ms_group->mg_vd;
 
-		msp->ms_condense_wanted = B_TRUE;
-		vdev_dirty(vd, VDD_METASLAB, msp, txg + 1);
-		spa_dbgmsg(spa, "txg %llu, requesting force condense: "
-		    "msp %p, vd %p", txg, msp, vd);
+		if (spa_writeable(spa)) {
+			msp->ms_condense_wanted = B_TRUE;
+			vdev_dirty(vd, VDD_METASLAB, msp, txg + 1);
+			spa_dbgmsg(spa, "txg %llu, requesting force condense: "
+			    "msp %p, vd %p", txg, msp, vd);
+		}
 		return (ZFS_FRAG_INVALID);
 	}
 
@@ -1828,6 +1830,15 @@ metaslab_sync(metaslab_t *msp, uint64_t txg)
 
 	mutex_enter(&msp->ms_lock);
 
+	/*
+	 * Note: metaslab_condense() clears the space_map's histogram.
+	 * Therefore we must verify and remove this histogram before
+	 * condensing.
+	 */
+	metaslab_group_histogram_verify(mg);
+	metaslab_class_histogram_verify(mg->mg_class);
+	metaslab_group_histogram_remove(mg, msp);
+
 	if (msp->ms_loaded && spa_sync_pass(spa) == 1 &&
 	    metaslab_should_condense(msp)) {
 		metaslab_condense(msp, txg, tx);
@@ -1836,9 +1847,6 @@ metaslab_sync(metaslab_t *msp, uint64_t txg)
 		space_map_write(msp->ms_sm, *freetree, SM_FREE, tx);
 	}
 
-	metaslab_group_histogram_verify(mg);
-	metaslab_class_histogram_verify(mg->mg_class);
-	metaslab_group_histogram_remove(mg, msp);
 	if (msp->ms_loaded) {
 		/*
 		 * When the space map is loaded, we have an accruate
diff --git a/uts/common/fs/zfs/space_map.c b/uts/common/fs/zfs/space_map.c
index d158b246d8c2..94790b9eaee5 100644
--- a/uts/common/fs/zfs/space_map.c
+++ b/uts/common/fs/zfs/space_map.c
@@ -38,15 +38,12 @@
 #include <sys/zfeature.h>
 
 /*
- * This value controls how the space map's block size is allowed to grow.
- * If the value is set to the same size as SPACE_MAP_INITIAL_BLOCKSIZE then
- * the space map block size will remain fixed. Setting this value to something
- * greater than SPACE_MAP_INITIAL_BLOCKSIZE will allow the space map to
- * increase its block size as needed. To maintain backwards compatibilty the
- * space map's block size must be a power of 2 and SPACE_MAP_INITIAL_BLOCKSIZE
- * or larger.
+ * The data for a given space map can be kept on blocks of any size.
+ * Larger blocks entail fewer i/o operations, but they also cause the
+ * DMU to keep more data in-core, and also to waste more i/o bandwidth
+ * when only a few blocks have changed since the last transaction group.
  */
-int space_map_max_blksz = (1 << 12);
+int space_map_blksz = (1 << 12);
 
 /*
  * Load the space map disk into the specified range tree. Segments of maptype
@@ -233,58 +230,6 @@ space_map_entries(space_map_t *sm, range_tree_t *rt)
 	return (entries);
 }
 
-void
-space_map_set_blocksize(space_map_t *sm, uint64_t size, dmu_tx_t *tx)
-{
-	uint32_t blksz;
-	u_longlong_t blocks;
-
-	ASSERT3U(sm->sm_blksz, !=, 0);
-	ASSERT3U(space_map_object(sm), !=, 0);
-	ASSERT(sm->sm_dbuf != NULL);
-	VERIFY(ISP2(space_map_max_blksz));
-
-	if (sm->sm_blksz >= space_map_max_blksz)
-		return;
-
-	/*
-	 * The object contains more than one block so we can't adjust
-	 * its size.
-	 */
-	if (sm->sm_phys->smp_objsize > sm->sm_blksz)
-		return;
-
-	if (size > sm->sm_blksz) {
-		uint64_t newsz;
-
-		/*
-		 * Older software versions treat space map blocks as fixed
-		 * entities. The DMU is capable of handling different block
-		 * sizes making it possible for us to increase the
-		 * block size and maintain backwards compatibility. The
-		 * caveat is that the new block sizes must be a
-		 * power of 2 so that old software can append to the file,
-		 * adding more blocks. The block size can grow until it
-		 * reaches space_map_max_blksz.
-		 */
-		newsz = ISP2(size) ? size : 1ULL << highbit64(size);
-		if (newsz > space_map_max_blksz)
-			newsz = space_map_max_blksz;
-
-		VERIFY0(dmu_object_set_blocksize(sm->sm_os,
-		    space_map_object(sm), newsz, 0, tx));
-		dmu_object_size_from_db(sm->sm_dbuf, &blksz, &blocks);
-
-		zfs_dbgmsg("txg %llu, spa %s, increasing blksz from %d to %d",
-		    dmu_tx_get_txg(tx), spa_name(dmu_objset_spa(sm->sm_os)),
-		    sm->sm_blksz, blksz);
-
-		VERIFY3U(newsz, ==, blksz);
-		VERIFY3U(sm->sm_blksz, <, blksz);
-		sm->sm_blksz = blksz;
-	}
-}
-
 /*
  * Note: space_map_write() will drop sm_lock across dmu_write() calls.
  */
@@ -298,7 +243,7 @@ space_map_write(space_map_t *sm, range_tree_t *rt, maptype_t maptype,
 	range_seg_t *rs;
 	uint64_t size, total, rt_space, nodes;
 	uint64_t *entry, *entry_map, *entry_map_end;
-	uint64_t newsz, expected_entries, actual_entries = 1;
+	uint64_t expected_entries, actual_entries = 1;
 
 	ASSERT(MUTEX_HELD(rt->rt_lock));
 	ASSERT(dsl_pool_sync_context(dmu_objset_pool(os)));
@@ -324,13 +269,6 @@ space_map_write(space_map_t *sm, range_tree_t *rt, maptype_t maptype,
 
 	expected_entries = space_map_entries(sm, rt);
 
-	/*
-	 * Calculate the new size for the space map on-disk and see if
-	 * we can grow the block size to accommodate the new size.
-	 */
-	newsz = sm->sm_phys->smp_objsize + expected_entries * sizeof (uint64_t);
-	space_map_set_blocksize(sm, newsz, tx);
-
 	entry_map = zio_buf_alloc(sm->sm_blksz);
 	entry_map_end = entry_map + (sm->sm_blksz / sizeof (uint64_t));
 	entry = entry_map;
@@ -457,46 +395,48 @@ space_map_close(space_map_t *sm)
 	kmem_free(sm, sizeof (*sm));
 }
 
-static void
-space_map_reallocate(space_map_t *sm, dmu_tx_t *tx)
-{
-	ASSERT(dmu_tx_is_syncing(tx));
-
-	space_map_free(sm, tx);
-	dmu_buf_rele(sm->sm_dbuf, sm);
-
-	sm->sm_object = space_map_alloc(sm->sm_os, tx);
-	VERIFY0(space_map_open_impl(sm));
-}
-
 void
 space_map_truncate(space_map_t *sm, dmu_tx_t *tx)
 {
 	objset_t *os = sm->sm_os;
 	spa_t *spa = dmu_objset_spa(os);
 	dmu_object_info_t doi;
-	int bonuslen;
 
 	ASSERT(dsl_pool_sync_context(dmu_objset_pool(os)));
 	ASSERT(dmu_tx_is_syncing(tx));
 
-	VERIFY0(dmu_free_range(os, space_map_object(sm), 0, -1ULL, tx));
 	dmu_object_info_from_db(sm->sm_dbuf, &doi);
 
-	if (spa_feature_is_enabled(spa, SPA_FEATURE_SPACEMAP_HISTOGRAM)) {
-		bonuslen = sizeof (space_map_phys_t);
-		ASSERT3U(bonuslen, <=, dmu_bonus_max());
-	} else {
-		bonuslen = SPACE_MAP_SIZE_V0;
-	}
-
-	if (bonuslen != doi.doi_bonus_size ||
-	    doi.doi_data_block_size != SPACE_MAP_INITIAL_BLOCKSIZE) {
+	/*
+	 * If the space map has the wrong bonus size (because
+	 * SPA_FEATURE_SPACEMAP_HISTOGRAM has recently been enabled), or
+	 * the wrong block size (because space_map_blksz has changed),
+	 * free and re-allocate its object with the updated sizes.
+	 *
+	 * Otherwise, just truncate the current object.
+	 */
+	if ((spa_feature_is_enabled(spa, SPA_FEATURE_SPACEMAP_HISTOGRAM) &&
+	    doi.doi_bonus_size != sizeof (space_map_phys_t)) ||
+	    doi.doi_data_block_size != space_map_blksz) {
 		zfs_dbgmsg("txg %llu, spa %s, reallocating: "
 		    "old bonus %u, old blocksz %u", dmu_tx_get_txg(tx),
 		    spa_name(spa), doi.doi_bonus_size, doi.doi_data_block_size);
-		space_map_reallocate(sm, tx);
-		VERIFY3U(sm->sm_blksz, ==, SPACE_MAP_INITIAL_BLOCKSIZE);
+
+		space_map_free(sm, tx);
+		dmu_buf_rele(sm->sm_dbuf, sm);
+
+		sm->sm_object = space_map_alloc(sm->sm_os, tx);
+		VERIFY0(space_map_open_impl(sm));
+	} else {
+		VERIFY0(dmu_free_range(os, space_map_object(sm), 0, -1ULL, tx));
+
+		/*
+		 * If the spacemap is reallocated, its histogram
+		 * will be reset.  Do the same in the common case so that
+		 * bugs related to the uncommon case do not go unnoticed.
+		 */
+		bzero(sm->sm_phys->smp_histogram,
+		    sizeof (sm->sm_phys->smp_histogram));
 	}
 
 	dmu_buf_will_dirty(sm->sm_dbuf, tx);
@@ -535,7 +475,7 @@ space_map_alloc(objset_t *os, dmu_tx_t *tx)
 	}
 
 	object = dmu_object_alloc(os,
-	    DMU_OT_SPACE_MAP, SPACE_MAP_INITIAL_BLOCKSIZE,
+	    DMU_OT_SPACE_MAP, space_map_blksz,
 	    DMU_OT_SPACE_MAP_HEADER, bonuslen, tx);
 
 	return (object);
diff --git a/uts/common/fs/zfs/sys/space_map.h b/uts/common/fs/zfs/sys/space_map.h
index 67fa2767b1f8..a59e6d37d43a 100644
--- a/uts/common/fs/zfs/sys/space_map.h
+++ b/uts/common/fs/zfs/sys/space_map.h
@@ -133,17 +133,6 @@ typedef enum {
 	SM_FREE
 } maptype_t;
 
-/*
- * The data for a given space map can be kept on blocks of any size.
- * Larger blocks entail fewer i/o operations, but they also cause the
- * DMU to keep more data in-core, and also to waste more i/o bandwidth
- * when only a few blocks have changed since the last transaction group.
- * Rather than having a fixed block size for all space maps the block size
- * can adjust as needed (see space_map_max_blksz). Set the initial block
- * size for the space map to 4k.
- */
-#define	SPACE_MAP_INITIAL_BLOCKSIZE	(1ULL << 12)
-
 int space_map_load(space_map_t *sm, range_tree_t *rt, maptype_t maptype);
 
 void space_map_histogram_clear(space_map_t *sm);

From 72d9eb68901b9fb7f4113feda97240f65ca6da01 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 4 Oct 2014 07:32:19 +0000
Subject: [PATCH 038/380] 5163 arc should reap range_seg_cache Reviewed by:
 Christopher Siden <christopher.siden@delphix.com> Reviewed by: Matthew Ahrens
 <mahrens@delphix.com> Reviewed by: Richard Elling <richard.elling@gmail.com>
 Reviewed by: Saso Kiselkov <skiselkov.ml@gmail.com> Approved by: Dan McDonald
 <danmcd@omniti.com> Author:	George Wilson <george.wilson@delphix.com>

illumos/illumos@83803b51e1fa57fbf4e86ca5aa58eab6c5822dae
---
 uts/common/fs/zfs/arc.c        | 2 ++
 uts/common/fs/zfs/range_tree.c | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index 2353ac67ce49..69d16af3b669 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -2303,6 +2303,7 @@ arc_kmem_reap_now(arc_reclaim_strategy_t strat)
 	kmem_cache_t		*prev_data_cache = NULL;
 	extern kmem_cache_t	*zio_buf_cache[];
 	extern kmem_cache_t	*zio_data_buf_cache[];
+	extern kmem_cache_t	*range_seg_cache;
 
 #ifdef _KERNEL
 	if (arc_meta_used >= arc_meta_limit) {
@@ -2339,6 +2340,7 @@ arc_kmem_reap_now(arc_reclaim_strategy_t strat)
 	}
 	kmem_cache_reap_now(buf_cache);
 	kmem_cache_reap_now(hdr_cache);
+	kmem_cache_reap_now(range_seg_cache);
 
 	/*
 	 * Ask the vmem areana to reclaim unused memory from its
diff --git a/uts/common/fs/zfs/range_tree.c b/uts/common/fs/zfs/range_tree.c
index 22175e06ab30..6422fd1c1fa6 100644
--- a/uts/common/fs/zfs/range_tree.c
+++ b/uts/common/fs/zfs/range_tree.c
@@ -33,7 +33,7 @@
 #include <sys/zio.h>
 #include <sys/range_tree.h>
 
-static kmem_cache_t *range_seg_cache;
+kmem_cache_t *range_seg_cache;
 
 void
 range_tree_init(void)

From ef4525212526c25694fce4193ea773d842771012 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 4 Oct 2014 07:34:50 +0000
Subject: [PATCH 039/380] 5161 add tunable for number of metaslabs per vdev
 Reviewed by: Alex Reece <alex.reece@delphix.com> Reviewed by: Christopher
 Siden <christopher.siden@delphix.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Reviewed by: Paul Dagnelie
 <paul.dagnelie@delphix.com> Reviewed by: Saso Kiselkov
 <skiselkov.ml@gmail.com> Reviewed by: Richard Elling
 <richard.elling@gmail.com> Approved by: Richard Lowe <richlowe@richlowe.net>
 Author:	Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@bf3e216c7efa56332b456f4cf19d208e21d63839
---
 uts/common/fs/zfs/vdev.c | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/uts/common/fs/zfs/vdev.c b/uts/common/fs/zfs/vdev.c
index 28ef0414a280..7571b21a5f8c 100644
--- a/uts/common/fs/zfs/vdev.c
+++ b/uts/common/fs/zfs/vdev.c
@@ -64,6 +64,12 @@ static vdev_ops_t *vdev_ops_table[] = {
 /* maximum scrub/resilver I/O queue per leaf vdev */
 int zfs_scrub_limit = 10;
 
+/*
+ * When a vdev is added, it will be divided into approximately (but no
+ * more than) this number of metaslabs.
+ */
+int metaslabs_per_vdev = 200;
+
 /*
  * Given a vdev type, return the appropriate ops vector.
  */
@@ -1551,9 +1557,9 @@ void
 vdev_metaslab_set_size(vdev_t *vd)
 {
 	/*
-	 * Aim for roughly 200 metaslabs per vdev.
+	 * Aim for roughly metaslabs_per_vdev (default 200) metaslabs per vdev.
 	 */
-	vd->vdev_ms_shift = highbit64(vd->vdev_asize / 200);
+	vd->vdev_ms_shift = highbit64(vd->vdev_asize / metaslabs_per_vdev);
 	vd->vdev_ms_shift = MAX(vd->vdev_ms_shift, SPA_MAXBLOCKSHIFT);
 }
 

From 67c85130622cf045b87f3995dfd581c59479e982 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 4 Oct 2014 07:35:50 +0000
Subject: [PATCH 040/380] 5148 zvol's DKIOCFREE holds zfsdev_state_lock too
 long Reviewed by: Matthew Ahrens <mahrens@delphix.com> Reviewed by: Paul
 Dagnelie <paul.dagnelie@delphix.com> Reviewed by: Sebastien Roy
 <sebastien.roy@delphix.com> Reviewed by: Josef 'Jeff' Sipek
 <jeffpc@josefsipek.net> Reviewed by: Dan McDonald <danmcd@omniti.com>
 Approved by: Richard Lowe <richlowe@richlowe.net> Author: George Wilson
 <george.wilson@delphix.com>

illumos/illumos-gate@574e2414ac6a50c9d6097c2e4a98eb9263f90c44
---
 uts/common/fs/zfs/zvol.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/uts/common/fs/zfs/zvol.c b/uts/common/fs/zfs/zvol.c
index 5b511ffa0224..d9a69470f948 100644
--- a/uts/common/fs/zfs/zvol.c
+++ b/uts/common/fs/zfs/zvol.c
@@ -1781,8 +1781,8 @@ zvol_ioctl(dev_t dev, int cmd, intptr_t arg, int flag, cred_t *cr, int *rvalp)
 		 */
 		if (df.df_start >= zv->zv_volsize)
 			break;	/* No need to do anything... */
-		if (df.df_start + df.df_length > zv->zv_volsize)
-			df.df_length = DMU_OBJECT_END;
+
+		mutex_exit(&zfsdev_state_lock);
 
 		rl = zfs_range_lock(&zv->zv_znode, df.df_start, df.df_length,
 		    RL_WRITER);
@@ -1821,7 +1821,7 @@ zvol_ioctl(dev_t dev, int cmd, intptr_t arg, int flag, cred_t *cr, int *rvalp)
 				    dmu_objset_pool(zv->zv_objset), 0);
 			}
 		}
-		break;
+		return (error);
 	}
 
 	default:

From efa412f974352c1ef2d850157bb245a0e80ad537 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 4 Oct 2014 07:37:17 +0000
Subject: [PATCH 041/380] 5149 zvols need a way to ignore DKIOCFREE Reviewed
 by: Adam Leventhal <ahl@delphix.com> Reviewed by: Matthew Ahrens
 <matthew.ahrens@delphix.com> Reviewed by: Sebastien Roy
 <sebastien.roy@delphix.com> Reviewed by: Dan McDonald <danmcd@omniti.com>
 Reviewed by: Saso Kiselkov <skiselkov.ml@gmail.com> Approved by: Robert
 Mustacchi <rm@joyent.com> Author: George Wilson <george.wilson@delphix.com>

illumos/illumos-gate@893c83ba3e1c87a785b5274ce2ef02f45fba6087
---
 uts/common/fs/zfs/zvol.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/uts/common/fs/zfs/zvol.c b/uts/common/fs/zfs/zvol.c
index d9a69470f948..ca05d5cf214c 100644
--- a/uts/common/fs/zfs/zvol.c
+++ b/uts/common/fs/zfs/zvol.c
@@ -142,6 +142,11 @@ typedef struct zvol_state {
  */
 int zvol_maxphys = DMU_MAX_ACCESS/2;
 
+/*
+ * Toggle unmap functionality.
+ */
+boolean_t zvol_unmap_enabled = B_TRUE;
+
 extern int zfs_set_prop_nvlist(const char *, zprop_source_t,
     nvlist_t *, nvlist_t *);
 static int zvol_remove_zv(zvol_state_t *);
@@ -1769,6 +1774,9 @@ zvol_ioctl(dev_t dev, int cmd, intptr_t arg, int flag, cred_t *cr, int *rvalp)
 		dkioc_free_t df;
 		dmu_tx_t *tx;
 
+		if (!zvol_unmap_enabled)
+			break;
+
 		if (ddi_copyin((void *)arg, &df, sizeof (df), flag)) {
 			error = SET_ERROR(EFAULT);
 			break;

From ad51a6a233f60cc9675bfd4c0f7cf06532563abc Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 4 Oct 2014 07:38:59 +0000
Subject: [PATCH 042/380] 5174 add sdt probe for blocked read in dbuf_read()
 Reviewed by: Basil Crow <basil.crow@delphix.com> Reviewed by: Matthew Ahrens
 <mahrens@delphix.com> Reviewed by: Steven Hartland <killing@multiplay.co.uk>
 Reviewed by: Richard Elling <richard.elling@gmail.com> Reviewed by: Boris
 Protopopov <bprotopopov@hotmail.com> Reviewed by: Steven Hartland
 <killing@multiplay.co.uk> Reviewed by: Garrett D'Amore <garrett@damore.org>
 Approved by: Robert Mustacchi <rm@joyent.com> Author: Adam H. Leventhal
 <ahl@delphix.com>

illumos/illumos-gate@f6164ad638e7346c4ae4ba393760a897cf6eb744
---
 uts/common/fs/zfs/dbuf.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/uts/common/fs/zfs/dbuf.c b/uts/common/fs/zfs/dbuf.c
index 4b0ec01a02cb..8d064daf25ce 100644
--- a/uts/common/fs/zfs/dbuf.c
+++ b/uts/common/fs/zfs/dbuf.c
@@ -671,6 +671,8 @@ dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
 			    db->db_state == DB_FILL) {
 				ASSERT(db->db_state == DB_READ ||
 				    (flags & DB_RF_HAVESTRUCT) == 0);
+				DTRACE_PROBE2(blocked__read, dmu_buf_impl_t *,
+				    db, zio_t *, zio);
 				cv_wait(&db->db_changed, &db->db_mtx);
 			}
 			if (db->db_state == DB_UNCACHED)

From 193c68c7e2243ddd8464108c5988fed8108d25f0 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 4 Oct 2014 07:49:06 +0000
Subject: [PATCH 043/380] 5150 zfs clone of a defer_destroy snapshot causes
 strangeness Reviewed by: Christopher Siden <christopher.siden@delphix.com>
 Reviewed by: George Wilson <george.wilson@delphix.com> Reviewed by: Max
 Grossman <max.grossman@delphix.com> Reviewed by: Saso Kiselkov
 <skiselkov.ml@gmail.com> Reviewed by: Richard Elling
 <richard.elling@gmail.com> Approved by: Robert Mustacchi <rm@joyent.com>
 Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@42fcb65ea4f2c6f8cc5a3c6142a486cb49871fd2
---
 uts/common/fs/zfs/dsl_dataset.c | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/uts/common/fs/zfs/dsl_dataset.c b/uts/common/fs/zfs/dsl_dataset.c
index c075c6ac3dd5..f1b92f3eaa33 100644
--- a/uts/common/fs/zfs/dsl_dataset.c
+++ b/uts/common/fs/zfs/dsl_dataset.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2014, Joyent, Inc. All rights reserved.
  * Copyright (c) 2014 RackTop Systems.
  */
@@ -692,7 +692,13 @@ dsl_dataset_create_sync_dd(dsl_dir_t *dd, dsl_dataset_t *origin,
 		dsphys->ds_uncompressed_bytes =
 		    origin->ds_phys->ds_uncompressed_bytes;
 		dsphys->ds_bp = origin->ds_phys->ds_bp;
-		dsphys->ds_flags |= origin->ds_phys->ds_flags;
+
+		/*
+		 * Inherit flags that describe the dataset's contents
+		 * (INCONSISTENT) or properties (Case Insensitive).
+		 */
+		dsphys->ds_flags |= origin->ds_phys->ds_flags &
+		    (DS_FLAG_INCONSISTENT | DS_FLAG_CI_DATASET);
 
 		dmu_buf_will_dirty(origin->ds_dbuf, tx);
 		origin->ds_phys->ds_num_children++;

From de77ba6df410495c7945124a8bcb0adb933560d5 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 4 Oct 2014 07:50:06 +0000
Subject: [PATCH 044/380] 5177 remove dead code from dsl_scan.c Reviewed by:
 Christopher Siden <christopher.siden@delphix.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Reviewed by: Richard Elling
 <richard.elling@gmail.com> Reviewed by: Richard Lowe <richlowe@richlowe.net>
 Approved by: Robert Mustacchi <rm@joyent.com> Author: Matthew Ahrens
 <mahrens@delphix.com>

illumos/illumos-gate@5f37736ac8f99922368294d745d3fefa22b49d11
---
 uts/common/fs/zfs/dsl_scan.c | 80 ++++++++++++++++--------------------
 1 file changed, 36 insertions(+), 44 deletions(-)

diff --git a/uts/common/fs/zfs/dsl_scan.c b/uts/common/fs/zfs/dsl_scan.c
index 7638ee183a6f..2392b7f33695 100644
--- a/uts/common/fs/zfs/dsl_scan.c
+++ b/uts/common/fs/zfs/dsl_scan.c
@@ -351,13 +351,12 @@ dsl_scan_cancel(dsl_pool_t *dp)
 	    dsl_scan_cancel_sync, NULL, 3, ZFS_SPACE_CHECK_RESERVED));
 }
 
-static void dsl_scan_visitbp(blkptr_t *bp,
-    const zbookmark_phys_t *zb, dnode_phys_t *dnp, arc_buf_t *pbuf,
-    dsl_dataset_t *ds, dsl_scan_t *scn, dmu_objset_type_t ostype,
-    dmu_tx_t *tx);
+static void dsl_scan_visitbp(blkptr_t *bp, const zbookmark_phys_t *zb,
+    dnode_phys_t *dnp, dsl_dataset_t *ds, dsl_scan_t *scn,
+    dmu_objset_type_t ostype, dmu_tx_t *tx);
 static void dsl_scan_visitdnode(dsl_scan_t *, dsl_dataset_t *ds,
     dmu_objset_type_t ostype,
-    dnode_phys_t *dnp, arc_buf_t *buf, uint64_t object, dmu_tx_t *tx);
+    dnode_phys_t *dnp, uint64_t object, dmu_tx_t *tx);
 
 void
 dsl_free(dsl_pool_t *dp, uint64_t txg, const blkptr_t *bp)
@@ -590,7 +589,7 @@ dsl_scan_check_resume(dsl_scan_t *scn, const dnode_phys_t *dnp,
 static int
 dsl_scan_recurse(dsl_scan_t *scn, dsl_dataset_t *ds, dmu_objset_type_t ostype,
     dnode_phys_t *dnp, const blkptr_t *bp,
-    const zbookmark_phys_t *zb, dmu_tx_t *tx, arc_buf_t **bufp)
+    const zbookmark_phys_t *zb, dmu_tx_t *tx)
 {
 	dsl_pool_t *dp = scn->scn_dp;
 	int zio_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_SCAN_THREAD;
@@ -601,76 +600,72 @@ dsl_scan_recurse(dsl_scan_t *scn, dsl_dataset_t *ds, dmu_objset_type_t ostype,
 		int i;
 		blkptr_t *cbp;
 		int epb = BP_GET_LSIZE(bp) >> SPA_BLKPTRSHIFT;
+		arc_buf_t *buf;
 
-		err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, bufp,
+		err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, &buf,
 		    ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb);
 		if (err) {
 			scn->scn_phys.scn_errors++;
 			return (err);
 		}
-		for (i = 0, cbp = (*bufp)->b_data; i < epb; i++, cbp++) {
-			dsl_scan_prefetch(scn, *bufp, cbp, zb->zb_objset,
+		for (i = 0, cbp = buf->b_data; i < epb; i++, cbp++) {
+			dsl_scan_prefetch(scn, buf, cbp, zb->zb_objset,
 			    zb->zb_object, zb->zb_blkid * epb + i);
 		}
-		for (i = 0, cbp = (*bufp)->b_data; i < epb; i++, cbp++) {
+		for (i = 0, cbp = buf->b_data; i < epb; i++, cbp++) {
 			zbookmark_phys_t czb;
 
 			SET_BOOKMARK(&czb, zb->zb_objset, zb->zb_object,
 			    zb->zb_level - 1,
 			    zb->zb_blkid * epb + i);
 			dsl_scan_visitbp(cbp, &czb, dnp,
-			    *bufp, ds, scn, ostype, tx);
-		}
-	} else if (BP_GET_TYPE(bp) == DMU_OT_USERGROUP_USED) {
-		uint32_t flags = ARC_WAIT;
-
-		err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, bufp,
-		    ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb);
-		if (err) {
-			scn->scn_phys.scn_errors++;
-			return (err);
+			    ds, scn, ostype, tx);
 		}
+		(void) arc_buf_remove_ref(buf, &buf);
 	} else if (BP_GET_TYPE(bp) == DMU_OT_DNODE) {
 		uint32_t flags = ARC_WAIT;
 		dnode_phys_t *cdnp;
 		int i, j;
 		int epb = BP_GET_LSIZE(bp) >> DNODE_SHIFT;
+		arc_buf_t *buf;
 
-		err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, bufp,
+		err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, &buf,
 		    ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb);
 		if (err) {
 			scn->scn_phys.scn_errors++;
 			return (err);
 		}
-		for (i = 0, cdnp = (*bufp)->b_data; i < epb; i++, cdnp++) {
+		for (i = 0, cdnp = buf->b_data; i < epb; i++, cdnp++) {
 			for (j = 0; j < cdnp->dn_nblkptr; j++) {
 				blkptr_t *cbp = &cdnp->dn_blkptr[j];
-				dsl_scan_prefetch(scn, *bufp, cbp,
+				dsl_scan_prefetch(scn, buf, cbp,
 				    zb->zb_objset, zb->zb_blkid * epb + i, j);
 			}
 		}
-		for (i = 0, cdnp = (*bufp)->b_data; i < epb; i++, cdnp++) {
+		for (i = 0, cdnp = buf->b_data; i < epb; i++, cdnp++) {
 			dsl_scan_visitdnode(scn, ds, ostype,
-			    cdnp, *bufp, zb->zb_blkid * epb + i, tx);
+			    cdnp, zb->zb_blkid * epb + i, tx);
 		}
 
+		(void) arc_buf_remove_ref(buf, &buf);
 	} else if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) {
 		uint32_t flags = ARC_WAIT;
 		objset_phys_t *osp;
+		arc_buf_t *buf;
 
-		err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, bufp,
+		err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, &buf,
 		    ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb);
 		if (err) {
 			scn->scn_phys.scn_errors++;
 			return (err);
 		}
 
-		osp = (*bufp)->b_data;
+		osp = buf->b_data;
 
 		dsl_scan_visitdnode(scn, ds, osp->os_type,
-		    &osp->os_meta_dnode, *bufp, DMU_META_DNODE_OBJECT, tx);
+		    &osp->os_meta_dnode, DMU_META_DNODE_OBJECT, tx);
 
-		if (OBJSET_BUF_HAS_USERUSED(*bufp)) {
+		if (OBJSET_BUF_HAS_USERUSED(buf)) {
 			/*
 			 * We also always visit user/group accounting
 			 * objects, and never skip them, even if we are
@@ -678,12 +673,13 @@ dsl_scan_recurse(dsl_scan_t *scn, dsl_dataset_t *ds, dmu_objset_type_t ostype,
 			 * deltas from this txg get integrated.
 			 */
 			dsl_scan_visitdnode(scn, ds, osp->os_type,
-			    &osp->os_groupused_dnode, *bufp,
+			    &osp->os_groupused_dnode,
 			    DMU_GROUPUSED_OBJECT, tx);
 			dsl_scan_visitdnode(scn, ds, osp->os_type,
-			    &osp->os_userused_dnode, *bufp,
+			    &osp->os_userused_dnode,
 			    DMU_USERUSED_OBJECT, tx);
 		}
+		(void) arc_buf_remove_ref(buf, &buf);
 	}
 
 	return (0);
@@ -691,7 +687,7 @@ dsl_scan_recurse(dsl_scan_t *scn, dsl_dataset_t *ds, dmu_objset_type_t ostype,
 
 static void
 dsl_scan_visitdnode(dsl_scan_t *scn, dsl_dataset_t *ds,
-    dmu_objset_type_t ostype, dnode_phys_t *dnp, arc_buf_t *buf,
+    dmu_objset_type_t ostype, dnode_phys_t *dnp,
     uint64_t object, dmu_tx_t *tx)
 {
 	int j;
@@ -702,7 +698,7 @@ dsl_scan_visitdnode(dsl_scan_t *scn, dsl_dataset_t *ds,
 		SET_BOOKMARK(&czb, ds ? ds->ds_object : 0, object,
 		    dnp->dn_nlevels - 1, j);
 		dsl_scan_visitbp(&dnp->dn_blkptr[j],
-		    &czb, dnp, buf, ds, scn, ostype, tx);
+		    &czb, dnp, ds, scn, ostype, tx);
 	}
 
 	if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
@@ -710,7 +706,7 @@ dsl_scan_visitdnode(dsl_scan_t *scn, dsl_dataset_t *ds,
 		SET_BOOKMARK(&czb, ds ? ds->ds_object : 0, object,
 		    0, DMU_SPILL_BLKID);
 		dsl_scan_visitbp(&dnp->dn_spill,
-		    &czb, dnp, buf, ds, scn, ostype, tx);
+		    &czb, dnp, ds, scn, ostype, tx);
 	}
 }
 
@@ -720,9 +716,8 @@ dsl_scan_visitdnode(dsl_scan_t *scn, dsl_dataset_t *ds,
  */
 static void
 dsl_scan_visitbp(blkptr_t *bp, const zbookmark_phys_t *zb,
-    dnode_phys_t *dnp, arc_buf_t *pbuf,
-    dsl_dataset_t *ds, dsl_scan_t *scn, dmu_objset_type_t ostype,
-    dmu_tx_t *tx)
+    dnode_phys_t *dnp, dsl_dataset_t *ds, dsl_scan_t *scn,
+    dmu_objset_type_t ostype, dmu_tx_t *tx)
 {
 	dsl_pool_t *dp = scn->scn_dp;
 	arc_buf_t *buf = NULL;
@@ -742,16 +737,15 @@ dsl_scan_visitbp(blkptr_t *bp, const zbookmark_phys_t *zb,
 	scn->scn_visited_this_txg++;
 
 	dprintf_bp(bp,
-	    "visiting ds=%p/%llu zb=%llx/%llx/%llx/%llx buf=%p bp=%p",
+	    "visiting ds=%p/%llu zb=%llx/%llx/%llx/%llx bp=%p",
 	    ds, ds ? ds->ds_object : 0,
 	    zb->zb_objset, zb->zb_object, zb->zb_level, zb->zb_blkid,
-	    pbuf, bp);
+	    bp);
 
 	if (bp->blk_birth <= scn->scn_phys.scn_cur_min_txg)
 		return;
 
-	if (dsl_scan_recurse(scn, ds, ostype, dnp, &bp_toread, zb, tx,
-	    &buf) != 0)
+	if (dsl_scan_recurse(scn, ds, ostype, dnp, &bp_toread, zb, tx) != 0)
 		return;
 
 	/*
@@ -775,8 +769,6 @@ dsl_scan_visitbp(blkptr_t *bp, const zbookmark_phys_t *zb,
 	if (BP_PHYSICAL_BIRTH(bp) <= scn->scn_phys.scn_cur_max_txg) {
 		scan_funcs[scn->scn_phys.scn_func](dp, bp, zb);
 	}
-	if (buf)
-		(void) arc_buf_remove_ref(buf, &buf);
 }
 
 static void
@@ -787,7 +779,7 @@ dsl_scan_visit_rootbp(dsl_scan_t *scn, dsl_dataset_t *ds, blkptr_t *bp,
 
 	SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET,
 	    ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID);
-	dsl_scan_visitbp(bp, &zb, NULL, NULL,
+	dsl_scan_visitbp(bp, &zb, NULL,
 	    ds, scn, DMU_OST_NONE, tx);
 
 	dprintf_ds(ds, "finished scan%s", "");

From 430ef8803890319e611bf387d03bdfc8a8596c24 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Mon, 6 Oct 2014 05:54:39 +0000
Subject: [PATCH 045/380] 5176 lock contention on godfather zio Reviewed by:
 Adam Leventhal <ahl@delphix.com> Reviewed by: Alex Reece
 <alex.reece@delphix.com> Reviewed by: Christopher Siden
 <christopher.siden@delphix.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Reviewed by: Richard Elling
 <richard.elling@gmail.com> Reviewed by: Bayard Bell <Bayard.Bell@nexenta.com>
 Approved by: Garrett D'Amore <garrett@damore.org> Author: Matthew Ahrens
 <mahrens@delphix.com>

illumos/illumos-gate@6f834bc197c703a6568554c889157fb345bac079
---
 uts/common/fs/zfs/spa.c          | 22 +++++++++++++++++-----
 uts/common/fs/zfs/sys/spa_impl.h |  3 ++-
 uts/common/fs/zfs/zio.c          |  2 +-
 3 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/uts/common/fs/zfs/spa.c b/uts/common/fs/zfs/spa.c
index 72870b589891..be308e2f8730 100644
--- a/uts/common/fs/zfs/spa.c
+++ b/uts/common/fs/zfs/spa.c
@@ -1222,7 +1222,9 @@ spa_unload(spa_t *spa)
 	 * Wait for any outstanding async I/O to complete.
 	 */
 	if (spa->spa_async_zio_root != NULL) {
-		(void) zio_wait(spa->spa_async_zio_root);
+		for (int i = 0; i < max_ncpus; i++)
+			(void) zio_wait(spa->spa_async_zio_root[i]);
+		kmem_free(spa->spa_async_zio_root, max_ncpus * sizeof (void *));
 		spa->spa_async_zio_root = NULL;
 	}
 
@@ -2141,8 +2143,13 @@ spa_load_impl(spa_t *spa, uint64_t pool_guid, nvlist_t *config,
 	/*
 	 * Create "The Godfather" zio to hold all async IOs
 	 */
-	spa->spa_async_zio_root = zio_root(spa, NULL, NULL,
-	    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | ZIO_FLAG_GODFATHER);
+	spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *),
+	    KM_SLEEP);
+	for (int i = 0; i < max_ncpus; i++) {
+		spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL,
+		    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE |
+		    ZIO_FLAG_GODFATHER);
+	}
 
 	/*
 	 * Parse the configuration into a vdev tree.  We explicitly set the
@@ -3481,8 +3488,13 @@ spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props,
 	/*
 	 * Create "The Godfather" zio to hold all async IOs
 	 */
-	spa->spa_async_zio_root = zio_root(spa, NULL, NULL,
-	    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | ZIO_FLAG_GODFATHER);
+	spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *),
+	    KM_SLEEP);
+	for (int i = 0; i < max_ncpus; i++) {
+		spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL,
+		    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE |
+		    ZIO_FLAG_GODFATHER);
+	}
 
 	/*
 	 * Create the root vdev.
diff --git a/uts/common/fs/zfs/sys/spa_impl.h b/uts/common/fs/zfs/sys/spa_impl.h
index 739b95988c25..48b28eb5a139 100644
--- a/uts/common/fs/zfs/sys/spa_impl.h
+++ b/uts/common/fs/zfs/sys/spa_impl.h
@@ -204,7 +204,8 @@ struct spa {
 	uint64_t	spa_failmode;		/* failure mode for the pool */
 	uint64_t	spa_delegation;		/* delegation on/off */
 	list_t		spa_config_list;	/* previous cache file(s) */
-	zio_t		*spa_async_zio_root;	/* root of all async I/O */
+	/* per-CPU array of root of async I/O: */
+	zio_t		**spa_async_zio_root;
 	zio_t		*spa_suspend_zio_root;	/* root of all suspended I/O */
 	kmutex_t	spa_suspend_lock;	/* protects suspend_zio_root */
 	kcondvar_t	spa_suspend_cv;		/* notification of resume */
diff --git a/uts/common/fs/zfs/zio.c b/uts/common/fs/zfs/zio.c
index d8833f67079d..b154cb639d5d 100644
--- a/uts/common/fs/zfs/zio.c
+++ b/uts/common/fs/zfs/zio.c
@@ -1367,7 +1367,7 @@ zio_nowait(zio_t *zio)
 		 */
 		spa_t *spa = zio->io_spa;
 
-		zio_add_child(spa->spa_async_zio_root, zio);
+		zio_add_child(spa->spa_async_zio_root[CPU_SEQID], zio);
 	}
 
 	zio_execute(zio);

From d8fa6fd7104b1d102478293169404d6990e6f195 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Mon, 6 Oct 2014 06:04:10 +0000
Subject: [PATCH 046/380] 5162 zfs recv should use loaned arc buffer to avoid
 copy Reviewed by: Christopher Siden <christopher.siden@delphix.com> Reviewed
 by: George Wilson <george.wilson@delphix.com> Reviewed by: Bayard Bell
 <Bayard.Bell@nexenta.com> Reviewed by: Richard Elling
 <richard.elling@gmail.com> Approved by: Garrett D'Amore <garrett@damore.org>
 Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@8a9047098ad8ce5afa38b6d012c8b509bb619f40
---
 uts/common/fs/zfs/dmu.c      |  9 ++++++-
 uts/common/fs/zfs/dmu_send.c | 46 +++++++++++++++++++++++-------------
 2 files changed, 37 insertions(+), 18 deletions(-)

diff --git a/uts/common/fs/zfs/dmu.c b/uts/common/fs/zfs/dmu.c
index 619b3b4aa458..4fdcb1475018 100644
--- a/uts/common/fs/zfs/dmu.c
+++ b/uts/common/fs/zfs/dmu.c
@@ -1264,7 +1264,14 @@ dmu_assign_arcbuf(dmu_buf_t *handle, uint64_t offset, arc_buf_t *buf,
 	rw_exit(&dn->dn_struct_rwlock);
 	DB_DNODE_EXIT(dbuf);
 
-	if (offset == db->db.db_offset && blksz == db->db.db_size) {
+	/*
+	 * We can only assign if the offset is aligned, the arc buf is the
+	 * same size as the dbuf, and the dbuf is not metadata.  It
+	 * can't be metadata because the loaned arc buf comes from the
+	 * user-data kmem arena.
+	 */
+	if (offset == db->db.db_offset && blksz == db->db.db_size &&
+	    DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA) {
 		dbuf_assign_arcbuf(db, buf, tx);
 		dbuf_rele(db, FTAG);
 	} else {
diff --git a/uts/common/fs/zfs/dmu_send.c b/uts/common/fs/zfs/dmu_send.c
index 97c9926bd79b..9b03eff7a06b 100644
--- a/uts/common/fs/zfs/dmu_send.c
+++ b/uts/common/fs/zfs/dmu_send.c
@@ -1212,11 +1212,13 @@ free_guid_map_onexit(void *arg)
 }
 
 static void *
-restore_read(struct restorearg *ra, int len)
+restore_read(struct restorearg *ra, int len, char *buf)
 {
-	void *rv;
 	int done = 0;
 
+	if (buf == NULL)
+		buf = ra->buf;
+
 	/* some things will require 8-byte alignment, so everything must */
 	ASSERT0(len % 8);
 
@@ -1224,7 +1226,7 @@ restore_read(struct restorearg *ra, int len)
 		ssize_t resid;
 
 		ra->err = vn_rdwr(UIO_READ, ra->vp,
-		    (caddr_t)ra->buf + done, len - done,
+		    buf + done, len - done,
 		    ra->voff, UIO_SYSSPACE, FAPPEND,
 		    RLIM64_INFINITY, CRED(), &resid);
 
@@ -1237,12 +1239,11 @@ restore_read(struct restorearg *ra, int len)
 	}
 
 	ASSERT3U(done, ==, len);
-	rv = ra->buf;
 	if (ra->byteswap)
-		fletcher_4_incremental_byteswap(rv, len, &ra->cksum);
+		fletcher_4_incremental_byteswap(buf, len, &ra->cksum);
 	else
-		fletcher_4_incremental_native(rv, len, &ra->cksum);
-	return (rv);
+		fletcher_4_incremental_native(buf, len, &ra->cksum);
+	return (buf);
 }
 
 static void
@@ -1357,7 +1358,7 @@ restore_object(struct restorearg *ra, objset_t *os, struct drr_object *drro)
 		return (SET_ERROR(EINVAL));
 
 	if (drro->drr_bonuslen) {
-		data = restore_read(ra, P2ROUNDUP(drro->drr_bonuslen, 8));
+		data = restore_read(ra, P2ROUNDUP(drro->drr_bonuslen, 8), NULL);
 		if (ra->err != 0)
 			return (ra->err);
 	}
@@ -1454,19 +1455,30 @@ restore_write(struct restorearg *ra, objset_t *os,
 	    !DMU_OT_IS_VALID(drrw->drr_type))
 		return (SET_ERROR(EINVAL));
 
-	data = restore_read(ra, drrw->drr_length);
-	if (data == NULL)
-		return (ra->err);
-
 	if (dmu_object_info(os, drrw->drr_object, NULL) != 0)
 		return (SET_ERROR(EINVAL));
 
+	dmu_buf_t *bonus;
+	if (dmu_bonus_hold(os, drrw->drr_object, FTAG, &bonus) != 0)
+		return (SET_ERROR(EINVAL));
+
+	arc_buf_t *abuf = dmu_request_arcbuf(bonus, drrw->drr_length);
+
+	data = restore_read(ra, drrw->drr_length, abuf->b_data);
+	if (data == NULL) {
+		dmu_return_arcbuf(abuf);
+		dmu_buf_rele(bonus, FTAG);
+		return (ra->err);
+	}
+
 	tx = dmu_tx_create(os);
 
 	dmu_tx_hold_write(tx, drrw->drr_object,
 	    drrw->drr_offset, drrw->drr_length);
 	err = dmu_tx_assign(tx, TXG_WAIT);
 	if (err != 0) {
+		dmu_return_arcbuf(abuf);
+		dmu_buf_rele(bonus, FTAG);
 		dmu_tx_abort(tx);
 		return (err);
 	}
@@ -1475,9 +1487,9 @@ restore_write(struct restorearg *ra, objset_t *os,
 		    DMU_OT_BYTESWAP(drrw->drr_type);
 		dmu_ot_byteswap[byteswap].ob_func(data, drrw->drr_length);
 	}
-	dmu_write(os, drrw->drr_object,
-	    drrw->drr_offset, drrw->drr_length, data, tx);
+	dmu_assign_arcbuf(bonus, drrw->drr_offset, abuf, tx);
 	dmu_tx_commit(tx);
+	dmu_buf_rele(bonus, FTAG);
 	return (0);
 }
 
@@ -1559,7 +1571,7 @@ restore_write_embedded(struct restorearg *ra, objset_t *os,
 	if (drrwnp->drr_compression >= ZIO_COMPRESS_FUNCTIONS)
 		return (EINVAL);
 
-	data = restore_read(ra, P2ROUNDUP(drrwnp->drr_psize, 8));
+	data = restore_read(ra, P2ROUNDUP(drrwnp->drr_psize, 8), NULL);
 	if (data == NULL)
 		return (ra->err);
 
@@ -1594,7 +1606,7 @@ restore_spill(struct restorearg *ra, objset_t *os, struct drr_spill *drrs)
 	    drrs->drr_length > SPA_MAXBLOCKSIZE)
 		return (SET_ERROR(EINVAL));
 
-	data = restore_read(ra, drrs->drr_length);
+	data = restore_read(ra, drrs->drr_length, NULL);
 	if (data == NULL)
 		return (ra->err);
 
@@ -1735,7 +1747,7 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, vnode_t *vp, offset_t *voffp,
 	 */
 	pcksum = ra.cksum;
 	while (ra.err == 0 &&
-	    NULL != (drr = restore_read(&ra, sizeof (*drr)))) {
+	    NULL != (drr = restore_read(&ra, sizeof (*drr), NULL))) {
 		if (issig(JUSTLOOKING) && issig(FORREAL)) {
 			ra.err = SET_ERROR(EINTR);
 			goto out;

From badbfa537f6af7cdd388377852997692931305e6 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Thu, 9 Oct 2014 05:50:23 +0000
Subject: [PATCH 047/380] 5175 implement dmu_read_uio_dbuf() to improve cached
 read performance Reviewed by: Adam Leventhal <ahl@delphix.com> Reviewed by:
 Alex Reece <alex.reece@delphix.com> Reviewed by: George Wilson
 <george@delphix.com> Reviewed by: Richard Elling <richard.elling@gmail.com>
 Approved by: Robert Mustacchi <rm@joyent.com> Author: Matthew Ahrens
 <mahrens@delphix.com>

illumos/illumos-gate@f8554bb92be6be88a684bea58293b41a6168a538
---
 uts/common/fs/zfs/dmu.c       | 74 +++++++++++++++++++++++++++++++++--
 uts/common/fs/zfs/sys/dmu.h   |  1 +
 uts/common/fs/zfs/zfs_vnops.c | 14 +++----
 3 files changed, 78 insertions(+), 11 deletions(-)

diff --git a/uts/common/fs/zfs/dmu.c b/uts/common/fs/zfs/dmu.c
index 4fdcb1475018..13c189831ff0 100644
--- a/uts/common/fs/zfs/dmu.c
+++ b/uts/common/fs/zfs/dmu.c
@@ -1015,8 +1015,8 @@ xuio_stat_wbuf_nocopy()
 }
 
 #ifdef _KERNEL
-int
-dmu_read_uio(objset_t *os, uint64_t object, uio_t *uio, uint64_t size)
+static int
+dmu_read_uio_dnode(dnode_t *dn, uio_t *uio, uint64_t size)
 {
 	dmu_buf_t **dbp;
 	int numbufs, i, err;
@@ -1026,8 +1026,8 @@ dmu_read_uio(objset_t *os, uint64_t object, uio_t *uio, uint64_t size)
 	 * NB: we could do this block-at-a-time, but it's nice
 	 * to be reading in parallel.
 	 */
-	err = dmu_buf_hold_array(os, object, uio->uio_loffset, size, TRUE, FTAG,
-	    &numbufs, &dbp);
+	err = dmu_buf_hold_array_by_dnode(dn, uio->uio_loffset, size,
+	    TRUE, FTAG, &numbufs, &dbp, 0);
 	if (err)
 		return (err);
 
@@ -1072,6 +1072,58 @@ dmu_read_uio(objset_t *os, uint64_t object, uio_t *uio, uint64_t size)
 	return (err);
 }
 
+/*
+ * Read 'size' bytes into the uio buffer.
+ * From object zdb->db_object.
+ * Starting at offset uio->uio_loffset.
+ *
+ * If the caller already has a dbuf in the target object
+ * (e.g. its bonus buffer), this routine is faster than dmu_read_uio(),
+ * because we don't have to find the dnode_t for the object.
+ */
+int
+dmu_read_uio_dbuf(dmu_buf_t *zdb, uio_t *uio, uint64_t size)
+{
+	dmu_buf_impl_t *db = (dmu_buf_impl_t *)zdb;
+	dnode_t *dn;
+	int err;
+
+	if (size == 0)
+		return (0);
+
+	DB_DNODE_ENTER(db);
+	dn = DB_DNODE(db);
+	err = dmu_read_uio_dnode(dn, uio, size);
+	DB_DNODE_EXIT(db);
+
+	return (err);
+}
+
+/*
+ * Read 'size' bytes into the uio buffer.
+ * From the specified object
+ * Starting at offset uio->uio_loffset.
+ */
+int
+dmu_read_uio(objset_t *os, uint64_t object, uio_t *uio, uint64_t size)
+{
+	dnode_t *dn;
+	int err;
+
+	if (size == 0)
+		return (0);
+
+	err = dnode_hold(os, object, FTAG, &dn);
+	if (err)
+		return (err);
+
+	err = dmu_read_uio_dnode(dn, uio, size);
+
+	dnode_rele(dn, FTAG);
+
+	return (err);
+}
+
 static int
 dmu_write_uio_dnode(dnode_t *dn, uio_t *uio, uint64_t size, dmu_tx_t *tx)
 {
@@ -1124,6 +1176,15 @@ dmu_write_uio_dnode(dnode_t *dn, uio_t *uio, uint64_t size, dmu_tx_t *tx)
 	return (err);
 }
 
+/*
+ * Write 'size' bytes from the uio buffer.
+ * To object zdb->db_object.
+ * Starting at offset uio->uio_loffset.
+ *
+ * If the caller already has a dbuf in the target object
+ * (e.g. its bonus buffer), this routine is faster than dmu_write_uio(),
+ * because we don't have to find the dnode_t for the object.
+ */
 int
 dmu_write_uio_dbuf(dmu_buf_t *zdb, uio_t *uio, uint64_t size,
     dmu_tx_t *tx)
@@ -1143,6 +1204,11 @@ dmu_write_uio_dbuf(dmu_buf_t *zdb, uio_t *uio, uint64_t size,
 	return (err);
 }
 
+/*
+ * Write 'size' bytes from the uio buffer.
+ * To the specified object.
+ * Starting at offset uio->uio_loffset.
+ */
 int
 dmu_write_uio(objset_t *os, uint64_t object, uio_t *uio, uint64_t size,
     dmu_tx_t *tx)
diff --git a/uts/common/fs/zfs/sys/dmu.h b/uts/common/fs/zfs/sys/dmu.h
index b73de0649377..ed374d5f8040 100644
--- a/uts/common/fs/zfs/sys/dmu.h
+++ b/uts/common/fs/zfs/sys/dmu.h
@@ -614,6 +614,7 @@ void dmu_write(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
 void dmu_prealloc(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
 	dmu_tx_t *tx);
 int dmu_read_uio(objset_t *os, uint64_t object, struct uio *uio, uint64_t size);
+int dmu_read_uio_dbuf(dmu_buf_t *zdb, struct uio *uio, uint64_t size);
 int dmu_write_uio(objset_t *os, uint64_t object, struct uio *uio, uint64_t size,
     dmu_tx_t *tx);
 int dmu_write_uio_dbuf(dmu_buf_t *zdb, struct uio *uio, uint64_t size,
diff --git a/uts/common/fs/zfs/zfs_vnops.c b/uts/common/fs/zfs/zfs_vnops.c
index e87c0027370e..c608a79fd2a0 100644
--- a/uts/common/fs/zfs/zfs_vnops.c
+++ b/uts/common/fs/zfs/zfs_vnops.c
@@ -406,7 +406,6 @@ static int
 mappedread(vnode_t *vp, int nbytes, uio_t *uio)
 {
 	znode_t *zp = VTOZ(vp);
-	objset_t *os = zp->z_zfsvfs->z_os;
 	int64_t	start, off;
 	int len = nbytes;
 	int error = 0;
@@ -425,7 +424,8 @@ mappedread(vnode_t *vp, int nbytes, uio_t *uio)
 			zfs_unmap_page(pp, va);
 			page_unlock(pp);
 		} else {
-			error = dmu_read_uio(os, zp->z_id, uio, bytes);
+			error = dmu_read_uio_dbuf(sa_get_db(zp->z_sa_hdl),
+			    uio, bytes);
 		}
 		len -= bytes;
 		off = 0;
@@ -460,7 +460,6 @@ zfs_read(vnode_t *vp, uio_t *uio, int ioflag, cred_t *cr, caller_context_t *ct)
 {
 	znode_t		*zp = VTOZ(vp);
 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
-	objset_t	*os;
 	ssize_t		n, nbytes;
 	int		error = 0;
 	rl_t		*rl;
@@ -468,7 +467,6 @@ zfs_read(vnode_t *vp, uio_t *uio, int ioflag, cred_t *cr, caller_context_t *ct)
 
 	ZFS_ENTER(zfsvfs);
 	ZFS_VERIFY_ZP(zp);
-	os = zfsvfs->z_os;
 
 	if (zp->z_pflags & ZFS_AV_QUARANTINED) {
 		ZFS_EXIT(zfsvfs);
@@ -558,10 +556,12 @@ zfs_read(vnode_t *vp, uio_t *uio, int ioflag, cred_t *cr, caller_context_t *ct)
 		nbytes = MIN(n, zfs_read_chunk_size -
 		    P2PHASE(uio->uio_loffset, zfs_read_chunk_size));
 
-		if (vn_has_cached_data(vp))
+		if (vn_has_cached_data(vp)) {
 			error = mappedread(vp, nbytes, uio);
-		else
-			error = dmu_read_uio(os, zp->z_id, uio, nbytes);
+		} else {
+			error = dmu_read_uio_dbuf(sa_get_db(zp->z_sa_hdl),
+			    uio, nbytes);
+		}
 		if (error) {
 			/* convert checksum errors into IO errors */
 			if (error == ECKSUM)

From cde5f074b0d3cac2a0c831dbd9b73ec9507fe458 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Thu, 9 Oct 2014 05:53:04 +0000
Subject: [PATCH 048/380] 3693 restore_object uses at least two transactions to
 restore an object Reviewed by: Christopher Siden
 <christopher.siden@delphix.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Reviewed by: Andriy Gapon
 <andriy.gapon@hybridcluster.com> Approved by: Robert Mustacchi
 <rm@joyent.com> Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@e77d42eaa49fe55bfae1e0e0065c6e99affc001b
---
 uts/common/fs/zfs/dmu.c        |  1 +
 uts/common/fs/zfs/dmu_object.c | 42 ++-----------------
 uts/common/fs/zfs/dmu_send.c   | 77 ++++++++++++++++++++++++----------
 uts/common/fs/zfs/sys/dmu.h    |  6 ++-
 4 files changed, 62 insertions(+), 64 deletions(-)

diff --git a/uts/common/fs/zfs/dmu.c b/uts/common/fs/zfs/dmu.c
index 13c189831ff0..c541e04d5429 100644
--- a/uts/common/fs/zfs/dmu.c
+++ b/uts/common/fs/zfs/dmu.c
@@ -1848,6 +1848,7 @@ dmu_object_info_from_dnode(dnode_t *dn, dmu_object_info_t *doi)
 	doi->doi_indirection = dn->dn_nlevels;
 	doi->doi_checksum = dn->dn_checksum;
 	doi->doi_compress = dn->dn_compress;
+	doi->doi_nblkptr = dn->dn_nblkptr;
 	doi->doi_physical_blocks_512 = (DN_USED_BYTES(dnp) + 256) >> 9;
 	doi->doi_max_offset = (dn->dn_maxblkid + 1) * dn->dn_datablksz;
 	doi->doi_fill_count = 0;
diff --git a/uts/common/fs/zfs/dmu_object.c b/uts/common/fs/zfs/dmu_object.c
index 5b512ad24e3c..808864a57607 100644
--- a/uts/common/fs/zfs/dmu_object.c
+++ b/uts/common/fs/zfs/dmu_object.c
@@ -20,7 +20,8 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
+ * Copyright 2014 HybridCluster. All rights reserved.
  */
 
 #include <sys/dmu.h>
@@ -107,11 +108,9 @@ dmu_object_claim(objset_t *os, uint64_t object, dmu_object_type_t ot,
 
 int
 dmu_object_reclaim(objset_t *os, uint64_t object, dmu_object_type_t ot,
-    int blocksize, dmu_object_type_t bonustype, int bonuslen)
+    int blocksize, dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx)
 {
 	dnode_t *dn;
-	dmu_tx_t *tx;
-	int nblkptr;
 	int err;
 
 	if (object == DMU_META_DNODE_OBJECT)
@@ -122,44 +121,9 @@ dmu_object_reclaim(objset_t *os, uint64_t object, dmu_object_type_t ot,
 	if (err)
 		return (err);
 
-	if (dn->dn_type == ot && dn->dn_datablksz == blocksize &&
-	    dn->dn_bonustype == bonustype && dn->dn_bonuslen == bonuslen) {
-		/* nothing is changing, this is a noop */
-		dnode_rele(dn, FTAG);
-		return (0);
-	}
-
-	if (bonustype == DMU_OT_SA) {
-		nblkptr = 1;
-	} else {
-		nblkptr = 1 + ((DN_MAX_BONUSLEN - bonuslen) >> SPA_BLKPTRSHIFT);
-	}
-
-	/*
-	 * If we are losing blkptrs or changing the block size this must
-	 * be a new file instance.   We must clear out the previous file
-	 * contents before we can change this type of metadata in the dnode.
-	 */
-	if (dn->dn_nblkptr > nblkptr || dn->dn_datablksz != blocksize) {
-		err = dmu_free_long_range(os, object, 0, DMU_OBJECT_END);
-		if (err)
-			goto out;
-	}
-
-	tx = dmu_tx_create(os);
-	dmu_tx_hold_bonus(tx, object);
-	err = dmu_tx_assign(tx, TXG_WAIT);
-	if (err) {
-		dmu_tx_abort(tx);
-		goto out;
-	}
-
 	dnode_reallocate(dn, ot, blocksize, bonustype, bonuslen, tx);
 
-	dmu_tx_commit(tx);
-out:
 	dnode_rele(dn, FTAG);
-
 	return (err);
 }
 
diff --git a/uts/common/fs/zfs/dmu_send.c b/uts/common/fs/zfs/dmu_send.c
index 9b03eff7a06b..fed1b86a1b1e 100644
--- a/uts/common/fs/zfs/dmu_send.c
+++ b/uts/common/fs/zfs/dmu_send.c
@@ -23,6 +23,7 @@
  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2014, Joyent, Inc. All rights reserved.
+ * Copyright 2014 HybridCluster. All rights reserved.
  */
 
 #include <sys/dmu.h>
@@ -1333,12 +1334,25 @@ backup_byteswap(dmu_replay_record_t *drr)
 #undef DO32
 }
 
+static inline uint8_t
+deduce_nblkptr(dmu_object_type_t bonus_type, uint64_t bonus_size)
+{
+	if (bonus_type == DMU_OT_SA) {
+		return (1);
+	} else {
+		return (1 +
+		    ((DN_MAX_BONUSLEN - bonus_size) >> SPA_BLKPTRSHIFT));
+	}
+}
+
 static int
 restore_object(struct restorearg *ra, objset_t *os, struct drr_object *drro)
 {
-	int err;
+	dmu_object_info_t doi;
 	dmu_tx_t *tx;
 	void *data = NULL;
+	uint64_t object;
+	int err;
 
 	if (drro->drr_type == DMU_OT_NONE ||
 	    !DMU_OT_IS_VALID(drro->drr_type) ||
@@ -1352,10 +1366,11 @@ restore_object(struct restorearg *ra, objset_t *os, struct drr_object *drro)
 		return (SET_ERROR(EINVAL));
 	}
 
-	err = dmu_object_info(os, drro->drr_object, NULL);
+	err = dmu_object_info(os, drro->drr_object, &doi);
 
 	if (err != 0 && err != ENOENT)
 		return (SET_ERROR(EINVAL));
+	object = err == 0 ? drro->drr_object : DMU_NEW_OBJECT;
 
 	if (drro->drr_bonuslen) {
 		data = restore_read(ra, P2ROUNDUP(drro->drr_bonuslen, 8), NULL);
@@ -1363,37 +1378,53 @@ restore_object(struct restorearg *ra, objset_t *os, struct drr_object *drro)
 			return (ra->err);
 	}
 
-	if (err == ENOENT) {
-		/* currently free, want to be allocated */
-		tx = dmu_tx_create(os);
-		dmu_tx_hold_bonus(tx, DMU_NEW_OBJECT);
-		err = dmu_tx_assign(tx, TXG_WAIT);
-		if (err != 0) {
-			dmu_tx_abort(tx);
-			return (err);
+	/*
+	 * If we are losing blkptrs or changing the block size this must
+	 * be a new file instance.  We must clear out the previous file
+	 * contents before we can change this type of metadata in the dnode.
+	 */
+	if (err == 0) {
+		int nblkptr;
+
+		nblkptr = deduce_nblkptr(drro->drr_bonustype,
+		    drro->drr_bonuslen);
+
+		if (drro->drr_blksz != doi.doi_data_block_size ||
+		    nblkptr < doi.doi_nblkptr) {
+			err = dmu_free_long_range(os, drro->drr_object,
+			    0, DMU_OBJECT_END);
+			if (err != 0)
+				return (SET_ERROR(EINVAL));
 		}
-		err = dmu_object_claim(os, drro->drr_object,
-		    drro->drr_type, drro->drr_blksz,
-		    drro->drr_bonustype, drro->drr_bonuslen, tx);
-		dmu_tx_commit(tx);
-	} else {
-		/* currently allocated, want to be allocated */
-		err = dmu_object_reclaim(os, drro->drr_object,
-		    drro->drr_type, drro->drr_blksz,
-		    drro->drr_bonustype, drro->drr_bonuslen);
-	}
-	if (err != 0) {
-		return (SET_ERROR(EINVAL));
 	}
 
 	tx = dmu_tx_create(os);
-	dmu_tx_hold_bonus(tx, drro->drr_object);
+	dmu_tx_hold_bonus(tx, object);
 	err = dmu_tx_assign(tx, TXG_WAIT);
 	if (err != 0) {
 		dmu_tx_abort(tx);
 		return (err);
 	}
 
+	if (object == DMU_NEW_OBJECT) {
+		/* currently free, want to be allocated */
+		err = dmu_object_claim(os, drro->drr_object,
+		    drro->drr_type, drro->drr_blksz,
+		    drro->drr_bonustype, drro->drr_bonuslen, tx);
+	} else if (drro->drr_type != doi.doi_type ||
+	    drro->drr_blksz != doi.doi_data_block_size ||
+	    drro->drr_bonustype != doi.doi_bonus_type ||
+	    drro->drr_bonuslen != doi.doi_bonus_size) {
+		/* currently allocated, but with different properties */
+		err = dmu_object_reclaim(os, drro->drr_object,
+		    drro->drr_type, drro->drr_blksz,
+		    drro->drr_bonustype, drro->drr_bonuslen, tx);
+	}
+	if (err != 0) {
+		dmu_tx_commit(tx);
+		return (SET_ERROR(EINVAL));
+	}
+
 	dmu_object_set_checksum(os, drro->drr_object, drro->drr_checksumtype,
 	    tx);
 	dmu_object_set_compress(os, drro->drr_object, drro->drr_compress, tx);
diff --git a/uts/common/fs/zfs/sys/dmu.h b/uts/common/fs/zfs/sys/dmu.h
index ed374d5f8040..e9fa39d5d6f1 100644
--- a/uts/common/fs/zfs/sys/dmu.h
+++ b/uts/common/fs/zfs/sys/dmu.h
@@ -25,6 +25,7 @@
  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
  * Copyright 2013 DEY Storage Systems, Inc.
+ * Copyright 2014 HybridCluster. All rights reserved.
  */
 
 /* Portions Copyright 2010 Robert Milkowski */
@@ -339,7 +340,7 @@ uint64_t dmu_object_alloc(objset_t *os, dmu_object_type_t ot,
 int dmu_object_claim(objset_t *os, uint64_t object, dmu_object_type_t ot,
     int blocksize, dmu_object_type_t bonus_type, int bonus_len, dmu_tx_t *tx);
 int dmu_object_reclaim(objset_t *os, uint64_t object, dmu_object_type_t ot,
-    int blocksize, dmu_object_type_t bonustype, int bonuslen);
+    int blocksize, dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *txp);
 
 /*
  * Free an object from this objset.
@@ -653,7 +654,8 @@ typedef struct dmu_object_info {
 	uint8_t doi_indirection;		/* 2 = dnode->indirect->data */
 	uint8_t doi_checksum;
 	uint8_t doi_compress;
-	uint8_t doi_pad[5];
+	uint8_t doi_nblkptr;
+	uint8_t doi_pad[4];
 	uint64_t doi_physical_blocks_512;	/* data + metadata, 512b blks */
 	uint64_t doi_max_offset;
 	uint64_t doi_fill_count;		/* number of non-empty blocks */

From 9fa8fe3325c6e22936fb5ec8f848b91d3278f528 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 8 Nov 2014 06:31:51 +0000
Subject: [PATCH 049/380] 5243 zdb -b could be much faster Reviewed by:
 Christopher Siden <christopher.siden@delphix.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Reviewed by: Richard Elling
 <richard.elling@gmail.com> Approved by: Dan McDonald <danmcd@omniti.com>
 Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@f7950bf1145637c6dc57742a8bb95631fd5c846f
---
 uts/common/fs/zfs/dmu_traverse.c | 41 +++++++++++++++++---------------
 1 file changed, 22 insertions(+), 19 deletions(-)

diff --git a/uts/common/fs/zfs/dmu_traverse.c b/uts/common/fs/zfs/dmu_traverse.c
index fb1ec0755cf1..5836549d200a 100644
--- a/uts/common/fs/zfs/dmu_traverse.c
+++ b/uts/common/fs/zfs/dmu_traverse.c
@@ -59,6 +59,7 @@ typedef struct traverse_data {
 	int td_flags;
 	prefetch_data_t *td_pfd;
 	boolean_t td_paused;
+	uint64_t td_hole_birth_enabled_txg;
 	blkptr_cb_t *td_func;
 	void *td_arg;
 } traverse_data_t;
@@ -229,25 +230,20 @@ traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
 	}
 
 	if (bp->blk_birth == 0) {
-		if (spa_feature_is_active(td->td_spa, SPA_FEATURE_HOLE_BIRTH)) {
-			/*
-			 * Since this block has a birth time of 0 it must be a
-			 * hole created before the SPA_FEATURE_HOLE_BIRTH
-			 * feature was enabled.  If SPA_FEATURE_HOLE_BIRTH
-			 * was enabled before the min_txg for this traveral we
-			 * know the hole must have been created before the
-			 * min_txg for this traveral, so we can skip it. If
-			 * SPA_FEATURE_HOLE_BIRTH was enabled after the min_txg
-			 * for this traveral we cannot tell if the hole was
-			 * created before or after the min_txg for this
-			 * traversal, so we cannot skip it.
-			 */
-			uint64_t hole_birth_enabled_txg;
-			VERIFY(spa_feature_enabled_txg(td->td_spa,
-			    SPA_FEATURE_HOLE_BIRTH, &hole_birth_enabled_txg));
-			if (hole_birth_enabled_txg < td->td_min_txg)
-				return (0);
-		}
+		/*
+		 * Since this block has a birth time of 0 it must be a
+		 * hole created before the SPA_FEATURE_HOLE_BIRTH
+		 * feature was enabled.  If SPA_FEATURE_HOLE_BIRTH
+		 * was enabled before the min_txg for this traveral we
+		 * know the hole must have been created before the
+		 * min_txg for this traveral, so we can skip it. If
+		 * SPA_FEATURE_HOLE_BIRTH was enabled after the min_txg
+		 * for this traveral we cannot tell if the hole was
+		 * created before or after the min_txg for this
+		 * traversal, so we cannot skip it.
+		 */
+		if (td->td_hole_birth_enabled_txg < td->td_min_txg)
+			return (0);
 	} else if (bp->blk_birth <= td->td_min_txg) {
 		return (0);
 	}
@@ -523,6 +519,13 @@ traverse_impl(spa_t *spa, dsl_dataset_t *ds, uint64_t objset, blkptr_t *rootbp,
 	td.td_flags = flags;
 	td.td_paused = B_FALSE;
 
+	if (spa_feature_is_active(spa, SPA_FEATURE_HOLE_BIRTH)) {
+		VERIFY(spa_feature_enabled_txg(spa,
+		    SPA_FEATURE_HOLE_BIRTH, &td.td_hole_birth_enabled_txg));
+	} else {
+		td.td_hole_birth_enabled_txg = 0;
+	}
+
 	pd.pd_blks_max = zfs_pd_blks_max;
 	pd.pd_flags = flags;
 	mutex_init(&pd.pd_mtx, NULL, MUTEX_DEFAULT, NULL);

From 34dc14e7e42fe4618495a32061f966359b46372d Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 8 Nov 2014 06:34:37 +0000
Subject: [PATCH 050/380] 5244 zio pipeline callers should explicitly invoke
 next stage Reviewed by: Adam Leventhal <ahl@delphix.com> Reviewed by: Alex
 Reece <alex.reece@delphix.com> Reviewed by: Christopher Siden
 <christopher.siden@delphix.com> Reviewed by: Matthew Ahrens
 <mahrens@delphix.com> Reviewed by: Richard Elling <richard.elling@gmail.com>
 Reviewed by: Dan McDonald <danmcd@omniti.com> Reviewed by: Steven Hartland
 <killing@multiplay.co.uk> Approved by: Gordon Ross <gwr@nexenta.com> Author:
 George Wilson <george.wilson@delphix.com>

illumos/illumos-gate@738f37bc3dcd61e8a893af0f2d466d76690b70ec
---
 uts/common/fs/zfs/sys/vdev_impl.h |  2 +-
 uts/common/fs/zfs/sys/zio.h       |  3 ---
 uts/common/fs/zfs/vdev_disk.c     | 15 ++++++++-------
 uts/common/fs/zfs/vdev_file.c     | 12 ++++++------
 uts/common/fs/zfs/vdev_mirror.c   |  9 +++++----
 uts/common/fs/zfs/vdev_missing.c  |  6 +++---
 uts/common/fs/zfs/vdev_raidz.c    |  9 +++++----
 uts/common/fs/zfs/zio.c           | 23 ++++++++++++++++++++---
 8 files changed, 48 insertions(+), 31 deletions(-)

diff --git a/uts/common/fs/zfs/sys/vdev_impl.h b/uts/common/fs/zfs/sys/vdev_impl.h
index 9c22384f733d..6d9bcb17d00f 100644
--- a/uts/common/fs/zfs/sys/vdev_impl.h
+++ b/uts/common/fs/zfs/sys/vdev_impl.h
@@ -60,7 +60,7 @@ typedef int	vdev_open_func_t(vdev_t *vd, uint64_t *size, uint64_t *max_size,
     uint64_t *ashift);
 typedef void	vdev_close_func_t(vdev_t *vd);
 typedef uint64_t vdev_asize_func_t(vdev_t *vd, uint64_t psize);
-typedef int	vdev_io_start_func_t(zio_t *zio);
+typedef void	vdev_io_start_func_t(zio_t *zio);
 typedef void	vdev_io_done_func_t(zio_t *zio);
 typedef void	vdev_state_change_func_t(vdev_t *vd, int, int);
 typedef void	vdev_hold_func_t(vdev_t *vd);
diff --git a/uts/common/fs/zfs/sys/zio.h b/uts/common/fs/zfs/sys/zio.h
index f158042eb40c..f6cf259bf71a 100644
--- a/uts/common/fs/zfs/sys/zio.h
+++ b/uts/common/fs/zfs/sys/zio.h
@@ -151,9 +151,6 @@ typedef enum zio_priority {
 	ZIO_PRIORITY_NOW		/* non-queued i/os (e.g. free) */
 } zio_priority_t;
 
-#define	ZIO_PIPELINE_CONTINUE		0x100
-#define	ZIO_PIPELINE_STOP		0x101
-
 enum zio_flag {
 	/*
 	 * Flags inherited by gang, ddt, and vdev children,
diff --git a/uts/common/fs/zfs/vdev_disk.c b/uts/common/fs/zfs/vdev_disk.c
index 3a4bc20f5be4..ed4a8b773bf4 100644
--- a/uts/common/fs/zfs/vdev_disk.c
+++ b/uts/common/fs/zfs/vdev_disk.c
@@ -715,7 +715,7 @@ vdev_disk_ioctl_done(void *zio_arg, int error)
 	zio_interrupt(zio);
 }
 
-static int
+static void
 vdev_disk_io_start(zio_t *zio)
 {
 	vdev_t *vd = zio->io_vd;
@@ -731,14 +731,16 @@ vdev_disk_io_start(zio_t *zio)
 	 */
 	if (dvd == NULL || (dvd->vd_ldi_offline && dvd->vd_lh == NULL)) {
 		zio->io_error = ENXIO;
-		return (ZIO_PIPELINE_CONTINUE);
+		zio_interrupt(zio);
+		return;
 	}
 
 	if (zio->io_type == ZIO_TYPE_IOCTL) {
 		/* XXPOLICY */
 		if (!vdev_readable(vd)) {
 			zio->io_error = SET_ERROR(ENXIO);
-			return (ZIO_PIPELINE_CONTINUE);
+			zio_interrupt(zio);
+			return;
 		}
 
 		switch (zio->io_cmd) {
@@ -769,7 +771,7 @@ vdev_disk_io_start(zio_t *zio)
 				 * and will call vdev_disk_ioctl_done()
 				 * upon completion.
 				 */
-				return (ZIO_PIPELINE_STOP);
+				return;
 			}
 
 			if (error == ENOTSUP || error == ENOTTY) {
@@ -790,7 +792,8 @@ vdev_disk_io_start(zio_t *zio)
 			zio->io_error = SET_ERROR(ENOTSUP);
 		}
 
-		return (ZIO_PIPELINE_CONTINUE);
+		zio_execute(zio);
+		return;
 	}
 
 	vb = kmem_alloc(sizeof (vdev_buf_t), KM_SLEEP);
@@ -811,8 +814,6 @@ vdev_disk_io_start(zio_t *zio)
 
 	/* ldi_strategy() will return non-zero only on programming errors */
 	VERIFY(ldi_strategy(dvd->vd_lh, bp) == 0);
-
-	return (ZIO_PIPELINE_STOP);
 }
 
 static void
diff --git a/uts/common/fs/zfs/vdev_file.c b/uts/common/fs/zfs/vdev_file.c
index a05abeb9d910..5dfc331d20a4 100644
--- a/uts/common/fs/zfs/vdev_file.c
+++ b/uts/common/fs/zfs/vdev_file.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -182,7 +182,7 @@ vdev_file_io_strategy(void *arg)
 	}
 }
 
-static int
+static void
 vdev_file_io_start(zio_t *zio)
 {
 	vdev_t *vd = zio->io_vd;
@@ -194,7 +194,8 @@ vdev_file_io_start(zio_t *zio)
 		/* XXPOLICY */
 		if (!vdev_readable(vd)) {
 			zio->io_error = SET_ERROR(ENXIO);
-			return (ZIO_PIPELINE_CONTINUE);
+			zio_interrupt(zio);
+			return;
 		}
 
 		switch (zio->io_cmd) {
@@ -206,7 +207,8 @@ vdev_file_io_start(zio_t *zio)
 			zio->io_error = SET_ERROR(ENOTSUP);
 		}
 
-		return (ZIO_PIPELINE_CONTINUE);
+		zio_execute(zio);
+		return;
 	}
 
 	vb = kmem_alloc(sizeof (vdev_buf_t), KM_SLEEP);
@@ -225,8 +227,6 @@ vdev_file_io_start(zio_t *zio)
 
 	VERIFY3U(taskq_dispatch(system_taskq, vdev_file_io_strategy, bp,
 	    TQ_SLEEP), !=, 0);
-
-	return (ZIO_PIPELINE_STOP);
 }
 
 /* ARGSUSED */
diff --git a/uts/common/fs/zfs/vdev_mirror.c b/uts/common/fs/zfs/vdev_mirror.c
index f62c1e3617d8..8749e539f466 100644
--- a/uts/common/fs/zfs/vdev_mirror.c
+++ b/uts/common/fs/zfs/vdev_mirror.c
@@ -24,7 +24,7 @@
  */
 
 /*
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -260,7 +260,7 @@ vdev_mirror_child_select(zio_t *zio)
 	return (-1);
 }
 
-static int
+static void
 vdev_mirror_io_start(zio_t *zio)
 {
 	mirror_map_t *mm;
@@ -285,7 +285,8 @@ vdev_mirror_io_start(zio_t *zio)
 				    zio->io_type, zio->io_priority, 0,
 				    vdev_mirror_scrub_done, mc));
 			}
-			return (ZIO_PIPELINE_CONTINUE);
+			zio_execute(zio);
+			return;
 		}
 		/*
 		 * For normal reads just pick one child.
@@ -311,7 +312,7 @@ vdev_mirror_io_start(zio_t *zio)
 		c++;
 	}
 
-	return (ZIO_PIPELINE_CONTINUE);
+	zio_execute(zio);
 }
 
 static int
diff --git a/uts/common/fs/zfs/vdev_missing.c b/uts/common/fs/zfs/vdev_missing.c
index b9eb99d18005..228757334234 100644
--- a/uts/common/fs/zfs/vdev_missing.c
+++ b/uts/common/fs/zfs/vdev_missing.c
@@ -24,7 +24,7 @@
  */
 
 /*
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 /*
@@ -66,11 +66,11 @@ vdev_missing_close(vdev_t *vd)
 }
 
 /* ARGSUSED */
-static int
+static void
 vdev_missing_io_start(zio_t *zio)
 {
 	zio->io_error = SET_ERROR(ENOTSUP);
-	return (ZIO_PIPELINE_CONTINUE);
+	zio_execute(zio);
 }
 
 /* ARGSUSED */
diff --git a/uts/common/fs/zfs/vdev_raidz.c b/uts/common/fs/zfs/vdev_raidz.c
index 480141dc6379..f686d56e9e5f 100644
--- a/uts/common/fs/zfs/vdev_raidz.c
+++ b/uts/common/fs/zfs/vdev_raidz.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  */
 
@@ -1711,7 +1711,7 @@ vdev_raidz_child_done(zio_t *zio)
  *      vdevs have had errors, then create zio read operations to the parity
  *      columns' VDevs as well.
  */
-static int
+static void
 vdev_raidz_io_start(zio_t *zio)
 {
 	vdev_t *vd = zio->io_vd;
@@ -1759,7 +1759,8 @@ vdev_raidz_io_start(zio_t *zio)
 			    ZIO_FLAG_NODATA | ZIO_FLAG_OPTIONAL, NULL, NULL));
 		}
 
-		return (ZIO_PIPELINE_CONTINUE);
+		zio_execute(zio);
+		return;
 	}
 
 	ASSERT(zio->io_type == ZIO_TYPE_READ);
@@ -1799,7 +1800,7 @@ vdev_raidz_io_start(zio_t *zio)
 		}
 	}
 
-	return (ZIO_PIPELINE_CONTINUE);
+	zio_execute(zio);
 }
 
 
diff --git a/uts/common/fs/zfs/zio.c b/uts/common/fs/zfs/zio.c
index b154cb639d5d..302d692c64a6 100644
--- a/uts/common/fs/zfs/zio.c
+++ b/uts/common/fs/zfs/zio.c
@@ -63,6 +63,9 @@ kmem_cache_t *zio_data_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT];
 extern vmem_t *zio_alloc_arena;
 #endif
 
+#define	ZIO_PIPELINE_CONTINUE		0x100
+#define	ZIO_PIPELINE_STOP		0x101
+
 /*
  * The following actions directly effect the spa's sync-to-convergence logic.
  * The values below define the sync pass when we start performing the action.
@@ -2466,6 +2469,18 @@ zio_free_zil(spa_t *spa, uint64_t txg, blkptr_t *bp)
  * Read and write to physical devices
  * ==========================================================================
  */
+
+
+/*
+ * Issue an I/O to the underlying vdev. Typically the issue pipeline
+ * stops after this stage and will resume upon I/O completion.
+ * However, there are instances where the vdev layer may need to
+ * continue the pipeline when an I/O was not issued. Since the I/O
+ * that was sent to the vdev layer might be different than the one
+ * currently active in the pipeline (see vdev_queue_io()), we explicitly
+ * force the underlying vdev layers to call either zio_execute() or
+ * zio_interrupt() to ensure that the pipeline continues with the correct I/O.
+ */
 static int
 zio_vdev_io_start(zio_t *zio)
 {
@@ -2483,7 +2498,8 @@ zio_vdev_io_start(zio_t *zio)
 		/*
 		 * The mirror_ops handle multiple DVAs in a single BP.
 		 */
-		return (vdev_mirror_ops.vdev_op_io_start(zio));
+		vdev_mirror_ops.vdev_op_io_start(zio);
+		return (ZIO_PIPELINE_STOP);
 	}
 
 	/*
@@ -2491,7 +2507,7 @@ zio_vdev_io_start(zio_t *zio)
 	 * can quickly react to certain workloads.  In particular, we care
 	 * about non-scrubbing, top-level reads and writes with the following
 	 * characteristics:
-	 * 	- synchronous writes of user data to non-slog devices
+	 *	- synchronous writes of user data to non-slog devices
 	 *	- any reads of user data
 	 * When these conditions are met, adjust the timestamp of spa_last_io
 	 * which allows the scan thread to adjust its workload accordingly.
@@ -2577,7 +2593,8 @@ zio_vdev_io_start(zio_t *zio)
 		}
 	}
 
-	return (vd->vdev_ops->vdev_op_io_start(zio));
+	vd->vdev_ops->vdev_op_io_start(zio);
+	return (ZIO_PIPELINE_STOP);
 }
 
 static int

From 049751b4fcb775b01d4d5e40f80e97e5375a88df Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 8 Nov 2014 06:43:33 +0000
Subject: [PATCH 051/380] 5027 zfs large block support Reviewed by: Alek
 Pinchuk <pinchuk.alek@gmail.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Reviewed by: Josef 'Jeff' Sipek
 <josef.sipek@nexenta.com> Reviewed by: Richard Elling
 <richard.elling@richardelling.com> Reviewed by: Saso Kiselkov
 <skiselkov.ml@gmail.com> Reviewed by: Brian Behlendorf <behlendorf1@llnl.gov>
 Approved by: Dan McDonald <danmcd@omniti.com> Author: Matthew Ahrens
 <matt@mahrens.org>

illumos/illumos-gate@b515258426fed6c7311fd3f1dea697cfbd4085c6
---
 common/zfs/zfeature_common.c        |  12 +++-
 common/zfs/zfeature_common.h        |   1 +
 common/zfs/zfs_prop.c               |   4 +-
 common/zfs/zpool_prop.c             |   2 +
 uts/common/fs/zfs/bpobj.c           |   5 +-
 uts/common/fs/zfs/bptree.c          |   2 +-
 uts/common/fs/zfs/dbuf.c            |   6 +-
 uts/common/fs/zfs/dmu_objset.c      |  16 +++++
 uts/common/fs/zfs/dmu_send.c        |  83 ++++++++++++++++------
 uts/common/fs/zfs/dmu_tx.c          |  24 ++++---
 uts/common/fs/zfs/dnode.c           |  10 +--
 uts/common/fs/zfs/dsl_dataset.c     | 105 +++++++++++++++++++++++++++-
 uts/common/fs/zfs/dsl_deadlist.c    |   8 +--
 uts/common/fs/zfs/dsl_destroy.c     |   7 ++
 uts/common/fs/zfs/dsl_pool.c        |   4 +-
 uts/common/fs/zfs/metaslab.c        |   2 +-
 uts/common/fs/zfs/sa.c              |   6 +-
 uts/common/fs/zfs/spa.c             |  26 +++++--
 uts/common/fs/zfs/spa_history.c     |   2 +-
 uts/common/fs/zfs/spa_misc.c        |   9 +++
 uts/common/fs/zfs/sys/dmu.h         |   3 +-
 uts/common/fs/zfs/sys/dmu_objset.h  |   1 +
 uts/common/fs/zfs/sys/dmu_send.h    |   6 +-
 uts/common/fs/zfs/sys/dsl_dataset.h |  11 +++
 uts/common/fs/zfs/sys/spa.h         |  22 ++++--
 uts/common/fs/zfs/sys/zap_impl.h    |   3 +-
 uts/common/fs/zfs/sys/zfs_ioctl.h   |   5 +-
 uts/common/fs/zfs/sys/zfs_znode.h   |   2 -
 uts/common/fs/zfs/sys/zil.h         |   1 -
 uts/common/fs/zfs/sys/zil_impl.h    |   2 +-
 uts/common/fs/zfs/vdev.c            |   6 +-
 uts/common/fs/zfs/vdev_queue.c      |   2 +-
 uts/common/fs/zfs/vdev_raidz.c      |   6 +-
 uts/common/fs/zfs/zap_micro.c       |  16 ++---
 uts/common/fs/zfs/zfs_ioctl.c       |  54 ++++++++++++--
 uts/common/fs/zfs/zfs_log.c         |   2 +-
 uts/common/fs/zfs/zfs_vfsops.c      |   9 ++-
 uts/common/fs/zfs/zfs_vnops.c       |   8 ++-
 uts/common/fs/zfs/zfs_znode.c       |   8 ++-
 uts/common/fs/zfs/zil.c             |  11 +--
 uts/common/fs/zfs/zio.c             |   9 +--
 uts/common/fs/zfs/zvol.c            |  11 +--
 uts/common/sys/fs/zfs.h             |   1 +
 43 files changed, 413 insertions(+), 120 deletions(-)

diff --git a/common/zfs/zfeature_common.c b/common/zfs/zfeature_common.c
index 9b046ab07db1..3358e5eb87ba 100644
--- a/common/zfs/zfeature_common.c
+++ b/common/zfs/zfeature_common.c
@@ -57,7 +57,8 @@ valid_char(char c, boolean_t after_colon)
 {
 	return ((c >= 'a' && c <= 'z') ||
 	    (c >= '0' && c <= '9') ||
-	    c == (after_colon ? '_' : '.'));
+	    (after_colon && c == '_') ||
+	    (!after_colon && (c == '.' || c == '-')));
 }
 
 /*
@@ -221,4 +222,13 @@ zpool_feature_init(void)
 	    "com.delphix:embedded_data", "embedded_data",
 	    "Blocks which compress very well use even less space.",
 	    B_FALSE, B_TRUE, B_TRUE, NULL);
+
+	static const spa_feature_t large_blocks_deps[] = {
+		SPA_FEATURE_EXTENSIBLE_DATASET,
+		SPA_FEATURE_NONE
+	};
+	zfeature_register(SPA_FEATURE_LARGE_BLOCKS,
+	    "org.open-zfs:large_blocks", "large_blocks",
+	    "Support for blocks larger than 128KB.", B_FALSE, B_FALSE, B_FALSE,
+	    large_blocks_deps);
 }
diff --git a/common/zfs/zfeature_common.h b/common/zfs/zfeature_common.h
index be2111be9184..3f54b392be42 100644
--- a/common/zfs/zfeature_common.h
+++ b/common/zfs/zfeature_common.h
@@ -51,6 +51,7 @@ typedef enum spa_feature {
 	SPA_FEATURE_EMBEDDED_DATA,
 	SPA_FEATURE_BOOKMARKS,
 	SPA_FEATURE_FS_SS_LIMIT,
+	SPA_FEATURE_LARGE_BLOCKS,
 	SPA_FEATURES
 } spa_feature_t;
 
diff --git a/common/zfs/zfs_prop.c b/common/zfs/zfs_prop.c
index 0b4927243fb6..e145b1c866e3 100644
--- a/common/zfs/zfs_prop.c
+++ b/common/zfs/zfs_prop.c
@@ -397,8 +397,8 @@ zfs_prop_init(void)
 
 	/* inherit number properties */
 	zprop_register_number(ZFS_PROP_RECORDSIZE, "recordsize",
-	    SPA_MAXBLOCKSIZE, PROP_INHERIT,
-	    ZFS_TYPE_FILESYSTEM, "512 to 128k, power of 2", "RECSIZE");
+	    SPA_OLD_MAXBLOCKSIZE, PROP_INHERIT,
+	    ZFS_TYPE_FILESYSTEM, "512 to 1M, power of 2", "RECSIZE");
 
 	/* hidden properties */
 	zprop_register_hidden(ZFS_PROP_CREATETXG, "createtxg", PROP_TYPE_NUMBER,
diff --git a/common/zfs/zpool_prop.c b/common/zfs/zpool_prop.c
index a400f821e2e2..4d906b02bc02 100644
--- a/common/zfs/zpool_prop.c
+++ b/common/zfs/zpool_prop.c
@@ -127,6 +127,8 @@ zpool_prop_init(void)
 	/* hidden properties */
 	zprop_register_hidden(ZPOOL_PROP_NAME, "name", PROP_TYPE_STRING,
 	    PROP_READONLY, ZFS_TYPE_POOL, "NAME");
+	zprop_register_hidden(ZPOOL_PROP_MAXBLOCKSIZE, "maxblocksize",
+	    PROP_TYPE_NUMBER, PROP_READONLY, ZFS_TYPE_POOL, "MAXBLOCKSIZE");
 }
 
 /*
diff --git a/uts/common/fs/zfs/bpobj.c b/uts/common/fs/zfs/bpobj.c
index e75ae72f9e9c..da4d38a3a9cf 100644
--- a/uts/common/fs/zfs/bpobj.c
+++ b/uts/common/fs/zfs/bpobj.c
@@ -43,7 +43,7 @@ bpobj_alloc_empty(objset_t *os, int blocksize, dmu_tx_t *tx)
 		if (!spa_feature_is_active(spa, SPA_FEATURE_EMPTY_BPOBJ)) {
 			ASSERT0(dp->dp_empty_bpobj);
 			dp->dp_empty_bpobj =
-			    bpobj_alloc(os, SPA_MAXBLOCKSIZE, tx);
+			    bpobj_alloc(os, SPA_OLD_MAXBLOCKSIZE, tx);
 			VERIFY(zap_add(os,
 			    DMU_POOL_DIRECTORY_OBJECT,
 			    DMU_POOL_EMPTY_BPOBJ, sizeof (uint64_t), 1,
@@ -396,7 +396,8 @@ bpobj_enqueue_subobj(bpobj_t *bpo, uint64_t subobj, dmu_tx_t *tx)
 	dmu_buf_will_dirty(bpo->bpo_dbuf, tx);
 	if (bpo->bpo_phys->bpo_subobjs == 0) {
 		bpo->bpo_phys->bpo_subobjs = dmu_object_alloc(bpo->bpo_os,
-		    DMU_OT_BPOBJ_SUBOBJ, SPA_MAXBLOCKSIZE, DMU_OT_NONE, 0, tx);
+		    DMU_OT_BPOBJ_SUBOBJ, SPA_OLD_MAXBLOCKSIZE,
+		    DMU_OT_NONE, 0, tx);
 	}
 
 	dmu_object_info_t doi;
diff --git a/uts/common/fs/zfs/bptree.c b/uts/common/fs/zfs/bptree.c
index c724ed074103..5f7d76f0e2ad 100644
--- a/uts/common/fs/zfs/bptree.c
+++ b/uts/common/fs/zfs/bptree.c
@@ -65,7 +65,7 @@ bptree_alloc(objset_t *os, dmu_tx_t *tx)
 	bptree_phys_t *bt;
 
 	obj = dmu_object_alloc(os, DMU_OTN_UINT64_METADATA,
-	    SPA_MAXBLOCKSIZE, DMU_OTN_UINT64_METADATA,
+	    SPA_OLD_MAXBLOCKSIZE, DMU_OTN_UINT64_METADATA,
 	    sizeof (bptree_phys_t), tx);
 
 	/*
diff --git a/uts/common/fs/zfs/dbuf.c b/uts/common/fs/zfs/dbuf.c
index 8d064daf25ce..a5816e12ccf0 100644
--- a/uts/common/fs/zfs/dbuf.c
+++ b/uts/common/fs/zfs/dbuf.c
@@ -2022,10 +2022,8 @@ dbuf_spill_set_blksz(dmu_buf_t *db_fake, uint64_t blksz, dmu_tx_t *tx)
 		return (SET_ERROR(ENOTSUP));
 	if (blksz == 0)
 		blksz = SPA_MINBLOCKSIZE;
-	if (blksz > SPA_MAXBLOCKSIZE)
-		blksz = SPA_MAXBLOCKSIZE;
-	else
-		blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE);
+	ASSERT3U(blksz, <=, spa_maxblocksize(dmu_objset_spa(db->db_objset)));
+	blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE);
 
 	DB_DNODE_ENTER(db);
 	dn = DB_DNODE(db);
diff --git a/uts/common/fs/zfs/dmu_objset.c b/uts/common/fs/zfs/dmu_objset.c
index 73b8e056cc8c..e7aeed17fb9d 100644
--- a/uts/common/fs/zfs/dmu_objset.c
+++ b/uts/common/fs/zfs/dmu_objset.c
@@ -255,6 +255,14 @@ logbias_changed_cb(void *arg, uint64_t newval)
 		zil_set_logbias(os->os_zil, newval);
 }
 
+static void
+recordsize_changed_cb(void *arg, uint64_t newval)
+{
+	objset_t *os = arg;
+
+	os->os_recordsize = newval;
+}
+
 void
 dmu_objset_byteswap(void *buf, size_t size)
 {
@@ -384,6 +392,11 @@ dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
 				    ZFS_PROP_REDUNDANT_METADATA),
 				    redundant_metadata_changed_cb, os);
 			}
+			if (err == 0) {
+				err = dsl_prop_register(ds,
+				    zfs_prop_to_name(ZFS_PROP_RECORDSIZE),
+				    recordsize_changed_cb, os);
+			}
 		}
 		if (err != 0) {
 			VERIFY(arc_buf_remove_ref(os->os_phys_buf,
@@ -642,6 +655,9 @@ dmu_objset_evict(objset_t *os)
 			VERIFY0(dsl_prop_unregister(ds,
 			    zfs_prop_to_name(ZFS_PROP_REDUNDANT_METADATA),
 			    redundant_metadata_changed_cb, os));
+			VERIFY0(dsl_prop_unregister(ds,
+			    zfs_prop_to_name(ZFS_PROP_RECORDSIZE),
+			    recordsize_changed_cb, os));
 		}
 		VERIFY0(dsl_prop_unregister(ds,
 		    zfs_prop_to_name(ZFS_PROP_PRIMARYCACHE),
diff --git a/uts/common/fs/zfs/dmu_send.c b/uts/common/fs/zfs/dmu_send.c
index fed1b86a1b1e..2c08e7075f66 100644
--- a/uts/common/fs/zfs/dmu_send.c
+++ b/uts/common/fs/zfs/dmu_send.c
@@ -206,11 +206,12 @@ dump_write(dmu_sendarg_t *dsp, dmu_object_type_t type,
 	drrw->drr_offset = offset;
 	drrw->drr_length = blksz;
 	drrw->drr_toguid = dsp->dsa_toguid;
-	if (BP_IS_EMBEDDED(bp)) {
+	if (bp == NULL || BP_IS_EMBEDDED(bp)) {
 		/*
-		 * There's no pre-computed checksum of embedded BP's, so
-		 * (like fletcher4-checkummed blocks) userland will have
-		 * to compute a dedup-capable checksum itself.
+		 * There's no pre-computed checksum for partial-block
+		 * writes or embedded BP's, so (like
+		 * fletcher4-checkummed blocks) userland will have to
+		 * compute a dedup-capable checksum itself.
 		 */
 		drrw->drr_checksumtype = ZIO_CHECKSUM_OFF;
 	} else {
@@ -372,6 +373,10 @@ dump_dnode(dmu_sendarg_t *dsp, uint64_t object, dnode_phys_t *dnp)
 	drro->drr_compress = dnp->dn_compress;
 	drro->drr_toguid = dsp->dsa_toguid;
 
+	if (!(dsp->dsa_featureflags & DMU_BACKUP_FEATURE_LARGE_BLOCKS) &&
+	    drro->drr_blksz > SPA_OLD_MAXBLOCKSIZE)
+		drro->drr_blksz = SPA_OLD_MAXBLOCKSIZE;
+
 	if (dump_bytes(dsp, dsp->dsa_drr, sizeof (dmu_replay_record_t)) != 0)
 		return (SET_ERROR(EINTR));
 
@@ -491,6 +496,7 @@ backup_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 		uint32_t aflags = ARC_WAIT;
 		arc_buf_t *abuf;
 		int blksz = BP_GET_LSIZE(bp);
+		uint64_t offset;
 
 		ASSERT3U(blksz, ==, dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT);
 		ASSERT0(zb->zb_level);
@@ -511,8 +517,24 @@ backup_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 			}
 		}
 
-		err = dump_write(dsp, type, zb->zb_object, zb->zb_blkid * blksz,
-		    blksz, bp, abuf->b_data);
+		offset = zb->zb_blkid * blksz;
+
+		if (!(dsp->dsa_featureflags &
+		    DMU_BACKUP_FEATURE_LARGE_BLOCKS) &&
+		    blksz > SPA_OLD_MAXBLOCKSIZE) {
+			char *buf = abuf->b_data;
+			while (blksz > 0 && err == 0) {
+				int n = MIN(blksz, SPA_OLD_MAXBLOCKSIZE);
+				err = dump_write(dsp, type, zb->zb_object,
+				    offset, n, NULL, buf);
+				offset += n;
+				buf += n;
+				blksz -= n;
+			}
+		} else {
+			err = dump_write(dsp, type, zb->zb_object,
+			    offset, blksz, bp, abuf->b_data);
+		}
 		(void) arc_buf_remove_ref(abuf, &abuf);
 	}
 
@@ -526,7 +548,7 @@ backup_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 static int
 dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *ds,
     zfs_bookmark_phys_t *fromzb, boolean_t is_clone, boolean_t embedok,
-    int outfd, vnode_t *vp, offset_t *off)
+    boolean_t large_block_ok, int outfd, vnode_t *vp, offset_t *off)
 {
 	objset_t *os;
 	dmu_replay_record_t *drr;
@@ -561,6 +583,8 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *ds,
 	}
 #endif
 
+	if (large_block_ok && ds->ds_large_blocks)
+		featureflags |= DMU_BACKUP_FEATURE_LARGE_BLOCKS;
 	if (embedok &&
 	    spa_feature_is_active(dp->dp_spa, SPA_FEATURE_EMBEDDED_DATA)) {
 		featureflags |= DMU_BACKUP_FEATURE_EMBED_DATA;
@@ -656,7 +680,8 @@ out:
 
 int
 dmu_send_obj(const char *pool, uint64_t tosnap, uint64_t fromsnap,
-    boolean_t embedok, int outfd, vnode_t *vp, offset_t *off)
+    boolean_t embedok, boolean_t large_block_ok,
+    int outfd, vnode_t *vp, offset_t *off)
 {
 	dsl_pool_t *dp;
 	dsl_dataset_t *ds;
@@ -690,18 +715,19 @@ dmu_send_obj(const char *pool, uint64_t tosnap, uint64_t fromsnap,
 		zb.zbm_guid = fromds->ds_phys->ds_guid;
 		is_clone = (fromds->ds_dir != ds->ds_dir);
 		dsl_dataset_rele(fromds, FTAG);
-		err = dmu_send_impl(FTAG, dp, ds, &zb, is_clone, embedok,
-		    outfd, vp, off);
+		err = dmu_send_impl(FTAG, dp, ds, &zb, is_clone,
+		    embedok, large_block_ok, outfd, vp, off);
 	} else {
-		err = dmu_send_impl(FTAG, dp, ds, NULL, B_FALSE, embedok,
-		    outfd, vp, off);
+		err = dmu_send_impl(FTAG, dp, ds, NULL, B_FALSE,
+		    embedok, large_block_ok, outfd, vp, off);
 	}
 	dsl_dataset_rele(ds, FTAG);
 	return (err);
 }
 
 int
-dmu_send(const char *tosnap, const char *fromsnap, boolean_t embedok,
+dmu_send(const char *tosnap, const char *fromsnap,
+    boolean_t embedok, boolean_t large_block_ok,
     int outfd, vnode_t *vp, offset_t *off)
 {
 	dsl_pool_t *dp;
@@ -768,11 +794,11 @@ dmu_send(const char *tosnap, const char *fromsnap, boolean_t embedok,
 			dsl_pool_rele(dp, FTAG);
 			return (err);
 		}
-		err = dmu_send_impl(FTAG, dp, ds, &zb, is_clone, embedok,
-		    outfd, vp, off);
+		err = dmu_send_impl(FTAG, dp, ds, &zb, is_clone,
+		    embedok, large_block_ok, outfd, vp, off);
 	} else {
-		err = dmu_send_impl(FTAG, dp, ds, NULL, B_FALSE, embedok,
-		    outfd, vp, off);
+		err = dmu_send_impl(FTAG, dp, ds, NULL, B_FALSE,
+		    embedok, large_block_ok, outfd, vp, off);
 	}
 	if (owned)
 		dsl_dataset_disown(ds, FTAG);
@@ -972,6 +998,15 @@ dmu_recv_begin_check(void *arg, dmu_tx_t *tx)
 	    !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_LZ4_COMPRESS))
 		return (SET_ERROR(ENOTSUP));
 
+	/*
+	 * The receiving code doesn't know how to translate large blocks
+	 * to smaller ones, so the pool must have the LARGE_BLOCKS
+	 * feature enabled if the stream has LARGE_BLOCKS.
+	 */
+	if ((featureflags & DMU_BACKUP_FEATURE_LARGE_BLOCKS) &&
+	    !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_LARGE_BLOCKS))
+		return (SET_ERROR(ENOTSUP));
+
 	error = dsl_dataset_hold(dp, tofs, FTAG, &ds);
 	if (error == 0) {
 		/* target fs already exists; recv into temp clone */
@@ -1097,6 +1132,13 @@ dmu_recv_begin_sync(void *arg, dmu_tx_t *tx)
 	}
 	VERIFY0(dsl_dataset_own_obj(dp, dsobj, dmu_recv_tag, &newds));
 
+	if ((DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) &
+	    DMU_BACKUP_FEATURE_LARGE_BLOCKS) &&
+	    !newds->ds_large_blocks) {
+		dsl_dataset_activate_large_blocks_sync_impl(dsobj, tx);
+		newds->ds_large_blocks = B_TRUE;
+	}
+
 	dmu_buf_will_dirty(newds->ds_dbuf, tx);
 	newds->ds_phys->ds_flags |= DS_FLAG_INCONSISTENT;
 
@@ -1222,6 +1264,7 @@ restore_read(struct restorearg *ra, int len, char *buf)
 
 	/* some things will require 8-byte alignment, so everything must */
 	ASSERT0(len % 8);
+	ASSERT3U(len, <=, ra->bufsize);
 
 	while (done < len) {
 		ssize_t resid;
@@ -1361,7 +1404,7 @@ restore_object(struct restorearg *ra, objset_t *os, struct drr_object *drro)
 	    drro->drr_compress >= ZIO_COMPRESS_FUNCTIONS ||
 	    P2PHASE(drro->drr_blksz, SPA_MINBLOCKSIZE) ||
 	    drro->drr_blksz < SPA_MINBLOCKSIZE ||
-	    drro->drr_blksz > SPA_MAXBLOCKSIZE ||
+	    drro->drr_blksz > spa_maxblocksize(dmu_objset_spa(os)) ||
 	    drro->drr_bonuslen > DN_MAX_BONUSLEN) {
 		return (SET_ERROR(EINVAL));
 	}
@@ -1634,7 +1677,7 @@ restore_spill(struct restorearg *ra, objset_t *os, struct drr_spill *drrs)
 	int err;
 
 	if (drrs->drr_length < SPA_MINBLOCKSIZE ||
-	    drrs->drr_length > SPA_MAXBLOCKSIZE)
+	    drrs->drr_length > spa_maxblocksize(dmu_objset_spa(os)))
 		return (SET_ERROR(EINVAL));
 
 	data = restore_read(ra, drrs->drr_length, NULL);
@@ -1721,7 +1764,7 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, vnode_t *vp, offset_t *voffp,
 	ra.cksum = drc->drc_cksum;
 	ra.vp = vp;
 	ra.voff = *voffp;
-	ra.bufsize = 1<<20;
+	ra.bufsize = SPA_MAXBLOCKSIZE;
 	ra.buf = kmem_alloc(ra.bufsize, KM_SLEEP);
 
 	/* these were verified in dmu_recv_begin */
diff --git a/uts/common/fs/zfs/dmu_tx.c b/uts/common/fs/zfs/dmu_tx.c
index 55ce31eda85c..d249762e5a2c 100644
--- a/uts/common/fs/zfs/dmu_tx.c
+++ b/uts/common/fs/zfs/dmu_tx.c
@@ -224,7 +224,7 @@ dmu_tx_count_write(dmu_tx_hold_t *txh, uint64_t off, uint64_t len)
 		return;
 
 	min_bs = SPA_MINBLOCKSHIFT;
-	max_bs = SPA_MAXBLOCKSHIFT;
+	max_bs = highbit64(txh->txh_tx->tx_objset->os_recordsize) - 1;
 	min_ibs = DN_MIN_INDBLKSHIFT;
 	max_ibs = DN_MAX_INDBLKSHIFT;
 
@@ -293,6 +293,14 @@ dmu_tx_count_write(dmu_tx_hold_t *txh, uint64_t off, uint64_t len)
 			 */
 			ASSERT(dn->dn_datablkshift != 0);
 			min_bs = max_bs = dn->dn_datablkshift;
+		} else {
+			/*
+			 * The blocksize can increase up to the recordsize,
+			 * or if it is already more than the recordsize,
+			 * up to the next power of 2.
+			 */
+			min_bs = highbit64(dn->dn_datablksz - 1);
+			max_bs = MAX(max_bs, highbit64(dn->dn_datablksz - 1));
 		}
 
 		/*
@@ -750,11 +758,11 @@ dmu_tx_hold_zap(dmu_tx_t *tx, uint64_t object, int add, const char *name)
 		bp = &dn->dn_phys->dn_blkptr[0];
 		if (dsl_dataset_block_freeable(dn->dn_objset->os_dsl_dataset,
 		    bp, bp->blk_birth))
-			txh->txh_space_tooverwrite += SPA_MAXBLOCKSIZE;
+			txh->txh_space_tooverwrite += MZAP_MAX_BLKSZ;
 		else
-			txh->txh_space_towrite += SPA_MAXBLOCKSIZE;
+			txh->txh_space_towrite += MZAP_MAX_BLKSZ;
 		if (!BP_IS_HOLE(bp))
-			txh->txh_space_tounref += SPA_MAXBLOCKSIZE;
+			txh->txh_space_tounref += MZAP_MAX_BLKSZ;
 		return;
 	}
 
@@ -1543,18 +1551,18 @@ dmu_tx_hold_spill(dmu_tx_t *tx, uint64_t object)
 
 	/* If blkptr doesn't exist then add space to towrite */
 	if (!(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) {
-		txh->txh_space_towrite += SPA_MAXBLOCKSIZE;
+		txh->txh_space_towrite += SPA_OLD_MAXBLOCKSIZE;
 	} else {
 		blkptr_t *bp;
 
 		bp = &dn->dn_phys->dn_spill;
 		if (dsl_dataset_block_freeable(dn->dn_objset->os_dsl_dataset,
 		    bp, bp->blk_birth))
-			txh->txh_space_tooverwrite += SPA_MAXBLOCKSIZE;
+			txh->txh_space_tooverwrite += SPA_OLD_MAXBLOCKSIZE;
 		else
-			txh->txh_space_towrite += SPA_MAXBLOCKSIZE;
+			txh->txh_space_towrite += SPA_OLD_MAXBLOCKSIZE;
 		if (!BP_IS_HOLE(bp))
-			txh->txh_space_tounref += SPA_MAXBLOCKSIZE;
+			txh->txh_space_tounref += SPA_OLD_MAXBLOCKSIZE;
 	}
 }
 
diff --git a/uts/common/fs/zfs/dnode.c b/uts/common/fs/zfs/dnode.c
index 175157714d06..9c70c2bfb1b0 100644
--- a/uts/common/fs/zfs/dnode.c
+++ b/uts/common/fs/zfs/dnode.c
@@ -510,10 +510,10 @@ dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs,
 {
 	int i;
 
+	ASSERT3U(blocksize, <=,
+	    spa_maxblocksize(dmu_objset_spa(dn->dn_objset)));
 	if (blocksize == 0)
 		blocksize = 1 << zfs_default_bs;
-	else if (blocksize > SPA_MAXBLOCKSIZE)
-		blocksize = SPA_MAXBLOCKSIZE;
 	else
 		blocksize = P2ROUNDUP(blocksize, SPA_MINBLOCKSIZE);
 
@@ -594,7 +594,8 @@ dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize,
 	int nblkptr;
 
 	ASSERT3U(blocksize, >=, SPA_MINBLOCKSIZE);
-	ASSERT3U(blocksize, <=, SPA_MAXBLOCKSIZE);
+	ASSERT3U(blocksize, <=,
+	    spa_maxblocksize(dmu_objset_spa(dn->dn_objset)));
 	ASSERT0(blocksize % SPA_MINBLOCKSIZE);
 	ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT || dmu_tx_private_ok(tx));
 	ASSERT(tx->tx_txg != 0);
@@ -1347,10 +1348,9 @@ dnode_set_blksz(dnode_t *dn, uint64_t size, int ibs, dmu_tx_t *tx)
 	dmu_buf_impl_t *db;
 	int err;
 
+	ASSERT3U(size, <=, spa_maxblocksize(dmu_objset_spa(dn->dn_objset)));
 	if (size == 0)
 		size = SPA_MINBLOCKSIZE;
-	if (size > SPA_MAXBLOCKSIZE)
-		size = SPA_MAXBLOCKSIZE;
 	else
 		size = P2ROUNDUP(size, SPA_MINBLOCKSIZE);
 
diff --git a/uts/common/fs/zfs/dsl_dataset.c b/uts/common/fs/zfs/dsl_dataset.c
index f1b92f3eaa33..e7ed75090208 100644
--- a/uts/common/fs/zfs/dsl_dataset.c
+++ b/uts/common/fs/zfs/dsl_dataset.c
@@ -50,6 +50,17 @@
 #include <sys/dsl_userhold.h>
 #include <sys/dsl_bookmark.h>
 
+/*
+ * The SPA supports block sizes up to 16MB.  However, very large blocks
+ * can have an impact on i/o latency (e.g. tying up a spinning disk for
+ * ~300ms), and also potentially on the memory allocator.  Therefore,
+ * we do not allow the recordsize to be set larger than zfs_max_recordsize
+ * (default 1MB).  Larger blocks can be created by changing this tunable,
+ * and pools with larger blocks can always be imported and used, regardless
+ * of this setting.
+ */
+int zfs_max_recordsize = 1 * 1024 * 1024;
+
 #define	SWITCH64(x, y) \
 	{ \
 		uint64_t __tmp = (x); \
@@ -59,8 +70,6 @@
 
 #define	DS_REF_MAX	(1ULL << 62)
 
-#define	DSL_DEADLIST_BLOCKSIZE	SPA_MAXBLOCKSIZE
-
 /*
  * Figure out how much of this delta should be propogated to the dsl_dir
  * layer.  If there's a refreservation, that space has already been
@@ -110,6 +119,8 @@ dsl_dataset_block_born(dsl_dataset_t *ds, const blkptr_t *bp, dmu_tx_t *tx)
 	ds->ds_phys->ds_compressed_bytes += compressed;
 	ds->ds_phys->ds_uncompressed_bytes += uncompressed;
 	ds->ds_phys->ds_unique_bytes += used;
+	if (BP_GET_LSIZE(bp) > SPA_OLD_MAXBLOCKSIZE)
+		ds->ds_need_large_blocks = B_TRUE;
 	mutex_exit(&ds->ds_lock);
 	dsl_dir_diduse_space(ds->ds_dir, DD_USED_HEAD, delta,
 	    compressed, uncompressed, tx);
@@ -387,6 +398,14 @@ dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
 		list_create(&ds->ds_sendstreams, sizeof (dmu_sendarg_t),
 		    offsetof(dmu_sendarg_t, dsa_link));
 
+		if (doi.doi_type == DMU_OTN_ZAP_METADATA) {
+			err = zap_contains(mos, dsobj, DS_FIELD_LARGE_BLOCKS);
+			if (err == 0)
+				ds->ds_large_blocks = B_TRUE;
+			else
+				ASSERT3U(err, ==, ENOENT);
+		}
+
 		if (err == 0) {
 			err = dsl_dir_hold_obj(dp,
 			    ds->ds_phys->ds_dir_obj, NULL, ds, &ds->ds_dir);
@@ -700,6 +719,9 @@ dsl_dataset_create_sync_dd(dsl_dir_t *dd, dsl_dataset_t *origin,
 		dsphys->ds_flags |= origin->ds_phys->ds_flags &
 		    (DS_FLAG_INCONSISTENT | DS_FLAG_CI_DATASET);
 
+		if (origin->ds_large_blocks)
+			dsl_dataset_activate_large_blocks_sync_impl(dsobj, tx);
+
 		dmu_buf_will_dirty(origin->ds_dbuf, tx);
 		origin->ds_phys->ds_num_children++;
 
@@ -1213,6 +1235,9 @@ dsl_dataset_snapshot_sync_impl(dsl_dataset_t *ds, const char *snapname,
 	dsphys->ds_bp = ds->ds_phys->ds_bp;
 	dmu_buf_rele(dbuf, FTAG);
 
+	if (ds->ds_large_blocks)
+		dsl_dataset_activate_large_blocks_sync_impl(dsobj, tx);
+
 	ASSERT3U(ds->ds_prev != 0, ==, ds->ds_phys->ds_prev_snap_obj != 0);
 	if (ds->ds_prev) {
 		uint64_t next_clones_obj =
@@ -1486,6 +1511,11 @@ dsl_dataset_sync(dsl_dataset_t *ds, zio_t *zio, dmu_tx_t *tx)
 	ds->ds_phys->ds_fsid_guid = ds->ds_fsid_guid;
 
 	dmu_objset_sync(ds->ds_objset, zio, tx);
+
+	if (ds->ds_need_large_blocks && !ds->ds_large_blocks) {
+		dsl_dataset_activate_large_blocks_sync_impl(ds->ds_object, tx);
+		ds->ds_large_blocks = B_TRUE;
+	}
 }
 
 static void
@@ -3128,6 +3158,77 @@ dsl_dataset_space_wouldfree(dsl_dataset_t *firstsnap,
 	return (err);
 }
 
+static int
+dsl_dataset_activate_large_blocks_check(void *arg, dmu_tx_t *tx)
+{
+	const char *dsname = arg;
+	dsl_dataset_t *ds;
+	dsl_pool_t *dp = dmu_tx_pool(tx);
+	int error = 0;
+
+	if (!spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_LARGE_BLOCKS))
+		return (SET_ERROR(ENOTSUP));
+
+	ASSERT(spa_feature_is_enabled(dp->dp_spa,
+	    SPA_FEATURE_EXTENSIBLE_DATASET));
+
+	error = dsl_dataset_hold(dp, dsname, FTAG, &ds);
+	if (error != 0)
+		return (error);
+
+	if (ds->ds_large_blocks)
+		error = EALREADY;
+	dsl_dataset_rele(ds, FTAG);
+
+	return (error);
+}
+
+void
+dsl_dataset_activate_large_blocks_sync_impl(uint64_t dsobj, dmu_tx_t *tx)
+{
+	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
+	objset_t *mos = dmu_tx_pool(tx)->dp_meta_objset;
+	uint64_t zero = 0;
+
+	spa_feature_incr(spa, SPA_FEATURE_LARGE_BLOCKS, tx);
+	dmu_object_zapify(mos, dsobj, DMU_OT_DSL_DATASET, tx);
+
+	VERIFY0(zap_add(mos, dsobj, DS_FIELD_LARGE_BLOCKS,
+	    sizeof (zero), 1, &zero, tx));
+}
+
+static void
+dsl_dataset_activate_large_blocks_sync(void *arg, dmu_tx_t *tx)
+{
+	const char *dsname = arg;
+	dsl_dataset_t *ds;
+
+	VERIFY0(dsl_dataset_hold(dmu_tx_pool(tx), dsname, FTAG, &ds));
+
+	dsl_dataset_activate_large_blocks_sync_impl(ds->ds_object, tx);
+	ASSERT(!ds->ds_large_blocks);
+	ds->ds_large_blocks = B_TRUE;
+	dsl_dataset_rele(ds, FTAG);
+}
+
+int
+dsl_dataset_activate_large_blocks(const char *dsname)
+{
+	int error;
+
+	error = dsl_sync_task(dsname,
+	    dsl_dataset_activate_large_blocks_check,
+	    dsl_dataset_activate_large_blocks_sync, (void *)dsname,
+	    1, ZFS_SPACE_CHECK_RESERVED);
+
+	/*
+	 * EALREADY indicates that this dataset already supports large blocks.
+	 */
+	if (error == EALREADY)
+		error = 0;
+	return (error);
+}
+
 /*
  * Return TRUE if 'earlier' is an earlier snapshot in 'later's timeline.
  * For example, they could both be snapshots of the same filesystem, and
diff --git a/uts/common/fs/zfs/dsl_deadlist.c b/uts/common/fs/zfs/dsl_deadlist.c
index 4f39c397a064..8c8e3746eecb 100644
--- a/uts/common/fs/zfs/dsl_deadlist.c
+++ b/uts/common/fs/zfs/dsl_deadlist.c
@@ -143,7 +143,7 @@ uint64_t
 dsl_deadlist_alloc(objset_t *os, dmu_tx_t *tx)
 {
 	if (spa_version(dmu_objset_spa(os)) < SPA_VERSION_DEADLISTS)
-		return (bpobj_alloc(os, SPA_MAXBLOCKSIZE, tx));
+		return (bpobj_alloc(os, SPA_OLD_MAXBLOCKSIZE, tx));
 	return (zap_create(os, DMU_OT_DEADLIST, DMU_OT_DEADLIST_HDR,
 	    sizeof (dsl_deadlist_phys_t), tx));
 }
@@ -180,7 +180,7 @@ dle_enqueue(dsl_deadlist_t *dl, dsl_deadlist_entry_t *dle,
 {
 	if (dle->dle_bpobj.bpo_object ==
 	    dmu_objset_pool(dl->dl_os)->dp_empty_bpobj) {
-		uint64_t obj = bpobj_alloc(dl->dl_os, SPA_MAXBLOCKSIZE, tx);
+		uint64_t obj = bpobj_alloc(dl->dl_os, SPA_OLD_MAXBLOCKSIZE, tx);
 		bpobj_close(&dle->dle_bpobj);
 		bpobj_decr_empty(dl->dl_os, tx);
 		VERIFY3U(0, ==, bpobj_open(&dle->dle_bpobj, dl->dl_os, obj));
@@ -254,7 +254,7 @@ dsl_deadlist_add_key(dsl_deadlist_t *dl, uint64_t mintxg, dmu_tx_t *tx)
 
 	dle = kmem_alloc(sizeof (*dle), KM_SLEEP);
 	dle->dle_mintxg = mintxg;
-	obj = bpobj_alloc_empty(dl->dl_os, SPA_MAXBLOCKSIZE, tx);
+	obj = bpobj_alloc_empty(dl->dl_os, SPA_OLD_MAXBLOCKSIZE, tx);
 	VERIFY3U(0, ==, bpobj_open(&dle->dle_bpobj, dl->dl_os, obj));
 	avl_add(&dl->dl_tree, dle);
 
@@ -338,7 +338,7 @@ dsl_deadlist_clone(dsl_deadlist_t *dl, uint64_t maxtxg,
 		if (dle->dle_mintxg >= maxtxg)
 			break;
 
-		obj = bpobj_alloc_empty(dl->dl_os, SPA_MAXBLOCKSIZE, tx);
+		obj = bpobj_alloc_empty(dl->dl_os, SPA_OLD_MAXBLOCKSIZE, tx);
 		VERIFY3U(0, ==, zap_add_int_key(dl->dl_os, newobj,
 		    dle->dle_mintxg, obj, tx));
 	}
diff --git a/uts/common/fs/zfs/dsl_destroy.c b/uts/common/fs/zfs/dsl_destroy.c
index f8a4546535e4..1237641583a1 100644
--- a/uts/common/fs/zfs/dsl_destroy.c
+++ b/uts/common/fs/zfs/dsl_destroy.c
@@ -264,6 +264,10 @@ dsl_destroy_snapshot_sync_impl(dsl_dataset_t *ds, boolean_t defer, dmu_tx_t *tx)
 
 	obj = ds->ds_object;
 
+	if (ds->ds_large_blocks) {
+		ASSERT0(zap_contains(mos, obj, DS_FIELD_LARGE_BLOCKS));
+		spa_feature_decr(dp->dp_spa, SPA_FEATURE_LARGE_BLOCKS, tx);
+	}
 	if (ds->ds_phys->ds_prev_snap_obj != 0) {
 		ASSERT3P(ds->ds_prev, ==, NULL);
 		VERIFY0(dsl_dataset_hold_obj(dp,
@@ -720,6 +724,9 @@ dsl_destroy_head_sync_impl(dsl_dataset_t *ds, dmu_tx_t *tx)
 		ASSERT0(ds->ds_reserved);
 	}
 
+	if (ds->ds_large_blocks)
+		spa_feature_decr(dp->dp_spa, SPA_FEATURE_LARGE_BLOCKS, tx);
+
 	dsl_scan_ds_destroyed(ds, tx);
 
 	obj = ds->ds_object;
diff --git a/uts/common/fs/zfs/dsl_pool.c b/uts/common/fs/zfs/dsl_pool.c
index ad26a9070b68..1d246b8e688d 100644
--- a/uts/common/fs/zfs/dsl_pool.c
+++ b/uts/common/fs/zfs/dsl_pool.c
@@ -367,7 +367,7 @@ dsl_pool_create(spa_t *spa, nvlist_t *zplprops, uint64_t txg)
 		    FREE_DIR_NAME, &dp->dp_free_dir));
 
 		/* create and open the free_bplist */
-		obj = bpobj_alloc(dp->dp_meta_objset, SPA_MAXBLOCKSIZE, tx);
+		obj = bpobj_alloc(dp->dp_meta_objset, SPA_OLD_MAXBLOCKSIZE, tx);
 		VERIFY(zap_add(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
 		    DMU_POOL_FREE_BPOBJ, sizeof (uint64_t), 1, &obj, tx) == 0);
 		VERIFY0(bpobj_open(&dp->dp_free_bpobj,
@@ -792,7 +792,7 @@ dsl_pool_upgrade_dir_clones(dsl_pool_t *dp, dmu_tx_t *tx)
 	 * subobj support.  So call dmu_object_alloc() directly.
 	 */
 	obj = dmu_object_alloc(dp->dp_meta_objset, DMU_OT_BPOBJ,
-	    SPA_MAXBLOCKSIZE, DMU_OT_BPOBJ_HDR, sizeof (bpobj_phys_t), tx);
+	    SPA_OLD_MAXBLOCKSIZE, DMU_OT_BPOBJ_HDR, sizeof (bpobj_phys_t), tx);
 	VERIFY0(zap_add(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
 	    DMU_POOL_FREE_BPOBJ, sizeof (uint64_t), 1, &obj, tx));
 	VERIFY0(bpobj_open(&dp->dp_free_bpobj, dp->dp_meta_objset, obj));
diff --git a/uts/common/fs/zfs/metaslab.c b/uts/common/fs/zfs/metaslab.c
index a33ec7f62810..0f05e8b00a87 100644
--- a/uts/common/fs/zfs/metaslab.c
+++ b/uts/common/fs/zfs/metaslab.c
@@ -125,7 +125,7 @@ int metaslab_debug_unload = 0;
  * an allocation of this size then it switches to using more
  * aggressive strategy (i.e search by size rather than offset).
  */
-uint64_t metaslab_df_alloc_threshold = SPA_MAXBLOCKSIZE;
+uint64_t metaslab_df_alloc_threshold = SPA_OLD_MAXBLOCKSIZE;
 
 /*
  * The minimum free space, in percent, which must be available
diff --git a/uts/common/fs/zfs/sa.c b/uts/common/fs/zfs/sa.c
index bd2ebce5df34..8b3963aed9f5 100644
--- a/uts/common/fs/zfs/sa.c
+++ b/uts/common/fs/zfs/sa.c
@@ -500,7 +500,7 @@ sa_resize_spill(sa_handle_t *hdl, uint32_t size, dmu_tx_t *tx)
 
 	if (size == 0) {
 		blocksize = SPA_MINBLOCKSIZE;
-	} else if (size > SPA_MAXBLOCKSIZE) {
+	} else if (size > SPA_OLD_MAXBLOCKSIZE) {
 		ASSERT(0);
 		return (SET_ERROR(EFBIG));
 	} else {
@@ -675,7 +675,7 @@ sa_build_layouts(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc, int attr_count,
 	hdrsize = sa_find_sizes(sa, attr_desc, attr_count, hdl->sa_bonus,
 	    SA_BONUS, &i, &used, &spilling);
 
-	if (used > SPA_MAXBLOCKSIZE)
+	if (used > SPA_OLD_MAXBLOCKSIZE)
 		return (SET_ERROR(EFBIG));
 
 	VERIFY(0 == dmu_set_bonus(hdl->sa_bonus, spilling ?
@@ -699,7 +699,7 @@ sa_build_layouts(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc, int attr_count,
 		    attr_count - i, hdl->sa_spill, SA_SPILL, &i,
 		    &spill_used, &dummy);
 
-		if (spill_used > SPA_MAXBLOCKSIZE)
+		if (spill_used > SPA_OLD_MAXBLOCKSIZE)
 			return (SET_ERROR(EFBIG));
 
 		buf_space = hdl->sa_spill->db_size - spillhdrsize;
diff --git a/uts/common/fs/zfs/spa.c b/uts/common/fs/zfs/spa.c
index be308e2f8730..634967c46f1b 100644
--- a/uts/common/fs/zfs/spa.c
+++ b/uts/common/fs/zfs/spa.c
@@ -267,6 +267,14 @@ spa_prop_get_config(spa_t *spa, nvlist_t **nvp)
 		spa_prop_add_list(*nvp, ZPOOL_PROP_ALTROOT, spa->spa_root,
 		    0, ZPROP_SRC_LOCAL);
 
+	if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) {
+		spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL,
+		    MIN(zfs_max_recordsize, SPA_MAXBLOCKSIZE), ZPROP_SRC_NONE);
+	} else {
+		spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL,
+		    SPA_OLD_MAXBLOCKSIZE, ZPROP_SRC_NONE);
+	}
+
 	if ((dp = list_head(&spa->spa_config_list)) != NULL) {
 		if (dp->scd_path == NULL) {
 			spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE,
@@ -481,7 +489,7 @@ spa_prop_validate(spa_t *spa, nvlist_t *props)
 
 			if (!error) {
 				objset_t *os;
-				uint64_t compress;
+				uint64_t propval;
 
 				if (strval == NULL || strval[0] == '\0') {
 					objnum = zpool_prop_default_numeric(
@@ -492,15 +500,25 @@ spa_prop_validate(spa_t *spa, nvlist_t *props)
 				if (error = dmu_objset_hold(strval, FTAG, &os))
 					break;
 
-				/* Must be ZPL and not gzip compressed. */
+				/*
+				 * Must be ZPL, and its property settings
+				 * must be supported by GRUB (compression
+				 * is not gzip, and large blocks are not used).
+				 */
 
 				if (dmu_objset_type(os) != DMU_OST_ZFS) {
 					error = SET_ERROR(ENOTSUP);
 				} else if ((error =
 				    dsl_prop_get_int_ds(dmu_objset_ds(os),
 				    zfs_prop_to_name(ZFS_PROP_COMPRESSION),
-				    &compress)) == 0 &&
-				    !BOOTFS_COMPRESS_VALID(compress)) {
+				    &propval)) == 0 &&
+				    !BOOTFS_COMPRESS_VALID(propval)) {
+					error = SET_ERROR(ENOTSUP);
+				} else if ((error =
+				    dsl_prop_get_int_ds(dmu_objset_ds(os),
+				    zfs_prop_to_name(ZFS_PROP_RECORDSIZE),
+				    &propval)) == 0 &&
+				    propval > SPA_OLD_MAXBLOCKSIZE) {
 					error = SET_ERROR(ENOTSUP);
 				} else {
 					objnum = dmu_objset_id(os);
diff --git a/uts/common/fs/zfs/spa_history.c b/uts/common/fs/zfs/spa_history.c
index cf72c8ad8827..ce64f70b28c3 100644
--- a/uts/common/fs/zfs/spa_history.c
+++ b/uts/common/fs/zfs/spa_history.c
@@ -90,7 +90,7 @@ spa_history_create_obj(spa_t *spa, dmu_tx_t *tx)
 
 	ASSERT(spa->spa_history == 0);
 	spa->spa_history = dmu_object_alloc(mos, DMU_OT_SPA_HISTORY,
-	    SPA_MAXBLOCKSIZE, DMU_OT_SPA_HISTORY_OFFSETS,
+	    SPA_OLD_MAXBLOCKSIZE, DMU_OT_SPA_HISTORY_OFFSETS,
 	    sizeof (spa_history_phys_t), tx);
 
 	VERIFY(zap_add(mos, DMU_POOL_DIRECTORY_OBJECT,
diff --git a/uts/common/fs/zfs/spa_misc.c b/uts/common/fs/zfs/spa_misc.c
index 050436272696..1729ba0503be 100644
--- a/uts/common/fs/zfs/spa_misc.c
+++ b/uts/common/fs/zfs/spa_misc.c
@@ -1963,3 +1963,12 @@ spa_debug_enabled(spa_t *spa)
 {
 	return (spa->spa_debug);
 }
+
+int
+spa_maxblocksize(spa_t *spa)
+{
+	if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS))
+		return (SPA_MAXBLOCKSIZE);
+	else
+		return (SPA_OLD_MAXBLOCKSIZE);
+}
diff --git a/uts/common/fs/zfs/sys/dmu.h b/uts/common/fs/zfs/sys/dmu.h
index e9fa39d5d6f1..93165a95951d 100644
--- a/uts/common/fs/zfs/sys/dmu.h
+++ b/uts/common/fs/zfs/sys/dmu.h
@@ -249,7 +249,7 @@ void zfs_znode_byteswap(void *buf, size_t size);
  * The maximum number of bytes that can be accessed as part of one
  * operation, including metadata.
  */
-#define	DMU_MAX_ACCESS (10<<20) /* 10MB */
+#define	DMU_MAX_ACCESS (32 * 1024 * 1024) /* 32MB */
 #define	DMU_MAX_DELETEBLKCNT (20480) /* ~5MB of indirect blocks */
 
 #define	DMU_USERUSED_OBJECT	(-1ULL)
@@ -637,6 +637,7 @@ void xuio_stat_wbuf_copied();
 void xuio_stat_wbuf_nocopy();
 
 extern int zfs_prefetch_disable;
+extern int zfs_max_recordsize;
 
 /*
  * Asynchronously try to read in the data.
diff --git a/uts/common/fs/zfs/sys/dmu_objset.h b/uts/common/fs/zfs/sys/dmu_objset.h
index 23d88fd048bd..804f0c182b6f 100644
--- a/uts/common/fs/zfs/sys/dmu_objset.h
+++ b/uts/common/fs/zfs/sys/dmu_objset.h
@@ -95,6 +95,7 @@ struct objset {
 	zfs_cache_type_t os_secondary_cache;
 	zfs_sync_type_t os_sync;
 	zfs_redundant_metadata_type_t os_redundant_metadata;
+	int os_recordsize;
 
 	/* no lock needed: */
 	struct dmu_tx *os_synctx; /* XXX sketchy */
diff --git a/uts/common/fs/zfs/sys/dmu_send.h b/uts/common/fs/zfs/sys/dmu_send.h
index dc183c02c350..3a8dc89abd4a 100644
--- a/uts/common/fs/zfs/sys/dmu_send.h
+++ b/uts/common/fs/zfs/sys/dmu_send.h
@@ -37,12 +37,14 @@ struct dsl_dataset;
 struct drr_begin;
 struct avl_tree;
 
-int dmu_send(const char *tosnap, const char *fromsnap, boolean_t embedok,
+int dmu_send(const char *tosnap, const char *fromsnap,
+    boolean_t embedok, boolean_t large_block_ok,
     int outfd, struct vnode *vp, offset_t *off);
 int dmu_send_estimate(struct dsl_dataset *ds, struct dsl_dataset *fromds,
     uint64_t *sizep);
 int dmu_send_obj(const char *pool, uint64_t tosnap, uint64_t fromsnap,
-    boolean_t embedok, int outfd, vnode_t *vp, offset_t *off);
+    boolean_t embedok, boolean_t large_block_ok,
+    int outfd, struct vnode *vp, offset_t *off);
 
 typedef struct dmu_recv_cookie {
 	struct dsl_dataset *drc_ds;
diff --git a/uts/common/fs/zfs/sys/dsl_dataset.h b/uts/common/fs/zfs/sys/dsl_dataset.h
index d9552b2260af..ff90f8b439cc 100644
--- a/uts/common/fs/zfs/sys/dsl_dataset.h
+++ b/uts/common/fs/zfs/sys/dsl_dataset.h
@@ -82,6 +82,13 @@ struct dsl_pool;
  */
 #define	DS_FIELD_BOOKMARK_NAMES "com.delphix:bookmarks"
 
+/*
+ * This field is present (with value=0) if this dataset may contain large
+ * blocks (>128KB).  If it is present, then this dataset
+ * is counted in the refcount of the SPA_FEATURE_LARGE_BLOCKS feature.
+ */
+#define	DS_FIELD_LARGE_BLOCKS "org.open-zfs:large_blocks"
+
 /*
  * DS_FLAG_CI_DATASET is set if the dataset contains a file system whose
  * name lookups should be performed case-insensitively.
@@ -135,6 +142,8 @@ typedef struct dsl_dataset {
 	/* only used in syncing context, only valid for non-snapshots: */
 	struct dsl_dataset *ds_prev;
 	uint64_t ds_bookmarks;  /* DMU_OTN_ZAP_METADATA */
+	boolean_t ds_large_blocks;
+	boolean_t ds_need_large_blocks;
 
 	/* has internal locking: */
 	dsl_deadlist_t ds_deadlist;
@@ -244,6 +253,8 @@ int dsl_dataset_space_written(dsl_dataset_t *oldsnap, dsl_dataset_t *new,
 int dsl_dataset_space_wouldfree(dsl_dataset_t *firstsnap, dsl_dataset_t *last,
     uint64_t *usedp, uint64_t *compp, uint64_t *uncompp);
 boolean_t dsl_dataset_is_dirty(dsl_dataset_t *ds);
+int dsl_dataset_activate_large_blocks(const char *dsname);
+void dsl_dataset_activate_large_blocks_sync_impl(uint64_t dsobj, dmu_tx_t *tx);
 
 int dsl_dsobj_to_dsname(char *pname, uint64_t obj, char *buf);
 
diff --git a/uts/common/fs/zfs/sys/spa.h b/uts/common/fs/zfs/sys/spa.h
index 6c7d34cb8235..e4731ae5a739 100644
--- a/uts/common/fs/zfs/sys/spa.h
+++ b/uts/common/fs/zfs/sys/spa.h
@@ -94,17 +94,26 @@ _NOTE(CONSTCOND) } while (0)
 _NOTE(CONSTCOND) } while (0)
 
 /*
- * We currently support nine block sizes, from 512 bytes to 128K.
- * We could go higher, but the benefits are near-zero and the cost
- * of COWing a giant block to modify one byte would become excessive.
+ * We currently support block sizes from 512 bytes to 16MB.
+ * The benefits of larger blocks, and thus larger IO, need to be weighed
+ * against the cost of COWing a giant block to modify one byte, and the
+ * large latency of reading or writing a large block.
+ *
+ * Note that although blocks up to 16MB are supported, the recordsize
+ * property can not be set larger than zfs_max_recordsize (default 1MB).
+ * See the comment near zfs_max_recordsize in dsl_dataset.c for details.
+ *
+ * Note that although the LSIZE field of the blkptr_t can store sizes up
+ * to 32MB, the dnode's dn_datablkszsec can only store sizes up to
+ * 32MB - 512 bytes.  Therefore, we limit SPA_MAXBLOCKSIZE to 16MB.
  */
 #define	SPA_MINBLOCKSHIFT	9
-#define	SPA_MAXBLOCKSHIFT	17
+#define	SPA_OLD_MAXBLOCKSHIFT	17
+#define	SPA_MAXBLOCKSHIFT	24
 #define	SPA_MINBLOCKSIZE	(1ULL << SPA_MINBLOCKSHIFT)
+#define	SPA_OLD_MAXBLOCKSIZE	(1ULL << SPA_OLD_MAXBLOCKSHIFT)
 #define	SPA_MAXBLOCKSIZE	(1ULL << SPA_MAXBLOCKSHIFT)
 
-#define	SPA_BLOCKSIZES		(SPA_MAXBLOCKSHIFT - SPA_MINBLOCKSHIFT + 1)
-
 /*
  * Size of block to hold the configuration data (a packed nvlist)
  */
@@ -781,6 +790,7 @@ extern boolean_t spa_has_slogs(spa_t *spa);
 extern boolean_t spa_is_root(spa_t *spa);
 extern boolean_t spa_writeable(spa_t *spa);
 extern boolean_t spa_has_pending_synctask(spa_t *spa);
+extern int spa_maxblocksize(spa_t *spa);
 
 extern int spa_mode(spa_t *spa);
 extern uint64_t strtonum(const char *str, char **nptr);
diff --git a/uts/common/fs/zfs/sys/zap_impl.h b/uts/common/fs/zfs/sys/zap_impl.h
index 466aab02bacb..8b4a8b2b56ed 100644
--- a/uts/common/fs/zfs/sys/zap_impl.h
+++ b/uts/common/fs/zfs/sys/zap_impl.h
@@ -42,8 +42,7 @@ extern int fzap_default_block_shift;
 
 #define	MZAP_ENT_LEN		64
 #define	MZAP_NAME_LEN		(MZAP_ENT_LEN - 8 - 4 - 2)
-#define	MZAP_MAX_BLKSHIFT	SPA_MAXBLOCKSHIFT
-#define	MZAP_MAX_BLKSZ		(1 << MZAP_MAX_BLKSHIFT)
+#define	MZAP_MAX_BLKSZ		SPA_OLD_MAXBLOCKSIZE
 
 #define	ZAP_NEED_CD		(-1U)
 
diff --git a/uts/common/fs/zfs/sys/zfs_ioctl.h b/uts/common/fs/zfs/sys/zfs_ioctl.h
index bf9f83c37672..62f6ff997d16 100644
--- a/uts/common/fs/zfs/sys/zfs_ioctl.h
+++ b/uts/common/fs/zfs/sys/zfs_ioctl.h
@@ -85,13 +85,16 @@ typedef enum drr_headertype {
 /* flags #3 - #15 are reserved for incompatible closed-source implementations */
 #define	DMU_BACKUP_FEATURE_EMBED_DATA		(1<<16)
 #define	DMU_BACKUP_FEATURE_EMBED_DATA_LZ4	(1<<17)
+/* flag #18 is reserved for a Delphix feature */
+#define	DMU_BACKUP_FEATURE_LARGE_BLOCKS		(1<<19)
 
 /*
  * Mask of all supported backup features
  */
 #define	DMU_BACKUP_FEATURE_MASK	(DMU_BACKUP_FEATURE_DEDUP | \
     DMU_BACKUP_FEATURE_DEDUPPROPS | DMU_BACKUP_FEATURE_SA_SPILL | \
-    DMU_BACKUP_FEATURE_EMBED_DATA | DMU_BACKUP_FEATURE_EMBED_DATA_LZ4)
+    DMU_BACKUP_FEATURE_EMBED_DATA | DMU_BACKUP_FEATURE_EMBED_DATA_LZ4 | \
+    DMU_BACKUP_FEATURE_LARGE_BLOCKS)
 
 /* Are all features in the given flag word currently supported? */
 #define	DMU_STREAM_SUPPORTED(x)	(!((x) & ~DMU_BACKUP_FEATURE_MASK))
diff --git a/uts/common/fs/zfs/sys/zfs_znode.h b/uts/common/fs/zfs/sys/zfs_znode.h
index bc389b4a4366..df08aad7b41e 100644
--- a/uts/common/fs/zfs/sys/zfs_znode.h
+++ b/uts/common/fs/zfs/sys/zfs_znode.h
@@ -136,8 +136,6 @@ extern "C" {
 #define	ZFS_SHARES_DIR		"SHARES"
 #define	ZFS_SA_ATTRS		"SA_ATTRS"
 
-#define	ZFS_MAX_BLOCKSIZE	(SPA_MAXBLOCKSIZE)
-
 /*
  * Path component length
  *
diff --git a/uts/common/fs/zfs/sys/zil.h b/uts/common/fs/zfs/sys/zil.h
index 15ef2aa8bf98..895d632a2620 100644
--- a/uts/common/fs/zfs/sys/zil.h
+++ b/uts/common/fs/zfs/sys/zil.h
@@ -90,7 +90,6 @@ typedef struct zil_chain {
 } zil_chain_t;
 
 #define	ZIL_MIN_BLKSZ	4096ULL
-#define	ZIL_MAX_BLKSZ	SPA_MAXBLOCKSIZE
 
 /*
  * The words of a log block checksum.
diff --git a/uts/common/fs/zfs/sys/zil_impl.h b/uts/common/fs/zfs/sys/zil_impl.h
index 58566203b697..b5c666c02b73 100644
--- a/uts/common/fs/zfs/sys/zil_impl.h
+++ b/uts/common/fs/zfs/sys/zil_impl.h
@@ -139,7 +139,7 @@ typedef struct zil_bp_node {
 	avl_node_t	zn_node;
 } zil_bp_node_t;
 
-#define	ZIL_MAX_LOG_DATA (SPA_MAXBLOCKSIZE - sizeof (zil_chain_t) - \
+#define	ZIL_MAX_LOG_DATA (SPA_OLD_MAXBLOCKSIZE - sizeof (zil_chain_t) - \
     sizeof (lr_write_t))
 
 #ifdef	__cplusplus
diff --git a/uts/common/fs/zfs/vdev.c b/uts/common/fs/zfs/vdev.c
index 7571b21a5f8c..67b58edeed0b 100644
--- a/uts/common/fs/zfs/vdev.c
+++ b/uts/common/fs/zfs/vdev.c
@@ -828,9 +828,9 @@ vdev_metaslab_init(vdev_t *vd, uint64_t txg)
 
 	/*
 	 * Compute the raidz-deflation ratio.  Note, we hard-code
-	 * in 128k (1 << 17) because it is the current "typical" blocksize.
-	 * Even if SPA_MAXBLOCKSIZE changes, this algorithm must never change,
-	 * or we will inconsistently account for existing bp's.
+	 * in 128k (1 << 17) because it is the "typical" blocksize.
+	 * Even though SPA_MAXBLOCKSIZE changed, this algorithm can not change,
+	 * otherwise it would inconsistently account for existing bp's.
 	 */
 	vd->vdev_deflate_ratio = (1 << 17) /
 	    (vdev_psize_to_asize(vd, 1 << 17) >> SPA_MINBLOCKSHIFT);
diff --git a/uts/common/fs/zfs/vdev_queue.c b/uts/common/fs/zfs/vdev_queue.c
index 561e6a4482eb..f47c4cd1e26d 100644
--- a/uts/common/fs/zfs/vdev_queue.c
+++ b/uts/common/fs/zfs/vdev_queue.c
@@ -162,7 +162,7 @@ int zfs_vdev_async_write_active_max_dirty_percent = 60;
  * we include spans of optional I/Os to aid aggregation at the disk even when
  * they aren't able to help us aggregate at this level.
  */
-int zfs_vdev_aggregation_limit = SPA_MAXBLOCKSIZE;
+int zfs_vdev_aggregation_limit = SPA_OLD_MAXBLOCKSIZE;
 int zfs_vdev_read_gap_limit = 32 << 10;
 int zfs_vdev_write_gap_limit = 4 << 10;
 
diff --git a/uts/common/fs/zfs/vdev_raidz.c b/uts/common/fs/zfs/vdev_raidz.c
index f686d56e9e5f..085d1250a1ad 100644
--- a/uts/common/fs/zfs/vdev_raidz.c
+++ b/uts/common/fs/zfs/vdev_raidz.c
@@ -1604,7 +1604,7 @@ vdev_raidz_physio(vdev_t *vd, caddr_t data, size_t size,
 	/*
 	 * Don't write past the end of the block
 	 */
-	VERIFY3U(offset + size, <=, origoffset + SPA_MAXBLOCKSIZE);
+	VERIFY3U(offset + size, <=, origoffset + SPA_OLD_MAXBLOCKSIZE);
 
 	start = offset;
 	end = start + size;
@@ -1619,8 +1619,8 @@ vdev_raidz_physio(vdev_t *vd, caddr_t data, size_t size,
 	 * KB size.
 	 */
 	rm = vdev_raidz_map_alloc(data - (offset - origoffset),
-	    SPA_MAXBLOCKSIZE, origoffset, tvd->vdev_ashift, vd->vdev_children,
-	    vd->vdev_nparity);
+	    SPA_OLD_MAXBLOCKSIZE, origoffset, tvd->vdev_ashift,
+	    vd->vdev_children, vd->vdev_nparity);
 
 	coloffset = origoffset;
 
diff --git a/uts/common/fs/zfs/zap_micro.c b/uts/common/fs/zfs/zap_micro.c
index 1152f9072f99..59a9f970448a 100644
--- a/uts/common/fs/zfs/zap_micro.c
+++ b/uts/common/fs/zfs/zap_micro.c
@@ -33,6 +33,7 @@
 #include <sys/zap_leaf.h>
 #include <sys/avl.h>
 #include <sys/arc.h>
+#include <sys/dmu_objset.h>
 
 #ifdef _KERNEL
 #include <sys/sunddi.h>
@@ -653,9 +654,9 @@ zap_create_flags(objset_t *os, int normflags, zap_flags_t flags,
 	uint64_t obj = dmu_object_alloc(os, ot, 0, bonustype, bonuslen, tx);
 
 	ASSERT(leaf_blockshift >= SPA_MINBLOCKSHIFT &&
-	    leaf_blockshift <= SPA_MAXBLOCKSHIFT &&
+	    leaf_blockshift <= SPA_OLD_MAXBLOCKSHIFT &&
 	    indirect_blockshift >= SPA_MINBLOCKSHIFT &&
-	    indirect_blockshift <= SPA_MAXBLOCKSHIFT);
+	    indirect_blockshift <= SPA_OLD_MAXBLOCKSHIFT);
 
 	VERIFY(dmu_object_set_blocksize(os, obj,
 	    1ULL << leaf_blockshift, indirect_blockshift, tx) == 0);
@@ -1345,7 +1346,6 @@ zap_count_write(objset_t *os, uint64_t zapobj, const char *name, int add,
 	zap_t *zap;
 	int err = 0;
 
-
 	/*
 	 * Since, we don't have a name, we cannot figure out which blocks will
 	 * be affected in this operation. So, account for the worst case :
@@ -1358,7 +1358,7 @@ zap_count_write(objset_t *os, uint64_t zapobj, const char *name, int add,
 	 * large microzap results in a promotion to fatzap.
 	 */
 	if (name == NULL) {
-		*towrite += (3 + (add ? 4 : 0)) * SPA_MAXBLOCKSIZE;
+		*towrite += (3 + (add ? 4 : 0)) * SPA_OLD_MAXBLOCKSIZE;
 		return (err);
 	}
 
@@ -1382,7 +1382,7 @@ zap_count_write(objset_t *os, uint64_t zapobj, const char *name, int add,
 			/*
 			 * We treat this case as similar to (name == NULL)
 			 */
-			*towrite += (3 + (add ? 4 : 0)) * SPA_MAXBLOCKSIZE;
+			*towrite += (3 + (add ? 4 : 0)) * SPA_OLD_MAXBLOCKSIZE;
 		}
 	} else {
 		/*
@@ -1401,12 +1401,12 @@ zap_count_write(objset_t *os, uint64_t zapobj, const char *name, int add,
 		 *			ptrtbl blocks
 		 */
 		if (dmu_buf_freeable(zap->zap_dbuf))
-			*tooverwrite += SPA_MAXBLOCKSIZE;
+			*tooverwrite += MZAP_MAX_BLKSZ;
 		else
-			*towrite += SPA_MAXBLOCKSIZE;
+			*towrite += MZAP_MAX_BLKSZ;
 
 		if (add) {
-			*towrite += 4 * SPA_MAXBLOCKSIZE;
+			*towrite += 4 * MZAP_MAX_BLKSZ;
 		}
 	}
 
diff --git a/uts/common/fs/zfs/zfs_ioctl.c b/uts/common/fs/zfs/zfs_ioctl.c
index 0cec1c2adad3..a05e7b2a29b3 100644
--- a/uts/common/fs/zfs/zfs_ioctl.c
+++ b/uts/common/fs/zfs/zfs_ioctl.c
@@ -2383,7 +2383,7 @@ zfs_prop_set_special(const char *dsname, zprop_source_t source,
 	const char *propname = nvpair_name(pair);
 	zfs_prop_t prop = zfs_name_to_prop(propname);
 	uint64_t intval;
-	int err;
+	int err = -1;
 
 	if (prop == ZPROP_INVAL) {
 		if (zfs_prop_userquota(propname))
@@ -3772,8 +3772,7 @@ zfs_check_settable(const char *dsname, nvpair_t *pair, cred_t *cr)
 		 * the SPA supports it. We ignore any errors here since
 		 * we'll catch them later.
 		 */
-		if (nvpair_type(pair) == DATA_TYPE_UINT64 &&
-		    nvpair_value_uint64(pair, &intval) == 0) {
+		if (nvpair_value_uint64(pair, &intval) == 0) {
 			if (intval >= ZIO_COMPRESS_GZIP_1 &&
 			    intval <= ZIO_COMPRESS_GZIP_9 &&
 			    zfs_earlier_version(dsname,
@@ -3824,6 +3823,42 @@ zfs_check_settable(const char *dsname, nvpair_t *pair, cred_t *cr)
 			return (SET_ERROR(ENOTSUP));
 		break;
 
+	case ZFS_PROP_RECORDSIZE:
+		/* Record sizes above 128k need the feature to be enabled */
+		if (nvpair_value_uint64(pair, &intval) == 0 &&
+		    intval > SPA_OLD_MAXBLOCKSIZE) {
+			spa_t *spa;
+
+			/*
+			 * If this is a bootable dataset then
+			 * the we don't allow large (>128K) blocks,
+			 * because GRUB doesn't support them.
+			 */
+			if (zfs_is_bootfs(dsname) &&
+			    intval > SPA_OLD_MAXBLOCKSIZE) {
+				return (SET_ERROR(EDOM));
+			}
+
+			/*
+			 * We don't allow setting the property above 1MB,
+			 * unless the tunable has been changed.
+			 */
+			if (intval > zfs_max_recordsize ||
+			    intval > SPA_MAXBLOCKSIZE)
+				return (SET_ERROR(EDOM));
+
+			if ((err = spa_open(dsname, &spa, FTAG)) != 0)
+				return (err);
+
+			if (!spa_feature_is_enabled(spa,
+			    SPA_FEATURE_LARGE_BLOCKS)) {
+				spa_close(spa, FTAG);
+				return (SET_ERROR(ENOTSUP));
+			}
+			spa_close(spa, FTAG);
+		}
+		break;
+
 	case ZFS_PROP_SHARESMB:
 		if (zpl_earlier_version(dsname, ZPL_VERSION_FUID))
 			return (SET_ERROR(ENOTSUP));
@@ -4245,7 +4280,7 @@ out:
  * zc_fromobj	objsetid of incremental fromsnap (may be zero)
  * zc_guid	if set, estimate size of stream only.  zc_cookie is ignored.
  *		output size in zc_objset_type.
- * zc_flags	if =1, WRITE_EMBEDDED records are permitted
+ * zc_flags	lzc_send_flags
  *
  * outputs:
  * zc_objset_type	estimated size, if zc_guid is set
@@ -4257,6 +4292,7 @@ zfs_ioc_send(zfs_cmd_t *zc)
 	offset_t off;
 	boolean_t estimate = (zc->zc_guid != 0);
 	boolean_t embedok = (zc->zc_flags & 0x1);
+	boolean_t large_block_ok = (zc->zc_flags & 0x2);
 
 	if (zc->zc_obj != 0) {
 		dsl_pool_t *dp;
@@ -4317,7 +4353,8 @@ zfs_ioc_send(zfs_cmd_t *zc)
 
 		off = fp->f_offset;
 		error = dmu_send_obj(zc->zc_name, zc->zc_sendobj,
-		    zc->zc_fromobj, embedok, zc->zc_cookie, fp->f_vnode, &off);
+		    zc->zc_fromobj, embedok, large_block_ok,
+		    zc->zc_cookie, fp->f_vnode, &off);
 
 		if (VOP_SEEK(fp->f_vnode, fp->f_offset, &off, NULL) == 0)
 			fp->f_offset = off;
@@ -5219,6 +5256,8 @@ zfs_ioc_space_snaps(const char *lastsnap, nvlist_t *innvl, nvlist_t *outnvl)
  * innvl: {
  *     "fd" -> file descriptor to write stream to (int32)
  *     (optional) "fromsnap" -> full snap name to send an incremental from
+ *     (optional) "largeblockok" -> (value ignored)
+ *         indicates that blocks > 128KB are permitted
  *     (optional) "embedok" -> (value ignored)
  *         presence indicates DRR_WRITE_EMBEDDED records are permitted
  * }
@@ -5233,6 +5272,7 @@ zfs_ioc_send_new(const char *snapname, nvlist_t *innvl, nvlist_t *outnvl)
 	offset_t off;
 	char *fromname = NULL;
 	int fd;
+	boolean_t largeblockok;
 	boolean_t embedok;
 
 	error = nvlist_lookup_int32(innvl, "fd", &fd);
@@ -5241,6 +5281,7 @@ zfs_ioc_send_new(const char *snapname, nvlist_t *innvl, nvlist_t *outnvl)
 
 	(void) nvlist_lookup_string(innvl, "fromsnap", &fromname);
 
+	largeblockok = nvlist_exists(innvl, "largeblockok");
 	embedok = nvlist_exists(innvl, "embedok");
 
 	file_t *fp = getf(fd);
@@ -5248,7 +5289,8 @@ zfs_ioc_send_new(const char *snapname, nvlist_t *innvl, nvlist_t *outnvl)
 		return (SET_ERROR(EBADF));
 
 	off = fp->f_offset;
-	error = dmu_send(snapname, fromname, embedok, fd, fp->f_vnode, &off);
+	error = dmu_send(snapname, fromname, embedok, largeblockok,
+	    fd, fp->f_vnode, &off);
 
 	if (VOP_SEEK(fp->f_vnode, fp->f_offset, &off, NULL) == 0)
 		fp->f_offset = off;
diff --git a/uts/common/fs/zfs/zfs_log.c b/uts/common/fs/zfs/zfs_log.c
index aeaba2233a19..47d32a45c39d 100644
--- a/uts/common/fs/zfs/zfs_log.c
+++ b/uts/common/fs/zfs/zfs_log.c
@@ -485,7 +485,7 @@ zfs_log_write(zilog_t *zilog, dmu_tx_t *tx, int txtype,
 		 * If the write would overflow the largest block then split it.
 		 */
 		if (write_state != WR_INDIRECT && resid > ZIL_MAX_LOG_DATA)
-			len = SPA_MAXBLOCKSIZE >> 1;
+			len = SPA_OLD_MAXBLOCKSIZE >> 1;
 		else
 			len = resid;
 
diff --git a/uts/common/fs/zfs/zfs_vfsops.c b/uts/common/fs/zfs/zfs_vfsops.c
index 0b0b0a99cc6e..8cf83b399a15 100644
--- a/uts/common/fs/zfs/zfs_vfsops.c
+++ b/uts/common/fs/zfs/zfs_vfsops.c
@@ -272,10 +272,9 @@ static void
 blksz_changed_cb(void *arg, uint64_t newval)
 {
 	zfsvfs_t *zfsvfs = arg;
-
-	if (newval < SPA_MINBLOCKSIZE ||
-	    newval > SPA_MAXBLOCKSIZE || !ISP2(newval))
-		newval = SPA_MAXBLOCKSIZE;
+	ASSERT3U(newval, <=, spa_maxblocksize(dmu_objset_spa(zfsvfs->z_os)));
+	ASSERT3U(newval, >=, SPA_MINBLOCKSIZE);
+	ASSERT(ISP2(newval));
 
 	zfsvfs->z_max_blksz = newval;
 	zfsvfs->z_vfs->vfs_bsize = newval;
@@ -906,7 +905,7 @@ zfsvfs_create(const char *osname, zfsvfs_t **zfvp)
 	 */
 	zfsvfs->z_vfs = NULL;
 	zfsvfs->z_parent = zfsvfs;
-	zfsvfs->z_max_blksz = SPA_MAXBLOCKSIZE;
+	zfsvfs->z_max_blksz = SPA_OLD_MAXBLOCKSIZE;
 	zfsvfs->z_show_ctldir = ZFS_SNAPDIR_VISIBLE;
 	zfsvfs->z_os = os;
 
diff --git a/uts/common/fs/zfs/zfs_vnops.c b/uts/common/fs/zfs/zfs_vnops.c
index c608a79fd2a0..5060412afd9b 100644
--- a/uts/common/fs/zfs/zfs_vnops.c
+++ b/uts/common/fs/zfs/zfs_vnops.c
@@ -817,8 +817,14 @@ zfs_write(vnode_t *vp, uio_t *uio, int ioflag, cred_t *cr, caller_context_t *ct)
 			uint64_t new_blksz;
 
 			if (zp->z_blksz > max_blksz) {
+				/*
+				 * File's blocksize is already larger than the
+				 * "recordsize" property.  Only let it grow to
+				 * the next power of 2.
+				 */
 				ASSERT(!ISP2(zp->z_blksz));
-				new_blksz = MIN(end_size, SPA_MAXBLOCKSIZE);
+				new_blksz = MIN(end_size,
+				    1 << highbit64(zp->z_blksz));
 			} else {
 				new_blksz = MIN(end_size, max_blksz);
 			}
diff --git a/uts/common/fs/zfs/zfs_znode.c b/uts/common/fs/zfs/zfs_znode.c
index 7577250408df..4664899d13f8 100644
--- a/uts/common/fs/zfs/zfs_znode.c
+++ b/uts/common/fs/zfs/zfs_znode.c
@@ -58,6 +58,7 @@
 #endif /* _KERNEL */
 
 #include <sys/dmu.h>
+#include <sys/dmu_objset.h>
 #include <sys/refcount.h>
 #include <sys/stat.h>
 #include <sys/zap.h>
@@ -1474,8 +1475,13 @@ zfs_extend(znode_t *zp, uint64_t end)
 		 * We are growing the file past the current block size.
 		 */
 		if (zp->z_blksz > zp->z_zfsvfs->z_max_blksz) {
+			/*
+			 * File's blocksize is already larger than the
+			 * "recordsize" property.  Only let it grow to
+			 * the next power of 2.
+			 */
 			ASSERT(!ISP2(zp->z_blksz));
-			newblksz = MIN(end, SPA_MAXBLOCKSIZE);
+			newblksz = MIN(end, 1 << highbit64(zp->z_blksz));
 		} else {
 			newblksz = MIN(end, zp->z_zfsvfs->z_max_blksz);
 		}
diff --git a/uts/common/fs/zfs/zil.c b/uts/common/fs/zfs/zil.c
index 6377285fe2a9..6beacadf710d 100644
--- a/uts/common/fs/zfs/zil.c
+++ b/uts/common/fs/zfs/zil.c
@@ -220,6 +220,7 @@ zil_read_log_block(zilog_t *zilog, const blkptr_t *bp, blkptr_t *nbp, void *dst,
 			    sizeof (cksum)) || BP_IS_HOLE(&zilc->zc_next_blk)) {
 				error = SET_ERROR(ECKSUM);
 			} else {
+				ASSERT3U(len, <=, SPA_OLD_MAXBLOCKSIZE);
 				bcopy(lr, dst, len);
 				*end = (char *)dst + len;
 				*nbp = zilc->zc_next_blk;
@@ -234,6 +235,8 @@ zil_read_log_block(zilog_t *zilog, const blkptr_t *bp, blkptr_t *nbp, void *dst,
 			    (zilc->zc_nused > (size - sizeof (*zilc)))) {
 				error = SET_ERROR(ECKSUM);
 			} else {
+				ASSERT3U(zilc->zc_nused, <=,
+				    SPA_OLD_MAXBLOCKSIZE);
 				bcopy(lr, dst, zilc->zc_nused);
 				*end = (char *)dst + zilc->zc_nused;
 				*nbp = zilc->zc_next_blk;
@@ -317,7 +320,7 @@ zil_parse(zilog_t *zilog, zil_parse_blk_func_t *parse_blk_func,
 	 * If the log has been claimed, stop if we encounter a sequence
 	 * number greater than the highest claimed sequence number.
 	 */
-	lrbuf = zio_buf_alloc(SPA_MAXBLOCKSIZE);
+	lrbuf = zio_buf_alloc(SPA_OLD_MAXBLOCKSIZE);
 	zil_bp_tree_init(zilog);
 
 	for (blk = zh->zh_log; !BP_IS_HOLE(&blk); blk = next_blk) {
@@ -364,7 +367,7 @@ done:
 	    (max_blk_seq == claim_blk_seq && max_lr_seq == claim_lr_seq));
 
 	zil_bp_tree_fini(zilog);
-	zio_buf_free(lrbuf, SPA_MAXBLOCKSIZE);
+	zio_buf_free(lrbuf, SPA_OLD_MAXBLOCKSIZE);
 
 	return (error);
 }
@@ -896,7 +899,7 @@ zil_lwb_write_init(zilog_t *zilog, lwb_t *lwb)
  *
  * These must be a multiple of 4KB. Note only the amount used (again
  * aligned to 4KB) actually gets written. However, we can't always just
- * allocate SPA_MAXBLOCKSIZE as the slog space could be exhausted.
+ * allocate SPA_OLD_MAXBLOCKSIZE as the slog space could be exhausted.
  */
 uint64_t zil_block_buckets[] = {
     4096,		/* non TX_WRITE */
@@ -978,7 +981,7 @@ zil_lwb_write_start(zilog_t *zilog, lwb_t *lwb)
 		continue;
 	zil_blksz = zil_block_buckets[i];
 	if (zil_blksz == UINT64_MAX)
-		zil_blksz = SPA_MAXBLOCKSIZE;
+		zil_blksz = SPA_OLD_MAXBLOCKSIZE;
 	zilog->zl_prev_blks[zilog->zl_prev_rotor] = zil_blksz;
 	for (i = 0; i < ZIL_PREV_BLKS; i++)
 		zil_blksz = MAX(zil_blksz, zilog->zl_prev_blks[i]);
diff --git a/uts/common/fs/zfs/zio.c b/uts/common/fs/zfs/zio.c
index 302d692c64a6..d67e1361b4f8 100644
--- a/uts/common/fs/zfs/zio.c
+++ b/uts/common/fs/zfs/zio.c
@@ -112,9 +112,8 @@ zio_init(void)
 
 	/*
 	 * For small buffers, we want a cache for each multiple of
-	 * SPA_MINBLOCKSIZE.  For medium-size buffers, we want a cache
-	 * for each quarter-power of 2.  For large buffers, we want
-	 * a cache for each multiple of PAGESIZE.
+	 * SPA_MINBLOCKSIZE.  For larger buffers, we want a cache
+	 * for each quarter-power of 2.
 	 */
 	for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) {
 		size_t size = (c + 1) << SPA_MINBLOCKSHIFT;
@@ -137,10 +136,8 @@ zio_init(void)
 #endif
 		if (size <= 4 * SPA_MINBLOCKSIZE) {
 			align = SPA_MINBLOCKSIZE;
-		} else if (IS_P2ALIGNED(size, PAGESIZE)) {
-			align = PAGESIZE;
 		} else if (IS_P2ALIGNED(size, p2 >> 2)) {
-			align = p2 >> 2;
+			align = MIN(p2 >> 2, PAGESIZE);
 		}
 
 		if (align != 0) {
diff --git a/uts/common/fs/zfs/zvol.c b/uts/common/fs/zfs/zvol.c
index ca05d5cf214c..80888103fe48 100644
--- a/uts/common/fs/zfs/zvol.c
+++ b/uts/common/fs/zfs/zvol.c
@@ -191,7 +191,7 @@ int
 zvol_check_volblocksize(uint64_t volblocksize)
 {
 	if (volblocksize < SPA_MINBLOCKSIZE ||
-	    volblocksize > SPA_MAXBLOCKSIZE ||
+	    volblocksize > SPA_OLD_MAXBLOCKSIZE ||
 	    !ISP2(volblocksize))
 		return (SET_ERROR(EDOM));
 
@@ -692,7 +692,7 @@ zvol_prealloc(zvol_state_t *zv)
 
 	while (resid != 0) {
 		int error;
-		uint64_t bytes = MIN(resid, SPA_MAXBLOCKSIZE);
+		uint64_t bytes = MIN(resid, SPA_OLD_MAXBLOCKSIZE);
 
 		tx = dmu_tx_create(os);
 		dmu_tx_hold_write(tx, ZVOL_OBJ, off, bytes);
@@ -1661,7 +1661,8 @@ zvol_ioctl(dev_t dev, int cmd, intptr_t arg, int flag, cred_t *cr, int *rvalp)
 		(void) strcpy(dki.dki_dname, "zvol");
 		dki.dki_ctype = DKC_UNKNOWN;
 		dki.dki_unit = getminor(dev);
-		dki.dki_maxtransfer = 1 << (SPA_MAXBLOCKSHIFT - zv->zv_min_bs);
+		dki.dki_maxtransfer =
+		    1 << (SPA_OLD_MAXBLOCKSHIFT - zv->zv_min_bs);
 		mutex_exit(&zfsdev_state_lock);
 		if (ddi_copyout(&dki, (void *)arg, sizeof (dki), flag))
 			error = SET_ERROR(EFAULT);
@@ -1978,14 +1979,14 @@ zvol_dump_init(zvol_state_t *zv, boolean_t resize)
 		    zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE), 8, 1,
 		    &vbs, tx);
 		error = error ? error : dmu_object_set_blocksize(
-		    os, ZVOL_OBJ, SPA_MAXBLOCKSIZE, 0, tx);
+		    os, ZVOL_OBJ, SPA_OLD_MAXBLOCKSIZE, 0, tx);
 		if (version >= SPA_VERSION_DEDUP) {
 			error = error ? error : zap_update(os, ZVOL_ZAP_OBJ,
 			    zfs_prop_to_name(ZFS_PROP_DEDUP), 8, 1,
 			    &dedup, tx);
 		}
 		if (error == 0)
-			zv->zv_volblocksize = SPA_MAXBLOCKSIZE;
+			zv->zv_volblocksize = SPA_OLD_MAXBLOCKSIZE;
 	}
 	dmu_tx_commit(tx);
 
diff --git a/uts/common/sys/fs/zfs.h b/uts/common/sys/fs/zfs.h
index 7155d9702bb2..569fae20915d 100644
--- a/uts/common/sys/fs/zfs.h
+++ b/uts/common/sys/fs/zfs.h
@@ -192,6 +192,7 @@ typedef enum {
 	ZPOOL_PROP_FREEING,
 	ZPOOL_PROP_FRAGMENTATION,
 	ZPOOL_PROP_LEAKED,
+	ZPOOL_PROP_MAXBLOCKSIZE,
 	ZPOOL_NUM_PROPS
 } zpool_prop_t;
 

From 047c407c383af64f7ef40ff769b4207ad12cfd02 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Fri, 5 Dec 2014 23:50:16 +0000
Subject: [PATCH 052/380] 5100 sparc build failed after 5004 Reviewed by:
 Garrett D'Amore <garrett@damore.org> Reviewed by: Gary Mills
 <gary_mills@fastmail.fm> Approved by: Robert Mustacchi <rm@joyent.com>
 Author: Igor Kozhukhov <ikozhukhov@gmail.com>

illumos/illuoms-gate@6481fd49a4385db15285ddf89f9c80b45eaaa407
---
 uts/common/sys/cpuvar.h | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/uts/common/sys/cpuvar.h b/uts/common/sys/cpuvar.h
index bdefd08e3aeb..166c7180a5ff 100644
--- a/uts/common/sys/cpuvar.h
+++ b/uts/common/sys/cpuvar.h
@@ -22,6 +22,7 @@
 /*
  * Copyright (c) 1989, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012 by Delphix. All rights reserved.
+ * Copyright 2014 Igor Kozhukhov <ikozhukhov@gmail.com>.
  */
 
 #ifndef _SYS_CPUVAR_H
@@ -32,6 +33,7 @@
 #include <sys/disp.h>
 #include <sys/processor.h>
 
+#include <sys/loadavg.h>
 #if (defined(_KERNEL) || defined(_KMEMUSER)) && defined(_MACHDEP)
 #include <sys/machcpuvar.h>
 #endif
@@ -53,15 +55,6 @@ extern "C" {
 struct squeue_set_s;
 
 #define	CPU_CACHE_COHERENCE_SIZE	64
-#define	S_LOADAVG_SZ	11
-#define	S_MOVAVG_SZ	10
-
-struct loadavg_s {
-	int lg_cur;		/* current loadavg entry */
-	unsigned int lg_len;	/* number entries recorded */
-	hrtime_t lg_total;	/* used to temporarily hold load totals */
-	hrtime_t lg_loads[S_LOADAVG_SZ];	/* table of recorded entries */
-};
 
 /*
  * For fast event tracing.

From 9e40ba56d50bbadcb89674e1bf20f667c91de9f7 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Fri, 5 Dec 2014 23:51:20 +0000
Subject: [PATCH 053/380] 5285 pass in cpu_pause_func via pause_cpus Reviewed
 by: Robert Mustacchi <rm@joyent.com> Reviewed by: Dan McDonald
 <danmcd@omniti.com> Approved by: Gordon Ross <gwr@nexenta.com> Author: Josef
 'Jeff' Sipek <josef.sipek@nexenta.com>

illumos/illumos-gate@0ed5c46e82c989cfa9726d9dae452e3d24ef83be
---
 uts/common/sys/cpuvar.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/uts/common/sys/cpuvar.h b/uts/common/sys/cpuvar.h
index 166c7180a5ff..7b333680c672 100644
--- a/uts/common/sys/cpuvar.h
+++ b/uts/common/sys/cpuvar.h
@@ -652,7 +652,7 @@ void	poke_cpu(int cpun);	 /* interrupt another CPU (to preempt) */
 
 void	mach_cpu_pause(volatile char *);
 
-void	pause_cpus(cpu_t *off_cp);
+void	pause_cpus(cpu_t *off_cp, void *(*func)(void *));
 void	start_cpus(void);
 int	cpus_paused(void);
 

From fcf634236b9453af2a264606d23f44d7eeb56d5f Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Fri, 5 Dec 2014 23:55:44 +0000
Subject: [PATCH 054/380] 5255 uts shouldn't open-code ISP2 Reviewed by: Marcel
 Telka <marcel@telka.sk> Reviewed by: Dan McDonald <danmcd@omniti.com>
 Approved by: Robert Mustacchi <rm@joyent.com> Author: Josef 'Jeff' Sipek
 <josef.sipek@nexenta.com>

illumos/illumos-gate@de710d24d2fae4468e64da999e1d952a247f142c
---
 uts/common/dtrace/dtrace.c   | 4 ++--
 uts/common/dtrace/fasttrap.c | 6 +++---
 uts/common/fs/zfs/zio.c      | 3 ++-
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/uts/common/dtrace/dtrace.c b/uts/common/dtrace/dtrace.c
index b9c5c1ddc7d0..0d2d6f524c2e 100644
--- a/uts/common/dtrace/dtrace.c
+++ b/uts/common/dtrace/dtrace.c
@@ -139,7 +139,7 @@ dtrace_optval_t dtrace_ustackframes_default = 20;
 dtrace_optval_t dtrace_jstackframes_default = 50;
 dtrace_optval_t dtrace_jstackstrsize_default = 512;
 int		dtrace_msgdsize_max = 128;
-hrtime_t	dtrace_chill_max = 500 * (NANOSEC / MILLISEC);	/* 500 ms */
+hrtime_t	dtrace_chill_max = MSEC2NSEC(500);		/* 500 ms */
 hrtime_t	dtrace_chill_interval = NANOSEC;		/* 1000 ms */
 int		dtrace_devdepth_max = 32;
 int		dtrace_err_verbose;
@@ -13039,7 +13039,7 @@ dtrace_dof_slurp(dof_hdr_t *dof, dtrace_vstate_t *vstate, cred_t *cr,
 		if (!(sec->dofs_flags & DOF_SECF_LOAD))
 			continue; /* just ignore non-loadable sections */
 
-		if (sec->dofs_align & (sec->dofs_align - 1)) {
+		if (!ISP2(sec->dofs_align)) {
 			dtrace_dof_error(dof, "bad section alignment");
 			return (-1);
 		}
diff --git a/uts/common/dtrace/fasttrap.c b/uts/common/dtrace/fasttrap.c
index ce7b1c40e61c..fc3d87574a29 100644
--- a/uts/common/dtrace/fasttrap.c
+++ b/uts/common/dtrace/fasttrap.c
@@ -2156,7 +2156,7 @@ fasttrap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
 	if (nent == 0 || nent > 0x1000000)
 		nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
 
-	if ((nent & (nent - 1)) == 0)
+	if (ISP2(nent))
 		fasttrap_tpoints.fth_nent = nent;
 	else
 		fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
@@ -2169,7 +2169,7 @@ fasttrap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
 	 * ... and the providers hash table...
 	 */
 	nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
-	if ((nent & (nent - 1)) == 0)
+	if (ISP2(nent))
 		fasttrap_provs.fth_nent = nent;
 	else
 		fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
@@ -2182,7 +2182,7 @@ fasttrap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
 	 * ... and the procs hash table.
 	 */
 	nent = FASTTRAP_PROCS_DEFAULT_SIZE;
-	if ((nent & (nent - 1)) == 0)
+	if (ISP2(nent))
 		fasttrap_procs.fth_nent = nent;
 	else
 		fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
diff --git a/uts/common/fs/zfs/zio.c b/uts/common/fs/zfs/zio.c
index d67e1361b4f8..7cb6953845eb 100644
--- a/uts/common/fs/zfs/zio.c
+++ b/uts/common/fs/zfs/zio.c
@@ -24,6 +24,7 @@
  * Copyright (c) 2011 Nexenta Systems, Inc. All rights reserved.
  */
 
+#include <sys/sysmacros.h>
 #include <sys/zfs_context.h>
 #include <sys/fm/fs/zfs.h>
 #include <sys/spa.h>
@@ -121,7 +122,7 @@ zio_init(void)
 		size_t align = 0;
 		size_t cflags = (size > zio_buf_debug_limit) ? KMC_NODEBUG : 0;
 
-		while (p2 & (p2 - 1))
+		while (!ISP2(p2))
 			p2 &= p2 - 1;
 
 #ifndef _KERNEL

From f48da8f9a6eccd715da75650a058569fc3315a3b Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 6 Dec 2014 00:12:58 +0000
Subject: [PATCH 055/380] 5179 Remove unused ZFS ARC functions Reviewed by:
 Josef 'Jeff' Sipek <josef.sipek@nexenta.com> Reviewed by: Steve Gonczi
 <gonczi@comcast.net> Reviewed by: Matthew Ahrens <mahrens@delphix.com>
 Reviewed by: Richard Elling <richard.elling@gmail.com> Reviewed by: George
 Wilson <george.wilson@delphix.com> Approved by: Richard Lowe
 <richlowe@richlowe.net> Author: Serapheim Dimitropoulos
 <serapheimd@gmail.com>

illumos/illumos-gate@fbefb14f62976763eeaa74a0c1ac68accb38cf44
---
 uts/common/fs/zfs/arc.c     | 17 -----------------
 uts/common/fs/zfs/sys/arc.h |  2 --
 2 files changed, 19 deletions(-)

diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index 69d16af3b669..c113b53a0175 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -1310,23 +1310,6 @@ arc_space_return(uint64_t space, arc_space_type_t type)
 	atomic_add_64(&arc_size, -space);
 }
 
-void *
-arc_data_buf_alloc(uint64_t size)
-{
-	if (arc_evict_needed(ARC_BUFC_DATA))
-		cv_signal(&arc_reclaim_thr_cv);
-	atomic_add_64(&arc_size, size);
-	return (zio_data_buf_alloc(size));
-}
-
-void
-arc_data_buf_free(void *buf, uint64_t size)
-{
-	zio_data_buf_free(buf, size);
-	ASSERT(arc_size >= size);
-	atomic_add_64(&arc_size, -size);
-}
-
 arc_buf_t *
 arc_buf_alloc(spa_t *spa, int size, void *tag, arc_buf_contents_t type)
 {
diff --git a/uts/common/fs/zfs/sys/arc.h b/uts/common/fs/zfs/sys/arc.h
index 04a20c13eeef..02643bd732dd 100644
--- a/uts/common/fs/zfs/sys/arc.h
+++ b/uts/common/fs/zfs/sys/arc.h
@@ -83,8 +83,6 @@ typedef enum arc_space_type {
 
 void arc_space_consume(uint64_t space, arc_space_type_t type);
 void arc_space_return(uint64_t space, arc_space_type_t type);
-void *arc_data_buf_alloc(uint64_t space);
-void arc_data_buf_free(void *buf, uint64_t space);
 arc_buf_t *arc_buf_alloc(spa_t *spa, int size, void *tag,
     arc_buf_contents_t type);
 arc_buf_t *arc_loan_buf(spa_t *spa, int size);

From 36a0906ffb7e26833824749cf469b407b7a4fd58 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 6 Dec 2014 00:14:38 +0000
Subject: [PATCH 056/380] 5213 panic in metaslab_init due to space_map_open
 returning ENXIO Reviewed by: Matthew Ahrens <mahrens@delphix.com> Reviewed
 by: George Wilson <george.wilson@delphix.com> Reviewed by: Bayard Bell
 <Bayard.Bell@nexenta.com> Reviewed by: Brian Behlendorf
 <behlendorf1@llnl.gov> Reviewed by: Steven Hartland
 <steven.hartland@multiplay.co.uk> Approved by: Dan McDonald
 <danmcd@omniti.com> Author: Prakash Surya <prakash.surya@delphix.com>

illumos/illumos-gate@1e9bd7ec42f2d3bf854c2da35310901194833267
---
 uts/common/fs/zfs/metaslab.c     | 56 +++++++++++++++++++-------------
 uts/common/fs/zfs/sys/metaslab.h |  4 +--
 uts/common/fs/zfs/vdev.c         |  6 +++-
 3 files changed, 40 insertions(+), 26 deletions(-)

diff --git a/uts/common/fs/zfs/metaslab.c b/uts/common/fs/zfs/metaslab.c
index 0f05e8b00a87..9ba9fd384141 100644
--- a/uts/common/fs/zfs/metaslab.c
+++ b/uts/common/fs/zfs/metaslab.c
@@ -1208,28 +1208,36 @@ metaslab_unload(metaslab_t *msp)
 	msp->ms_weight &= ~METASLAB_ACTIVE_MASK;
 }
 
-metaslab_t *
-metaslab_init(metaslab_group_t *mg, uint64_t id, uint64_t object, uint64_t txg)
+int
+metaslab_init(metaslab_group_t *mg, uint64_t id, uint64_t object, uint64_t txg,
+    metaslab_t **msp)
 {
 	vdev_t *vd = mg->mg_vd;
 	objset_t *mos = vd->vdev_spa->spa_meta_objset;
-	metaslab_t *msp;
+	metaslab_t *ms;
+	int error;
 
-	msp = kmem_zalloc(sizeof (metaslab_t), KM_SLEEP);
-	mutex_init(&msp->ms_lock, NULL, MUTEX_DEFAULT, NULL);
-	cv_init(&msp->ms_load_cv, NULL, CV_DEFAULT, NULL);
-	msp->ms_id = id;
-	msp->ms_start = id << vd->vdev_ms_shift;
-	msp->ms_size = 1ULL << vd->vdev_ms_shift;
+	ms = kmem_zalloc(sizeof (metaslab_t), KM_SLEEP);
+	mutex_init(&ms->ms_lock, NULL, MUTEX_DEFAULT, NULL);
+	cv_init(&ms->ms_load_cv, NULL, CV_DEFAULT, NULL);
+	ms->ms_id = id;
+	ms->ms_start = id << vd->vdev_ms_shift;
+	ms->ms_size = 1ULL << vd->vdev_ms_shift;
 
 	/*
 	 * We only open space map objects that already exist. All others
 	 * will be opened when we finally allocate an object for it.
 	 */
 	if (object != 0) {
-		VERIFY0(space_map_open(&msp->ms_sm, mos, object, msp->ms_start,
-		    msp->ms_size, vd->vdev_ashift, &msp->ms_lock));
-		ASSERT(msp->ms_sm != NULL);
+		error = space_map_open(&ms->ms_sm, mos, object, ms->ms_start,
+		    ms->ms_size, vd->vdev_ashift, &ms->ms_lock);
+
+		if (error != 0) {
+			kmem_free(ms, sizeof (metaslab_t));
+			return (error);
+		}
+
+		ASSERT(ms->ms_sm != NULL);
 	}
 
 	/*
@@ -1239,11 +1247,11 @@ metaslab_init(metaslab_group_t *mg, uint64_t id, uint64_t object, uint64_t txg)
 	 * addition of new space; and for debugging, it ensures that we'd
 	 * data fault on any attempt to use this metaslab before it's ready.
 	 */
-	msp->ms_tree = range_tree_create(&metaslab_rt_ops, msp, &msp->ms_lock);
-	metaslab_group_add(mg, msp);
+	ms->ms_tree = range_tree_create(&metaslab_rt_ops, ms, &ms->ms_lock);
+	metaslab_group_add(mg, ms);
 
-	msp->ms_fragmentation = metaslab_fragmentation(msp);
-	msp->ms_ops = mg->mg_class->mc_ops;
+	ms->ms_fragmentation = metaslab_fragmentation(ms);
+	ms->ms_ops = mg->mg_class->mc_ops;
 
 	/*
 	 * If we're opening an existing pool (txg == 0) or creating
@@ -1252,25 +1260,27 @@ metaslab_init(metaslab_group_t *mg, uint64_t id, uint64_t object, uint64_t txg)
 	 * does not become available until after this txg has synced.
 	 */
 	if (txg <= TXG_INITIAL)
-		metaslab_sync_done(msp, 0);
+		metaslab_sync_done(ms, 0);
 
 	/*
 	 * If metaslab_debug_load is set and we're initializing a metaslab
 	 * that has an allocated space_map object then load the its space
 	 * map so that can verify frees.
 	 */
-	if (metaslab_debug_load && msp->ms_sm != NULL) {
-		mutex_enter(&msp->ms_lock);
-		VERIFY0(metaslab_load(msp));
-		mutex_exit(&msp->ms_lock);
+	if (metaslab_debug_load && ms->ms_sm != NULL) {
+		mutex_enter(&ms->ms_lock);
+		VERIFY0(metaslab_load(ms));
+		mutex_exit(&ms->ms_lock);
 	}
 
 	if (txg != 0) {
 		vdev_dirty(vd, 0, NULL, txg);
-		vdev_dirty(vd, VDD_METASLAB, msp, txg);
+		vdev_dirty(vd, VDD_METASLAB, ms, txg);
 	}
 
-	return (msp);
+	*msp = ms;
+
+	return (0);
 }
 
 void
diff --git a/uts/common/fs/zfs/sys/metaslab.h b/uts/common/fs/zfs/sys/metaslab.h
index 7fd47c08c9df..b3b9374c779f 100644
--- a/uts/common/fs/zfs/sys/metaslab.h
+++ b/uts/common/fs/zfs/sys/metaslab.h
@@ -42,8 +42,8 @@ typedef struct metaslab_ops {
 
 extern metaslab_ops_t *zfs_metaslab_ops;
 
-metaslab_t *metaslab_init(metaslab_group_t *, uint64_t,
-    uint64_t, uint64_t);
+int metaslab_init(metaslab_group_t *, uint64_t, uint64_t, uint64_t,
+    metaslab_t **);
 void metaslab_fini(metaslab_t *);
 
 void metaslab_load_wait(metaslab_t *);
diff --git a/uts/common/fs/zfs/vdev.c b/uts/common/fs/zfs/vdev.c
index 67b58edeed0b..6ee2a91dbd42 100644
--- a/uts/common/fs/zfs/vdev.c
+++ b/uts/common/fs/zfs/vdev.c
@@ -857,7 +857,11 @@ vdev_metaslab_init(vdev_t *vd, uint64_t txg)
 			if (error)
 				return (error);
 		}
-		vd->vdev_ms[m] = metaslab_init(vd->vdev_mg, m, object, txg);
+
+		error = metaslab_init(vd->vdev_mg, m, object, txg,
+		    &(vd->vdev_ms[m]));
+		if (error)
+			return (error);
 	}
 
 	if (txg == 0)

From 8526785526ca8643e4382b8b0fb51f7f7cfb5cc1 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 6 Dec 2014 00:18:53 +0000
Subject: [PATCH 057/380] 5311 traverse_dnode may report success when it should
 not Reviewed by: Matthew Ahrens <mahrens@delphix.com> Reviewed by: Andriy
 Gapon <avg@FreeBSD.org> Reviewed by: Will Andrews <willa@spectralogic.com>
 Approved by: Dan McDonald <danmcd@omniti.com> Author: Justin T. Gibbs
 <justing@spectralogic.com>

illumos/illumos-gate@2a89c2c59b7c2beb2373c14368cbe7e32af6ffc1
---
 uts/common/fs/zfs/dmu_traverse.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/uts/common/fs/zfs/dmu_traverse.c b/uts/common/fs/zfs/dmu_traverse.c
index 5836549d200a..5523f6206fbc 100644
--- a/uts/common/fs/zfs/dmu_traverse.c
+++ b/uts/common/fs/zfs/dmu_traverse.c
@@ -429,7 +429,7 @@ traverse_dnode(traverse_data_t *td, const dnode_phys_t *dnp,
 			break;
 	}
 
-	if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
+	if (err == 0 && dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
 		SET_BOOKMARK(&czb, objset, object, 0, DMU_SPILL_BLKID);
 		err = traverse_visitbp(td, dnp, &dnp->dn_spill, &czb);
 	}

From 34f74501212cb870cd25a63c46565012fb030067 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 6 Dec 2014 00:31:58 +0000
Subject: [PATCH 058/380] 5310 Remove always true tests for non-NULL
 ds->ds_phys Reviewed by: Matthew Ahrens <mahrens@delphix.com> Reviewed by:
 Will Andrews <willa@spectralogic.com> Reviewed by: Andriy Gapon
 <avg@FreeBSD.org> Approved by: Dan McDonald <danmcd@omniti.com> Author:
 Justin T. Gibbs <justing@spectralogic.com>

illumos/illumos-gate@d808a4fc6ac40e878a28e96f1ad7dd2ec439bfbf
---
 uts/common/fs/zfs/dsl_dataset.c | 8 +++-----
 uts/common/fs/zfs/dsl_prop.c    | 8 ++++----
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/uts/common/fs/zfs/dsl_dataset.c b/uts/common/fs/zfs/dsl_dataset.c
index e7ed75090208..5ac0050aef42 100644
--- a/uts/common/fs/zfs/dsl_dataset.c
+++ b/uts/common/fs/zfs/dsl_dataset.c
@@ -634,16 +634,14 @@ dsl_dataset_rele(dsl_dataset_t *ds, void *tag)
 void
 dsl_dataset_disown(dsl_dataset_t *ds, void *tag)
 {
-	ASSERT(ds->ds_owner == tag && ds->ds_dbuf != NULL);
+	ASSERT3P(ds->ds_owner, ==, tag);
+	ASSERT(ds->ds_dbuf != NULL);
 
 	mutex_enter(&ds->ds_lock);
 	ds->ds_owner = NULL;
 	mutex_exit(&ds->ds_lock);
 	dsl_dataset_long_rele(ds, tag);
-	if (ds->ds_dbuf != NULL)
-		dsl_dataset_rele(ds, tag);
-	else
-		dsl_dataset_evict(NULL, ds);
+	dsl_dataset_rele(ds, tag);
 }
 
 boolean_t
diff --git a/uts/common/fs/zfs/dsl_prop.c b/uts/common/fs/zfs/dsl_prop.c
index 7e62635176df..5907ea281b0e 100644
--- a/uts/common/fs/zfs/dsl_prop.c
+++ b/uts/common/fs/zfs/dsl_prop.c
@@ -168,8 +168,8 @@ dsl_prop_get_ds(dsl_dataset_t *ds, const char *propname,
 
 	ASSERT(dsl_pool_config_held(ds->ds_dir->dd_pool));
 	inheritable = (prop == ZPROP_INVAL || zfs_prop_inheritable(prop));
-	snapshot = (ds->ds_phys != NULL && dsl_dataset_is_snapshot(ds));
-	zapobj = (ds->ds_phys == NULL ? 0 : ds->ds_phys->ds_props_obj);
+	snapshot = dsl_dataset_is_snapshot(ds);
+	zapobj = ds->ds_phys->ds_props_obj;
 
 	if (zapobj != 0) {
 		objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset;
@@ -544,7 +544,7 @@ dsl_prop_set_sync_impl(dsl_dataset_t *ds, const char *propname,
 
 	isint = (dodefault(propname, 8, 1, &intval) == 0);
 
-	if (ds->ds_phys != NULL && dsl_dataset_is_snapshot(ds)) {
+	if (dsl_dataset_is_snapshot(ds)) {
 		ASSERT(version >= SPA_VERSION_SNAP_PROPS);
 		if (ds->ds_phys->ds_props_obj == 0) {
 			dmu_buf_will_dirty(ds->ds_dbuf, tx);
@@ -641,7 +641,7 @@ dsl_prop_set_sync_impl(dsl_dataset_t *ds, const char *propname,
 	if (isint) {
 		VERIFY0(dsl_prop_get_int_ds(ds, propname, &intval));
 
-		if (ds->ds_phys != NULL && dsl_dataset_is_snapshot(ds)) {
+		if (dsl_dataset_is_snapshot(ds)) {
 			dsl_prop_cb_record_t *cbr;
 			/*
 			 * It's a snapshot; nothing can inherit this

From de0d944949cdcb94754ae79ae04f3e262dc6200f Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 6 Dec 2014 00:33:09 +0000
Subject: [PATCH 059/380] 5350 clean up code in dnode_sync() Reviewed by: Alex
 Reece <alex.reece@delphix.com> Reviewed by: Christopher Siden
 <christopher.siden@delphix.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Reviewed by: Richard Elling
 <richard.elling@richardelling.com> Approved by: Dan McDonald
 <danmcd@omniti.com> Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@e6518318428d2be1962bf2d47fd83ebfe8cb2736
---
 uts/common/fs/zfs/dnode_sync.c | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/uts/common/fs/zfs/dnode_sync.c b/uts/common/fs/zfs/dnode_sync.c
index 2a57aee904fd..63bfc94f9aee 100644
--- a/uts/common/fs/zfs/dnode_sync.c
+++ b/uts/common/fs/zfs/dnode_sync.c
@@ -633,12 +633,11 @@ dnode_sync(dnode_t *dn, dmu_tx_t *tx)
 	    dn->dn_free_txg <= tx->tx_txg;
 
 	/*
-	 * We will either remove a spill block when a file is being removed
-	 * or we have been asked to remove it.
+	 * Remove the spill block if we have been explicitly asked to
+	 * remove it, or if the object is being removed.
 	 */
-	if (dn->dn_rm_spillblk[txgoff] ||
-	    ((dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) && freeing_dnode)) {
-		if ((dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR))
+	if (dn->dn_rm_spillblk[txgoff] || freeing_dnode) {
+		if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR)
 			kill_spill = B_TRUE;
 		dn->dn_rm_spillblk[txgoff] = 0;
 	}

From c3067c5bbbe8f88d09031a408fd737986999e872 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 6 Dec 2014 00:34:25 +0000
Subject: [PATCH 060/380] 5348 zio_checksum_error() only fills in info if
 ECKSUM Reviewed by: Alex Reece <alex.reece@delphix.com> Reviewed by:
 Christopher Siden <christopher.siden@delphix.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Reviewed by: Steven Hartland
 <killing@multiplay.co.uk> Approved by: Dan McDonald <danmcd@omniti.com>
 Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@373dc1cf9a4e0791397a9b268cdac1f664af58a8
---
 uts/common/fs/zfs/zio.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/uts/common/fs/zfs/zio.c b/uts/common/fs/zfs/zio.c
index 7cb6953845eb..583718c98c85 100644
--- a/uts/common/fs/zfs/zio.c
+++ b/uts/common/fs/zfs/zio.c
@@ -2817,7 +2817,8 @@ zio_checksum_verify(zio_t *zio)
 
 	if ((error = zio_checksum_error(zio, &info)) != 0) {
 		zio->io_error = error;
-		if (!(zio->io_flags & ZIO_FLAG_SPECULATIVE)) {
+		if (error == ECKSUM &&
+		    !(zio->io_flags & ZIO_FLAG_SPECULATIVE)) {
 			zfs_ereport_start_checksum(zio->io_spa,
 			    zio->io_vd, zio, zio->io_offset,
 			    zio->io_size, NULL, &info);

From e613515c26dc38900aa830718b7d7c3d8b49494a Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 6 Dec 2014 00:37:00 +0000
Subject: [PATCH 061/380] 5351 scrub goes for an extra second each txg 5352
 scrub should pause when there is some dirty data Reviewed by: Alex Reece
 <alex.reece@delphix.com> Reviewed by: Christopher Siden
 <christopher.siden@delphix.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Reviewed by: Richard Elling
 <richard.elling@richardelling.com> Approved by: Dan McDonald
 <danmcd@omniti.com> Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@6f6a76adacda33b10633476dc6c5d66d7f17dd94
---
 uts/common/fs/zfs/dsl_scan.c | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/uts/common/fs/zfs/dsl_scan.c b/uts/common/fs/zfs/dsl_scan.c
index 2392b7f33695..1f47589bfe0e 100644
--- a/uts/common/fs/zfs/dsl_scan.c
+++ b/uts/common/fs/zfs/dsl_scan.c
@@ -389,12 +389,11 @@ dsl_scan_sync_state(dsl_scan_t *scn, dmu_tx_t *tx)
 	    &scn->scn_phys, tx));
 }
 
+extern int zfs_vdev_async_write_active_min_dirty_percent;
+
 static boolean_t
 dsl_scan_check_pause(dsl_scan_t *scn, const zbookmark_phys_t *zb)
 {
-	uint64_t elapsed_nanosecs;
-	int mintime;
-
 	/* we never skip user/group accounting objects */
 	if (zb && (int64_t)zb->zb_object < 0)
 		return (B_FALSE);
@@ -409,12 +408,28 @@ dsl_scan_check_pause(dsl_scan_t *scn, const zbookmark_phys_t *zb)
 	if (zb && zb->zb_level != 0)
 		return (B_FALSE);
 
-	mintime = (scn->scn_phys.scn_func == POOL_SCAN_RESILVER) ?
+	/*
+	 * We pause if:
+	 *  - we have scanned for the maximum time: an entire txg
+	 *    timeout (default 5 sec)
+	 *  or
+	 *  - we have scanned for at least the minimum time (default 1 sec
+	 *    for scrub, 3 sec for resilver), and either we have sufficient
+	 *    dirty data that we are starting to write more quickly
+	 *    (default 30%), or someone is explicitly waiting for this txg
+	 *    to complete.
+	 *  or
+	 *  - the spa is shutting down because this pool is being exported
+	 *    or the machine is rebooting.
+	 */
+	int mintime = (scn->scn_phys.scn_func == POOL_SCAN_RESILVER) ?
 	    zfs_resilver_min_time_ms : zfs_scan_min_time_ms;
-	elapsed_nanosecs = gethrtime() - scn->scn_sync_start_time;
-	if (elapsed_nanosecs / NANOSEC > zfs_txg_timeout ||
+	uint64_t elapsed_nanosecs = gethrtime() - scn->scn_sync_start_time;
+	int dirty_pct = scn->scn_dp->dp_dirty_total * 100 / zfs_dirty_data_max;
+	if (elapsed_nanosecs / NANOSEC >= zfs_txg_timeout ||
 	    (NSEC2MSEC(elapsed_nanosecs) > mintime &&
-	    txg_sync_waiting(scn->scn_dp)) ||
+	    (txg_sync_waiting(scn->scn_dp) ||
+	    dirty_pct >= zfs_vdev_async_write_active_min_dirty_percent)) ||
 	    spa_shutting_down(scn->scn_dp->dp_spa)) {
 		if (zb) {
 			dprintf("pausing at bookmark %llx/%llx/%llx/%llx\n",

From da204d72009bb96b2f87f151b62850992d810d6e Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 6 Dec 2014 00:42:30 +0000
Subject: [PATCH 062/380] 5349 verify that block pointer is plausible before
 reading Reviewed by: Alex Reece <alex.reece@delphix.com> Reviewed by:
 Christopher Siden <christopher.siden@delphix.com> Reviewed by: Dan McDonald
 <danmcd@omniti.com> Reviewed by: George Wilson <george.wilson@delphix.com>
 Reviewed by: Richard Lowe <richlowe@richlowe.net> Reviewed by: Xin Li
 <delphij@FreeBSD.org> Reviewed by: Josef 'Jeff' Sipek
 <josef.sipek@nexenta.com> Approved by: Gordon Ross <gwr@nexenta.com> Author:
 Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@f63ab3d5a84a12b474655fc7e700db3efba6c4c9
---
 uts/common/fs/zfs/sys/spa.h |  1 +
 uts/common/fs/zfs/zio.c     | 90 +++++++++++++++++++++++++++++++++++--
 2 files changed, 87 insertions(+), 4 deletions(-)

diff --git a/uts/common/fs/zfs/sys/spa.h b/uts/common/fs/zfs/sys/spa.h
index e4731ae5a739..cf307a2ec977 100644
--- a/uts/common/fs/zfs/sys/spa.h
+++ b/uts/common/fs/zfs/sys/spa.h
@@ -791,6 +791,7 @@ extern boolean_t spa_is_root(spa_t *spa);
 extern boolean_t spa_writeable(spa_t *spa);
 extern boolean_t spa_has_pending_synctask(spa_t *spa);
 extern int spa_maxblocksize(spa_t *spa);
+extern void zfs_blkptr_verify(spa_t *spa, const blkptr_t *bp);
 
 extern int spa_mode(spa_t *spa);
 extern uint64_t strtonum(const char *str, char **nptr);
diff --git a/uts/common/fs/zfs/zio.c b/uts/common/fs/zfs/zio.c
index 583718c98c85..b9b31b375d3e 100644
--- a/uts/common/fs/zfs/zio.c
+++ b/uts/common/fs/zfs/zio.c
@@ -216,7 +216,7 @@ zio_buf_alloc(size_t size)
 {
 	size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;
 
-	ASSERT3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
+	VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
 
 	return (kmem_cache_alloc(zio_buf_cache[c], KM_PUSHPAGE));
 }
@@ -232,7 +232,7 @@ zio_data_buf_alloc(size_t size)
 {
 	size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;
 
-	ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
+	VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
 
 	return (kmem_cache_alloc(zio_data_buf_cache[c], KM_PUSHPAGE));
 }
@@ -242,7 +242,7 @@ zio_buf_free(void *buf, size_t size)
 {
 	size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;
 
-	ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
+	VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
 
 	kmem_cache_free(zio_buf_cache[c], buf);
 }
@@ -252,7 +252,7 @@ zio_data_buf_free(void *buf, size_t size)
 {
 	size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;
 
-	ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
+	VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
 
 	kmem_cache_free(zio_data_buf_cache[c], buf);
 }
@@ -596,6 +596,86 @@ zio_root(spa_t *spa, zio_done_func_t *done, void *private, enum zio_flag flags)
 	return (zio_null(NULL, spa, NULL, done, private, flags));
 }
 
+void
+zfs_blkptr_verify(spa_t *spa, const blkptr_t *bp)
+{
+	if (!DMU_OT_IS_VALID(BP_GET_TYPE(bp))) {
+		zfs_panic_recover("blkptr at %p has invalid TYPE %llu",
+		    bp, (longlong_t)BP_GET_TYPE(bp));
+	}
+	if (BP_GET_CHECKSUM(bp) >= ZIO_CHECKSUM_FUNCTIONS ||
+	    BP_GET_CHECKSUM(bp) <= ZIO_CHECKSUM_ON) {
+		zfs_panic_recover("blkptr at %p has invalid CHECKSUM %llu",
+		    bp, (longlong_t)BP_GET_CHECKSUM(bp));
+	}
+	if (BP_GET_COMPRESS(bp) >= ZIO_COMPRESS_FUNCTIONS ||
+	    BP_GET_COMPRESS(bp) <= ZIO_COMPRESS_ON) {
+		zfs_panic_recover("blkptr at %p has invalid COMPRESS %llu",
+		    bp, (longlong_t)BP_GET_COMPRESS(bp));
+	}
+	if (BP_GET_LSIZE(bp) > SPA_MAXBLOCKSIZE) {
+		zfs_panic_recover("blkptr at %p has invalid LSIZE %llu",
+		    bp, (longlong_t)BP_GET_LSIZE(bp));
+	}
+	if (BP_GET_PSIZE(bp) > SPA_MAXBLOCKSIZE) {
+		zfs_panic_recover("blkptr at %p has invalid PSIZE %llu",
+		    bp, (longlong_t)BP_GET_PSIZE(bp));
+	}
+
+	if (BP_IS_EMBEDDED(bp)) {
+		if (BPE_GET_ETYPE(bp) > NUM_BP_EMBEDDED_TYPES) {
+			zfs_panic_recover("blkptr at %p has invalid ETYPE %llu",
+			    bp, (longlong_t)BPE_GET_ETYPE(bp));
+		}
+	}
+
+	/*
+	 * Pool-specific checks.
+	 *
+	 * Note: it would be nice to verify that the blk_birth and
+	 * BP_PHYSICAL_BIRTH() are not too large.  However, spa_freeze()
+	 * allows the birth time of log blocks (and dmu_sync()-ed blocks
+	 * that are in the log) to be arbitrarily large.
+	 */
+	for (int i = 0; i < BP_GET_NDVAS(bp); i++) {
+		uint64_t vdevid = DVA_GET_VDEV(&bp->blk_dva[i]);
+		if (vdevid >= spa->spa_root_vdev->vdev_children) {
+			zfs_panic_recover("blkptr at %p DVA %u has invalid "
+			    "VDEV %llu",
+			    bp, i, (longlong_t)vdevid);
+		}
+		vdev_t *vd = spa->spa_root_vdev->vdev_child[vdevid];
+		if (vd == NULL) {
+			zfs_panic_recover("blkptr at %p DVA %u has invalid "
+			    "VDEV %llu",
+			    bp, i, (longlong_t)vdevid);
+		}
+		if (vd->vdev_ops == &vdev_hole_ops) {
+			zfs_panic_recover("blkptr at %p DVA %u has hole "
+			    "VDEV %llu",
+			    bp, i, (longlong_t)vdevid);
+
+		}
+		if (vd->vdev_ops == &vdev_missing_ops) {
+			/*
+			 * "missing" vdevs are valid during import, but we
+			 * don't have their detailed info (e.g. asize), so
+			 * we can't perform any more checks on them.
+			 */
+			continue;
+		}
+		uint64_t offset = DVA_GET_OFFSET(&bp->blk_dva[i]);
+		uint64_t asize = DVA_GET_ASIZE(&bp->blk_dva[i]);
+		if (BP_IS_GANG(bp))
+			asize = vdev_psize_to_asize(vd, SPA_GANGBLOCKSIZE);
+		if (offset + asize > vd->vdev_asize) {
+			zfs_panic_recover("blkptr at %p DVA %u has invalid "
+			    "OFFSET %llu",
+			    bp, i, (longlong_t)offset);
+		}
+	}
+}
+
 zio_t *
 zio_read(zio_t *pio, spa_t *spa, const blkptr_t *bp,
     void *data, uint64_t size, zio_done_func_t *done, void *private,
@@ -603,6 +683,8 @@ zio_read(zio_t *pio, spa_t *spa, const blkptr_t *bp,
 {
 	zio_t *zio;
 
+	zfs_blkptr_verify(spa, bp);
+
 	zio = zio_create(pio, spa, BP_PHYSICAL_BIRTH(bp), bp,
 	    data, size, done, private,
 	    ZIO_TYPE_READ, priority, flags, NULL, 0, zb,

From 20bd6b890faf51e512758f36471d5353fcaa27a4 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 6 Dec 2014 00:45:27 +0000
Subject: [PATCH 063/380] 5368 ARC should cache more metadata Reviewed by: Alex
 Reece <alex.reece@delphix.com> Reviewed by: Christopher Siden
 <christopher.siden@delphix.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Reviewed by: Richard Elling
 <richard.elling@richardelling.com> Approved by: Dan McDonald
 <danmcd@omniti.com> Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@3a5286a1cffceafcd8cf79c4156fad605129bf50
---
 uts/common/fs/zfs/arc.c | 57 +++++++++++++++++++++++++++++++++++++++--
 1 file changed, 55 insertions(+), 2 deletions(-)

diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index c113b53a0175..57171d7d21be 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -196,6 +196,7 @@ static boolean_t arc_warm;
 uint64_t zfs_arc_max;
 uint64_t zfs_arc_min;
 uint64_t zfs_arc_meta_limit = 0;
+uint64_t zfs_arc_meta_min = 0;
 int zfs_arc_grow_retry = 0;
 int zfs_arc_shrink_shift = 0;
 int zfs_arc_p_min_shift = 0;
@@ -324,6 +325,7 @@ typedef struct arc_stats {
 	kstat_named_t arcstat_meta_used;
 	kstat_named_t arcstat_meta_limit;
 	kstat_named_t arcstat_meta_max;
+	kstat_named_t arcstat_meta_min;
 } arc_stats_t;
 
 static arc_stats_t arc_stats = {
@@ -389,7 +391,8 @@ static arc_stats_t arc_stats = {
 	{ "duplicate_reads",		KSTAT_DATA_UINT64 },
 	{ "arc_meta_used",		KSTAT_DATA_UINT64 },
 	{ "arc_meta_limit",		KSTAT_DATA_UINT64 },
-	{ "arc_meta_max",		KSTAT_DATA_UINT64 }
+	{ "arc_meta_max",		KSTAT_DATA_UINT64 },
+	{ "arc_meta_min",		KSTAT_DATA_UINT64 }
 };
 
 #define	ARCSTAT(stat)	(arc_stats.stat.value.ui64)
@@ -452,6 +455,7 @@ static arc_state_t	*arc_l2c_only;
 #define	arc_c_min	ARCSTAT(arcstat_c_min)	/* min target cache size */
 #define	arc_c_max	ARCSTAT(arcstat_c_max)	/* max target cache size */
 #define	arc_meta_limit	ARCSTAT(arcstat_meta_limit) /* max size for metadata */
+#define	arc_meta_min	ARCSTAT(arcstat_meta_min) /* min size for metadata */
 #define	arc_meta_used	ARCSTAT(arcstat_meta_used) /* size of metadata */
 #define	arc_meta_max	ARCSTAT(arcstat_meta_max) /* max size of metadata */
 
@@ -1779,7 +1783,6 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 	arc_state_t *evicted_state;
 	uint64_t bytes_evicted = 0, skipped = 0, missed = 0;
 	arc_buf_hdr_t *ab, *ab_prev = NULL;
-	list_t *list = &state->arcs_list[type];
 	kmutex_t *hash_lock;
 	boolean_t have_lock;
 	void *stolen = NULL;
@@ -1793,6 +1796,50 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 	mutex_enter(&state->arcs_mtx);
 	mutex_enter(&evicted_state->arcs_mtx);
 
+	/*
+	 * Decide which "type" (data vs metadata) to recycle from.
+	 *
+	 * If we are over the metadata limit, recycle from metadata.
+	 * If we are under the metadata minimum, recycle from data.
+	 * Otherwise, recycle from whichever type has the oldest (least
+	 * recently accessed) header.
+	 */
+	if (recycle) {
+		arc_buf_hdr_t *data_hdr =
+		    list_tail(&state->arcs_list[ARC_BUFC_DATA]);
+		arc_buf_hdr_t *metadata_hdr =
+		    list_tail(&state->arcs_list[ARC_BUFC_METADATA]);
+		arc_buf_contents_t realtype;
+		if (data_hdr == NULL) {
+			realtype = ARC_BUFC_METADATA;
+		} else if (metadata_hdr == NULL) {
+			realtype = ARC_BUFC_DATA;
+		} else if (arc_meta_used >= arc_meta_limit) {
+			realtype = ARC_BUFC_METADATA;
+		} else if (arc_meta_used <= arc_meta_min) {
+			realtype = ARC_BUFC_DATA;
+		} else {
+			if (data_hdr->b_arc_access <
+			    metadata_hdr->b_arc_access) {
+				realtype = ARC_BUFC_DATA;
+			} else {
+				realtype = ARC_BUFC_METADATA;
+			}
+		}
+		if (realtype != type) {
+			/*
+			 * If we want to evict from a different list,
+			 * we can not recycle, because DATA vs METADATA
+			 * buffers are segregated into different kmem
+			 * caches (and vmem arenas).
+			 */
+			type = realtype;
+			recycle = B_FALSE;
+		}
+	}
+
+	list_t *list = &state->arcs_list[type];
+
 	for (ab = list_tail(list); ab; ab = ab_prev) {
 		ab_prev = list_prev(list, ab);
 		/* prefetch buffers have a minimum lifespan */
@@ -3755,6 +3802,12 @@ arc_init(void)
 	if (arc_c_min < arc_meta_limit / 2 && zfs_arc_min == 0)
 		arc_c_min = arc_meta_limit / 2;
 
+	if (zfs_arc_meta_min > 0) {
+		arc_meta_min = zfs_arc_meta_min;
+	} else {
+		arc_meta_min = arc_c_min / 2;
+	}
+
 	if (zfs_arc_grow_retry > 0)
 		arc_grow_retry = zfs_arc_grow_retry;
 

From 8aa8f11ed4b8fb0e9bc2e2ac0622b4870bbd8794 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 6 Dec 2014 00:47:31 +0000
Subject: [PATCH 064/380] 5347 idle pool may run itself out of space Reviewed
 by: Alex Reece <alex.reece@delphix.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Reviewed by: Steven Hartland
 <killing@multiplay.co.uk> Reviewed by: Richard Elling
 <richard.elling@richardelling.com> Approved by: Dan McDonald
 <danmcd@omniti.com> Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@231aab857f14a3e7a0ed5f2879399c3cd6ae92ea
---
 uts/common/fs/zfs/dsl_scan.c      | 19 +++++++-----
 uts/common/fs/zfs/spa.c           | 51 +++++++++++++++++++++----------
 uts/common/fs/zfs/sys/uberblock.h |  7 +++--
 uts/common/fs/zfs/uberblock.c     |  8 ++---
 4 files changed, 56 insertions(+), 29 deletions(-)

diff --git a/uts/common/fs/zfs/dsl_scan.c b/uts/common/fs/zfs/dsl_scan.c
index 1f47589bfe0e..9dacf88a4eb3 100644
--- a/uts/common/fs/zfs/dsl_scan.c
+++ b/uts/common/fs/zfs/dsl_scan.c
@@ -1449,13 +1449,6 @@ dsl_scan_sync(dsl_pool_t *dp, dmu_tx_t *tx)
 			    "traverse_dataset_destroyed()", err);
 		}
 
-		/*
-		 * If we didn't make progress, mark the async destroy as
-		 * stalled, so that we will not initiate a spa_sync() on
-		 * its behalf.
-		 */
-		scn->scn_async_stalled = (scn->scn_visited_this_txg == 0);
-
 		if (bptree_is_empty(dp->dp_meta_objset, dp->dp_bptree_obj)) {
 			/* finished; deactivate async destroy feature */
 			spa_feature_decr(spa, SPA_FEATURE_ASYNC_DESTROY, tx);
@@ -1468,6 +1461,18 @@ dsl_scan_sync(dsl_pool_t *dp, dmu_tx_t *tx)
 			    dp->dp_bptree_obj, tx));
 			dp->dp_bptree_obj = 0;
 			scn->scn_async_destroying = B_FALSE;
+			scn->scn_async_stalled = B_FALSE;
+		} else {
+			/*
+			 * If we didn't make progress, mark the async
+			 * destroy as stalled, so that we will not initiate
+			 * a spa_sync() on its behalf.  Note that we only
+			 * check this if we are not finished, because if the
+			 * bptree had no blocks for us to visit, we can
+			 * finish without "making progress".
+			 */
+			scn->scn_async_stalled =
+			    (scn->scn_visited_this_txg == 0);
 		}
 	}
 	if (scn->scn_visited_this_txg) {
diff --git a/uts/common/fs/zfs/spa.c b/uts/common/fs/zfs/spa.c
index 634967c46f1b..a61117812e66 100644
--- a/uts/common/fs/zfs/spa.c
+++ b/uts/common/fs/zfs/spa.c
@@ -6261,21 +6261,6 @@ spa_sync(spa_t *spa, uint64_t txg)
 		}
 	}
 
-	/*
-	 * If anything has changed in this txg, or if someone is waiting
-	 * for this txg to sync (eg, spa_vdev_remove()), push the
-	 * deferred frees from the previous txg.  If not, leave them
-	 * alone so that we don't generate work on an otherwise idle
-	 * system.
-	 */
-	if (!txg_list_empty(&dp->dp_dirty_datasets, txg) ||
-	    !txg_list_empty(&dp->dp_dirty_dirs, txg) ||
-	    !txg_list_empty(&dp->dp_sync_tasks, txg) ||
-	    ((dsl_scan_active(dp->dp_scan) ||
-	    txg_sync_waiting(dp)) && !spa_shutting_down(spa))) {
-		spa_sync_deferred_frees(spa, tx);
-	}
-
 	/*
 	 * Iterate to convergence.
 	 */
@@ -6293,6 +6278,11 @@ spa_sync(spa_t *spa, uint64_t txg)
 		if (pass < zfs_sync_pass_deferred_free) {
 			spa_sync_frees(spa, free_bpl, tx);
 		} else {
+			/*
+			 * We can not defer frees in pass 1, because
+			 * we sync the deferred frees later in pass 1.
+			 */
+			ASSERT3U(pass, >, 1);
 			bplist_iterate(free_bpl, bpobj_enqueue_cb,
 			    &spa->spa_deferred_bpobj, tx);
 		}
@@ -6303,8 +6293,37 @@ spa_sync(spa_t *spa, uint64_t txg)
 		while (vd = txg_list_remove(&spa->spa_vdev_txg_list, txg))
 			vdev_sync(vd, txg);
 
-		if (pass == 1)
+		if (pass == 1) {
 			spa_sync_upgrades(spa, tx);
+			ASSERT3U(txg, >=,
+			    spa->spa_uberblock.ub_rootbp.blk_birth);
+			/*
+			 * Note: We need to check if the MOS is dirty
+			 * because we could have marked the MOS dirty
+			 * without updating the uberblock (e.g. if we
+			 * have sync tasks but no dirty user data).  We
+			 * need to check the uberblock's rootbp because
+			 * it is updated if we have synced out dirty
+			 * data (though in this case the MOS will most
+			 * likely also be dirty due to second order
+			 * effects, we don't want to rely on that here).
+			 */
+			if (spa->spa_uberblock.ub_rootbp.blk_birth < txg &&
+			    !dmu_objset_is_dirty(mos, txg)) {
+				/*
+				 * Nothing changed on the first pass,
+				 * therefore this TXG is a no-op.  Avoid
+				 * syncing deferred frees, so that we
+				 * can keep this TXG as a no-op.
+				 */
+				ASSERT(txg_list_empty(&dp->dp_dirty_datasets,
+				    txg));
+				ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg));
+				ASSERT(txg_list_empty(&dp->dp_sync_tasks, txg));
+				break;
+			}
+			spa_sync_deferred_frees(spa, tx);
+		}
 
 	} while (dmu_objset_is_dirty(mos, txg));
 
diff --git a/uts/common/fs/zfs/sys/uberblock.h b/uts/common/fs/zfs/sys/uberblock.h
index b5bb91573145..21e7ae0de7a7 100644
--- a/uts/common/fs/zfs/sys/uberblock.h
+++ b/uts/common/fs/zfs/sys/uberblock.h
@@ -22,6 +22,9 @@
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright (c) 2014 by Delphix. All rights reserved.
+ */
 
 #ifndef _SYS_UBERBLOCK_H
 #define	_SYS_UBERBLOCK_H
@@ -36,8 +39,8 @@ extern "C" {
 
 typedef struct uberblock uberblock_t;
 
-extern int uberblock_verify(uberblock_t *ub);
-extern int uberblock_update(uberblock_t *ub, vdev_t *rvd, uint64_t txg);
+extern int uberblock_verify(uberblock_t *);
+extern boolean_t uberblock_update(uberblock_t *, vdev_t *, uint64_t);
 
 #ifdef	__cplusplus
 }
diff --git a/uts/common/fs/zfs/uberblock.c b/uts/common/fs/zfs/uberblock.c
index a07dc00ae19a..f8bdecdf5749 100644
--- a/uts/common/fs/zfs/uberblock.c
+++ b/uts/common/fs/zfs/uberblock.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -40,10 +40,10 @@ uberblock_verify(uberblock_t *ub)
 }
 
 /*
- * Update the uberblock and return a boolean value indicating whether
- * anything changed in this transaction group.
+ * Update the uberblock and return TRUE if anything changed in this
+ * transaction group.
  */
-int
+boolean_t
 uberblock_update(uberblock_t *ub, vdev_t *rvd, uint64_t txg)
 {
 	ASSERT(ub->ub_txg < txg);

From 0f4f297768e7accfbc28de32e7f620d9e883f2de Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 6 Dec 2014 00:50:57 +0000
Subject: [PATCH 065/380] 5314 Remove "dbuf phys" db->db_data pointer aliases
 in ZFS Reviewed by: Andriy Gapon <avg@freebsd.org> Reviewed by: Matthew
 Ahrens <mahrens@delphix.com> Reviewed by: Will Andrews
 <willa@spectralogic.com> Approved by: Dan McDonald <danmcd@omniti.com>
 Author: Justin T. Gibbs <justing@spectralogic.com>

illumos/illumos-gate@c1379625401dfbe1c39b79136dd384a571d47fde
---
 uts/common/fs/zfs/dbuf.c            |  32 +-
 uts/common/fs/zfs/dmu_diff.c        |   2 +-
 uts/common/fs/zfs/dmu_objset.c      |  30 +-
 uts/common/fs/zfs/dmu_send.c        |  86 +++--
 uts/common/fs/zfs/dmu_traverse.c    |   6 +-
 uts/common/fs/zfs/dmu_tx.c          |   4 +-
 uts/common/fs/zfs/dnode.c           |   2 +-
 uts/common/fs/zfs/dsl_bookmark.c    |  11 +-
 uts/common/fs/zfs/dsl_dataset.c     | 521 +++++++++++++++-------------
 uts/common/fs/zfs/dsl_deadlist.c    |   5 +-
 uts/common/fs/zfs/dsl_deleg.c       |  22 +-
 uts/common/fs/zfs/dsl_destroy.c     | 219 ++++++------
 uts/common/fs/zfs/dsl_dir.c         | 189 +++++-----
 uts/common/fs/zfs/dsl_pool.c        |  58 ++--
 uts/common/fs/zfs/dsl_prop.c        |  37 +-
 uts/common/fs/zfs/dsl_scan.c        |  98 +++---
 uts/common/fs/zfs/dsl_synctask.c    |   2 +-
 uts/common/fs/zfs/dsl_userhold.c    |  25 +-
 uts/common/fs/zfs/sa.c              |   7 +-
 uts/common/fs/zfs/spa.c             |   6 +-
 uts/common/fs/zfs/spa_history.c     |   2 +-
 uts/common/fs/zfs/sys/dbuf.h        |   1 -
 uts/common/fs/zfs/sys/dmu.h         |  13 +-
 uts/common/fs/zfs/sys/dsl_dataset.h |  20 +-
 uts/common/fs/zfs/sys/dsl_dir.h     |  11 +-
 uts/common/fs/zfs/sys/zap_impl.h    |  19 +-
 uts/common/fs/zfs/sys/zap_leaf.h    |   8 +-
 uts/common/fs/zfs/zap.c             | 158 +++++----
 uts/common/fs/zfs/zap_leaf.c        |  91 ++---
 uts/common/fs/zfs/zap_micro.c       |  26 +-
 uts/common/fs/zfs/zfs_ioctl.c       |   7 +-
 31 files changed, 912 insertions(+), 806 deletions(-)

diff --git a/uts/common/fs/zfs/dbuf.c b/uts/common/fs/zfs/dbuf.c
index a5816e12ccf0..bcf66bc53d7b 100644
--- a/uts/common/fs/zfs/dbuf.c
+++ b/uts/common/fs/zfs/dbuf.c
@@ -223,11 +223,8 @@ dbuf_evict_user(dmu_buf_impl_t *db)
 	if (db->db_level != 0 || db->db_evict_func == NULL)
 		return;
 
-	if (db->db_user_data_ptr_ptr)
-		*db->db_user_data_ptr_ptr = db->db.db_data;
 	db->db_evict_func(&db->db, db->db_user_ptr);
 	db->db_user_ptr = NULL;
-	db->db_user_data_ptr_ptr = NULL;
 	db->db_evict_func = NULL;
 }
 
@@ -417,16 +414,6 @@ dbuf_verify(dmu_buf_impl_t *db)
 }
 #endif
 
-static void
-dbuf_update_data(dmu_buf_impl_t *db)
-{
-	ASSERT(MUTEX_HELD(&db->db_mtx));
-	if (db->db_level == 0 && db->db_user_data_ptr_ptr) {
-		ASSERT(!refcount_is_zero(&db->db_holds));
-		*db->db_user_data_ptr_ptr = db->db.db_data;
-	}
-}
-
 static void
 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf)
 {
@@ -437,7 +424,6 @@ dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf)
 		db->db.db_data = buf->b_data;
 		if (!arc_released(buf))
 			arc_set_callback(buf, dbuf_do_evict, db);
-		dbuf_update_data(db);
 	} else {
 		dbuf_evict_user(db);
 		db->db.db_data = NULL;
@@ -543,7 +529,6 @@ dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags)
 		if (bonuslen)
 			bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen);
 		DB_DNODE_EXIT(db);
-		dbuf_update_data(db);
 		db->db_state = DB_CACHED;
 		mutex_exit(&db->db_mtx);
 		return;
@@ -1726,7 +1711,6 @@ dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid,
 	db->db_blkptr = blkptr;
 
 	db->db_user_ptr = NULL;
-	db->db_user_data_ptr_ptr = NULL;
 	db->db_evict_func = NULL;
 	db->db_immediate_evict = 0;
 	db->db_freed_in_flight = 0;
@@ -1971,7 +1955,6 @@ top:
 	}
 
 	(void) refcount_add(&db->db_holds, tag);
-	dbuf_update_data(db);
 	DBUF_VERIFY(db);
 	mutex_exit(&db->db_mtx);
 
@@ -2182,27 +2165,25 @@ dbuf_refcount(dmu_buf_impl_t *db)
 }
 
 void *
-dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr,
+dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr,
     dmu_buf_evict_func_t *evict_func)
 {
-	return (dmu_buf_update_user(db_fake, NULL, user_ptr,
-	    user_data_ptr_ptr, evict_func));
+	return (dmu_buf_update_user(db_fake, NULL, user_ptr, evict_func));
 }
 
 void *
-dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr,
+dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr,
     dmu_buf_evict_func_t *evict_func)
 {
 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
 
 	db->db_immediate_evict = TRUE;
-	return (dmu_buf_update_user(db_fake, NULL, user_ptr,
-	    user_data_ptr_ptr, evict_func));
+	return (dmu_buf_update_user(db_fake, NULL, user_ptr, evict_func));
 }
 
 void *
 dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr,
-    void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func)
+    dmu_buf_evict_func_t *evict_func)
 {
 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
 	ASSERT(db->db_level == 0);
@@ -2213,10 +2194,7 @@ dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr,
 
 	if (db->db_user_ptr == old_user_ptr) {
 		db->db_user_ptr = user_ptr;
-		db->db_user_data_ptr_ptr = user_data_ptr_ptr;
 		db->db_evict_func = evict_func;
-
-		dbuf_update_data(db);
 	} else {
 		old_user_ptr = db->db_user_ptr;
 	}
diff --git a/uts/common/fs/zfs/dmu_diff.c b/uts/common/fs/zfs/dmu_diff.c
index 30fabbb07957..32e451a77773 100644
--- a/uts/common/fs/zfs/dmu_diff.c
+++ b/uts/common/fs/zfs/dmu_diff.c
@@ -194,7 +194,7 @@ dmu_diff(const char *tosnap_name, const char *fromsnap_name,
 		return (SET_ERROR(EXDEV));
 	}
 
-	fromtxg = fromsnap->ds_phys->ds_creation_txg;
+	fromtxg = dsl_dataset_phys(fromsnap)->ds_creation_txg;
 	dsl_dataset_rele(fromsnap, FTAG);
 
 	dsl_dataset_long_hold(tosnap, FTAG);
diff --git a/uts/common/fs/zfs/dmu_objset.c b/uts/common/fs/zfs/dmu_objset.c
index e7aeed17fb9d..e39264cc2f93 100644
--- a/uts/common/fs/zfs/dmu_objset.c
+++ b/uts/common/fs/zfs/dmu_objset.c
@@ -1479,12 +1479,12 @@ dmu_snapshot_realname(objset_t *os, char *name, char *real, int maxlen,
 	dsl_dataset_t *ds = os->os_dsl_dataset;
 	uint64_t ignored;
 
-	if (ds->ds_phys->ds_snapnames_zapobj == 0)
+	if (dsl_dataset_phys(ds)->ds_snapnames_zapobj == 0)
 		return (SET_ERROR(ENOENT));
 
 	return (zap_lookup_norm(ds->ds_dir->dd_pool->dp_meta_objset,
-	    ds->ds_phys->ds_snapnames_zapobj, name, 8, 1, &ignored, MT_FIRST,
-	    real, maxlen, conflict));
+	    dsl_dataset_phys(ds)->ds_snapnames_zapobj, name, 8, 1, &ignored,
+	    MT_FIRST, real, maxlen, conflict));
 }
 
 int
@@ -1497,12 +1497,12 @@ dmu_snapshot_list_next(objset_t *os, int namelen, char *name,
 
 	ASSERT(dsl_pool_config_held(dmu_objset_pool(os)));
 
-	if (ds->ds_phys->ds_snapnames_zapobj == 0)
+	if (dsl_dataset_phys(ds)->ds_snapnames_zapobj == 0)
 		return (SET_ERROR(ENOENT));
 
 	zap_cursor_init_serialized(&cursor,
 	    ds->ds_dir->dd_pool->dp_meta_objset,
-	    ds->ds_phys->ds_snapnames_zapobj, *offp);
+	    dsl_dataset_phys(ds)->ds_snapnames_zapobj, *offp);
 
 	if (zap_cursor_retrieve(&cursor, &attr) != 0) {
 		zap_cursor_fini(&cursor);
@@ -1536,12 +1536,12 @@ dmu_dir_list_next(objset_t *os, int namelen, char *name,
 
 	/* there is no next dir on a snapshot! */
 	if (os->os_dsl_dataset->ds_object !=
-	    dd->dd_phys->dd_head_dataset_obj)
+	    dsl_dir_phys(dd)->dd_head_dataset_obj)
 		return (SET_ERROR(ENOENT));
 
 	zap_cursor_init_serialized(&cursor,
 	    dd->dd_pool->dp_meta_objset,
-	    dd->dd_phys->dd_child_dir_zapobj, *offp);
+	    dsl_dir_phys(dd)->dd_child_dir_zapobj, *offp);
 
 	if (zap_cursor_retrieve(&cursor, &attr) != 0) {
 		zap_cursor_fini(&cursor);
@@ -1589,7 +1589,7 @@ dmu_objset_find_dp(dsl_pool_t *dp, uint64_t ddobj,
 		return (0);
 	}
 
-	thisobj = dd->dd_phys->dd_head_dataset_obj;
+	thisobj = dsl_dir_phys(dd)->dd_head_dataset_obj;
 	attr = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP);
 
 	/*
@@ -1597,7 +1597,7 @@ dmu_objset_find_dp(dsl_pool_t *dp, uint64_t ddobj,
 	 */
 	if (flags & DS_FIND_CHILDREN) {
 		for (zap_cursor_init(&zc, dp->dp_meta_objset,
-		    dd->dd_phys->dd_child_dir_zapobj);
+		    dsl_dir_phys(dd)->dd_child_dir_zapobj);
 		    zap_cursor_retrieve(&zc, attr) == 0;
 		    (void) zap_cursor_advance(&zc)) {
 			ASSERT3U(attr->za_integer_length, ==,
@@ -1626,7 +1626,9 @@ dmu_objset_find_dp(dsl_pool_t *dp, uint64_t ddobj,
 		err = dsl_dataset_hold_obj(dp, thisobj, FTAG, &ds);
 
 		if (err == 0) {
-			uint64_t snapobj = ds->ds_phys->ds_snapnames_zapobj;
+			uint64_t snapobj;
+
+			snapobj = dsl_dataset_phys(ds)->ds_snapnames_zapobj;
 			dsl_dataset_rele(ds, FTAG);
 
 			for (zap_cursor_init(&zc, dp->dp_meta_objset, snapobj);
@@ -1701,7 +1703,7 @@ dmu_objset_find_impl(spa_t *spa, const char *name,
 		return (0);
 	}
 
-	thisobj = dd->dd_phys->dd_head_dataset_obj;
+	thisobj = dsl_dir_phys(dd)->dd_head_dataset_obj;
 	attr = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP);
 
 	/*
@@ -1709,7 +1711,7 @@ dmu_objset_find_impl(spa_t *spa, const char *name,
 	 */
 	if (flags & DS_FIND_CHILDREN) {
 		for (zap_cursor_init(&zc, dp->dp_meta_objset,
-		    dd->dd_phys->dd_child_dir_zapobj);
+		    dsl_dir_phys(dd)->dd_child_dir_zapobj);
 		    zap_cursor_retrieve(&zc, attr) == 0;
 		    (void) zap_cursor_advance(&zc)) {
 			ASSERT3U(attr->za_integer_length, ==,
@@ -1742,7 +1744,9 @@ dmu_objset_find_impl(spa_t *spa, const char *name,
 		err = dsl_dataset_hold_obj(dp, thisobj, FTAG, &ds);
 
 		if (err == 0) {
-			uint64_t snapobj = ds->ds_phys->ds_snapnames_zapobj;
+			uint64_t snapobj;
+
+			snapobj = dsl_dataset_phys(ds)->ds_snapnames_zapobj;
 			dsl_dataset_rele(ds, FTAG);
 
 			for (zap_cursor_init(&zc, dp->dp_meta_objset, snapobj);
diff --git a/uts/common/fs/zfs/dmu_send.c b/uts/common/fs/zfs/dmu_send.c
index 2c08e7075f66..dc9f81cd2f28 100644
--- a/uts/common/fs/zfs/dmu_send.c
+++ b/uts/common/fs/zfs/dmu_send.c
@@ -598,12 +598,12 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *ds,
 	    featureflags);
 
 	drr->drr_u.drr_begin.drr_creation_time =
-	    ds->ds_phys->ds_creation_time;
+	    dsl_dataset_phys(ds)->ds_creation_time;
 	drr->drr_u.drr_begin.drr_type = dmu_objset_type(os);
 	if (is_clone)
 		drr->drr_u.drr_begin.drr_flags |= DRR_FLAG_CLONE;
-	drr->drr_u.drr_begin.drr_toguid = ds->ds_phys->ds_guid;
-	if (ds->ds_phys->ds_flags & DS_FLAG_CI_DATASET)
+	drr->drr_u.drr_begin.drr_toguid = dsl_dataset_phys(ds)->ds_guid;
+	if (dsl_dataset_phys(ds)->ds_flags & DS_FLAG_CI_DATASET)
 		drr->drr_u.drr_begin.drr_flags |= DRR_FLAG_CI_DATA;
 
 	if (fromzb != NULL) {
@@ -624,7 +624,7 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *ds,
 	dsp->dsa_proc = curproc;
 	dsp->dsa_os = os;
 	dsp->dsa_off = off;
-	dsp->dsa_toguid = ds->ds_phys->ds_guid;
+	dsp->dsa_toguid = dsl_dataset_phys(ds)->ds_guid;
 	ZIO_SET_CHECKSUM(&dsp->dsa_zc, 0, 0, 0, 0);
 	dsp->dsa_pending_op = PENDING_NONE;
 	dsp->dsa_incremental = (fromzb != NULL);
@@ -710,9 +710,10 @@ dmu_send_obj(const char *pool, uint64_t tosnap, uint64_t fromsnap,
 		}
 		if (!dsl_dataset_is_before(ds, fromds, 0))
 			err = SET_ERROR(EXDEV);
-		zb.zbm_creation_time = fromds->ds_phys->ds_creation_time;
-		zb.zbm_creation_txg = fromds->ds_phys->ds_creation_txg;
-		zb.zbm_guid = fromds->ds_phys->ds_guid;
+		zb.zbm_creation_time =
+		    dsl_dataset_phys(fromds)->ds_creation_time;
+		zb.zbm_creation_txg = dsl_dataset_phys(fromds)->ds_creation_txg;
+		zb.zbm_guid = dsl_dataset_phys(fromds)->ds_guid;
 		is_clone = (fromds->ds_dir != ds->ds_dir);
 		dsl_dataset_rele(fromds, FTAG);
 		err = dmu_send_impl(FTAG, dp, ds, &zb, is_clone,
@@ -779,10 +780,10 @@ dmu_send(const char *tosnap, const char *fromsnap,
 				if (!dsl_dataset_is_before(ds, fromds, 0))
 					err = SET_ERROR(EXDEV);
 				zb.zbm_creation_time =
-				    fromds->ds_phys->ds_creation_time;
+				    dsl_dataset_phys(fromds)->ds_creation_time;
 				zb.zbm_creation_txg =
-				    fromds->ds_phys->ds_creation_txg;
-				zb.zbm_guid = fromds->ds_phys->ds_guid;
+				    dsl_dataset_phys(fromds)->ds_creation_txg;
+				zb.zbm_guid = dsl_dataset_phys(fromds)->ds_guid;
 				is_clone = (ds->ds_dir != fromds->ds_dir);
 				dsl_dataset_rele(fromds, FTAG);
 			}
@@ -829,7 +830,7 @@ dmu_send_estimate(dsl_dataset_t *ds, dsl_dataset_t *fromds, uint64_t *sizep)
 
 	/* Get uncompressed size estimate of changed data. */
 	if (fromds == NULL) {
-		size = ds->ds_phys->ds_uncompressed_bytes;
+		size = dsl_dataset_phys(ds)->ds_uncompressed_bytes;
 	} else {
 		uint64_t used, comp;
 		err = dsl_dataset_space_written(fromds, ds,
@@ -884,15 +885,15 @@ recv_begin_check_existing_impl(dmu_recv_begin_arg_t *drba, dsl_dataset_t *ds,
 
 	/* temporary clone name must not exist */
 	error = zap_lookup(dp->dp_meta_objset,
-	    ds->ds_dir->dd_phys->dd_child_dir_zapobj, recv_clone_name,
+	    dsl_dir_phys(ds->ds_dir)->dd_child_dir_zapobj, recv_clone_name,
 	    8, 1, &val);
 	if (error != ENOENT)
 		return (error == 0 ? EBUSY : error);
 
 	/* new snapshot name must not exist */
 	error = zap_lookup(dp->dp_meta_objset,
-	    ds->ds_phys->ds_snapnames_zapobj, drba->drba_cookie->drc_tosnap,
-	    8, 1, &val);
+	    dsl_dataset_phys(ds)->ds_snapnames_zapobj,
+	    drba->drba_cookie->drc_tosnap, 8, 1, &val);
 	if (error != ENOENT)
 		return (error == 0 ? EEXIST : error);
 
@@ -912,7 +913,7 @@ recv_begin_check_existing_impl(dmu_recv_begin_arg_t *drba, dsl_dataset_t *ds,
 
 	if (fromguid != 0) {
 		dsl_dataset_t *snap;
-		uint64_t obj = ds->ds_phys->ds_prev_snap_obj;
+		uint64_t obj = dsl_dataset_phys(ds)->ds_prev_snap_obj;
 
 		/* Find snapshot in this dir that matches fromguid. */
 		while (obj != 0) {
@@ -924,9 +925,9 @@ recv_begin_check_existing_impl(dmu_recv_begin_arg_t *drba, dsl_dataset_t *ds,
 				dsl_dataset_rele(snap, FTAG);
 				return (SET_ERROR(ENODEV));
 			}
-			if (snap->ds_phys->ds_guid == fromguid)
+			if (dsl_dataset_phys(snap)->ds_guid == fromguid)
 				break;
-			obj = snap->ds_phys->ds_prev_snap_obj;
+			obj = dsl_dataset_phys(snap)->ds_prev_snap_obj;
 			dsl_dataset_rele(snap, FTAG);
 		}
 		if (obj == 0)
@@ -949,9 +950,9 @@ recv_begin_check_existing_impl(dmu_recv_begin_arg_t *drba, dsl_dataset_t *ds,
 		dsl_dataset_rele(snap, FTAG);
 	} else {
 		/* if full, most recent snapshot must be $ORIGIN */
-		if (ds->ds_phys->ds_prev_snap_txg >= TXG_INITIAL)
+		if (dsl_dataset_phys(ds)->ds_prev_snap_txg >= TXG_INITIAL)
 			return (SET_ERROR(ENODEV));
-		drba->drba_snapobj = ds->ds_phys->ds_prev_snap_obj;
+		drba->drba_snapobj = dsl_dataset_phys(ds)->ds_prev_snap_obj;
 	}
 
 	return (0);
@@ -1069,7 +1070,7 @@ dmu_recv_begin_check(void *arg, dmu_tx_t *tx)
 				dsl_dataset_rele(ds, FTAG);
 				return (SET_ERROR(EINVAL));
 			}
-			if (origin->ds_phys->ds_guid != fromguid) {
+			if (dsl_dataset_phys(origin)->ds_guid != fromguid) {
 				dsl_dataset_rele(origin, FTAG);
 				dsl_dataset_rele(ds, FTAG);
 				return (SET_ERROR(ENODEV));
@@ -1140,7 +1141,7 @@ dmu_recv_begin_sync(void *arg, dmu_tx_t *tx)
 	}
 
 	dmu_buf_will_dirty(newds->ds_dbuf, tx);
-	newds->ds_phys->ds_flags |= DS_FLAG_INCONSISTENT;
+	dsl_dataset_phys(newds)->ds_flags |= DS_FLAG_INCONSISTENT;
 
 	/*
 	 * If we actually created a non-clone, we need to create the
@@ -1777,7 +1778,7 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, vnode_t *vp, offset_t *voffp,
 	 */
 	VERIFY0(dmu_objset_from_ds(drc->drc_ds, &os));
 
-	ASSERT(drc->drc_ds->ds_phys->ds_flags & DS_FLAG_INCONSISTENT);
+	ASSERT(dsl_dataset_phys(drc->drc_ds)->ds_flags & DS_FLAG_INCONSISTENT);
 
 	featureflags = DMU_GET_FEATUREFLAGS(drc->drc_drrb->drr_versioninfo);
 
@@ -1940,8 +1941,11 @@ dmu_recv_end_check(void *arg, dmu_tx_t *tx)
 			 * the snap before drc_ds, because drc_ds can not
 			 * have any snaps of its own).
 			 */
-			uint64_t obj = origin_head->ds_phys->ds_prev_snap_obj;
-			while (obj != drc->drc_ds->ds_phys->ds_prev_snap_obj) {
+			uint64_t obj;
+
+			obj = dsl_dataset_phys(origin_head)->ds_prev_snap_obj;
+			while (obj !=
+			    dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj) {
 				dsl_dataset_t *snap;
 				error = dsl_dataset_hold_obj(dp, obj, FTAG,
 				    &snap);
@@ -1953,7 +1957,7 @@ dmu_recv_end_check(void *arg, dmu_tx_t *tx)
 					error = dsl_destroy_snapshot_check_impl(
 					    snap, B_FALSE);
 				}
-				obj = snap->ds_phys->ds_prev_snap_obj;
+				obj = dsl_dataset_phys(snap)->ds_prev_snap_obj;
 				dsl_dataset_rele(snap, FTAG);
 				if (error != 0)
 					return (error);
@@ -1999,13 +2003,16 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx)
 			 * Destroy any snapshots of drc_tofs (origin_head)
 			 * after the origin (the snap before drc_ds).
 			 */
-			uint64_t obj = origin_head->ds_phys->ds_prev_snap_obj;
-			while (obj != drc->drc_ds->ds_phys->ds_prev_snap_obj) {
+			uint64_t obj;
+
+			obj = dsl_dataset_phys(origin_head)->ds_prev_snap_obj;
+			while (obj !=
+			    dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj) {
 				dsl_dataset_t *snap;
 				VERIFY0(dsl_dataset_hold_obj(dp, obj, FTAG,
 				    &snap));
 				ASSERT3P(snap->ds_dir, ==, origin_head->ds_dir);
-				obj = snap->ds_phys->ds_prev_snap_obj;
+				obj = dsl_dataset_phys(snap)->ds_prev_snap_obj;
 				dsl_destroy_snapshot_sync_impl(snap,
 				    B_FALSE, tx);
 				dsl_dataset_rele(snap, FTAG);
@@ -2021,15 +2028,16 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx)
 
 		/* set snapshot's creation time and guid */
 		dmu_buf_will_dirty(origin_head->ds_prev->ds_dbuf, tx);
-		origin_head->ds_prev->ds_phys->ds_creation_time =
+		dsl_dataset_phys(origin_head->ds_prev)->ds_creation_time =
 		    drc->drc_drrb->drr_creation_time;
-		origin_head->ds_prev->ds_phys->ds_guid =
+		dsl_dataset_phys(origin_head->ds_prev)->ds_guid =
 		    drc->drc_drrb->drr_toguid;
-		origin_head->ds_prev->ds_phys->ds_flags &=
+		dsl_dataset_phys(origin_head->ds_prev)->ds_flags &=
 		    ~DS_FLAG_INCONSISTENT;
 
 		dmu_buf_will_dirty(origin_head->ds_dbuf, tx);
-		origin_head->ds_phys->ds_flags &= ~DS_FLAG_INCONSISTENT;
+		dsl_dataset_phys(origin_head)->ds_flags &=
+		    ~DS_FLAG_INCONSISTENT;
 
 		dsl_dataset_rele(origin_head, FTAG);
 		dsl_destroy_head_sync_impl(drc->drc_ds, tx);
@@ -2043,15 +2051,17 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx)
 
 		/* set snapshot's creation time and guid */
 		dmu_buf_will_dirty(ds->ds_prev->ds_dbuf, tx);
-		ds->ds_prev->ds_phys->ds_creation_time =
+		dsl_dataset_phys(ds->ds_prev)->ds_creation_time =
 		    drc->drc_drrb->drr_creation_time;
-		ds->ds_prev->ds_phys->ds_guid = drc->drc_drrb->drr_toguid;
-		ds->ds_prev->ds_phys->ds_flags &= ~DS_FLAG_INCONSISTENT;
+		dsl_dataset_phys(ds->ds_prev)->ds_guid =
+		    drc->drc_drrb->drr_toguid;
+		dsl_dataset_phys(ds->ds_prev)->ds_flags &=
+		    ~DS_FLAG_INCONSISTENT;
 
 		dmu_buf_will_dirty(ds->ds_dbuf, tx);
-		ds->ds_phys->ds_flags &= ~DS_FLAG_INCONSISTENT;
+		dsl_dataset_phys(ds)->ds_flags &= ~DS_FLAG_INCONSISTENT;
 	}
-	drc->drc_newsnapobj = drc->drc_ds->ds_phys->ds_prev_snap_obj;
+	drc->drc_newsnapobj = dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj;
 	/*
 	 * Release the hold from dmu_recv_begin.  This must be done before
 	 * we return to open context, so that when we free the dataset's dnode,
@@ -2077,7 +2087,7 @@ add_ds_to_guidmap(const char *name, avl_tree_t *guid_map, uint64_t snapobj)
 	gmep = kmem_alloc(sizeof (*gmep), KM_SLEEP);
 	err = dsl_dataset_hold_obj(dp, snapobj, gmep, &snapds);
 	if (err == 0) {
-		gmep->guid = snapds->ds_phys->ds_guid;
+		gmep->guid = dsl_dataset_phys(snapds)->ds_guid;
 		gmep->gme_ds = snapds;
 		avl_add(guid_map, gmep);
 		dsl_dataset_long_hold(snapds, gmep);
diff --git a/uts/common/fs/zfs/dmu_traverse.c b/uts/common/fs/zfs/dmu_traverse.c
index 5523f6206fbc..bffdff65d627 100644
--- a/uts/common/fs/zfs/dmu_traverse.c
+++ b/uts/common/fs/zfs/dmu_traverse.c
@@ -579,7 +579,7 @@ traverse_dataset(dsl_dataset_t *ds, uint64_t txg_start, int flags,
     blkptr_cb_t func, void *arg)
 {
 	return (traverse_impl(ds->ds_dir->dd_pool->dp_spa, ds, ds->ds_object,
-	    &ds->ds_phys->ds_bp, txg_start, NULL, flags, func, arg));
+	    &dsl_dataset_phys(ds)->ds_bp, txg_start, NULL, flags, func, arg));
 }
 
 int
@@ -634,8 +634,8 @@ traverse_pool(spa_t *spa, uint64_t txg_start, int flags,
 					continue;
 				break;
 			}
-			if (ds->ds_phys->ds_prev_snap_txg > txg)
-				txg = ds->ds_phys->ds_prev_snap_txg;
+			if (dsl_dataset_phys(ds)->ds_prev_snap_txg > txg)
+				txg = dsl_dataset_phys(ds)->ds_prev_snap_txg;
 			err = traverse_dataset(ds, txg, flags, func, arg);
 			dsl_dataset_rele(ds, FTAG);
 			if (err != 0)
diff --git a/uts/common/fs/zfs/dmu_tx.c b/uts/common/fs/zfs/dmu_tx.c
index d249762e5a2c..56ce2f0c279f 100644
--- a/uts/common/fs/zfs/dmu_tx.c
+++ b/uts/common/fs/zfs/dmu_tx.c
@@ -713,6 +713,7 @@ dmu_tx_hold_zap(dmu_tx_t *tx, uint64_t object, int add, const char *name)
 {
 	dmu_tx_hold_t *txh;
 	dnode_t *dn;
+	dsl_dataset_phys_t *ds_phys;
 	uint64_t nblocks;
 	int epbs, err;
 
@@ -787,8 +788,9 @@ dmu_tx_hold_zap(dmu_tx_t *tx, uint64_t object, int add, const char *name)
 	 * we'll have to modify an indirect twig for each.
 	 */
 	epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
+	ds_phys = dsl_dataset_phys(dn->dn_objset->os_dsl_dataset);
 	for (nblocks = dn->dn_maxblkid >> epbs; nblocks != 0; nblocks >>= epbs)
-		if (dn->dn_objset->os_dsl_dataset->ds_phys->ds_prev_snap_obj)
+		if (ds_phys->ds_prev_snap_obj)
 			txh->txh_space_towrite += 3 << dn->dn_indblkshift;
 		else
 			txh->txh_space_tooverwrite += 3 << dn->dn_indblkshift;
diff --git a/uts/common/fs/zfs/dnode.c b/uts/common/fs/zfs/dnode.c
index 9c70c2bfb1b0..250dd9f9415e 100644
--- a/uts/common/fs/zfs/dnode.c
+++ b/uts/common/fs/zfs/dnode.c
@@ -1116,7 +1116,7 @@ dnode_hold_impl(objset_t *os, uint64_t object, int flag,
 			zrl_init(&dnh[i].dnh_zrlock);
 			dnh[i].dnh_dnode = NULL;
 		}
-		if (winner = dmu_buf_set_user(&db->db, children_dnodes, NULL,
+		if (winner = dmu_buf_set_user(&db->db, children_dnodes,
 		    dnode_buf_pageout)) {
 
 			for (i = 0; i < epb; i++) {
diff --git a/uts/common/fs/zfs/dsl_bookmark.c b/uts/common/fs/zfs/dsl_bookmark.c
index 21784630da6b..5fb7f96600dc 100644
--- a/uts/common/fs/zfs/dsl_bookmark.c
+++ b/uts/common/fs/zfs/dsl_bookmark.c
@@ -65,7 +65,7 @@ dsl_dataset_bmark_lookup(dsl_dataset_t *ds, const char *shortname,
 	if (bmark_zapobj == 0)
 		return (SET_ERROR(ESRCH));
 
-	if (ds->ds_phys->ds_flags & DS_FLAG_CI_DATASET)
+	if (dsl_dataset_phys(ds)->ds_flags & DS_FLAG_CI_DATASET)
 		mt = MT_FIRST;
 	else
 		mt = MT_EXACT;
@@ -208,10 +208,11 @@ dsl_bookmark_create_sync(void *arg, dmu_tx_t *tx)
 			    &bmark_fs->ds_bookmarks, tx));
 		}
 
-		bmark_phys.zbm_guid = snapds->ds_phys->ds_guid;
-		bmark_phys.zbm_creation_txg = snapds->ds_phys->ds_creation_txg;
+		bmark_phys.zbm_guid = dsl_dataset_phys(snapds)->ds_guid;
+		bmark_phys.zbm_creation_txg =
+		    dsl_dataset_phys(snapds)->ds_creation_txg;
 		bmark_phys.zbm_creation_time =
-		    snapds->ds_phys->ds_creation_time;
+		    dsl_dataset_phys(snapds)->ds_creation_time;
 
 		VERIFY0(zap_add(mos, bmark_fs->ds_bookmarks,
 		    shortname, sizeof (uint64_t),
@@ -340,7 +341,7 @@ dsl_dataset_bookmark_remove(dsl_dataset_t *ds, const char *name, dmu_tx_t *tx)
 	uint64_t bmark_zapobj = ds->ds_bookmarks;
 	matchtype_t mt;
 
-	if (ds->ds_phys->ds_flags & DS_FLAG_CI_DATASET)
+	if (dsl_dataset_phys(ds)->ds_flags & DS_FLAG_CI_DATASET)
 		mt = MT_FIRST;
 	else
 		mt = MT_EXACT;
diff --git a/uts/common/fs/zfs/dsl_dataset.c b/uts/common/fs/zfs/dsl_dataset.c
index 5ac0050aef42..5baf5c3c0a11 100644
--- a/uts/common/fs/zfs/dsl_dataset.c
+++ b/uts/common/fs/zfs/dsl_dataset.c
@@ -70,6 +70,9 @@ int zfs_max_recordsize = 1 * 1024 * 1024;
 
 #define	DS_REF_MAX	(1ULL << 62)
 
+extern inline dsl_dataset_phys_t *dsl_dataset_phys(dsl_dataset_t *ds);
+extern inline boolean_t dsl_dataset_is_snapshot(dsl_dataset_t *ds);
+
 /*
  * Figure out how much of this delta should be propogated to the dsl_dir
  * layer.  If there's a refreservation, that space has already been
@@ -78,13 +81,15 @@ int zfs_max_recordsize = 1 * 1024 * 1024;
 static int64_t
 parent_delta(dsl_dataset_t *ds, int64_t delta)
 {
+	dsl_dataset_phys_t *ds_phys;
 	uint64_t old_bytes, new_bytes;
 
 	if (ds->ds_reserved == 0)
 		return (delta);
 
-	old_bytes = MAX(ds->ds_phys->ds_unique_bytes, ds->ds_reserved);
-	new_bytes = MAX(ds->ds_phys->ds_unique_bytes + delta, ds->ds_reserved);
+	ds_phys = dsl_dataset_phys(ds);
+	old_bytes = MAX(ds_phys->ds_unique_bytes, ds->ds_reserved);
+	new_bytes = MAX(ds_phys->ds_unique_bytes + delta, ds->ds_reserved);
 
 	ASSERT3U(ABS((int64_t)(new_bytes - old_bytes)), <=, ABS(delta));
 	return (new_bytes - old_bytes);
@@ -115,10 +120,10 @@ dsl_dataset_block_born(dsl_dataset_t *ds, const blkptr_t *bp, dmu_tx_t *tx)
 	dmu_buf_will_dirty(ds->ds_dbuf, tx);
 	mutex_enter(&ds->ds_lock);
 	delta = parent_delta(ds, used);
-	ds->ds_phys->ds_referenced_bytes += used;
-	ds->ds_phys->ds_compressed_bytes += compressed;
-	ds->ds_phys->ds_uncompressed_bytes += uncompressed;
-	ds->ds_phys->ds_unique_bytes += used;
+	dsl_dataset_phys(ds)->ds_referenced_bytes += used;
+	dsl_dataset_phys(ds)->ds_compressed_bytes += compressed;
+	dsl_dataset_phys(ds)->ds_uncompressed_bytes += uncompressed;
+	dsl_dataset_phys(ds)->ds_unique_bytes += used;
 	if (BP_GET_LSIZE(bp) > SPA_OLD_MAXBLOCKSIZE)
 		ds->ds_need_large_blocks = B_TRUE;
 	mutex_exit(&ds->ds_lock);
@@ -153,17 +158,17 @@ dsl_dataset_block_kill(dsl_dataset_t *ds, const blkptr_t *bp, dmu_tx_t *tx,
 	ASSERT(!dsl_dataset_is_snapshot(ds));
 	dmu_buf_will_dirty(ds->ds_dbuf, tx);
 
-	if (bp->blk_birth > ds->ds_phys->ds_prev_snap_txg) {
+	if (bp->blk_birth > dsl_dataset_phys(ds)->ds_prev_snap_txg) {
 		int64_t delta;
 
 		dprintf_bp(bp, "freeing ds=%llu", ds->ds_object);
 		dsl_free(tx->tx_pool, tx->tx_txg, bp);
 
 		mutex_enter(&ds->ds_lock);
-		ASSERT(ds->ds_phys->ds_unique_bytes >= used ||
+		ASSERT(dsl_dataset_phys(ds)->ds_unique_bytes >= used ||
 		    !DS_UNIQUE_IS_ACCURATE(ds));
 		delta = parent_delta(ds, -used);
-		ds->ds_phys->ds_unique_bytes -= used;
+		dsl_dataset_phys(ds)->ds_unique_bytes -= used;
 		mutex_exit(&ds->ds_lock);
 		dsl_dir_diduse_space(ds->ds_dir, DD_USED_HEAD,
 		    delta, -compressed, -uncompressed, tx);
@@ -184,15 +189,15 @@ dsl_dataset_block_kill(dsl_dataset_t *ds, const blkptr_t *bp, dmu_tx_t *tx,
 			dsl_deadlist_insert(&ds->ds_deadlist, bp, tx);
 		}
 		ASSERT3U(ds->ds_prev->ds_object, ==,
-		    ds->ds_phys->ds_prev_snap_obj);
-		ASSERT(ds->ds_prev->ds_phys->ds_num_children > 0);
+		    dsl_dataset_phys(ds)->ds_prev_snap_obj);
+		ASSERT(dsl_dataset_phys(ds->ds_prev)->ds_num_children > 0);
 		/* if (bp->blk_birth > prev prev snap txg) prev unique += bs */
-		if (ds->ds_prev->ds_phys->ds_next_snap_obj ==
+		if (dsl_dataset_phys(ds->ds_prev)->ds_next_snap_obj ==
 		    ds->ds_object && bp->blk_birth >
-		    ds->ds_prev->ds_phys->ds_prev_snap_txg) {
+		    dsl_dataset_phys(ds->ds_prev)->ds_prev_snap_txg) {
 			dmu_buf_will_dirty(ds->ds_prev->ds_dbuf, tx);
 			mutex_enter(&ds->ds_prev->ds_lock);
-			ds->ds_prev->ds_phys->ds_unique_bytes += used;
+			dsl_dataset_phys(ds->ds_prev)->ds_unique_bytes += used;
 			mutex_exit(&ds->ds_prev->ds_lock);
 		}
 		if (bp->blk_birth > ds->ds_dir->dd_origin_txg) {
@@ -201,12 +206,12 @@ dsl_dataset_block_kill(dsl_dataset_t *ds, const blkptr_t *bp, dmu_tx_t *tx,
 		}
 	}
 	mutex_enter(&ds->ds_lock);
-	ASSERT3U(ds->ds_phys->ds_referenced_bytes, >=, used);
-	ds->ds_phys->ds_referenced_bytes -= used;
-	ASSERT3U(ds->ds_phys->ds_compressed_bytes, >=, compressed);
-	ds->ds_phys->ds_compressed_bytes -= compressed;
-	ASSERT3U(ds->ds_phys->ds_uncompressed_bytes, >=, uncompressed);
-	ds->ds_phys->ds_uncompressed_bytes -= uncompressed;
+	ASSERT3U(dsl_dataset_phys(ds)->ds_referenced_bytes, >=, used);
+	dsl_dataset_phys(ds)->ds_referenced_bytes -= used;
+	ASSERT3U(dsl_dataset_phys(ds)->ds_compressed_bytes, >=, compressed);
+	dsl_dataset_phys(ds)->ds_compressed_bytes -= compressed;
+	ASSERT3U(dsl_dataset_phys(ds)->ds_uncompressed_bytes, >=, uncompressed);
+	dsl_dataset_phys(ds)->ds_uncompressed_bytes -= uncompressed;
 	mutex_exit(&ds->ds_lock);
 
 	return (used);
@@ -232,7 +237,7 @@ dsl_dataset_prev_snap_txg(dsl_dataset_t *ds)
 	if (ds->ds_trysnap_txg >
 	    spa_last_synced_txg(ds->ds_dir->dd_pool->dp_spa))
 		trysnap = ds->ds_trysnap_txg;
-	return (MAX(ds->ds_phys->ds_prev_snap_txg, trysnap));
+	return (MAX(dsl_dataset_phys(ds)->ds_prev_snap_txg, trysnap));
 }
 
 boolean_t
@@ -267,7 +272,7 @@ dsl_dataset_evict(dmu_buf_t *db, void *dsv)
 	}
 
 	bplist_destroy(&ds->ds_pending_deadlist);
-	if (ds->ds_phys->ds_deadlist_obj != 0)
+	if (dsl_dataset_phys(ds)->ds_deadlist_obj != 0)
 		dsl_deadlist_close(&ds->ds_deadlist);
 	if (ds->ds_dir)
 		dsl_dir_rele(ds->ds_dir, ds);
@@ -293,10 +298,10 @@ dsl_dataset_get_snapname(dsl_dataset_t *ds)
 
 	if (ds->ds_snapname[0])
 		return (0);
-	if (ds->ds_phys->ds_next_snap_obj == 0)
+	if (dsl_dataset_phys(ds)->ds_next_snap_obj == 0)
 		return (0);
 
-	err = dmu_bonus_hold(mos, ds->ds_dir->dd_phys->dd_head_dataset_obj,
+	err = dmu_bonus_hold(mos, dsl_dir_phys(ds->ds_dir)->dd_head_dataset_obj,
 	    FTAG, &headdbuf);
 	if (err != 0)
 		return (err);
@@ -311,11 +316,11 @@ int
 dsl_dataset_snap_lookup(dsl_dataset_t *ds, const char *name, uint64_t *value)
 {
 	objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset;
-	uint64_t snapobj = ds->ds_phys->ds_snapnames_zapobj;
+	uint64_t snapobj = dsl_dataset_phys(ds)->ds_snapnames_zapobj;
 	matchtype_t mt;
 	int err;
 
-	if (ds->ds_phys->ds_flags & DS_FLAG_CI_DATASET)
+	if (dsl_dataset_phys(ds)->ds_flags & DS_FLAG_CI_DATASET)
 		mt = MT_FIRST;
 	else
 		mt = MT_EXACT;
@@ -332,13 +337,13 @@ dsl_dataset_snap_remove(dsl_dataset_t *ds, const char *name, dmu_tx_t *tx,
     boolean_t adj_cnt)
 {
 	objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset;
-	uint64_t snapobj = ds->ds_phys->ds_snapnames_zapobj;
+	uint64_t snapobj = dsl_dataset_phys(ds)->ds_snapnames_zapobj;
 	matchtype_t mt;
 	int err;
 
 	dsl_dir_snap_cmtime_update(ds->ds_dir);
 
-	if (ds->ds_phys->ds_flags & DS_FLAG_CI_DATASET)
+	if (dsl_dataset_phys(ds)->ds_flags & DS_FLAG_CI_DATASET)
 		mt = MT_FIRST;
 	else
 		mt = MT_EXACT;
@@ -384,7 +389,6 @@ dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
 		ds = kmem_zalloc(sizeof (dsl_dataset_t), KM_SLEEP);
 		ds->ds_dbuf = dbuf;
 		ds->ds_object = dsobj;
-		ds->ds_phys = dbuf->db_data;
 
 		mutex_init(&ds->ds_lock, NULL, MUTEX_DEFAULT, NULL);
 		mutex_init(&ds->ds_opening_lock, NULL, MUTEX_DEFAULT, NULL);
@@ -393,7 +397,7 @@ dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
 
 		bplist_create(&ds->ds_pending_deadlist);
 		dsl_deadlist_open(&ds->ds_deadlist,
-		    mos, ds->ds_phys->ds_deadlist_obj);
+		    mos, dsl_dataset_phys(ds)->ds_deadlist_obj);
 
 		list_create(&ds->ds_sendstreams, sizeof (dmu_sendarg_t),
 		    offsetof(dmu_sendarg_t, dsa_link));
@@ -408,7 +412,8 @@ dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
 
 		if (err == 0) {
 			err = dsl_dir_hold_obj(dp,
-			    ds->ds_phys->ds_dir_obj, NULL, ds, &ds->ds_dir);
+			    dsl_dataset_phys(ds)->ds_dir_obj, NULL, ds,
+			    &ds->ds_dir);
 		}
 		if (err != 0) {
 			mutex_destroy(&ds->ds_lock);
@@ -424,9 +429,9 @@ dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
 
 		if (!dsl_dataset_is_snapshot(ds)) {
 			ds->ds_snapname[0] = '\0';
-			if (ds->ds_phys->ds_prev_snap_obj != 0) {
+			if (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) {
 				err = dsl_dataset_hold_obj(dp,
-				    ds->ds_phys->ds_prev_snap_obj,
+				    dsl_dataset_phys(ds)->ds_prev_snap_obj,
 				    ds, &ds->ds_prev);
 			}
 			if (doi.doi_type == DMU_OTN_ZAP_METADATA) {
@@ -440,10 +445,11 @@ dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
 		} else {
 			if (zfs_flags & ZFS_DEBUG_SNAPNAMES)
 				err = dsl_dataset_get_snapname(ds);
-			if (err == 0 && ds->ds_phys->ds_userrefs_obj != 0) {
+			if (err == 0 &&
+			    dsl_dataset_phys(ds)->ds_userrefs_obj != 0) {
 				err = zap_count(
 				    ds->ds_dir->dd_pool->dp_meta_objset,
-				    ds->ds_phys->ds_userrefs_obj,
+				    dsl_dataset_phys(ds)->ds_userrefs_obj,
 				    &ds->ds_userrefs);
 			}
 		}
@@ -462,7 +468,7 @@ dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
 		}
 
 		if (err != 0 || (winner = dmu_buf_set_user_ie(dbuf, ds,
-		    &ds->ds_phys, dsl_dataset_evict)) != NULL) {
+		    dsl_dataset_evict)) != NULL) {
 			bplist_destroy(&ds->ds_pending_deadlist);
 			dsl_deadlist_close(&ds->ds_deadlist);
 			if (ds->ds_prev)
@@ -480,12 +486,12 @@ dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
 			ds = winner;
 		} else {
 			ds->ds_fsid_guid =
-			    unique_insert(ds->ds_phys->ds_fsid_guid);
+			    unique_insert(dsl_dataset_phys(ds)->ds_fsid_guid);
 		}
 	}
 	ASSERT3P(ds->ds_dbuf, ==, dbuf);
-	ASSERT3P(ds->ds_phys, ==, dbuf->db_data);
-	ASSERT(ds->ds_phys->ds_prev_snap_obj != 0 ||
+	ASSERT3P(dsl_dataset_phys(ds), ==, dbuf->db_data);
+	ASSERT(dsl_dataset_phys(ds)->ds_prev_snap_obj != 0 ||
 	    spa_version(dp->dp_spa) < SPA_VERSION_ORIGIN ||
 	    dp->dp_origin_snap == NULL || ds == dp->dp_origin_snap);
 	*dsp = ds;
@@ -506,7 +512,7 @@ dsl_dataset_hold(dsl_pool_t *dp, const char *name,
 		return (err);
 
 	ASSERT(dsl_pool_config_held(dp));
-	obj = dd->dd_phys->dd_head_dataset_obj;
+	obj = dsl_dir_phys(dd)->dd_head_dataset_obj;
 	if (obj != 0)
 		err = dsl_dataset_hold_obj(dp, obj, tag, dsp);
 	else
@@ -673,9 +679,9 @@ dsl_dataset_create_sync_dd(dsl_dir_t *dd, dsl_dataset_t *origin,
 		origin = dp->dp_origin_snap;
 
 	ASSERT(origin == NULL || origin->ds_dir->dd_pool == dp);
-	ASSERT(origin == NULL || origin->ds_phys->ds_num_children > 0);
+	ASSERT(origin == NULL || dsl_dataset_phys(origin)->ds_num_children > 0);
 	ASSERT(dmu_tx_is_syncing(tx));
-	ASSERT(dd->dd_phys->dd_head_dataset_obj == 0);
+	ASSERT(dsl_dir_phys(dd)->dd_head_dataset_obj == 0);
 
 	dsobj = dmu_object_alloc(mos, DMU_OT_DSL_DATASET, 0,
 	    DMU_OT_DSL_DATASET, sizeof (dsl_dataset_phys_t), tx);
@@ -701,55 +707,58 @@ dsl_dataset_create_sync_dd(dsl_dir_t *dd, dsl_dataset_t *origin,
 
 		dsphys->ds_prev_snap_obj = origin->ds_object;
 		dsphys->ds_prev_snap_txg =
-		    origin->ds_phys->ds_creation_txg;
+		    dsl_dataset_phys(origin)->ds_creation_txg;
 		dsphys->ds_referenced_bytes =
-		    origin->ds_phys->ds_referenced_bytes;
+		    dsl_dataset_phys(origin)->ds_referenced_bytes;
 		dsphys->ds_compressed_bytes =
-		    origin->ds_phys->ds_compressed_bytes;
+		    dsl_dataset_phys(origin)->ds_compressed_bytes;
 		dsphys->ds_uncompressed_bytes =
-		    origin->ds_phys->ds_uncompressed_bytes;
-		dsphys->ds_bp = origin->ds_phys->ds_bp;
+		    dsl_dataset_phys(origin)->ds_uncompressed_bytes;
+		dsphys->ds_bp = dsl_dataset_phys(origin)->ds_bp;
 
 		/*
 		 * Inherit flags that describe the dataset's contents
 		 * (INCONSISTENT) or properties (Case Insensitive).
 		 */
-		dsphys->ds_flags |= origin->ds_phys->ds_flags &
+		dsphys->ds_flags |= dsl_dataset_phys(origin)->ds_flags &
 		    (DS_FLAG_INCONSISTENT | DS_FLAG_CI_DATASET);
 
 		if (origin->ds_large_blocks)
 			dsl_dataset_activate_large_blocks_sync_impl(dsobj, tx);
 
 		dmu_buf_will_dirty(origin->ds_dbuf, tx);
-		origin->ds_phys->ds_num_children++;
+		dsl_dataset_phys(origin)->ds_num_children++;
 
 		VERIFY0(dsl_dataset_hold_obj(dp,
-		    origin->ds_dir->dd_phys->dd_head_dataset_obj, FTAG, &ohds));
+		    dsl_dir_phys(origin->ds_dir)->dd_head_dataset_obj,
+		    FTAG, &ohds));
 		dsphys->ds_deadlist_obj = dsl_deadlist_clone(&ohds->ds_deadlist,
 		    dsphys->ds_prev_snap_txg, dsphys->ds_prev_snap_obj, tx);
 		dsl_dataset_rele(ohds, FTAG);
 
 		if (spa_version(dp->dp_spa) >= SPA_VERSION_NEXT_CLONES) {
-			if (origin->ds_phys->ds_next_clones_obj == 0) {
-				origin->ds_phys->ds_next_clones_obj =
+			if (dsl_dataset_phys(origin)->ds_next_clones_obj == 0) {
+				dsl_dataset_phys(origin)->ds_next_clones_obj =
 				    zap_create(mos,
 				    DMU_OT_NEXT_CLONES, DMU_OT_NONE, 0, tx);
 			}
 			VERIFY0(zap_add_int(mos,
-			    origin->ds_phys->ds_next_clones_obj, dsobj, tx));
+			    dsl_dataset_phys(origin)->ds_next_clones_obj,
+			    dsobj, tx));
 		}
 
 		dmu_buf_will_dirty(dd->dd_dbuf, tx);
-		dd->dd_phys->dd_origin_obj = origin->ds_object;
+		dsl_dir_phys(dd)->dd_origin_obj = origin->ds_object;
 		if (spa_version(dp->dp_spa) >= SPA_VERSION_DIR_CLONES) {
-			if (origin->ds_dir->dd_phys->dd_clones == 0) {
+			if (dsl_dir_phys(origin->ds_dir)->dd_clones == 0) {
 				dmu_buf_will_dirty(origin->ds_dir->dd_dbuf, tx);
-				origin->ds_dir->dd_phys->dd_clones =
+				dsl_dir_phys(origin->ds_dir)->dd_clones =
 				    zap_create(mos,
 				    DMU_OT_DSL_CLONES, DMU_OT_NONE, 0, tx);
 			}
 			VERIFY0(zap_add_int(mos,
-			    origin->ds_dir->dd_phys->dd_clones, dsobj, tx));
+			    dsl_dir_phys(origin->ds_dir)->dd_clones,
+			    dsobj, tx));
 		}
 	}
 
@@ -759,7 +768,7 @@ dsl_dataset_create_sync_dd(dsl_dir_t *dd, dsl_dataset_t *origin,
 	dmu_buf_rele(dbuf, FTAG);
 
 	dmu_buf_will_dirty(dd->dd_dbuf, tx);
-	dd->dd_phys->dd_head_dataset_obj = dsobj;
+	dsl_dir_phys(dd)->dd_head_dataset_obj = dsobj;
 
 	return (dsobj);
 }
@@ -841,20 +850,20 @@ dsl_dataset_recalc_head_uniq(dsl_dataset_t *ds)
 
 	ASSERT(!dsl_dataset_is_snapshot(ds));
 
-	if (ds->ds_phys->ds_prev_snap_obj != 0)
-		mrs_used = ds->ds_prev->ds_phys->ds_referenced_bytes;
+	if (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0)
+		mrs_used = dsl_dataset_phys(ds->ds_prev)->ds_referenced_bytes;
 	else
 		mrs_used = 0;
 
 	dsl_deadlist_space(&ds->ds_deadlist, &dlused, &dlcomp, &dluncomp);
 
 	ASSERT3U(dlused, <=, mrs_used);
-	ds->ds_phys->ds_unique_bytes =
-	    ds->ds_phys->ds_referenced_bytes - (mrs_used - dlused);
+	dsl_dataset_phys(ds)->ds_unique_bytes =
+	    dsl_dataset_phys(ds)->ds_referenced_bytes - (mrs_used - dlused);
 
 	if (spa_version(ds->ds_dir->dd_pool->dp_spa) >=
 	    SPA_VERSION_UNIQUE_ACCURATE)
-		ds->ds_phys->ds_flags |= DS_FLAG_UNIQUE_ACCURATE;
+		dsl_dataset_phys(ds)->ds_flags |= DS_FLAG_UNIQUE_ACCURATE;
 }
 
 void
@@ -865,8 +874,9 @@ dsl_dataset_remove_from_next_clones(dsl_dataset_t *ds, uint64_t obj,
 	uint64_t count;
 	int err;
 
-	ASSERT(ds->ds_phys->ds_num_children >= 2);
-	err = zap_remove_int(mos, ds->ds_phys->ds_next_clones_obj, obj, tx);
+	ASSERT(dsl_dataset_phys(ds)->ds_num_children >= 2);
+	err = zap_remove_int(mos, dsl_dataset_phys(ds)->ds_next_clones_obj,
+	    obj, tx);
 	/*
 	 * The err should not be ENOENT, but a bug in a previous version
 	 * of the code could cause upgrade_clones_cb() to not set
@@ -879,16 +889,16 @@ dsl_dataset_remove_from_next_clones(dsl_dataset_t *ds, uint64_t obj,
 	 */
 	if (err != ENOENT)
 		VERIFY0(err);
-	ASSERT0(zap_count(mos, ds->ds_phys->ds_next_clones_obj,
+	ASSERT0(zap_count(mos, dsl_dataset_phys(ds)->ds_next_clones_obj,
 	    &count));
-	ASSERT3U(count, <=, ds->ds_phys->ds_num_children - 2);
+	ASSERT3U(count, <=, dsl_dataset_phys(ds)->ds_num_children - 2);
 }
 
 
 blkptr_t *
 dsl_dataset_get_blkptr(dsl_dataset_t *ds)
 {
-	return (&ds->ds_phys->ds_bp);
+	return (&dsl_dataset_phys(ds)->ds_bp);
 }
 
 void
@@ -900,7 +910,7 @@ dsl_dataset_set_blkptr(dsl_dataset_t *ds, blkptr_t *bp, dmu_tx_t *tx)
 		tx->tx_pool->dp_meta_rootbp = *bp;
 	} else {
 		dmu_buf_will_dirty(ds->ds_dbuf, tx);
-		ds->ds_phys->ds_bp = *bp;
+		dsl_dataset_phys(ds)->ds_bp = *bp;
 	}
 }
 
@@ -920,7 +930,7 @@ dsl_dataset_dirty(dsl_dataset_t *ds, dmu_tx_t *tx)
 
 	ASSERT(ds->ds_objset != NULL);
 
-	if (ds->ds_phys->ds_next_snap_obj != 0)
+	if (dsl_dataset_phys(ds)->ds_next_snap_obj != 0)
 		panic("dirtying snapshot!");
 
 	dp = ds->ds_dir->dd_pool;
@@ -956,7 +966,7 @@ dsl_dataset_snapshot_reserve_space(dsl_dataset_t *ds, dmu_tx_t *tx)
 	 * outside of the reservation.
 	 */
 	ASSERT(ds->ds_reserved == 0 || DS_UNIQUE_IS_ACCURATE(ds));
-	asize = MIN(ds->ds_phys->ds_unique_bytes, ds->ds_reserved);
+	asize = MIN(dsl_dataset_phys(ds)->ds_unique_bytes, ds->ds_reserved);
 	if (asize > dsl_dir_space_available(ds->ds_dir, NULL, 0, TRUE))
 		return (SET_ERROR(ENOSPC));
 
@@ -993,7 +1003,7 @@ dsl_dataset_snapshot_check_impl(dsl_dataset_t *ds, const char *snapname,
 	 * We don't allow multiple snapshots of the same txg.  If there
 	 * is already one, try again.
 	 */
-	if (ds->ds_phys->ds_prev_snap_txg >= tx->tx_txg)
+	if (dsl_dataset_phys(ds)->ds_prev_snap_txg >= tx->tx_txg)
 		return (SET_ERROR(EAGAIN));
 
 	/*
@@ -1219,35 +1229,38 @@ dsl_dataset_snapshot_sync_impl(dsl_dataset_t *ds, const char *snapname,
 	dsphys->ds_fsid_guid = unique_create();
 	(void) random_get_pseudo_bytes((void*)&dsphys->ds_guid,
 	    sizeof (dsphys->ds_guid));
-	dsphys->ds_prev_snap_obj = ds->ds_phys->ds_prev_snap_obj;
-	dsphys->ds_prev_snap_txg = ds->ds_phys->ds_prev_snap_txg;
+	dsphys->ds_prev_snap_obj = dsl_dataset_phys(ds)->ds_prev_snap_obj;
+	dsphys->ds_prev_snap_txg = dsl_dataset_phys(ds)->ds_prev_snap_txg;
 	dsphys->ds_next_snap_obj = ds->ds_object;
 	dsphys->ds_num_children = 1;
 	dsphys->ds_creation_time = gethrestime_sec();
 	dsphys->ds_creation_txg = crtxg;
-	dsphys->ds_deadlist_obj = ds->ds_phys->ds_deadlist_obj;
-	dsphys->ds_referenced_bytes = ds->ds_phys->ds_referenced_bytes;
-	dsphys->ds_compressed_bytes = ds->ds_phys->ds_compressed_bytes;
-	dsphys->ds_uncompressed_bytes = ds->ds_phys->ds_uncompressed_bytes;
-	dsphys->ds_flags = ds->ds_phys->ds_flags;
-	dsphys->ds_bp = ds->ds_phys->ds_bp;
+	dsphys->ds_deadlist_obj = dsl_dataset_phys(ds)->ds_deadlist_obj;
+	dsphys->ds_referenced_bytes = dsl_dataset_phys(ds)->ds_referenced_bytes;
+	dsphys->ds_compressed_bytes = dsl_dataset_phys(ds)->ds_compressed_bytes;
+	dsphys->ds_uncompressed_bytes =
+	    dsl_dataset_phys(ds)->ds_uncompressed_bytes;
+	dsphys->ds_flags = dsl_dataset_phys(ds)->ds_flags;
+	dsphys->ds_bp = dsl_dataset_phys(ds)->ds_bp;
 	dmu_buf_rele(dbuf, FTAG);
 
 	if (ds->ds_large_blocks)
 		dsl_dataset_activate_large_blocks_sync_impl(dsobj, tx);
 
-	ASSERT3U(ds->ds_prev != 0, ==, ds->ds_phys->ds_prev_snap_obj != 0);
+	ASSERT3U(ds->ds_prev != 0, ==,
+	    dsl_dataset_phys(ds)->ds_prev_snap_obj != 0);
 	if (ds->ds_prev) {
 		uint64_t next_clones_obj =
-		    ds->ds_prev->ds_phys->ds_next_clones_obj;
-		ASSERT(ds->ds_prev->ds_phys->ds_next_snap_obj ==
+		    dsl_dataset_phys(ds->ds_prev)->ds_next_clones_obj;
+		ASSERT(dsl_dataset_phys(ds->ds_prev)->ds_next_snap_obj ==
 		    ds->ds_object ||
-		    ds->ds_prev->ds_phys->ds_num_children > 1);
-		if (ds->ds_prev->ds_phys->ds_next_snap_obj == ds->ds_object) {
+		    dsl_dataset_phys(ds->ds_prev)->ds_num_children > 1);
+		if (dsl_dataset_phys(ds->ds_prev)->ds_next_snap_obj ==
+		    ds->ds_object) {
 			dmu_buf_will_dirty(ds->ds_prev->ds_dbuf, tx);
-			ASSERT3U(ds->ds_phys->ds_prev_snap_txg, ==,
-			    ds->ds_prev->ds_phys->ds_creation_txg);
-			ds->ds_prev->ds_phys->ds_next_snap_obj = dsobj;
+			ASSERT3U(dsl_dataset_phys(ds)->ds_prev_snap_txg, ==,
+			    dsl_dataset_phys(ds->ds_prev)->ds_creation_txg);
+			dsl_dataset_phys(ds->ds_prev)->ds_next_snap_obj = dsobj;
 		} else if (next_clones_obj != 0) {
 			dsl_dataset_remove_from_next_clones(ds->ds_prev,
 			    dsphys->ds_next_snap_obj, tx);
@@ -1264,33 +1277,36 @@ dsl_dataset_snapshot_sync_impl(dsl_dataset_t *ds, const char *snapname,
 	if (ds->ds_reserved) {
 		int64_t delta;
 		ASSERT(DS_UNIQUE_IS_ACCURATE(ds));
-		delta = MIN(ds->ds_phys->ds_unique_bytes, ds->ds_reserved);
+		delta = MIN(dsl_dataset_phys(ds)->ds_unique_bytes,
+		    ds->ds_reserved);
 		dsl_dir_diduse_space(ds->ds_dir, DD_USED_REFRSRV,
 		    delta, 0, 0, tx);
 	}
 
 	dmu_buf_will_dirty(ds->ds_dbuf, tx);
-	ds->ds_phys->ds_deadlist_obj = dsl_deadlist_clone(&ds->ds_deadlist,
-	    UINT64_MAX, ds->ds_phys->ds_prev_snap_obj, tx);
+	dsl_dataset_phys(ds)->ds_deadlist_obj =
+	    dsl_deadlist_clone(&ds->ds_deadlist, UINT64_MAX,
+	    dsl_dataset_phys(ds)->ds_prev_snap_obj, tx);
 	dsl_deadlist_close(&ds->ds_deadlist);
-	dsl_deadlist_open(&ds->ds_deadlist, mos, ds->ds_phys->ds_deadlist_obj);
+	dsl_deadlist_open(&ds->ds_deadlist, mos,
+	    dsl_dataset_phys(ds)->ds_deadlist_obj);
 	dsl_deadlist_add_key(&ds->ds_deadlist,
-	    ds->ds_phys->ds_prev_snap_txg, tx);
+	    dsl_dataset_phys(ds)->ds_prev_snap_txg, tx);
 
-	ASSERT3U(ds->ds_phys->ds_prev_snap_txg, <, tx->tx_txg);
-	ds->ds_phys->ds_prev_snap_obj = dsobj;
-	ds->ds_phys->ds_prev_snap_txg = crtxg;
-	ds->ds_phys->ds_unique_bytes = 0;
+	ASSERT3U(dsl_dataset_phys(ds)->ds_prev_snap_txg, <, tx->tx_txg);
+	dsl_dataset_phys(ds)->ds_prev_snap_obj = dsobj;
+	dsl_dataset_phys(ds)->ds_prev_snap_txg = crtxg;
+	dsl_dataset_phys(ds)->ds_unique_bytes = 0;
 	if (spa_version(dp->dp_spa) >= SPA_VERSION_UNIQUE_ACCURATE)
-		ds->ds_phys->ds_flags |= DS_FLAG_UNIQUE_ACCURATE;
+		dsl_dataset_phys(ds)->ds_flags |= DS_FLAG_UNIQUE_ACCURATE;
 
-	VERIFY0(zap_add(mos, ds->ds_phys->ds_snapnames_zapobj,
+	VERIFY0(zap_add(mos, dsl_dataset_phys(ds)->ds_snapnames_zapobj,
 	    snapname, 8, 1, &dsobj, tx));
 
 	if (ds->ds_prev)
 		dsl_dataset_rele(ds->ds_prev, ds);
 	VERIFY0(dsl_dataset_hold_obj(dp,
-	    ds->ds_phys->ds_prev_snap_obj, ds, &ds->ds_prev));
+	    dsl_dataset_phys(ds)->ds_prev_snap_obj, ds, &ds->ds_prev));
 
 	dsl_scan_ds_snapshotted(ds, tx);
 
@@ -1499,14 +1515,14 @@ dsl_dataset_sync(dsl_dataset_t *ds, zio_t *zio, dmu_tx_t *tx)
 {
 	ASSERT(dmu_tx_is_syncing(tx));
 	ASSERT(ds->ds_objset != NULL);
-	ASSERT(ds->ds_phys->ds_next_snap_obj == 0);
+	ASSERT(dsl_dataset_phys(ds)->ds_next_snap_obj == 0);
 
 	/*
 	 * in case we had to change ds_fsid_guid when we opened it,
 	 * sync it out now.
 	 */
 	dmu_buf_will_dirty(ds->ds_dbuf, tx);
-	ds->ds_phys->ds_fsid_guid = ds->ds_fsid_guid;
+	dsl_dataset_phys(ds)->ds_fsid_guid = ds->ds_fsid_guid;
 
 	dmu_objset_sync(ds->ds_objset, zio, tx);
 
@@ -1533,13 +1549,14 @@ get_clones_stat(dsl_dataset_t *ds, nvlist_t *nv)
 	 * due to a bug in a previous version of the code.
 	 * Only trust it if it has the right number of entries.
 	 */
-	if (ds->ds_phys->ds_next_clones_obj != 0) {
-		VERIFY0(zap_count(mos, ds->ds_phys->ds_next_clones_obj,
+	if (dsl_dataset_phys(ds)->ds_next_clones_obj != 0) {
+		VERIFY0(zap_count(mos, dsl_dataset_phys(ds)->ds_next_clones_obj,
 		    &count));
 	}
-	if (count != ds->ds_phys->ds_num_children - 1)
+	if (count != dsl_dataset_phys(ds)->ds_num_children - 1)
 		goto fail;
-	for (zap_cursor_init(&zc, mos, ds->ds_phys->ds_next_clones_obj);
+	for (zap_cursor_init(&zc, mos,
+	    dsl_dataset_phys(ds)->ds_next_clones_obj);
 	    zap_cursor_retrieve(&zc, &za) == 0;
 	    zap_cursor_advance(&zc)) {
 		dsl_dataset_t *clone;
@@ -1566,18 +1583,18 @@ dsl_dataset_stats(dsl_dataset_t *ds, nvlist_t *nv)
 
 	ASSERT(dsl_pool_config_held(dp));
 
-	ratio = ds->ds_phys->ds_compressed_bytes == 0 ? 100 :
-	    (ds->ds_phys->ds_uncompressed_bytes * 100 /
-	    ds->ds_phys->ds_compressed_bytes);
+	ratio = dsl_dataset_phys(ds)->ds_compressed_bytes == 0 ? 100 :
+	    (dsl_dataset_phys(ds)->ds_uncompressed_bytes * 100 /
+	    dsl_dataset_phys(ds)->ds_compressed_bytes);
 
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_REFRATIO, ratio);
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_LOGICALREFERENCED,
-	    ds->ds_phys->ds_uncompressed_bytes);
+	    dsl_dataset_phys(ds)->ds_uncompressed_bytes);
 
 	if (dsl_dataset_is_snapshot(ds)) {
 		dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_COMPRESSRATIO, ratio);
 		dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_USED,
-		    ds->ds_phys->ds_unique_bytes);
+		    dsl_dataset_phys(ds)->ds_unique_bytes);
 		get_clones_stat(ds, nv);
 	} else {
 		if (ds->ds_prev != NULL && ds->ds_prev != dp->dp_origin_snap) {
@@ -1594,17 +1611,17 @@ dsl_dataset_stats(dsl_dataset_t *ds, nvlist_t *nv)
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_REFERENCED, refd);
 
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_CREATION,
-	    ds->ds_phys->ds_creation_time);
+	    dsl_dataset_phys(ds)->ds_creation_time);
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_CREATETXG,
-	    ds->ds_phys->ds_creation_txg);
+	    dsl_dataset_phys(ds)->ds_creation_txg);
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_REFQUOTA,
 	    ds->ds_quota);
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_REFRESERVATION,
 	    ds->ds_reserved);
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_GUID,
-	    ds->ds_phys->ds_guid);
+	    dsl_dataset_phys(ds)->ds_guid);
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_UNIQUE,
-	    ds->ds_phys->ds_unique_bytes);
+	    dsl_dataset_phys(ds)->ds_unique_bytes);
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_OBJSETID,
 	    ds->ds_object);
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_USERREFS,
@@ -1612,13 +1629,13 @@ dsl_dataset_stats(dsl_dataset_t *ds, nvlist_t *nv)
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_DEFER_DESTROY,
 	    DS_IS_DEFER_DESTROY(ds) ? 1 : 0);
 
-	if (ds->ds_phys->ds_prev_snap_obj != 0) {
+	if (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) {
 		uint64_t written, comp, uncomp;
 		dsl_pool_t *dp = ds->ds_dir->dd_pool;
 		dsl_dataset_t *prev;
 
 		int err = dsl_dataset_hold_obj(dp,
-		    ds->ds_phys->ds_prev_snap_obj, FTAG, &prev);
+		    dsl_dataset_phys(ds)->ds_prev_snap_obj, FTAG, &prev);
 		if (err == 0) {
 			err = dsl_dataset_space_written(prev, ds, &written,
 			    &comp, &uncomp);
@@ -1637,13 +1654,15 @@ dsl_dataset_fast_stat(dsl_dataset_t *ds, dmu_objset_stats_t *stat)
 	dsl_pool_t *dp = ds->ds_dir->dd_pool;
 	ASSERT(dsl_pool_config_held(dp));
 
-	stat->dds_creation_txg = ds->ds_phys->ds_creation_txg;
-	stat->dds_inconsistent = ds->ds_phys->ds_flags & DS_FLAG_INCONSISTENT;
-	stat->dds_guid = ds->ds_phys->ds_guid;
+	stat->dds_creation_txg = dsl_dataset_phys(ds)->ds_creation_txg;
+	stat->dds_inconsistent =
+	    dsl_dataset_phys(ds)->ds_flags & DS_FLAG_INCONSISTENT;
+	stat->dds_guid = dsl_dataset_phys(ds)->ds_guid;
 	stat->dds_origin[0] = '\0';
 	if (dsl_dataset_is_snapshot(ds)) {
 		stat->dds_is_snapshot = B_TRUE;
-		stat->dds_num_clones = ds->ds_phys->ds_num_children - 1;
+		stat->dds_num_clones =
+		    dsl_dataset_phys(ds)->ds_num_children - 1;
 	} else {
 		stat->dds_is_snapshot = B_FALSE;
 		stat->dds_num_clones = 0;
@@ -1652,7 +1671,8 @@ dsl_dataset_fast_stat(dsl_dataset_t *ds, dmu_objset_stats_t *stat)
 			dsl_dataset_t *ods;
 
 			VERIFY0(dsl_dataset_hold_obj(dp,
-			    ds->ds_dir->dd_phys->dd_origin_obj, FTAG, &ods));
+			    dsl_dir_phys(ds->ds_dir)->dd_origin_obj,
+			    FTAG, &ods));
 			dsl_dataset_name(ods, stat->dds_origin);
 			dsl_dataset_rele(ods, FTAG);
 		}
@@ -1670,10 +1690,11 @@ dsl_dataset_space(dsl_dataset_t *ds,
     uint64_t *refdbytesp, uint64_t *availbytesp,
     uint64_t *usedobjsp, uint64_t *availobjsp)
 {
-	*refdbytesp = ds->ds_phys->ds_referenced_bytes;
+	*refdbytesp = dsl_dataset_phys(ds)->ds_referenced_bytes;
 	*availbytesp = dsl_dir_space_available(ds->ds_dir, NULL, 0, TRUE);
-	if (ds->ds_reserved > ds->ds_phys->ds_unique_bytes)
-		*availbytesp += ds->ds_reserved - ds->ds_phys->ds_unique_bytes;
+	if (ds->ds_reserved > dsl_dataset_phys(ds)->ds_unique_bytes)
+		*availbytesp +=
+		    ds->ds_reserved - dsl_dataset_phys(ds)->ds_unique_bytes;
 	if (ds->ds_quota != 0) {
 		/*
 		 * Adjust available bytes according to refquota
@@ -1684,7 +1705,7 @@ dsl_dataset_space(dsl_dataset_t *ds,
 		else
 			*availbytesp = 0;
 	}
-	*usedobjsp = BP_GET_FILL(&ds->ds_phys->ds_bp);
+	*usedobjsp = BP_GET_FILL(&dsl_dataset_phys(ds)->ds_bp);
 	*availobjsp = DN_MAX_OBJECT - *usedobjsp;
 }
 
@@ -1696,8 +1717,8 @@ dsl_dataset_modified_since_snap(dsl_dataset_t *ds, dsl_dataset_t *snap)
 	ASSERT(dsl_pool_config_held(dp));
 	if (snap == NULL)
 		return (B_FALSE);
-	if (ds->ds_phys->ds_bp.blk_birth >
-	    snap->ds_phys->ds_creation_txg) {
+	if (dsl_dataset_phys(ds)->ds_bp.blk_birth >
+	    dsl_dataset_phys(snap)->ds_creation_txg) {
 		objset_t *os, *os_snap;
 		/*
 		 * It may be that only the ZIL differs, because it was
@@ -1804,7 +1825,8 @@ dsl_dataset_rename_snapshot_sync_impl(dsl_pool_t *dp,
 	mutex_enter(&ds->ds_lock);
 	(void) strcpy(ds->ds_snapname, ddrsa->ddrsa_newsnapname);
 	mutex_exit(&ds->ds_lock);
-	VERIFY0(zap_add(dp->dp_meta_objset, hds->ds_phys->ds_snapnames_zapobj,
+	VERIFY0(zap_add(dp->dp_meta_objset,
+	    dsl_dataset_phys(hds)->ds_snapnames_zapobj,
 	    ds->ds_snapname, 8, 1, &ds->ds_object, tx));
 
 	dsl_dataset_rele(ds, FTAG);
@@ -1903,7 +1925,7 @@ dsl_dataset_rollback_check(void *arg, dmu_tx_t *tx)
 	}
 
 	/* must have a most recent snapshot */
-	if (ds->ds_phys->ds_prev_snap_txg < TXG_INITIAL) {
+	if (dsl_dataset_phys(ds)->ds_prev_snap_txg < TXG_INITIAL) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(EINVAL));
 	}
@@ -1922,7 +1944,7 @@ dsl_dataset_rollback_check(void *arg, dmu_tx_t *tx)
 		    fnvlist_lookup_nvlist(fnvpair_value_nvlist(pair),
 		    zfs_prop_to_name(ZFS_PROP_CREATETXG));
 		uint64_t createtxg = fnvlist_lookup_uint64(valuenv, "value");
-		if (createtxg > ds->ds_phys->ds_prev_snap_txg) {
+		if (createtxg > dsl_dataset_phys(ds)->ds_prev_snap_txg) {
 			fnvlist_free(bookmarks);
 			dsl_dataset_rele(ds, FTAG);
 			return (SET_ERROR(EEXIST));
@@ -1941,7 +1963,7 @@ dsl_dataset_rollback_check(void *arg, dmu_tx_t *tx)
 	 * the refquota.
 	 */
 	if (ds->ds_quota != 0 &&
-	    ds->ds_prev->ds_phys->ds_referenced_bytes > ds->ds_quota) {
+	    dsl_dataset_phys(ds->ds_prev)->ds_referenced_bytes > ds->ds_quota) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(EDQUOT));
 	}
@@ -1954,7 +1976,7 @@ dsl_dataset_rollback_check(void *arg, dmu_tx_t *tx)
 	 * this space, but the freeing happens over many txg's.
 	 */
 	unused_refres_delta = (int64_t)MIN(ds->ds_reserved,
-	    ds->ds_phys->ds_unique_bytes);
+	    dsl_dataset_phys(ds)->ds_unique_bytes);
 
 	if (unused_refres_delta > 0 &&
 	    unused_refres_delta >
@@ -2060,7 +2082,7 @@ dsl_dataset_promote_check(void *arg, dmu_tx_t *tx)
 
 	hds = ddpa->ddpa_clone;
 
-	if (hds->ds_phys->ds_flags & DS_FLAG_NOPROMOTE) {
+	if (dsl_dataset_phys(hds)->ds_flags & DS_FLAG_NOPROMOTE) {
 		promote_rele(ddpa, FTAG);
 		return (SET_ERROR(EXDEV));
 	}
@@ -2079,9 +2101,10 @@ dsl_dataset_promote_check(void *arg, dmu_tx_t *tx)
 
 	/* compute origin's new unique space */
 	snap = list_tail(&ddpa->clone_snaps);
-	ASSERT3U(snap->ds->ds_phys->ds_prev_snap_obj, ==, origin_ds->ds_object);
+	ASSERT3U(dsl_dataset_phys(snap->ds)->ds_prev_snap_obj, ==,
+	    origin_ds->ds_object);
 	dsl_deadlist_space_range(&snap->ds->ds_deadlist,
-	    origin_ds->ds_phys->ds_prev_snap_txg, UINT64_MAX,
+	    dsl_dataset_phys(origin_ds)->ds_prev_snap_txg, UINT64_MAX,
 	    &ddpa->unique, &unused, &unused);
 
 	/*
@@ -2100,9 +2123,9 @@ dsl_dataset_promote_check(void *arg, dmu_tx_t *tx)
 	 * uN + kN + kN-1 + ... + kM - uM-1
 	 */
 	ss_mv_cnt = 0;
-	ddpa->used = origin_ds->ds_phys->ds_referenced_bytes;
-	ddpa->comp = origin_ds->ds_phys->ds_compressed_bytes;
-	ddpa->uncomp = origin_ds->ds_phys->ds_uncompressed_bytes;
+	ddpa->used = dsl_dataset_phys(origin_ds)->ds_referenced_bytes;
+	ddpa->comp = dsl_dataset_phys(origin_ds)->ds_compressed_bytes;
+	ddpa->uncomp = dsl_dataset_phys(origin_ds)->ds_uncompressed_bytes;
 	for (snap = list_head(&ddpa->shared_snaps); snap;
 	    snap = list_next(&ddpa->shared_snaps, snap)) {
 		uint64_t val, dlused, dlcomp, dluncomp;
@@ -2131,7 +2154,7 @@ dsl_dataset_promote_check(void *arg, dmu_tx_t *tx)
 			goto out;
 
 		/* The very first snapshot does not have a deadlist */
-		if (ds->ds_phys->ds_prev_snap_obj == 0)
+		if (dsl_dataset_phys(ds)->ds_prev_snap_obj == 0)
 			continue;
 
 		dsl_deadlist_space(&ds->ds_deadlist,
@@ -2146,10 +2169,13 @@ dsl_dataset_promote_check(void *arg, dmu_tx_t *tx)
 	 * so we need to subtract out the clone origin's used space.
 	 */
 	if (ddpa->origin_origin) {
-		ddpa->used -= ddpa->origin_origin->ds_phys->ds_referenced_bytes;
-		ddpa->comp -= ddpa->origin_origin->ds_phys->ds_compressed_bytes;
+		ddpa->used -=
+		    dsl_dataset_phys(ddpa->origin_origin)->ds_referenced_bytes;
+		ddpa->comp -=
+		    dsl_dataset_phys(ddpa->origin_origin)->ds_compressed_bytes;
 		ddpa->uncomp -=
-		    ddpa->origin_origin->ds_phys->ds_uncompressed_bytes;
+		    dsl_dataset_phys(ddpa->origin_origin)->
+		    ds_uncompressed_bytes;
 	}
 
 	/* Check that there is enough space and limit headroom here */
@@ -2164,7 +2190,7 @@ dsl_dataset_promote_check(void *arg, dmu_tx_t *tx)
 	 * it is the amount of space that will be on all of their
 	 * deadlists (that was not born before their new origin).
 	 */
-	if (hds->ds_dir->dd_phys->dd_flags & DD_FLAG_USED_BREAKDOWN) {
+	if (dsl_dir_phys(hds->ds_dir)->dd_flags & DD_FLAG_USED_BREAKDOWN) {
 		uint64_t space;
 
 		/*
@@ -2186,9 +2212,11 @@ dsl_dataset_promote_check(void *arg, dmu_tx_t *tx)
 			goto out;
 		ddpa->cloneusedsnap += space;
 	}
-	if (origin_ds->ds_dir->dd_phys->dd_flags & DD_FLAG_USED_BREAKDOWN) {
+	if (dsl_dir_phys(origin_ds->ds_dir)->dd_flags &
+	    DD_FLAG_USED_BREAKDOWN) {
 		err = snaplist_space(&ddpa->origin_snaps,
-		    origin_ds->ds_phys->ds_creation_txg, &ddpa->originusedsnap);
+		    dsl_dataset_phys(origin_ds)->ds_creation_txg,
+		    &ddpa->originusedsnap);
 		if (err != 0)
 			goto out;
 	}
@@ -2215,7 +2243,7 @@ dsl_dataset_promote_sync(void *arg, dmu_tx_t *tx)
 	VERIFY0(promote_hold(ddpa, dp, FTAG));
 	hds = ddpa->ddpa_clone;
 
-	ASSERT0(hds->ds_phys->ds_flags & DS_FLAG_NOPROMOTE);
+	ASSERT0(dsl_dataset_phys(hds)->ds_flags & DS_FLAG_NOPROMOTE);
 
 	snap = list_head(&ddpa->shared_snaps);
 	origin_ds = snap->ds;
@@ -2233,47 +2261,49 @@ dsl_dataset_promote_sync(void *arg, dmu_tx_t *tx)
 
 	/* change origin's next snap */
 	dmu_buf_will_dirty(origin_ds->ds_dbuf, tx);
-	oldnext_obj = origin_ds->ds_phys->ds_next_snap_obj;
+	oldnext_obj = dsl_dataset_phys(origin_ds)->ds_next_snap_obj;
 	snap = list_tail(&ddpa->clone_snaps);
-	ASSERT3U(snap->ds->ds_phys->ds_prev_snap_obj, ==, origin_ds->ds_object);
-	origin_ds->ds_phys->ds_next_snap_obj = snap->ds->ds_object;
+	ASSERT3U(dsl_dataset_phys(snap->ds)->ds_prev_snap_obj, ==,
+	    origin_ds->ds_object);
+	dsl_dataset_phys(origin_ds)->ds_next_snap_obj = snap->ds->ds_object;
 
 	/* change the origin's next clone */
-	if (origin_ds->ds_phys->ds_next_clones_obj) {
+	if (dsl_dataset_phys(origin_ds)->ds_next_clones_obj) {
 		dsl_dataset_remove_from_next_clones(origin_ds,
 		    snap->ds->ds_object, tx);
 		VERIFY0(zap_add_int(dp->dp_meta_objset,
-		    origin_ds->ds_phys->ds_next_clones_obj,
+		    dsl_dataset_phys(origin_ds)->ds_next_clones_obj,
 		    oldnext_obj, tx));
 	}
 
 	/* change origin */
 	dmu_buf_will_dirty(dd->dd_dbuf, tx);
-	ASSERT3U(dd->dd_phys->dd_origin_obj, ==, origin_ds->ds_object);
-	dd->dd_phys->dd_origin_obj = odd->dd_phys->dd_origin_obj;
+	ASSERT3U(dsl_dir_phys(dd)->dd_origin_obj, ==, origin_ds->ds_object);
+	dsl_dir_phys(dd)->dd_origin_obj = dsl_dir_phys(odd)->dd_origin_obj;
 	dd->dd_origin_txg = origin_head->ds_dir->dd_origin_txg;
 	dmu_buf_will_dirty(odd->dd_dbuf, tx);
-	odd->dd_phys->dd_origin_obj = origin_ds->ds_object;
+	dsl_dir_phys(odd)->dd_origin_obj = origin_ds->ds_object;
 	origin_head->ds_dir->dd_origin_txg =
-	    origin_ds->ds_phys->ds_creation_txg;
+	    dsl_dataset_phys(origin_ds)->ds_creation_txg;
 
 	/* change dd_clone entries */
 	if (spa_version(dp->dp_spa) >= SPA_VERSION_DIR_CLONES) {
 		VERIFY0(zap_remove_int(dp->dp_meta_objset,
-		    odd->dd_phys->dd_clones, hds->ds_object, tx));
+		    dsl_dir_phys(odd)->dd_clones, hds->ds_object, tx));
 		VERIFY0(zap_add_int(dp->dp_meta_objset,
-		    ddpa->origin_origin->ds_dir->dd_phys->dd_clones,
+		    dsl_dir_phys(ddpa->origin_origin->ds_dir)->dd_clones,
 		    hds->ds_object, tx));
 
 		VERIFY0(zap_remove_int(dp->dp_meta_objset,
-		    ddpa->origin_origin->ds_dir->dd_phys->dd_clones,
+		    dsl_dir_phys(ddpa->origin_origin->ds_dir)->dd_clones,
 		    origin_head->ds_object, tx));
-		if (dd->dd_phys->dd_clones == 0) {
-			dd->dd_phys->dd_clones = zap_create(dp->dp_meta_objset,
-			    DMU_OT_DSL_CLONES, DMU_OT_NONE, 0, tx);
+		if (dsl_dir_phys(dd)->dd_clones == 0) {
+			dsl_dir_phys(dd)->dd_clones =
+			    zap_create(dp->dp_meta_objset, DMU_OT_DSL_CLONES,
+			    DMU_OT_NONE, 0, tx);
 		}
 		VERIFY0(zap_add_int(dp->dp_meta_objset,
-		    dd->dd_phys->dd_clones, origin_head->ds_object, tx));
+		    dsl_dir_phys(dd)->dd_clones, origin_head->ds_object, tx));
 	}
 
 	/* move snapshots to this dir */
@@ -2296,28 +2326,28 @@ dsl_dataset_promote_sync(void *arg, dmu_tx_t *tx)
 		VERIFY0(dsl_dataset_snap_remove(origin_head,
 		    ds->ds_snapname, tx, B_TRUE));
 		VERIFY0(zap_add(dp->dp_meta_objset,
-		    hds->ds_phys->ds_snapnames_zapobj, ds->ds_snapname,
+		    dsl_dataset_phys(hds)->ds_snapnames_zapobj, ds->ds_snapname,
 		    8, 1, &ds->ds_object, tx));
 		dsl_fs_ss_count_adjust(hds->ds_dir, 1,
 		    DD_FIELD_SNAPSHOT_COUNT, tx);
 
 		/* change containing dsl_dir */
 		dmu_buf_will_dirty(ds->ds_dbuf, tx);
-		ASSERT3U(ds->ds_phys->ds_dir_obj, ==, odd->dd_object);
-		ds->ds_phys->ds_dir_obj = dd->dd_object;
+		ASSERT3U(dsl_dataset_phys(ds)->ds_dir_obj, ==, odd->dd_object);
+		dsl_dataset_phys(ds)->ds_dir_obj = dd->dd_object;
 		ASSERT3P(ds->ds_dir, ==, odd);
 		dsl_dir_rele(ds->ds_dir, ds);
 		VERIFY0(dsl_dir_hold_obj(dp, dd->dd_object,
 		    NULL, ds, &ds->ds_dir));
 
 		/* move any clone references */
-		if (ds->ds_phys->ds_next_clones_obj &&
+		if (dsl_dataset_phys(ds)->ds_next_clones_obj &&
 		    spa_version(dp->dp_spa) >= SPA_VERSION_DIR_CLONES) {
 			zap_cursor_t zc;
 			zap_attribute_t za;
 
 			for (zap_cursor_init(&zc, dp->dp_meta_objset,
-			    ds->ds_phys->ds_next_clones_obj);
+			    dsl_dataset_phys(ds)->ds_next_clones_obj);
 			    zap_cursor_retrieve(&zc, &za) == 0;
 			    zap_cursor_advance(&zc)) {
 				dsl_dataset_t *cnds;
@@ -2333,12 +2363,13 @@ dsl_dataset_promote_sync(void *arg, dmu_tx_t *tx)
 
 				VERIFY0(dsl_dataset_hold_obj(dp,
 				    za.za_first_integer, FTAG, &cnds));
-				o = cnds->ds_dir->dd_phys->dd_head_dataset_obj;
+				o = dsl_dir_phys(cnds->ds_dir)->
+				    dd_head_dataset_obj;
 
 				VERIFY0(zap_remove_int(dp->dp_meta_objset,
-				    odd->dd_phys->dd_clones, o, tx));
+				    dsl_dir_phys(odd)->dd_clones, o, tx));
 				VERIFY0(zap_add_int(dp->dp_meta_objset,
-				    dd->dd_phys->dd_clones, o, tx));
+				    dsl_dir_phys(dd)->dd_clones, o, tx));
 				dsl_dataset_rele(cnds, FTAG);
 			}
 			zap_cursor_fini(&zc);
@@ -2355,7 +2386,7 @@ dsl_dataset_promote_sync(void *arg, dmu_tx_t *tx)
 	 */
 
 	delta = ddpa->cloneusedsnap -
-	    dd->dd_phys->dd_used_breakdown[DD_USED_SNAP];
+	    dsl_dir_phys(dd)->dd_used_breakdown[DD_USED_SNAP];
 	ASSERT3S(delta, >=, 0);
 	ASSERT3U(ddpa->used, >=, delta);
 	dsl_dir_diduse_space(dd, DD_USED_SNAP, delta, 0, 0, tx);
@@ -2363,14 +2394,14 @@ dsl_dataset_promote_sync(void *arg, dmu_tx_t *tx)
 	    ddpa->used - delta, ddpa->comp, ddpa->uncomp, tx);
 
 	delta = ddpa->originusedsnap -
-	    odd->dd_phys->dd_used_breakdown[DD_USED_SNAP];
+	    dsl_dir_phys(odd)->dd_used_breakdown[DD_USED_SNAP];
 	ASSERT3S(delta, <=, 0);
 	ASSERT3U(ddpa->used, >=, -delta);
 	dsl_dir_diduse_space(odd, DD_USED_SNAP, delta, 0, 0, tx);
 	dsl_dir_diduse_space(odd, DD_USED_HEAD,
 	    -ddpa->used - delta, -ddpa->comp, -ddpa->uncomp, tx);
 
-	origin_ds->ds_phys->ds_unique_bytes = ddpa->unique;
+	dsl_dataset_phys(origin_ds)->ds_unique_bytes = ddpa->unique;
 
 	/* log history record */
 	spa_history_log_internal_ds(hds, "promote", tx, "");
@@ -2405,12 +2436,12 @@ snaplist_make(dsl_pool_t *dp,
 			return (err);
 
 		if (first_obj == 0)
-			first_obj = ds->ds_dir->dd_phys->dd_origin_obj;
+			first_obj = dsl_dir_phys(ds->ds_dir)->dd_origin_obj;
 
 		snap = kmem_alloc(sizeof (*snap), KM_SLEEP);
 		snap->ds = ds;
 		list_insert_tail(l, snap);
-		obj = ds->ds_phys->ds_prev_snap_obj;
+		obj = dsl_dataset_phys(ds)->ds_prev_snap_obj;
 	}
 
 	return (0);
@@ -2466,7 +2497,7 @@ promote_hold(dsl_dataset_promote_arg_t *ddpa, dsl_pool_t *dp, void *tag)
 		return (SET_ERROR(EINVAL));
 	}
 
-	error = snaplist_make(dp, 0, dd->dd_phys->dd_origin_obj,
+	error = snaplist_make(dp, 0, dsl_dir_phys(dd)->dd_origin_obj,
 	    &ddpa->shared_snaps, tag);
 	if (error != 0)
 		goto out;
@@ -2477,16 +2508,16 @@ promote_hold(dsl_dataset_promote_arg_t *ddpa, dsl_pool_t *dp, void *tag)
 		goto out;
 
 	snap = list_head(&ddpa->shared_snaps);
-	ASSERT3U(snap->ds->ds_object, ==, dd->dd_phys->dd_origin_obj);
-	error = snaplist_make(dp, dd->dd_phys->dd_origin_obj,
-	    snap->ds->ds_dir->dd_phys->dd_head_dataset_obj,
+	ASSERT3U(snap->ds->ds_object, ==, dsl_dir_phys(dd)->dd_origin_obj);
+	error = snaplist_make(dp, dsl_dir_phys(dd)->dd_origin_obj,
+	    dsl_dir_phys(snap->ds->ds_dir)->dd_head_dataset_obj,
 	    &ddpa->origin_snaps, tag);
 	if (error != 0)
 		goto out;
 
-	if (snap->ds->ds_dir->dd_phys->dd_origin_obj != 0) {
+	if (dsl_dir_phys(snap->ds->ds_dir)->dd_origin_obj != 0) {
 		error = dsl_dataset_hold_obj(dp,
-		    snap->ds->ds_dir->dd_phys->dd_origin_obj,
+		    dsl_dir_phys(snap->ds->ds_dir)->dd_origin_obj,
 		    tag, &ddpa->origin_origin);
 		if (error != 0)
 			goto out;
@@ -2530,7 +2561,8 @@ dsl_dataset_promote(const char *name, char *conflsnap)
 	if (error != 0)
 		return (error);
 	error = zap_count(dmu_objset_pool(os)->dp_meta_objset,
-	    dmu_objset_ds(os)->ds_phys->ds_snapnames_zapobj, &numsnaps);
+	    dsl_dataset_phys(dmu_objset_ds(os))->ds_snapnames_zapobj,
+	    &numsnaps);
 	dmu_objset_rele(os, FTAG);
 	if (error != 0)
 		return (error);
@@ -2581,9 +2613,9 @@ dsl_dataset_clone_swap_check_impl(dsl_dataset_t *clone,
 	/* check amount of any unconsumed refreservation */
 	unused_refres_delta =
 	    (int64_t)MIN(origin_head->ds_reserved,
-	    origin_head->ds_phys->ds_unique_bytes) -
+	    dsl_dataset_phys(origin_head)->ds_unique_bytes) -
 	    (int64_t)MIN(origin_head->ds_reserved,
-	    clone->ds_phys->ds_unique_bytes);
+	    dsl_dataset_phys(clone)->ds_unique_bytes);
 
 	if (unused_refres_delta > 0 &&
 	    unused_refres_delta >
@@ -2592,7 +2624,8 @@ dsl_dataset_clone_swap_check_impl(dsl_dataset_t *clone,
 
 	/* clone can't be over the head's refquota */
 	if (origin_head->ds_quota != 0 &&
-	    clone->ds_phys->ds_referenced_bytes > origin_head->ds_quota)
+	    dsl_dataset_phys(clone)->ds_referenced_bytes >
+	    origin_head->ds_quota)
 		return (SET_ERROR(EDQUOT));
 
 	return (0);
@@ -2607,7 +2640,7 @@ dsl_dataset_clone_swap_sync_impl(dsl_dataset_t *clone,
 
 	ASSERT(clone->ds_reserved == 0);
 	ASSERT(origin_head->ds_quota == 0 ||
-	    clone->ds_phys->ds_unique_bytes <= origin_head->ds_quota);
+	    dsl_dataset_phys(clone)->ds_unique_bytes <= origin_head->ds_quota);
 	ASSERT3P(clone->ds_prev, ==, origin_head->ds_prev);
 
 	dmu_buf_will_dirty(clone->ds_dbuf, tx);
@@ -2625,9 +2658,9 @@ dsl_dataset_clone_swap_sync_impl(dsl_dataset_t *clone,
 
 	unused_refres_delta =
 	    (int64_t)MIN(origin_head->ds_reserved,
-	    origin_head->ds_phys->ds_unique_bytes) -
+	    dsl_dataset_phys(origin_head)->ds_unique_bytes) -
 	    (int64_t)MIN(origin_head->ds_reserved,
-	    clone->ds_phys->ds_unique_bytes);
+	    dsl_dataset_phys(clone)->ds_unique_bytes);
 
 	/*
 	 * Reset origin's unique bytes, if it exists.
@@ -2638,16 +2671,17 @@ dsl_dataset_clone_swap_sync_impl(dsl_dataset_t *clone,
 
 		dmu_buf_will_dirty(origin->ds_dbuf, tx);
 		dsl_deadlist_space_range(&clone->ds_deadlist,
-		    origin->ds_phys->ds_prev_snap_txg, UINT64_MAX,
-		    &origin->ds_phys->ds_unique_bytes, &comp, &uncomp);
+		    dsl_dataset_phys(origin)->ds_prev_snap_txg, UINT64_MAX,
+		    &dsl_dataset_phys(origin)->ds_unique_bytes, &comp, &uncomp);
 	}
 
 	/* swap blkptrs */
 	{
 		blkptr_t tmp;
-		tmp = origin_head->ds_phys->ds_bp;
-		origin_head->ds_phys->ds_bp = clone->ds_phys->ds_bp;
-		clone->ds_phys->ds_bp = tmp;
+		tmp = dsl_dataset_phys(origin_head)->ds_bp;
+		dsl_dataset_phys(origin_head)->ds_bp =
+		    dsl_dataset_phys(clone)->ds_bp;
+		dsl_dataset_phys(clone)->ds_bp = tmp;
 	}
 
 	/* set dd_*_bytes */
@@ -2656,7 +2690,7 @@ dsl_dataset_clone_swap_sync_impl(dsl_dataset_t *clone,
 		uint64_t cdl_used, cdl_comp, cdl_uncomp;
 		uint64_t odl_used, odl_comp, odl_uncomp;
 
-		ASSERT3U(clone->ds_dir->dd_phys->
+		ASSERT3U(dsl_dir_phys(clone->ds_dir)->
 		    dd_used_breakdown[DD_USED_SNAP], ==, 0);
 
 		dsl_deadlist_space(&clone->ds_deadlist,
@@ -2664,13 +2698,18 @@ dsl_dataset_clone_swap_sync_impl(dsl_dataset_t *clone,
 		dsl_deadlist_space(&origin_head->ds_deadlist,
 		    &odl_used, &odl_comp, &odl_uncomp);
 
-		dused = clone->ds_phys->ds_referenced_bytes + cdl_used -
-		    (origin_head->ds_phys->ds_referenced_bytes + odl_used);
-		dcomp = clone->ds_phys->ds_compressed_bytes + cdl_comp -
-		    (origin_head->ds_phys->ds_compressed_bytes + odl_comp);
-		duncomp = clone->ds_phys->ds_uncompressed_bytes +
+		dused = dsl_dataset_phys(clone)->ds_referenced_bytes +
+		    cdl_used -
+		    (dsl_dataset_phys(origin_head)->ds_referenced_bytes +
+		    odl_used);
+		dcomp = dsl_dataset_phys(clone)->ds_compressed_bytes +
+		    cdl_comp -
+		    (dsl_dataset_phys(origin_head)->ds_compressed_bytes +
+		    odl_comp);
+		duncomp = dsl_dataset_phys(clone)->ds_uncompressed_bytes +
 		    cdl_uncomp -
-		    (origin_head->ds_phys->ds_uncompressed_bytes + odl_uncomp);
+		    (dsl_dataset_phys(origin_head)->ds_uncompressed_bytes +
+		    odl_uncomp);
 
 		dsl_dir_diduse_space(origin_head->ds_dir, DD_USED_HEAD,
 		    dused, dcomp, duncomp, tx);
@@ -2694,14 +2733,14 @@ dsl_dataset_clone_swap_sync_impl(dsl_dataset_t *clone,
 	}
 
 	/* swap ds_*_bytes */
-	SWITCH64(origin_head->ds_phys->ds_referenced_bytes,
-	    clone->ds_phys->ds_referenced_bytes);
-	SWITCH64(origin_head->ds_phys->ds_compressed_bytes,
-	    clone->ds_phys->ds_compressed_bytes);
-	SWITCH64(origin_head->ds_phys->ds_uncompressed_bytes,
-	    clone->ds_phys->ds_uncompressed_bytes);
-	SWITCH64(origin_head->ds_phys->ds_unique_bytes,
-	    clone->ds_phys->ds_unique_bytes);
+	SWITCH64(dsl_dataset_phys(origin_head)->ds_referenced_bytes,
+	    dsl_dataset_phys(clone)->ds_referenced_bytes);
+	SWITCH64(dsl_dataset_phys(origin_head)->ds_compressed_bytes,
+	    dsl_dataset_phys(clone)->ds_compressed_bytes);
+	SWITCH64(dsl_dataset_phys(origin_head)->ds_uncompressed_bytes,
+	    dsl_dataset_phys(clone)->ds_uncompressed_bytes);
+	SWITCH64(dsl_dataset_phys(origin_head)->ds_unique_bytes,
+	    dsl_dataset_phys(clone)->ds_unique_bytes);
 
 	/* apply any parent delta for change in unconsumed refreservation */
 	dsl_dir_diduse_space(origin_head->ds_dir, DD_USED_REFRSRV,
@@ -2712,12 +2751,12 @@ dsl_dataset_clone_swap_sync_impl(dsl_dataset_t *clone,
 	 */
 	dsl_deadlist_close(&clone->ds_deadlist);
 	dsl_deadlist_close(&origin_head->ds_deadlist);
-	SWITCH64(origin_head->ds_phys->ds_deadlist_obj,
-	    clone->ds_phys->ds_deadlist_obj);
+	SWITCH64(dsl_dataset_phys(origin_head)->ds_deadlist_obj,
+	    dsl_dataset_phys(clone)->ds_deadlist_obj);
 	dsl_deadlist_open(&clone->ds_deadlist, dp->dp_meta_objset,
-	    clone->ds_phys->ds_deadlist_obj);
+	    dsl_dataset_phys(clone)->ds_deadlist_obj);
 	dsl_deadlist_open(&origin_head->ds_deadlist, dp->dp_meta_objset,
-	    origin_head->ds_phys->ds_deadlist_obj);
+	    dsl_dataset_phys(origin_head)->ds_deadlist_obj);
 
 	dsl_scan_ds_clone_swapped(origin_head, clone, tx);
 
@@ -2768,10 +2807,11 @@ dsl_dataset_check_quota(dsl_dataset_t *ds, boolean_t check_quota,
 	/*
 	 * Make a space adjustment for reserved bytes.
 	 */
-	if (ds->ds_reserved > ds->ds_phys->ds_unique_bytes) {
+	if (ds->ds_reserved > dsl_dataset_phys(ds)->ds_unique_bytes) {
 		ASSERT3U(*used, >=,
-		    ds->ds_reserved - ds->ds_phys->ds_unique_bytes);
-		*used -= (ds->ds_reserved - ds->ds_phys->ds_unique_bytes);
+		    ds->ds_reserved - dsl_dataset_phys(ds)->ds_unique_bytes);
+		*used -=
+		    (ds->ds_reserved - dsl_dataset_phys(ds)->ds_unique_bytes);
 		*ref_rsrv =
 		    asize - MIN(asize, parent_delta(ds, asize + inflight));
 	}
@@ -2786,9 +2826,10 @@ dsl_dataset_check_quota(dsl_dataset_t *ds, boolean_t check_quota,
 	 * on-disk is over quota and there are no pending changes (which
 	 * may free up space for us).
 	 */
-	if (ds->ds_phys->ds_referenced_bytes + inflight >= ds->ds_quota) {
+	if (dsl_dataset_phys(ds)->ds_referenced_bytes + inflight >=
+	    ds->ds_quota) {
 		if (inflight > 0 ||
-		    ds->ds_phys->ds_referenced_bytes < ds->ds_quota)
+		    dsl_dataset_phys(ds)->ds_referenced_bytes < ds->ds_quota)
 			error = SET_ERROR(ERESTART);
 		else
 			error = SET_ERROR(EDQUOT);
@@ -2840,7 +2881,7 @@ dsl_dataset_set_refquota_check(void *arg, dmu_tx_t *tx)
 		return (0);
 	}
 
-	if (newval < ds->ds_phys->ds_referenced_bytes ||
+	if (newval < dsl_dataset_phys(ds)->ds_referenced_bytes ||
 	    newval < ds->ds_reserved) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(ENOSPC));
@@ -2930,7 +2971,7 @@ dsl_dataset_set_refreservation_check(void *arg, dmu_tx_t *tx)
 	mutex_enter(&ds->ds_lock);
 	if (!DS_UNIQUE_IS_ACCURATE(ds))
 		dsl_dataset_recalc_head_uniq(ds);
-	unique = ds->ds_phys->ds_unique_bytes;
+	unique = dsl_dataset_phys(ds)->ds_unique_bytes;
 	mutex_exit(&ds->ds_lock);
 
 	if (MAX(unique, newval) > MAX(unique, ds->ds_reserved)) {
@@ -2967,7 +3008,7 @@ dsl_dataset_set_refreservation_sync_impl(dsl_dataset_t *ds,
 	mutex_enter(&ds->ds_dir->dd_lock);
 	mutex_enter(&ds->ds_lock);
 	ASSERT(DS_UNIQUE_IS_ACCURATE(ds));
-	unique = ds->ds_phys->ds_unique_bytes;
+	unique = dsl_dataset_phys(ds)->ds_unique_bytes;
 	delta = MAX(0, (int64_t)(newval - unique)) -
 	    MAX(0, (int64_t)(ds->ds_reserved - unique));
 	ds->ds_reserved = newval;
@@ -3033,16 +3074,16 @@ dsl_dataset_space_written(dsl_dataset_t *oldsnap, dsl_dataset_t *new,
 	ASSERT(dsl_pool_config_held(dp));
 
 	*usedp = 0;
-	*usedp += new->ds_phys->ds_referenced_bytes;
-	*usedp -= oldsnap->ds_phys->ds_referenced_bytes;
+	*usedp += dsl_dataset_phys(new)->ds_referenced_bytes;
+	*usedp -= dsl_dataset_phys(oldsnap)->ds_referenced_bytes;
 
 	*compp = 0;
-	*compp += new->ds_phys->ds_compressed_bytes;
-	*compp -= oldsnap->ds_phys->ds_compressed_bytes;
+	*compp += dsl_dataset_phys(new)->ds_compressed_bytes;
+	*compp -= dsl_dataset_phys(oldsnap)->ds_compressed_bytes;
 
 	*uncompp = 0;
-	*uncompp += new->ds_phys->ds_uncompressed_bytes;
-	*uncompp -= oldsnap->ds_phys->ds_uncompressed_bytes;
+	*uncompp += dsl_dataset_phys(new)->ds_uncompressed_bytes;
+	*uncompp -= dsl_dataset_phys(oldsnap)->ds_uncompressed_bytes;
 
 	snapobj = new->ds_object;
 	while (snapobj != oldsnap->ds_object) {
@@ -3057,8 +3098,8 @@ dsl_dataset_space_written(dsl_dataset_t *oldsnap, dsl_dataset_t *new,
 				break;
 		}
 
-		if (snap->ds_phys->ds_prev_snap_txg ==
-		    oldsnap->ds_phys->ds_creation_txg) {
+		if (dsl_dataset_phys(snap)->ds_prev_snap_txg ==
+		    dsl_dataset_phys(oldsnap)->ds_creation_txg) {
 			/*
 			 * The blocks in the deadlist can not be born after
 			 * ds_prev_snap_txg, so get the whole deadlist space,
@@ -3071,7 +3112,7 @@ dsl_dataset_space_written(dsl_dataset_t *oldsnap, dsl_dataset_t *new,
 			    &used, &comp, &uncomp);
 		} else {
 			dsl_deadlist_space_range(&snap->ds_deadlist,
-			    0, oldsnap->ds_phys->ds_creation_txg,
+			    0, dsl_dataset_phys(oldsnap)->ds_creation_txg,
 			    &used, &comp, &uncomp);
 		}
 		*usedp += used;
@@ -3083,7 +3124,7 @@ dsl_dataset_space_written(dsl_dataset_t *oldsnap, dsl_dataset_t *new,
 		 * (ds_prev_snap_obj == 0) before oldsnap, then oldsnap
 		 * was not a snapshot of/before new.
 		 */
-		snapobj = snap->ds_phys->ds_prev_snap_obj;
+		snapobj = dsl_dataset_phys(snap)->ds_prev_snap_obj;
 		if (snap != new)
 			dsl_dataset_rele(snap, FTAG);
 		if (snapobj == 0) {
@@ -3127,13 +3168,13 @@ dsl_dataset_space_wouldfree(dsl_dataset_t *firstsnap,
 	 * is before lastsnap.
 	 */
 	if (firstsnap->ds_dir != lastsnap->ds_dir ||
-	    firstsnap->ds_phys->ds_creation_txg >
-	    lastsnap->ds_phys->ds_creation_txg)
+	    dsl_dataset_phys(firstsnap)->ds_creation_txg >
+	    dsl_dataset_phys(lastsnap)->ds_creation_txg)
 		return (SET_ERROR(EINVAL));
 
 	*usedp = *compp = *uncompp = 0;
 
-	snapobj = lastsnap->ds_phys->ds_next_snap_obj;
+	snapobj = dsl_dataset_phys(lastsnap)->ds_next_snap_obj;
 	while (snapobj != firstsnap->ds_object) {
 		dsl_dataset_t *ds;
 		uint64_t used, comp, uncomp;
@@ -3143,13 +3184,13 @@ dsl_dataset_space_wouldfree(dsl_dataset_t *firstsnap,
 			break;
 
 		dsl_deadlist_space_range(&ds->ds_deadlist,
-		    firstsnap->ds_phys->ds_prev_snap_txg, UINT64_MAX,
+		    dsl_dataset_phys(firstsnap)->ds_prev_snap_txg, UINT64_MAX,
 		    &used, &comp, &uncomp);
 		*usedp += used;
 		*compp += comp;
 		*uncompp += uncomp;
 
-		snapobj = ds->ds_phys->ds_prev_snap_obj;
+		snapobj = dsl_dataset_phys(ds)->ds_prev_snap_obj;
 		ASSERT3U(snapobj, !=, 0);
 		dsl_dataset_rele(ds, FTAG);
 	}
@@ -3248,10 +3289,10 @@ dsl_dataset_is_before(dsl_dataset_t *later, dsl_dataset_t *earlier,
 	ASSERT(dsl_dataset_is_snapshot(earlier) || earlier_txg != 0);
 
 	if (earlier_txg == 0)
-		earlier_txg = earlier->ds_phys->ds_creation_txg;
+		earlier_txg = dsl_dataset_phys(earlier)->ds_creation_txg;
 
 	if (dsl_dataset_is_snapshot(later) &&
-	    earlier_txg >= later->ds_phys->ds_creation_txg)
+	    earlier_txg >= dsl_dataset_phys(later)->ds_creation_txg)
 		return (B_FALSE);
 
 	if (later->ds_dir == earlier->ds_dir)
@@ -3259,11 +3300,11 @@ dsl_dataset_is_before(dsl_dataset_t *later, dsl_dataset_t *earlier,
 	if (!dsl_dir_is_clone(later->ds_dir))
 		return (B_FALSE);
 
-	if (later->ds_dir->dd_phys->dd_origin_obj == earlier->ds_object)
+	if (dsl_dir_phys(later->ds_dir)->dd_origin_obj == earlier->ds_object)
 		return (B_TRUE);
 	dsl_dataset_t *origin;
 	error = dsl_dataset_hold_obj(dp,
-	    later->ds_dir->dd_phys->dd_origin_obj, FTAG, &origin);
+	    dsl_dir_phys(later->ds_dir)->dd_origin_obj, FTAG, &origin);
 	if (error != 0)
 		return (B_FALSE);
 	ret = dsl_dataset_is_before(origin, earlier, earlier_txg);
diff --git a/uts/common/fs/zfs/dsl_deadlist.c b/uts/common/fs/zfs/dsl_deadlist.c
index 8c8e3746eecb..4ac562bfdba3 100644
--- a/uts/common/fs/zfs/dsl_deadlist.c
+++ b/uts/common/fs/zfs/dsl_deadlist.c
@@ -308,8 +308,9 @@ dsl_deadlist_regenerate(objset_t *os, uint64_t dlobj,
 	while (mrs_obj != 0) {
 		dsl_dataset_t *ds;
 		VERIFY3U(0, ==, dsl_dataset_hold_obj(dp, mrs_obj, FTAG, &ds));
-		dsl_deadlist_add_key(&dl, ds->ds_phys->ds_prev_snap_txg, tx);
-		mrs_obj = ds->ds_phys->ds_prev_snap_obj;
+		dsl_deadlist_add_key(&dl,
+		    dsl_dataset_phys(ds)->ds_prev_snap_txg, tx);
+		mrs_obj = dsl_dataset_phys(ds)->ds_prev_snap_obj;
 		dsl_dataset_rele(ds, FTAG);
 	}
 	dsl_deadlist_close(&dl);
diff --git a/uts/common/fs/zfs/dsl_deleg.c b/uts/common/fs/zfs/dsl_deleg.c
index 9c6f38a6bdea..dc405ebe58b5 100644
--- a/uts/common/fs/zfs/dsl_deleg.c
+++ b/uts/common/fs/zfs/dsl_deleg.c
@@ -164,10 +164,10 @@ dsl_deleg_set_sync(void *arg, dmu_tx_t *tx)
 
 	VERIFY0(dsl_dir_hold(dp, dda->dda_name, FTAG, &dd, NULL));
 
-	zapobj = dd->dd_phys->dd_deleg_zapobj;
+	zapobj = dsl_dir_phys(dd)->dd_deleg_zapobj;
 	if (zapobj == 0) {
 		dmu_buf_will_dirty(dd->dd_dbuf, tx);
-		zapobj = dd->dd_phys->dd_deleg_zapobj = zap_create(mos,
+		zapobj = dsl_dir_phys(dd)->dd_deleg_zapobj = zap_create(mos,
 		    DMU_OT_DSL_PERMS, DMU_OT_NONE, 0, tx);
 	}
 
@@ -208,7 +208,7 @@ dsl_deleg_unset_sync(void *arg, dmu_tx_t *tx)
 	uint64_t zapobj;
 
 	VERIFY0(dsl_dir_hold(dp, dda->dda_name, FTAG, &dd, NULL));
-	zapobj = dd->dd_phys->dd_deleg_zapobj;
+	zapobj = dsl_dir_phys(dd)->dd_deleg_zapobj;
 	if (zapobj == 0) {
 		dsl_dir_rele(dd, FTAG);
 		return;
@@ -332,14 +332,14 @@ dsl_deleg_get(const char *ddname, nvlist_t **nvp)
 		uint64_t n;
 		char source[MAXNAMELEN];
 
-		if (dd->dd_phys->dd_deleg_zapobj == 0 ||
-		    zap_count(mos, dd->dd_phys->dd_deleg_zapobj, &n) != 0 ||
-		    n == 0)
+		if (dsl_dir_phys(dd)->dd_deleg_zapobj == 0 ||
+		    zap_count(mos,
+		    dsl_dir_phys(dd)->dd_deleg_zapobj, &n) != 0 || n == 0)
 			continue;
 
 		sp_nvp = fnvlist_alloc();
 		for (zap_cursor_init(&basezc, mos,
-		    dd->dd_phys->dd_deleg_zapobj);
+		    dsl_dir_phys(dd)->dd_deleg_zapobj);
 		    zap_cursor_retrieve(&basezc, &baseza) == 0;
 		    zap_cursor_advance(&basezc)) {
 			zap_cursor_t zc;
@@ -594,7 +594,7 @@ dsl_deleg_access_impl(dsl_dataset_t *ds, const char *perm, cred_t *cr)
 			if (!zoned)
 				break;
 		}
-		zapobj = dd->dd_phys->dd_deleg_zapobj;
+		zapobj = dsl_dir_phys(dd)->dd_deleg_zapobj;
 
 		if (zapobj == 0)
 			continue;
@@ -673,7 +673,7 @@ copy_create_perms(dsl_dir_t *dd, uint64_t pzapobj,
 {
 	objset_t *mos = dd->dd_pool->dp_meta_objset;
 	uint64_t jumpobj, pjumpobj;
-	uint64_t zapobj = dd->dd_phys->dd_deleg_zapobj;
+	uint64_t zapobj = dsl_dir_phys(dd)->dd_deleg_zapobj;
 	zap_cursor_t zc;
 	zap_attribute_t za;
 	char whokey[ZFS_MAX_DELEG_NAME];
@@ -686,7 +686,7 @@ copy_create_perms(dsl_dir_t *dd, uint64_t pzapobj,
 
 	if (zapobj == 0) {
 		dmu_buf_will_dirty(dd->dd_dbuf, tx);
-		zapobj = dd->dd_phys->dd_deleg_zapobj = zap_create(mos,
+		zapobj = dsl_dir_phys(dd)->dd_deleg_zapobj = zap_create(mos,
 		    DMU_OT_DSL_PERMS, DMU_OT_NONE, 0, tx);
 	}
 
@@ -724,7 +724,7 @@ dsl_deleg_set_create_perms(dsl_dir_t *sdd, dmu_tx_t *tx, cred_t *cr)
 		return;
 
 	for (dd = sdd->dd_parent; dd != NULL; dd = dd->dd_parent) {
-		uint64_t pzapobj = dd->dd_phys->dd_deleg_zapobj;
+		uint64_t pzapobj = dsl_dir_phys(dd)->dd_deleg_zapobj;
 
 		if (pzapobj == 0)
 			continue;
diff --git a/uts/common/fs/zfs/dsl_destroy.c b/uts/common/fs/zfs/dsl_destroy.c
index 1237641583a1..a776e144f1d2 100644
--- a/uts/common/fs/zfs/dsl_destroy.c
+++ b/uts/common/fs/zfs/dsl_destroy.c
@@ -78,7 +78,7 @@ dsl_destroy_snapshot_check_impl(dsl_dataset_t *ds, boolean_t defer)
 	/*
 	 * Can't delete a branch point.
 	 */
-	if (ds->ds_phys->ds_num_children > 1)
+	if (dsl_dataset_phys(ds)->ds_num_children > 1)
 		return (SET_ERROR(EEXIST));
 
 	return (0);
@@ -147,12 +147,12 @@ process_old_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
 
 	ASSERT(!BP_IS_HOLE(bp));
 
-	if (bp->blk_birth <= poa->ds->ds_phys->ds_prev_snap_txg) {
+	if (bp->blk_birth <= dsl_dataset_phys(poa->ds)->ds_prev_snap_txg) {
 		dsl_deadlist_insert(&poa->ds->ds_deadlist, bp, tx);
 		if (poa->ds_prev && !poa->after_branch_point &&
 		    bp->blk_birth >
-		    poa->ds_prev->ds_phys->ds_prev_snap_txg) {
-			poa->ds_prev->ds_phys->ds_unique_bytes +=
+		    dsl_dataset_phys(poa->ds_prev)->ds_prev_snap_txg) {
+			dsl_dataset_phys(poa->ds_prev)->ds_unique_bytes +=
 			    bp_get_dsize_sync(dp->dp_spa, bp);
 		}
 	} else {
@@ -183,7 +183,7 @@ process_old_deadlist(dsl_dataset_t *ds, dsl_dataset_t *ds_prev,
 	VERIFY0(bpobj_iterate(&ds_next->ds_deadlist.dl_bpobj,
 	    process_old_cb, &poa, tx));
 	VERIFY0(zio_wait(poa.pio));
-	ASSERT3U(poa.used, ==, ds->ds_phys->ds_unique_bytes);
+	ASSERT3U(poa.used, ==, dsl_dataset_phys(ds)->ds_unique_bytes);
 
 	/* change snapused */
 	dsl_dir_diduse_space(ds->ds_dir, DD_USED_SNAP,
@@ -192,12 +192,14 @@ process_old_deadlist(dsl_dataset_t *ds, dsl_dataset_t *ds_prev,
 	/* swap next's deadlist to our deadlist */
 	dsl_deadlist_close(&ds->ds_deadlist);
 	dsl_deadlist_close(&ds_next->ds_deadlist);
-	deadlist_obj = ds->ds_phys->ds_deadlist_obj;
-	ds->ds_phys->ds_deadlist_obj = ds_next->ds_phys->ds_deadlist_obj;
-	ds_next->ds_phys->ds_deadlist_obj = deadlist_obj;
-	dsl_deadlist_open(&ds->ds_deadlist, mos, ds->ds_phys->ds_deadlist_obj);
+	deadlist_obj = dsl_dataset_phys(ds)->ds_deadlist_obj;
+	dsl_dataset_phys(ds)->ds_deadlist_obj =
+	    dsl_dataset_phys(ds_next)->ds_deadlist_obj;
+	dsl_dataset_phys(ds_next)->ds_deadlist_obj = deadlist_obj;
+	dsl_deadlist_open(&ds->ds_deadlist, mos,
+	    dsl_dataset_phys(ds)->ds_deadlist_obj);
 	dsl_deadlist_open(&ds_next->ds_deadlist, mos,
-	    ds_next->ds_phys->ds_deadlist_obj);
+	    dsl_dataset_phys(ds_next)->ds_deadlist_obj);
 }
 
 static void
@@ -212,10 +214,10 @@ dsl_dataset_remove_clones_key(dsl_dataset_t *ds, uint64_t mintxg, dmu_tx_t *tx)
 	 * find the clones, but dsl_deadlist_remove_key() is a no-op so it
 	 * doesn't matter.
 	 */
-	if (ds->ds_dir->dd_phys->dd_clones == 0)
+	if (dsl_dir_phys(ds->ds_dir)->dd_clones == 0)
 		return;
 
-	for (zap_cursor_init(&zc, mos, ds->ds_dir->dd_phys->dd_clones);
+	for (zap_cursor_init(&zc, mos, dsl_dir_phys(ds->ds_dir)->dd_clones);
 	    zap_cursor_retrieve(&zc, &za) == 0;
 	    zap_cursor_advance(&zc)) {
 		dsl_dataset_t *clone;
@@ -243,19 +245,20 @@ dsl_destroy_snapshot_sync_impl(dsl_dataset_t *ds, boolean_t defer, dmu_tx_t *tx)
 	uint64_t obj;
 
 	ASSERT(RRW_WRITE_HELD(&dp->dp_config_rwlock));
-	ASSERT3U(ds->ds_phys->ds_bp.blk_birth, <=, tx->tx_txg);
+	ASSERT3U(dsl_dataset_phys(ds)->ds_bp.blk_birth, <=, tx->tx_txg);
 	ASSERT(refcount_is_zero(&ds->ds_longholds));
 
 	if (defer &&
-	    (ds->ds_userrefs > 0 || ds->ds_phys->ds_num_children > 1)) {
+	    (ds->ds_userrefs > 0 ||
+	    dsl_dataset_phys(ds)->ds_num_children > 1)) {
 		ASSERT(spa_version(dp->dp_spa) >= SPA_VERSION_USERREFS);
 		dmu_buf_will_dirty(ds->ds_dbuf, tx);
-		ds->ds_phys->ds_flags |= DS_FLAG_DEFER_DESTROY;
+		dsl_dataset_phys(ds)->ds_flags |= DS_FLAG_DEFER_DESTROY;
 		spa_history_log_internal_ds(ds, "defer_destroy", tx, "");
 		return;
 	}
 
-	ASSERT3U(ds->ds_phys->ds_num_children, <=, 1);
+	ASSERT3U(dsl_dataset_phys(ds)->ds_num_children, <=, 1);
 
 	/* We need to log before removing it from the namespace. */
 	spa_history_log_internal_ds(ds, "destroy", tx, "");
@@ -268,26 +271,28 @@ dsl_destroy_snapshot_sync_impl(dsl_dataset_t *ds, boolean_t defer, dmu_tx_t *tx)
 		ASSERT0(zap_contains(mos, obj, DS_FIELD_LARGE_BLOCKS));
 		spa_feature_decr(dp->dp_spa, SPA_FEATURE_LARGE_BLOCKS, tx);
 	}
-	if (ds->ds_phys->ds_prev_snap_obj != 0) {
+	if (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) {
 		ASSERT3P(ds->ds_prev, ==, NULL);
 		VERIFY0(dsl_dataset_hold_obj(dp,
-		    ds->ds_phys->ds_prev_snap_obj, FTAG, &ds_prev));
+		    dsl_dataset_phys(ds)->ds_prev_snap_obj, FTAG, &ds_prev));
 		after_branch_point =
-		    (ds_prev->ds_phys->ds_next_snap_obj != obj);
+		    (dsl_dataset_phys(ds_prev)->ds_next_snap_obj != obj);
 
 		dmu_buf_will_dirty(ds_prev->ds_dbuf, tx);
 		if (after_branch_point &&
-		    ds_prev->ds_phys->ds_next_clones_obj != 0) {
+		    dsl_dataset_phys(ds_prev)->ds_next_clones_obj != 0) {
 			dsl_dataset_remove_from_next_clones(ds_prev, obj, tx);
-			if (ds->ds_phys->ds_next_snap_obj != 0) {
+			if (dsl_dataset_phys(ds)->ds_next_snap_obj != 0) {
 				VERIFY0(zap_add_int(mos,
-				    ds_prev->ds_phys->ds_next_clones_obj,
-				    ds->ds_phys->ds_next_snap_obj, tx));
+				    dsl_dataset_phys(ds_prev)->
+				    ds_next_clones_obj,
+				    dsl_dataset_phys(ds)->ds_next_snap_obj,
+				    tx));
 			}
 		}
 		if (!after_branch_point) {
-			ds_prev->ds_phys->ds_next_snap_obj =
-			    ds->ds_phys->ds_next_snap_obj;
+			dsl_dataset_phys(ds_prev)->ds_next_snap_obj =
+			    dsl_dataset_phys(ds)->ds_next_snap_obj;
 		}
 	}
 
@@ -296,18 +301,18 @@ dsl_destroy_snapshot_sync_impl(dsl_dataset_t *ds, boolean_t defer, dmu_tx_t *tx)
 	uint64_t used = 0, comp = 0, uncomp = 0;
 
 	VERIFY0(dsl_dataset_hold_obj(dp,
-	    ds->ds_phys->ds_next_snap_obj, FTAG, &ds_next));
-	ASSERT3U(ds_next->ds_phys->ds_prev_snap_obj, ==, obj);
+	    dsl_dataset_phys(ds)->ds_next_snap_obj, FTAG, &ds_next));
+	ASSERT3U(dsl_dataset_phys(ds_next)->ds_prev_snap_obj, ==, obj);
 
-	old_unique = ds_next->ds_phys->ds_unique_bytes;
+	old_unique = dsl_dataset_phys(ds_next)->ds_unique_bytes;
 
 	dmu_buf_will_dirty(ds_next->ds_dbuf, tx);
-	ds_next->ds_phys->ds_prev_snap_obj =
-	    ds->ds_phys->ds_prev_snap_obj;
-	ds_next->ds_phys->ds_prev_snap_txg =
-	    ds->ds_phys->ds_prev_snap_txg;
-	ASSERT3U(ds->ds_phys->ds_prev_snap_txg, ==,
-	    ds_prev ? ds_prev->ds_phys->ds_creation_txg : 0);
+	dsl_dataset_phys(ds_next)->ds_prev_snap_obj =
+	    dsl_dataset_phys(ds)->ds_prev_snap_obj;
+	dsl_dataset_phys(ds_next)->ds_prev_snap_txg =
+	    dsl_dataset_phys(ds)->ds_prev_snap_txg;
+	ASSERT3U(dsl_dataset_phys(ds)->ds_prev_snap_txg, ==,
+	    ds_prev ? dsl_dataset_phys(ds_prev)->ds_creation_txg : 0);
 
 	if (ds_next->ds_deadlist.dl_oldfmt) {
 		process_old_deadlist(ds, ds_prev, ds_next,
@@ -316,38 +321,38 @@ dsl_destroy_snapshot_sync_impl(dsl_dataset_t *ds, boolean_t defer, dmu_tx_t *tx)
 		/* Adjust prev's unique space. */
 		if (ds_prev && !after_branch_point) {
 			dsl_deadlist_space_range(&ds_next->ds_deadlist,
-			    ds_prev->ds_phys->ds_prev_snap_txg,
-			    ds->ds_phys->ds_prev_snap_txg,
+			    dsl_dataset_phys(ds_prev)->ds_prev_snap_txg,
+			    dsl_dataset_phys(ds)->ds_prev_snap_txg,
 			    &used, &comp, &uncomp);
-			ds_prev->ds_phys->ds_unique_bytes += used;
+			dsl_dataset_phys(ds_prev)->ds_unique_bytes += used;
 		}
 
 		/* Adjust snapused. */
 		dsl_deadlist_space_range(&ds_next->ds_deadlist,
-		    ds->ds_phys->ds_prev_snap_txg, UINT64_MAX,
+		    dsl_dataset_phys(ds)->ds_prev_snap_txg, UINT64_MAX,
 		    &used, &comp, &uncomp);
 		dsl_dir_diduse_space(ds->ds_dir, DD_USED_SNAP,
 		    -used, -comp, -uncomp, tx);
 
 		/* Move blocks to be freed to pool's free list. */
 		dsl_deadlist_move_bpobj(&ds_next->ds_deadlist,
-		    &dp->dp_free_bpobj, ds->ds_phys->ds_prev_snap_txg,
+		    &dp->dp_free_bpobj, dsl_dataset_phys(ds)->ds_prev_snap_txg,
 		    tx);
 		dsl_dir_diduse_space(tx->tx_pool->dp_free_dir,
 		    DD_USED_HEAD, used, comp, uncomp, tx);
 
 		/* Merge our deadlist into next's and free it. */
 		dsl_deadlist_merge(&ds_next->ds_deadlist,
-		    ds->ds_phys->ds_deadlist_obj, tx);
+		    dsl_dataset_phys(ds)->ds_deadlist_obj, tx);
 	}
 	dsl_deadlist_close(&ds->ds_deadlist);
-	dsl_deadlist_free(mos, ds->ds_phys->ds_deadlist_obj, tx);
+	dsl_deadlist_free(mos, dsl_dataset_phys(ds)->ds_deadlist_obj, tx);
 	dmu_buf_will_dirty(ds->ds_dbuf, tx);
-	ds->ds_phys->ds_deadlist_obj = 0;
+	dsl_dataset_phys(ds)->ds_deadlist_obj = 0;
 
 	/* Collapse range in clone heads */
 	dsl_dataset_remove_clones_key(ds,
-	    ds->ds_phys->ds_creation_txg, tx);
+	    dsl_dataset_phys(ds)->ds_creation_txg, tx);
 
 	if (dsl_dataset_is_snapshot(ds_next)) {
 		dsl_dataset_t *ds_nextnext;
@@ -362,21 +367,22 @@ dsl_destroy_snapshot_sync_impl(dsl_dataset_t *ds, boolean_t defer, dmu_tx_t *tx)
 		 * deadlist).
 		 */
 		VERIFY0(dsl_dataset_hold_obj(dp,
-		    ds_next->ds_phys->ds_next_snap_obj, FTAG, &ds_nextnext));
+		    dsl_dataset_phys(ds_next)->ds_next_snap_obj,
+		    FTAG, &ds_nextnext));
 		dsl_deadlist_space_range(&ds_nextnext->ds_deadlist,
-		    ds->ds_phys->ds_prev_snap_txg,
-		    ds->ds_phys->ds_creation_txg,
+		    dsl_dataset_phys(ds)->ds_prev_snap_txg,
+		    dsl_dataset_phys(ds)->ds_creation_txg,
 		    &used, &comp, &uncomp);
-		ds_next->ds_phys->ds_unique_bytes += used;
+		dsl_dataset_phys(ds_next)->ds_unique_bytes += used;
 		dsl_dataset_rele(ds_nextnext, FTAG);
 		ASSERT3P(ds_next->ds_prev, ==, NULL);
 
 		/* Collapse range in this head. */
 		dsl_dataset_t *hds;
 		VERIFY0(dsl_dataset_hold_obj(dp,
-		    ds->ds_dir->dd_phys->dd_head_dataset_obj, FTAG, &hds));
+		    dsl_dir_phys(ds->ds_dir)->dd_head_dataset_obj, FTAG, &hds));
 		dsl_deadlist_remove_key(&hds->ds_deadlist,
-		    ds->ds_phys->ds_creation_txg, tx);
+		    dsl_dataset_phys(ds)->ds_creation_txg, tx);
 		dsl_dataset_rele(hds, FTAG);
 
 	} else {
@@ -385,7 +391,7 @@ dsl_destroy_snapshot_sync_impl(dsl_dataset_t *ds, boolean_t defer, dmu_tx_t *tx)
 		ds_next->ds_prev = NULL;
 		if (ds_prev) {
 			VERIFY0(dsl_dataset_hold_obj(dp,
-			    ds->ds_phys->ds_prev_snap_obj,
+			    dsl_dataset_phys(ds)->ds_prev_snap_obj,
 			    ds_next, &ds_next->ds_prev));
 		}
 
@@ -399,7 +405,7 @@ dsl_destroy_snapshot_sync_impl(dsl_dataset_t *ds, boolean_t defer, dmu_tx_t *tx)
 		if (old_unique < ds_next->ds_reserved) {
 			int64_t mrsdelta;
 			uint64_t new_unique =
-			    ds_next->ds_phys->ds_unique_bytes;
+			    dsl_dataset_phys(ds_next)->ds_unique_bytes;
 
 			ASSERT(old_unique <= new_unique);
 			mrsdelta = MIN(new_unique - old_unique,
@@ -421,9 +427,9 @@ dsl_destroy_snapshot_sync_impl(dsl_dataset_t *ds, boolean_t defer, dmu_tx_t *tx)
 
 	/* remove from snapshot namespace */
 	dsl_dataset_t *ds_head;
-	ASSERT(ds->ds_phys->ds_snapnames_zapobj == 0);
+	ASSERT(dsl_dataset_phys(ds)->ds_snapnames_zapobj == 0);
 	VERIFY0(dsl_dataset_hold_obj(dp,
-	    ds->ds_dir->dd_phys->dd_head_dataset_obj, FTAG, &ds_head));
+	    dsl_dir_phys(ds->ds_dir)->dd_head_dataset_obj, FTAG, &ds_head));
 	VERIFY0(dsl_dataset_get_snapname(ds));
 #ifdef ZFS_DEBUG
 	{
@@ -443,17 +449,20 @@ dsl_destroy_snapshot_sync_impl(dsl_dataset_t *ds, boolean_t defer, dmu_tx_t *tx)
 
 	spa_prop_clear_bootfs(dp->dp_spa, ds->ds_object, tx);
 
-	if (ds->ds_phys->ds_next_clones_obj != 0) {
+	if (dsl_dataset_phys(ds)->ds_next_clones_obj != 0) {
 		uint64_t count;
 		ASSERT0(zap_count(mos,
-		    ds->ds_phys->ds_next_clones_obj, &count) && count == 0);
+		    dsl_dataset_phys(ds)->ds_next_clones_obj, &count) &&
+		    count == 0);
 		VERIFY0(dmu_object_free(mos,
-		    ds->ds_phys->ds_next_clones_obj, tx));
+		    dsl_dataset_phys(ds)->ds_next_clones_obj, tx));
 	}
-	if (ds->ds_phys->ds_props_obj != 0)
-		VERIFY0(zap_destroy(mos, ds->ds_phys->ds_props_obj, tx));
-	if (ds->ds_phys->ds_userrefs_obj != 0)
-		VERIFY0(zap_destroy(mos, ds->ds_phys->ds_userrefs_obj, tx));
+	if (dsl_dataset_phys(ds)->ds_props_obj != 0)
+		VERIFY0(zap_destroy(mos, dsl_dataset_phys(ds)->ds_props_obj,
+		    tx));
+	if (dsl_dataset_phys(ds)->ds_userrefs_obj != 0)
+		VERIFY0(zap_destroy(mos, dsl_dataset_phys(ds)->ds_userrefs_obj,
+		    tx));
 	dsl_dir_rele(ds->ds_dir, ds);
 	ds->ds_dir = NULL;
 	dmu_object_free_zapified(mos, obj, tx);
@@ -555,7 +564,8 @@ kill_blkptr(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 		dsl_free(ka->tx->tx_pool, ka->tx->tx_txg, bp);
 	} else {
 		ASSERT(zilog == NULL);
-		ASSERT3U(bp->blk_birth, >, ka->ds->ds_phys->ds_prev_snap_txg);
+		ASSERT3U(bp->blk_birth, >,
+		    dsl_dataset_phys(ka->ds)->ds_prev_snap_txg);
 		(void) dsl_dataset_block_kill(ka->ds, bp, tx, B_FALSE);
 	}
 
@@ -577,9 +587,10 @@ old_synchronous_dataset_destroy(dsl_dataset_t *ds, dmu_tx_t *tx)
 	ka.ds = ds;
 	ka.tx = tx;
 	VERIFY0(traverse_dataset(ds,
-	    ds->ds_phys->ds_prev_snap_txg, TRAVERSE_POST,
+	    dsl_dataset_phys(ds)->ds_prev_snap_txg, TRAVERSE_POST,
 	    kill_blkptr, &ka));
-	ASSERT(!DS_UNIQUE_IS_ACCURATE(ds) || ds->ds_phys->ds_unique_bytes == 0);
+	ASSERT(!DS_UNIQUE_IS_ACCURATE(ds) ||
+	    dsl_dataset_phys(ds)->ds_unique_bytes == 0);
 }
 
 typedef struct dsl_destroy_head_arg {
@@ -608,21 +619,21 @@ dsl_destroy_head_check_impl(dsl_dataset_t *ds, int expected_holds)
 	 * from.)
 	 */
 	if (ds->ds_prev != NULL &&
-	    ds->ds_prev->ds_phys->ds_next_snap_obj == ds->ds_object)
+	    dsl_dataset_phys(ds->ds_prev)->ds_next_snap_obj == ds->ds_object)
 		return (SET_ERROR(EBUSY));
 
 	/*
 	 * Can't delete if there are children of this fs.
 	 */
 	error = zap_count(mos,
-	    ds->ds_dir->dd_phys->dd_child_dir_zapobj, &count);
+	    dsl_dir_phys(ds->ds_dir)->dd_child_dir_zapobj, &count);
 	if (error != 0)
 		return (error);
 	if (count != 0)
 		return (SET_ERROR(EEXIST));
 
 	if (dsl_dir_is_clone(ds->ds_dir) && DS_IS_DEFER_DESTROY(ds->ds_prev) &&
-	    ds->ds_prev->ds_phys->ds_num_children == 2 &&
+	    dsl_dataset_phys(ds->ds_prev)->ds_num_children == 2 &&
 	    ds->ds_prev->ds_userrefs == 0) {
 		/* We need to remove the origin snapshot as well. */
 		if (!refcount_is_zero(&ds->ds_prev->ds_longholds))
@@ -660,7 +671,7 @@ dsl_dir_destroy_sync(uint64_t ddobj, dmu_tx_t *tx)
 
 	VERIFY0(dsl_dir_hold_obj(dp, ddobj, NULL, FTAG, &dd));
 
-	ASSERT0(dd->dd_phys->dd_head_dataset_obj);
+	ASSERT0(dsl_dir_phys(dd)->dd_head_dataset_obj);
 
 	/*
 	 * Decrement the filesystem count for all parent filesystems.
@@ -679,16 +690,17 @@ dsl_dir_destroy_sync(uint64_t ddobj, dmu_tx_t *tx)
 	 */
 	dsl_dir_set_reservation_sync_impl(dd, 0, tx);
 
-	ASSERT0(dd->dd_phys->dd_used_bytes);
-	ASSERT0(dd->dd_phys->dd_reserved);
+	ASSERT0(dsl_dir_phys(dd)->dd_used_bytes);
+	ASSERT0(dsl_dir_phys(dd)->dd_reserved);
 	for (t = 0; t < DD_USED_NUM; t++)
-		ASSERT0(dd->dd_phys->dd_used_breakdown[t]);
+		ASSERT0(dsl_dir_phys(dd)->dd_used_breakdown[t]);
 
-	VERIFY0(zap_destroy(mos, dd->dd_phys->dd_child_dir_zapobj, tx));
-	VERIFY0(zap_destroy(mos, dd->dd_phys->dd_props_zapobj, tx));
-	VERIFY0(dsl_deleg_destroy(mos, dd->dd_phys->dd_deleg_zapobj, tx));
+	VERIFY0(zap_destroy(mos, dsl_dir_phys(dd)->dd_child_dir_zapobj, tx));
+	VERIFY0(zap_destroy(mos, dsl_dir_phys(dd)->dd_props_zapobj, tx));
+	VERIFY0(dsl_deleg_destroy(mos, dsl_dir_phys(dd)->dd_deleg_zapobj, tx));
 	VERIFY0(zap_remove(mos,
-	    dd->dd_parent->dd_phys->dd_child_dir_zapobj, dd->dd_myname, tx));
+	    dsl_dir_phys(dd->dd_parent)->dd_child_dir_zapobj,
+	    dd->dd_myname, tx));
 
 	dsl_dir_rele(dd, FTAG);
 	dmu_object_free_zapified(mos, ddobj, tx);
@@ -702,10 +714,10 @@ dsl_destroy_head_sync_impl(dsl_dataset_t *ds, dmu_tx_t *tx)
 	uint64_t obj, ddobj, prevobj = 0;
 	boolean_t rmorigin;
 
-	ASSERT3U(ds->ds_phys->ds_num_children, <=, 1);
+	ASSERT3U(dsl_dataset_phys(ds)->ds_num_children, <=, 1);
 	ASSERT(ds->ds_prev == NULL ||
-	    ds->ds_prev->ds_phys->ds_next_snap_obj != ds->ds_object);
-	ASSERT3U(ds->ds_phys->ds_bp.blk_birth, <=, tx->tx_txg);
+	    dsl_dataset_phys(ds->ds_prev)->ds_next_snap_obj != ds->ds_object);
+	ASSERT3U(dsl_dataset_phys(ds)->ds_bp.blk_birth, <=, tx->tx_txg);
 	ASSERT(RRW_WRITE_HELD(&dp->dp_config_rwlock));
 
 	/* We need to log before removing it from the namespace. */
@@ -713,7 +725,7 @@ dsl_destroy_head_sync_impl(dsl_dataset_t *ds, dmu_tx_t *tx)
 
 	rmorigin = (dsl_dir_is_clone(ds->ds_dir) &&
 	    DS_IS_DEFER_DESTROY(ds->ds_prev) &&
-	    ds->ds_prev->ds_phys->ds_num_children == 2 &&
+	    dsl_dataset_phys(ds->ds_prev)->ds_num_children == 2 &&
 	    ds->ds_prev->ds_userrefs == 0);
 
 	/* Remove our reservation. */
@@ -731,20 +743,21 @@ dsl_destroy_head_sync_impl(dsl_dataset_t *ds, dmu_tx_t *tx)
 
 	obj = ds->ds_object;
 
-	if (ds->ds_phys->ds_prev_snap_obj != 0) {
+	if (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) {
 		/* This is a clone */
 		ASSERT(ds->ds_prev != NULL);
-		ASSERT3U(ds->ds_prev->ds_phys->ds_next_snap_obj, !=, obj);
-		ASSERT0(ds->ds_phys->ds_next_snap_obj);
+		ASSERT3U(dsl_dataset_phys(ds->ds_prev)->ds_next_snap_obj, !=,
+		    obj);
+		ASSERT0(dsl_dataset_phys(ds)->ds_next_snap_obj);
 
 		dmu_buf_will_dirty(ds->ds_prev->ds_dbuf, tx);
-		if (ds->ds_prev->ds_phys->ds_next_clones_obj != 0) {
+		if (dsl_dataset_phys(ds->ds_prev)->ds_next_clones_obj != 0) {
 			dsl_dataset_remove_from_next_clones(ds->ds_prev,
 			    obj, tx);
 		}
 
-		ASSERT3U(ds->ds_prev->ds_phys->ds_num_children, >, 1);
-		ds->ds_prev->ds_phys->ds_num_children--;
+		ASSERT3U(dsl_dataset_phys(ds->ds_prev)->ds_num_children, >, 1);
+		dsl_dataset_phys(ds->ds_prev)->ds_num_children--;
 	}
 
 	/*
@@ -753,9 +766,9 @@ dsl_destroy_head_sync_impl(dsl_dataset_t *ds, dmu_tx_t *tx)
 	 * safe to ignore the deadlist contents.)
 	 */
 	dsl_deadlist_close(&ds->ds_deadlist);
-	dsl_deadlist_free(mos, ds->ds_phys->ds_deadlist_obj, tx);
+	dsl_deadlist_free(mos, dsl_dataset_phys(ds)->ds_deadlist_obj, tx);
 	dmu_buf_will_dirty(ds->ds_dbuf, tx);
-	ds->ds_phys->ds_deadlist_obj = 0;
+	dsl_dataset_phys(ds)->ds_deadlist_obj = 0;
 
 	objset_t *os;
 	VERIFY0(dmu_objset_from_ds(ds, &os));
@@ -785,15 +798,16 @@ dsl_destroy_head_sync_impl(dsl_dataset_t *ds, dmu_tx_t *tx)
 			scn->scn_async_destroying = B_TRUE;
 		}
 
-		used = ds->ds_dir->dd_phys->dd_used_bytes;
-		comp = ds->ds_dir->dd_phys->dd_compressed_bytes;
-		uncomp = ds->ds_dir->dd_phys->dd_uncompressed_bytes;
+		used = dsl_dir_phys(ds->ds_dir)->dd_used_bytes;
+		comp = dsl_dir_phys(ds->ds_dir)->dd_compressed_bytes;
+		uncomp = dsl_dir_phys(ds->ds_dir)->dd_uncompressed_bytes;
 
 		ASSERT(!DS_UNIQUE_IS_ACCURATE(ds) ||
-		    ds->ds_phys->ds_unique_bytes == used);
+		    dsl_dataset_phys(ds)->ds_unique_bytes == used);
 
 		bptree_add(mos, dp->dp_bptree_obj,
-		    &ds->ds_phys->ds_bp, ds->ds_phys->ds_prev_snap_txg,
+		    &dsl_dataset_phys(ds)->ds_bp,
+		    dsl_dataset_phys(ds)->ds_prev_snap_txg,
 		    used, comp, uncomp, tx);
 		dsl_dir_diduse_space(ds->ds_dir, DD_USED_HEAD,
 		    -used, -comp, -uncomp, tx);
@@ -804,7 +818,7 @@ dsl_destroy_head_sync_impl(dsl_dataset_t *ds, dmu_tx_t *tx)
 	if (ds->ds_prev != NULL) {
 		if (spa_version(dp->dp_spa) >= SPA_VERSION_DIR_CLONES) {
 			VERIFY0(zap_remove_int(mos,
-			    ds->ds_prev->ds_dir->dd_phys->dd_clones,
+			    dsl_dir_phys(ds->ds_prev->ds_dir)->dd_clones,
 			    ds->ds_object, tx));
 		}
 		prevobj = ds->ds_prev->ds_object;
@@ -823,22 +837,22 @@ dsl_destroy_head_sync_impl(dsl_dataset_t *ds, dmu_tx_t *tx)
 
 	/* Erase the link in the dir */
 	dmu_buf_will_dirty(ds->ds_dir->dd_dbuf, tx);
-	ds->ds_dir->dd_phys->dd_head_dataset_obj = 0;
+	dsl_dir_phys(ds->ds_dir)->dd_head_dataset_obj = 0;
 	ddobj = ds->ds_dir->dd_object;
-	ASSERT(ds->ds_phys->ds_snapnames_zapobj != 0);
-	VERIFY0(zap_destroy(mos, ds->ds_phys->ds_snapnames_zapobj, tx));
+	ASSERT(dsl_dataset_phys(ds)->ds_snapnames_zapobj != 0);
+	VERIFY0(zap_destroy(mos,
+	    dsl_dataset_phys(ds)->ds_snapnames_zapobj, tx));
 
 	if (ds->ds_bookmarks != 0) {
-		VERIFY0(zap_destroy(mos,
-		    ds->ds_bookmarks, tx));
+		VERIFY0(zap_destroy(mos, ds->ds_bookmarks, tx));
 		spa_feature_decr(dp->dp_spa, SPA_FEATURE_BOOKMARKS, tx);
 	}
 
 	spa_prop_clear_bootfs(dp->dp_spa, ds->ds_object, tx);
 
-	ASSERT0(ds->ds_phys->ds_next_clones_obj);
-	ASSERT0(ds->ds_phys->ds_props_obj);
-	ASSERT0(ds->ds_phys->ds_userrefs_obj);
+	ASSERT0(dsl_dataset_phys(ds)->ds_next_clones_obj);
+	ASSERT0(dsl_dataset_phys(ds)->ds_props_obj);
+	ASSERT0(dsl_dataset_phys(ds)->ds_userrefs_obj);
 	dsl_dir_rele(ds->ds_dir, ds);
 	ds->ds_dir = NULL;
 	dmu_object_free_zapified(mos, obj, tx);
@@ -876,7 +890,7 @@ dsl_destroy_head_begin_sync(void *arg, dmu_tx_t *tx)
 
 	/* Mark it as inconsistent on-disk, in case we crash */
 	dmu_buf_will_dirty(ds->ds_dbuf, tx);
-	ds->ds_phys->ds_flags |= DS_FLAG_INCONSISTENT;
+	dsl_dataset_phys(ds)->ds_flags |= DS_FLAG_INCONSISTENT;
 
 	spa_history_log_internal_ds(ds, "destroy begin", tx, "");
 	dsl_dataset_rele(ds, FTAG);
@@ -919,7 +933,8 @@ dsl_destroy_head(const char *name)
 		error = dmu_objset_own(name, DMU_OST_ANY, B_FALSE, FTAG, &os);
 		if (error == 0) {
 			uint64_t prev_snap_txg =
-			    dmu_objset_ds(os)->ds_phys->ds_prev_snap_txg;
+			    dsl_dataset_phys(dmu_objset_ds(os))->
+			    ds_prev_snap_txg;
 			for (uint64_t obj = 0; error == 0;
 			    error = dmu_object_next(os, &obj, FALSE,
 			    prev_snap_txg))
diff --git a/uts/common/fs/zfs/dsl_dir.c b/uts/common/fs/zfs/dsl_dir.c
index 415bd91e9041..f49584168344 100644
--- a/uts/common/fs/zfs/dsl_dir.c
+++ b/uts/common/fs/zfs/dsl_dir.c
@@ -121,6 +121,8 @@
  * such as those created by zfs diff.
  */
 
+extern inline dsl_dir_phys_t *dsl_dir_phys(dsl_dir_t *dd);
+
 static uint64_t dsl_dir_space_towrite(dsl_dir_t *dd);
 
 /* ARGSUSED */
@@ -180,7 +182,6 @@ dsl_dir_hold_obj(dsl_pool_t *dp, uint64_t ddobj,
 		dd->dd_object = ddobj;
 		dd->dd_dbuf = dbuf;
 		dd->dd_pool = dp;
-		dd->dd_phys = dbuf->db_data;
 		mutex_init(&dd->dd_lock, NULL, MUTEX_DEFAULT, NULL);
 
 		list_create(&dd->dd_prop_cbs, sizeof (dsl_prop_cb_record_t),
@@ -188,9 +189,10 @@ dsl_dir_hold_obj(dsl_pool_t *dp, uint64_t ddobj,
 
 		dsl_dir_snap_cmtime_update(dd);
 
-		if (dd->dd_phys->dd_parent_obj) {
-			err = dsl_dir_hold_obj(dp, dd->dd_phys->dd_parent_obj,
-			    NULL, dd, &dd->dd_parent);
+		if (dsl_dir_phys(dd)->dd_parent_obj) {
+			err = dsl_dir_hold_obj(dp,
+			    dsl_dir_phys(dd)->dd_parent_obj, NULL, dd,
+			    &dd->dd_parent);
 			if (err != 0)
 				goto errout;
 			if (tail) {
@@ -198,14 +200,16 @@ dsl_dir_hold_obj(dsl_pool_t *dp, uint64_t ddobj,
 				uint64_t foundobj;
 
 				err = zap_lookup(dp->dp_meta_objset,
-				    dd->dd_parent->dd_phys->dd_child_dir_zapobj,
-				    tail, sizeof (foundobj), 1, &foundobj);
+				    dsl_dir_phys(dd->dd_parent)->
+				    dd_child_dir_zapobj, tail,
+				    sizeof (foundobj), 1, &foundobj);
 				ASSERT(err || foundobj == ddobj);
 #endif
 				(void) strcpy(dd->dd_myname, tail);
 			} else {
 				err = zap_value_search(dp->dp_meta_objset,
-				    dd->dd_parent->dd_phys->dd_child_dir_zapobj,
+				    dsl_dir_phys(dd->dd_parent)->
+				    dd_child_dir_zapobj,
 				    ddobj, 0, dd->dd_myname);
 			}
 			if (err != 0)
@@ -224,7 +228,8 @@ dsl_dir_hold_obj(dsl_pool_t *dp, uint64_t ddobj,
 			 * Just look at its phys directly instead.
 			 */
 			err = dmu_bonus_hold(dp->dp_meta_objset,
-			    dd->dd_phys->dd_origin_obj, FTAG, &origin_bonus);
+			    dsl_dir_phys(dd)->dd_origin_obj, FTAG,
+			    &origin_bonus);
 			if (err != 0)
 				goto errout;
 			origin_phys = origin_bonus->db_data;
@@ -233,8 +238,7 @@ dsl_dir_hold_obj(dsl_pool_t *dp, uint64_t ddobj,
 			dmu_buf_rele(origin_bonus, FTAG);
 		}
 
-		winner = dmu_buf_set_user_ie(dbuf, dd, &dd->dd_phys,
-		    dsl_dir_evict);
+		winner = dmu_buf_set_user_ie(dbuf, dd, dsl_dir_evict);
 		if (winner) {
 			if (dd->dd_parent)
 				dsl_dir_rele(dd->dd_parent, dd);
@@ -417,10 +421,10 @@ dsl_dir_hold(dsl_pool_t *dp, const char *name, void *tag,
 		if (next[0] == '@')
 			break;
 		dprintf("looking up %s in obj%lld\n",
-		    buf, dd->dd_phys->dd_child_dir_zapobj);
+		    buf, dsl_dir_phys(dd)->dd_child_dir_zapobj);
 
 		err = zap_lookup(dp->dp_meta_objset,
-		    dd->dd_phys->dd_child_dir_zapobj,
+		    dsl_dir_phys(dd)->dd_child_dir_zapobj,
 		    buf, sizeof (ddobj), 1, &ddobj);
 		if (err != 0) {
 			if (err == ENOENT)
@@ -501,7 +505,7 @@ dsl_dir_init_fs_ss_count(dsl_dir_t *dd, dmu_tx_t *tx)
 	za = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP);
 
 	/* Iterate my child dirs */
-	for (zap_cursor_init(zc, os, dd->dd_phys->dd_child_dir_zapobj);
+	for (zap_cursor_init(zc, os, dsl_dir_phys(dd)->dd_child_dir_zapobj);
 	    zap_cursor_retrieve(zc, za) == 0; zap_cursor_advance(zc)) {
 		dsl_dir_t *chld_dd;
 		uint64_t count;
@@ -535,9 +539,9 @@ dsl_dir_init_fs_ss_count(dsl_dir_t *dd, dmu_tx_t *tx)
 	zap_cursor_fini(zc);
 	/* Count my snapshots (we counted children's snapshots above) */
 	VERIFY0(dsl_dataset_hold_obj(dd->dd_pool,
-	    dd->dd_phys->dd_head_dataset_obj, FTAG, &ds));
+	    dsl_dir_phys(dd)->dd_head_dataset_obj, FTAG, &ds));
 
-	for (zap_cursor_init(zc, os, ds->ds_phys->ds_snapnames_zapobj);
+	for (zap_cursor_init(zc, os, dsl_dataset_phys(ds)->ds_snapnames_zapobj);
 	    zap_cursor_retrieve(zc, za) == 0;
 	    zap_cursor_advance(zc)) {
 		/* Don't count temporary snapshots */
@@ -686,7 +690,7 @@ dsl_enforce_ds_ss_limits(dsl_dir_t *dd, zfs_prop_t prop, cred_t *cr)
 		return (ENFORCE_NEVER);
 #endif
 
-	if ((obj = dd->dd_phys->dd_head_dataset_obj) == 0)
+	if ((obj = dsl_dir_phys(dd)->dd_head_dataset_obj) == 0)
 		return (ENFORCE_ALWAYS);
 
 	ASSERT(dsl_pool_config_held(dd->dd_pool));
@@ -864,7 +868,7 @@ dsl_dir_create_sync(dsl_pool_t *dp, dsl_dir_t *pds, const char *name,
 	ddobj = dmu_object_alloc(mos, DMU_OT_DSL_DIR, 0,
 	    DMU_OT_DSL_DIR, sizeof (dsl_dir_phys_t), tx);
 	if (pds) {
-		VERIFY(0 == zap_add(mos, pds->dd_phys->dd_child_dir_zapobj,
+		VERIFY(0 == zap_add(mos, dsl_dir_phys(pds)->dd_child_dir_zapobj,
 		    name, sizeof (uint64_t), 1, &ddobj, tx));
 	} else {
 		/* it's the root dir */
@@ -896,9 +900,9 @@ dsl_dir_create_sync(dsl_pool_t *dp, dsl_dir_t *pds, const char *name,
 boolean_t
 dsl_dir_is_clone(dsl_dir_t *dd)
 {
-	return (dd->dd_phys->dd_origin_obj &&
+	return (dsl_dir_phys(dd)->dd_origin_obj &&
 	    (dd->dd_pool->dp_origin_snap == NULL ||
-	    dd->dd_phys->dd_origin_obj !=
+	    dsl_dir_phys(dd)->dd_origin_obj !=
 	    dd->dd_pool->dp_origin_snap->ds_object));
 }
 
@@ -907,26 +911,27 @@ dsl_dir_stats(dsl_dir_t *dd, nvlist_t *nv)
 {
 	mutex_enter(&dd->dd_lock);
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_USED,
-	    dd->dd_phys->dd_used_bytes);
-	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_QUOTA, dd->dd_phys->dd_quota);
+	    dsl_dir_phys(dd)->dd_used_bytes);
+	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_QUOTA,
+	    dsl_dir_phys(dd)->dd_quota);
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_RESERVATION,
-	    dd->dd_phys->dd_reserved);
+	    dsl_dir_phys(dd)->dd_reserved);
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_COMPRESSRATIO,
-	    dd->dd_phys->dd_compressed_bytes == 0 ? 100 :
-	    (dd->dd_phys->dd_uncompressed_bytes * 100 /
-	    dd->dd_phys->dd_compressed_bytes));
+	    dsl_dir_phys(dd)->dd_compressed_bytes == 0 ? 100 :
+	    (dsl_dir_phys(dd)->dd_uncompressed_bytes * 100 /
+	    dsl_dir_phys(dd)->dd_compressed_bytes));
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_LOGICALUSED,
-	    dd->dd_phys->dd_uncompressed_bytes);
-	if (dd->dd_phys->dd_flags & DD_FLAG_USED_BREAKDOWN) {
+	    dsl_dir_phys(dd)->dd_uncompressed_bytes);
+	if (dsl_dir_phys(dd)->dd_flags & DD_FLAG_USED_BREAKDOWN) {
 		dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_USEDSNAP,
-		    dd->dd_phys->dd_used_breakdown[DD_USED_SNAP]);
+		    dsl_dir_phys(dd)->dd_used_breakdown[DD_USED_SNAP]);
 		dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_USEDDS,
-		    dd->dd_phys->dd_used_breakdown[DD_USED_HEAD]);
+		    dsl_dir_phys(dd)->dd_used_breakdown[DD_USED_HEAD]);
 		dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_USEDREFRESERV,
-		    dd->dd_phys->dd_used_breakdown[DD_USED_REFRSRV]);
+		    dsl_dir_phys(dd)->dd_used_breakdown[DD_USED_REFRSRV]);
 		dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_USEDCHILD,
-		    dd->dd_phys->dd_used_breakdown[DD_USED_CHILD] +
-		    dd->dd_phys->dd_used_breakdown[DD_USED_CHILD_RSRV]);
+		    dsl_dir_phys(dd)->dd_used_breakdown[DD_USED_CHILD] +
+		    dsl_dir_phys(dd)->dd_used_breakdown[DD_USED_CHILD_RSRV]);
 	}
 	mutex_exit(&dd->dd_lock);
 
@@ -951,7 +956,7 @@ dsl_dir_stats(dsl_dir_t *dd, nvlist_t *nv)
 		char buf[MAXNAMELEN];
 
 		VERIFY0(dsl_dataset_hold_obj(dd->dd_pool,
-		    dd->dd_phys->dd_origin_obj, FTAG, &ds));
+		    dsl_dir_phys(dd)->dd_origin_obj, FTAG, &ds));
 		dsl_dataset_name(ds, buf);
 		dsl_dataset_rele(ds, FTAG);
 		dsl_prop_nvlist_add_string(nv, ZFS_PROP_ORIGIN, buf);
@@ -963,7 +968,7 @@ dsl_dir_dirty(dsl_dir_t *dd, dmu_tx_t *tx)
 {
 	dsl_pool_t *dp = dd->dd_pool;
 
-	ASSERT(dd->dd_phys);
+	ASSERT(dsl_dir_phys(dd));
 
 	if (txg_list_add(&dp->dp_dirty_dirs, dd, tx->tx_txg)) {
 		/* up the hold count until we can be written out */
@@ -974,8 +979,9 @@ dsl_dir_dirty(dsl_dir_t *dd, dmu_tx_t *tx)
 static int64_t
 parent_delta(dsl_dir_t *dd, uint64_t used, int64_t delta)
 {
-	uint64_t old_accounted = MAX(used, dd->dd_phys->dd_reserved);
-	uint64_t new_accounted = MAX(used + delta, dd->dd_phys->dd_reserved);
+	uint64_t old_accounted = MAX(used, dsl_dir_phys(dd)->dd_reserved);
+	uint64_t new_accounted =
+	    MAX(used + delta, dsl_dir_phys(dd)->dd_reserved);
 	return (new_accounted - old_accounted);
 }
 
@@ -1034,9 +1040,9 @@ dsl_dir_space_available(dsl_dir_t *dd,
 	}
 
 	mutex_enter(&dd->dd_lock);
-	if (dd->dd_phys->dd_quota != 0)
-		quota = dd->dd_phys->dd_quota;
-	used = dd->dd_phys->dd_used_bytes;
+	if (dsl_dir_phys(dd)->dd_quota != 0)
+		quota = dsl_dir_phys(dd)->dd_quota;
+	used = dsl_dir_phys(dd)->dd_used_bytes;
 	if (!ondiskonly)
 		used += dsl_dir_space_towrite(dd);
 
@@ -1045,12 +1051,12 @@ dsl_dir_space_available(dsl_dir_t *dd,
 		quota = MIN(quota, poolsize);
 	}
 
-	if (dd->dd_phys->dd_reserved > used && parentspace != UINT64_MAX) {
+	if (dsl_dir_phys(dd)->dd_reserved > used && parentspace != UINT64_MAX) {
 		/*
 		 * We have some space reserved, in addition to what our
 		 * parent gave us.
 		 */
-		parentspace += dd->dd_phys->dd_reserved - used;
+		parentspace += dsl_dir_phys(dd)->dd_reserved - used;
 	}
 
 	if (dd == ancestor) {
@@ -1109,7 +1115,7 @@ dsl_dir_tempreserve_impl(dsl_dir_t *dd, uint64_t asize, boolean_t netfree,
 	est_inflight = dsl_dir_space_towrite(dd);
 	for (i = 0; i < TXG_SIZE; i++)
 		est_inflight += dd->dd_tempreserved[i];
-	used_on_disk = dd->dd_phys->dd_used_bytes;
+	used_on_disk = dsl_dir_phys(dd)->dd_used_bytes;
 
 	/*
 	 * On the first iteration, fetch the dataset's used-on-disk and
@@ -1132,10 +1138,10 @@ dsl_dir_tempreserve_impl(dsl_dir_t *dd, uint64_t asize, boolean_t netfree,
 	 * If this transaction will result in a net free of space,
 	 * we want to let it through.
 	 */
-	if (ignorequota || netfree || dd->dd_phys->dd_quota == 0)
+	if (ignorequota || netfree || dsl_dir_phys(dd)->dd_quota == 0)
 		quota = UINT64_MAX;
 	else
-		quota = dd->dd_phys->dd_quota;
+		quota = dsl_dir_phys(dd)->dd_quota;
 
 	/*
 	 * Adjust the quota against the actual pool size at the root
@@ -1189,7 +1195,7 @@ dsl_dir_tempreserve_impl(dsl_dir_t *dd, uint64_t asize, boolean_t netfree,
 
 	/* see if it's OK with our parent */
 	if (dd->dd_parent && parent_rsrv) {
-		boolean_t ismos = (dd->dd_phys->dd_head_dataset_obj == 0);
+		boolean_t ismos = (dsl_dir_phys(dd)->dd_head_dataset_obj == 0);
 
 		return (dsl_dir_tempreserve_impl(dd->dd_parent,
 		    parent_rsrv, netfree, ismos, TRUE, tr_list, tx, FALSE));
@@ -1308,7 +1314,7 @@ dsl_dir_willuse_space(dsl_dir_t *dd, int64_t space, dmu_tx_t *tx)
 	if (space > 0)
 		dd->dd_space_towrite[tx->tx_txg & TXG_MASK] += space;
 
-	est_used = dsl_dir_space_towrite(dd) + dd->dd_phys->dd_used_bytes;
+	est_used = dsl_dir_space_towrite(dd) + dsl_dir_phys(dd)->dd_used_bytes;
 	parent_space = parent_delta(dd, est_used, space);
 	mutex_exit(&dd->dd_lock);
 
@@ -1343,26 +1349,27 @@ dsl_dir_diduse_space(dsl_dir_t *dd, dd_used_t type,
 
 	if (needlock)
 		mutex_enter(&dd->dd_lock);
-	accounted_delta = parent_delta(dd, dd->dd_phys->dd_used_bytes, used);
-	ASSERT(used >= 0 || dd->dd_phys->dd_used_bytes >= -used);
+	accounted_delta =
+	    parent_delta(dd, dsl_dir_phys(dd)->dd_used_bytes, used);
+	ASSERT(used >= 0 || dsl_dir_phys(dd)->dd_used_bytes >= -used);
 	ASSERT(compressed >= 0 ||
-	    dd->dd_phys->dd_compressed_bytes >= -compressed);
+	    dsl_dir_phys(dd)->dd_compressed_bytes >= -compressed);
 	ASSERT(uncompressed >= 0 ||
-	    dd->dd_phys->dd_uncompressed_bytes >= -uncompressed);
-	dd->dd_phys->dd_used_bytes += used;
-	dd->dd_phys->dd_uncompressed_bytes += uncompressed;
-	dd->dd_phys->dd_compressed_bytes += compressed;
+	    dsl_dir_phys(dd)->dd_uncompressed_bytes >= -uncompressed);
+	dsl_dir_phys(dd)->dd_used_bytes += used;
+	dsl_dir_phys(dd)->dd_uncompressed_bytes += uncompressed;
+	dsl_dir_phys(dd)->dd_compressed_bytes += compressed;
 
-	if (dd->dd_phys->dd_flags & DD_FLAG_USED_BREAKDOWN) {
+	if (dsl_dir_phys(dd)->dd_flags & DD_FLAG_USED_BREAKDOWN) {
 		ASSERT(used > 0 ||
-		    dd->dd_phys->dd_used_breakdown[type] >= -used);
-		dd->dd_phys->dd_used_breakdown[type] += used;
+		    dsl_dir_phys(dd)->dd_used_breakdown[type] >= -used);
+		dsl_dir_phys(dd)->dd_used_breakdown[type] += used;
 #ifdef DEBUG
 		dd_used_t t;
 		uint64_t u = 0;
 		for (t = 0; t < DD_USED_NUM; t++)
-			u += dd->dd_phys->dd_used_breakdown[t];
-		ASSERT3U(u, ==, dd->dd_phys->dd_used_bytes);
+			u += dsl_dir_phys(dd)->dd_used_breakdown[t];
+		ASSERT3U(u, ==, dsl_dir_phys(dd)->dd_used_bytes);
 #endif
 	}
 	if (needlock)
@@ -1385,17 +1392,18 @@ dsl_dir_transfer_space(dsl_dir_t *dd, int64_t delta,
 	ASSERT(oldtype < DD_USED_NUM);
 	ASSERT(newtype < DD_USED_NUM);
 
-	if (delta == 0 || !(dd->dd_phys->dd_flags & DD_FLAG_USED_BREAKDOWN))
+	if (delta == 0 ||
+	    !(dsl_dir_phys(dd)->dd_flags & DD_FLAG_USED_BREAKDOWN))
 		return;
 
 	dmu_buf_will_dirty(dd->dd_dbuf, tx);
 	mutex_enter(&dd->dd_lock);
 	ASSERT(delta > 0 ?
-	    dd->dd_phys->dd_used_breakdown[oldtype] >= delta :
-	    dd->dd_phys->dd_used_breakdown[newtype] >= -delta);
-	ASSERT(dd->dd_phys->dd_used_bytes >= ABS(delta));
-	dd->dd_phys->dd_used_breakdown[oldtype] -= delta;
-	dd->dd_phys->dd_used_breakdown[newtype] += delta;
+	    dsl_dir_phys(dd)->dd_used_breakdown[oldtype] >= delta :
+	    dsl_dir_phys(dd)->dd_used_breakdown[newtype] >= -delta);
+	ASSERT(dsl_dir_phys(dd)->dd_used_bytes >= ABS(delta));
+	dsl_dir_phys(dd)->dd_used_breakdown[oldtype] -= delta;
+	dsl_dir_phys(dd)->dd_used_breakdown[newtype] += delta;
 	mutex_exit(&dd->dd_lock);
 }
 
@@ -1439,8 +1447,8 @@ dsl_dir_set_quota_check(void *arg, dmu_tx_t *tx)
 	 */
 	towrite = dsl_dir_space_towrite(ds->ds_dir);
 	if ((dmu_tx_is_syncing(tx) || towrite == 0) &&
-	    (newval < ds->ds_dir->dd_phys->dd_reserved ||
-	    newval < ds->ds_dir->dd_phys->dd_used_bytes + towrite)) {
+	    (newval < dsl_dir_phys(ds->ds_dir)->dd_reserved ||
+	    newval < dsl_dir_phys(ds->ds_dir)->dd_used_bytes + towrite)) {
 		error = SET_ERROR(ENOSPC);
 	}
 	mutex_exit(&ds->ds_dir->dd_lock);
@@ -1473,7 +1481,7 @@ dsl_dir_set_quota_sync(void *arg, dmu_tx_t *tx)
 
 	dmu_buf_will_dirty(ds->ds_dir->dd_dbuf, tx);
 	mutex_enter(&ds->ds_dir->dd_lock);
-	ds->ds_dir->dd_phys->dd_quota = newval;
+	dsl_dir_phys(ds->ds_dir)->dd_quota = newval;
 	mutex_exit(&ds->ds_dir->dd_lock);
 	dsl_dataset_rele(ds, FTAG);
 }
@@ -1524,7 +1532,7 @@ dsl_dir_set_reservation_check(void *arg, dmu_tx_t *tx)
 	}
 
 	mutex_enter(&dd->dd_lock);
-	used = dd->dd_phys->dd_used_bytes;
+	used = dsl_dir_phys(dd)->dd_used_bytes;
 	mutex_exit(&dd->dd_lock);
 
 	if (dd->dd_parent) {
@@ -1534,13 +1542,13 @@ dsl_dir_set_reservation_check(void *arg, dmu_tx_t *tx)
 		avail = dsl_pool_adjustedsize(dd->dd_pool, B_FALSE) - used;
 	}
 
-	if (MAX(used, newval) > MAX(used, dd->dd_phys->dd_reserved)) {
+	if (MAX(used, newval) > MAX(used, dsl_dir_phys(dd)->dd_reserved)) {
 		uint64_t delta = MAX(used, newval) -
-		    MAX(used, dd->dd_phys->dd_reserved);
+		    MAX(used, dsl_dir_phys(dd)->dd_reserved);
 
 		if (delta > avail ||
-		    (dd->dd_phys->dd_quota > 0 &&
-		    newval > dd->dd_phys->dd_quota))
+		    (dsl_dir_phys(dd)->dd_quota > 0 &&
+		    newval > dsl_dir_phys(dd)->dd_quota))
 			error = SET_ERROR(ENOSPC);
 	}
 
@@ -1557,9 +1565,9 @@ dsl_dir_set_reservation_sync_impl(dsl_dir_t *dd, uint64_t value, dmu_tx_t *tx)
 	dmu_buf_will_dirty(dd->dd_dbuf, tx);
 
 	mutex_enter(&dd->dd_lock);
-	used = dd->dd_phys->dd_used_bytes;
-	delta = MAX(used, value) - MAX(used, dd->dd_phys->dd_reserved);
-	dd->dd_phys->dd_reserved = value;
+	used = dsl_dir_phys(dd)->dd_used_bytes;
+	delta = MAX(used, value) - MAX(used, dsl_dir_phys(dd)->dd_reserved);
+	dsl_dir_phys(dd)->dd_reserved = value;
 
 	if (dd->dd_parent != NULL) {
 		/* Roll up this additional usage into our ancestors */
@@ -1637,7 +1645,7 @@ would_change(dsl_dir_t *dd, int64_t delta, dsl_dir_t *ancestor)
 		return (delta);
 
 	mutex_enter(&dd->dd_lock);
-	delta = parent_delta(dd, dd->dd_phys->dd_used_bytes, delta);
+	delta = parent_delta(dd, dsl_dir_phys(dd)->dd_used_bytes, delta);
 	mutex_exit(&dd->dd_lock);
 	return (would_change(dd->dd_parent, delta, ancestor));
 }
@@ -1728,7 +1736,8 @@ dsl_dir_rename_check(void *arg, dmu_tx_t *tx)
 	if (newparent != dd->dd_parent) {
 		/* is there enough space? */
 		uint64_t myspace =
-		    MAX(dd->dd_phys->dd_used_bytes, dd->dd_phys->dd_reserved);
+		    MAX(dsl_dir_phys(dd)->dd_used_bytes,
+		    dsl_dir_phys(dd)->dd_reserved);
 		objset_t *os = dd->dd_pool->dp_meta_objset;
 		uint64_t fs_cnt = 0;
 		uint64_t ss_cnt = 0;
@@ -1833,17 +1842,18 @@ dsl_dir_rename_sync(void *arg, dmu_tx_t *tx)
 		    DD_FIELD_SNAPSHOT_COUNT, tx);
 
 		dsl_dir_diduse_space(dd->dd_parent, DD_USED_CHILD,
-		    -dd->dd_phys->dd_used_bytes,
-		    -dd->dd_phys->dd_compressed_bytes,
-		    -dd->dd_phys->dd_uncompressed_bytes, tx);
+		    -dsl_dir_phys(dd)->dd_used_bytes,
+		    -dsl_dir_phys(dd)->dd_compressed_bytes,
+		    -dsl_dir_phys(dd)->dd_uncompressed_bytes, tx);
 		dsl_dir_diduse_space(newparent, DD_USED_CHILD,
-		    dd->dd_phys->dd_used_bytes,
-		    dd->dd_phys->dd_compressed_bytes,
-		    dd->dd_phys->dd_uncompressed_bytes, tx);
+		    dsl_dir_phys(dd)->dd_used_bytes,
+		    dsl_dir_phys(dd)->dd_compressed_bytes,
+		    dsl_dir_phys(dd)->dd_uncompressed_bytes, tx);
 
-		if (dd->dd_phys->dd_reserved > dd->dd_phys->dd_used_bytes) {
-			uint64_t unused_rsrv = dd->dd_phys->dd_reserved -
-			    dd->dd_phys->dd_used_bytes;
+		if (dsl_dir_phys(dd)->dd_reserved >
+		    dsl_dir_phys(dd)->dd_used_bytes) {
+			uint64_t unused_rsrv = dsl_dir_phys(dd)->dd_reserved -
+			    dsl_dir_phys(dd)->dd_used_bytes;
 
 			dsl_dir_diduse_space(dd->dd_parent, DD_USED_CHILD_RSRV,
 			    -unused_rsrv, 0, 0, tx);
@@ -1855,18 +1865,19 @@ dsl_dir_rename_sync(void *arg, dmu_tx_t *tx)
 	dmu_buf_will_dirty(dd->dd_dbuf, tx);
 
 	/* remove from old parent zapobj */
-	error = zap_remove(mos, dd->dd_parent->dd_phys->dd_child_dir_zapobj,
+	error = zap_remove(mos,
+	    dsl_dir_phys(dd->dd_parent)->dd_child_dir_zapobj,
 	    dd->dd_myname, tx);
 	ASSERT0(error);
 
 	(void) strcpy(dd->dd_myname, mynewname);
 	dsl_dir_rele(dd->dd_parent, dd);
-	dd->dd_phys->dd_parent_obj = newparent->dd_object;
+	dsl_dir_phys(dd)->dd_parent_obj = newparent->dd_object;
 	VERIFY0(dsl_dir_hold_obj(dp,
 	    newparent->dd_object, NULL, dd, &dd->dd_parent));
 
 	/* add to new parent zapobj */
-	VERIFY0(zap_add(mos, newparent->dd_phys->dd_child_dir_zapobj,
+	VERIFY0(zap_add(mos, dsl_dir_phys(newparent)->dd_child_dir_zapobj,
 	    dd->dd_myname, 8, 1, &dd->dd_object, tx));
 
 	dsl_prop_notify_all(dd);
diff --git a/uts/common/fs/zfs/dsl_pool.c b/uts/common/fs/zfs/dsl_pool.c
index 1d246b8e688d..39a797c27958 100644
--- a/uts/common/fs/zfs/dsl_pool.c
+++ b/uts/common/fs/zfs/dsl_pool.c
@@ -135,7 +135,7 @@ dsl_pool_open_special_dir(dsl_pool_t *dp, const char *name, dsl_dir_t **ddp)
 	int err;
 
 	err = zap_lookup(dp->dp_meta_objset,
-	    dp->dp_root_dir->dd_phys->dd_child_dir_zapobj,
+	    dsl_dir_phys(dp->dp_root_dir)->dd_child_dir_zapobj,
 	    name, sizeof (obj), 1, &obj);
 	if (err)
 		return (err);
@@ -217,11 +217,11 @@ dsl_pool_open(dsl_pool_t *dp)
 		err = dsl_pool_open_special_dir(dp, ORIGIN_DIR_NAME, &dd);
 		if (err)
 			goto out;
-		err = dsl_dataset_hold_obj(dp, dd->dd_phys->dd_head_dataset_obj,
-		    FTAG, &ds);
+		err = dsl_dataset_hold_obj(dp,
+		    dsl_dir_phys(dd)->dd_head_dataset_obj, FTAG, &ds);
 		if (err == 0) {
 			err = dsl_dataset_hold_obj(dp,
-			    ds->ds_phys->ds_prev_snap_obj, dp,
+			    dsl_dataset_phys(ds)->ds_prev_snap_obj, dp,
 			    &dp->dp_origin_snap);
 			dsl_dataset_rele(ds, FTAG);
 		}
@@ -674,15 +674,15 @@ upgrade_clones_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg)
 	if (err)
 		return (err);
 
-	while (ds->ds_phys->ds_prev_snap_obj != 0) {
-		err = dsl_dataset_hold_obj(dp, ds->ds_phys->ds_prev_snap_obj,
-		    FTAG, &prev);
+	while (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) {
+		err = dsl_dataset_hold_obj(dp,
+		    dsl_dataset_phys(ds)->ds_prev_snap_obj, FTAG, &prev);
 		if (err) {
 			dsl_dataset_rele(ds, FTAG);
 			return (err);
 		}
 
-		if (prev->ds_phys->ds_next_snap_obj != ds->ds_object)
+		if (dsl_dataset_phys(prev)->ds_next_snap_obj != ds->ds_object)
 			break;
 		dsl_dataset_rele(ds, FTAG);
 		ds = prev;
@@ -696,7 +696,7 @@ upgrade_clones_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg)
 		 * The $ORIGIN can't have any data, or the accounting
 		 * will be wrong.
 		 */
-		ASSERT0(prev->ds_phys->ds_bp.blk_birth);
+		ASSERT0(dsl_dataset_phys(prev)->ds_bp.blk_birth);
 
 		/* The origin doesn't get attached to itself */
 		if (ds->ds_object == prev->ds_object) {
@@ -705,33 +705,35 @@ upgrade_clones_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg)
 		}
 
 		dmu_buf_will_dirty(ds->ds_dbuf, tx);
-		ds->ds_phys->ds_prev_snap_obj = prev->ds_object;
-		ds->ds_phys->ds_prev_snap_txg = prev->ds_phys->ds_creation_txg;
+		dsl_dataset_phys(ds)->ds_prev_snap_obj = prev->ds_object;
+		dsl_dataset_phys(ds)->ds_prev_snap_txg =
+		    dsl_dataset_phys(prev)->ds_creation_txg;
 
 		dmu_buf_will_dirty(ds->ds_dir->dd_dbuf, tx);
-		ds->ds_dir->dd_phys->dd_origin_obj = prev->ds_object;
+		dsl_dir_phys(ds->ds_dir)->dd_origin_obj = prev->ds_object;
 
 		dmu_buf_will_dirty(prev->ds_dbuf, tx);
-		prev->ds_phys->ds_num_children++;
+		dsl_dataset_phys(prev)->ds_num_children++;
 
-		if (ds->ds_phys->ds_next_snap_obj == 0) {
+		if (dsl_dataset_phys(ds)->ds_next_snap_obj == 0) {
 			ASSERT(ds->ds_prev == NULL);
 			VERIFY0(dsl_dataset_hold_obj(dp,
-			    ds->ds_phys->ds_prev_snap_obj, ds, &ds->ds_prev));
+			    dsl_dataset_phys(ds)->ds_prev_snap_obj,
+			    ds, &ds->ds_prev));
 		}
 	}
 
-	ASSERT3U(ds->ds_dir->dd_phys->dd_origin_obj, ==, prev->ds_object);
-	ASSERT3U(ds->ds_phys->ds_prev_snap_obj, ==, prev->ds_object);
+	ASSERT3U(dsl_dir_phys(ds->ds_dir)->dd_origin_obj, ==, prev->ds_object);
+	ASSERT3U(dsl_dataset_phys(ds)->ds_prev_snap_obj, ==, prev->ds_object);
 
-	if (prev->ds_phys->ds_next_clones_obj == 0) {
+	if (dsl_dataset_phys(prev)->ds_next_clones_obj == 0) {
 		dmu_buf_will_dirty(prev->ds_dbuf, tx);
-		prev->ds_phys->ds_next_clones_obj =
+		dsl_dataset_phys(prev)->ds_next_clones_obj =
 		    zap_create(dp->dp_meta_objset,
 		    DMU_OT_NEXT_CLONES, DMU_OT_NONE, 0, tx);
 	}
 	VERIFY0(zap_add_int(dp->dp_meta_objset,
-	    prev->ds_phys->ds_next_clones_obj, ds->ds_object, tx));
+	    dsl_dataset_phys(prev)->ds_next_clones_obj, ds->ds_object, tx));
 
 	dsl_dataset_rele(ds, FTAG);
 	if (prev != dp->dp_origin_snap)
@@ -756,20 +758,22 @@ upgrade_dir_clones_cb(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg)
 	dmu_tx_t *tx = arg;
 	objset_t *mos = dp->dp_meta_objset;
 
-	if (ds->ds_dir->dd_phys->dd_origin_obj != 0) {
+	if (dsl_dir_phys(ds->ds_dir)->dd_origin_obj != 0) {
 		dsl_dataset_t *origin;
 
 		VERIFY0(dsl_dataset_hold_obj(dp,
-		    ds->ds_dir->dd_phys->dd_origin_obj, FTAG, &origin));
+		    dsl_dir_phys(ds->ds_dir)->dd_origin_obj, FTAG, &origin));
 
-		if (origin->ds_dir->dd_phys->dd_clones == 0) {
+		if (dsl_dir_phys(origin->ds_dir)->dd_clones == 0) {
 			dmu_buf_will_dirty(origin->ds_dir->dd_dbuf, tx);
-			origin->ds_dir->dd_phys->dd_clones = zap_create(mos,
-			    DMU_OT_DSL_CLONES, DMU_OT_NONE, 0, tx);
+			dsl_dir_phys(origin->ds_dir)->dd_clones =
+			    zap_create(mos, DMU_OT_DSL_CLONES, DMU_OT_NONE,
+			    0, tx);
 		}
 
 		VERIFY0(zap_add_int(dp->dp_meta_objset,
-		    origin->ds_dir->dd_phys->dd_clones, ds->ds_object, tx));
+		    dsl_dir_phys(origin->ds_dir)->dd_clones,
+		    ds->ds_object, tx));
 
 		dsl_dataset_rele(origin, FTAG);
 	}
@@ -816,7 +820,7 @@ dsl_pool_create_origin(dsl_pool_t *dp, dmu_tx_t *tx)
 	    NULL, 0, kcred, tx);
 	VERIFY0(dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds));
 	dsl_dataset_snapshot_sync_impl(ds, ORIGIN_DIR_NAME, tx);
-	VERIFY0(dsl_dataset_hold_obj(dp, ds->ds_phys->ds_prev_snap_obj,
+	VERIFY0(dsl_dataset_hold_obj(dp, dsl_dataset_phys(ds)->ds_prev_snap_obj,
 	    dp, &dp->dp_origin_snap));
 	dsl_dataset_rele(ds, FTAG);
 }
diff --git a/uts/common/fs/zfs/dsl_prop.c b/uts/common/fs/zfs/dsl_prop.c
index 5907ea281b0e..4e93e43f9698 100644
--- a/uts/common/fs/zfs/dsl_prop.c
+++ b/uts/common/fs/zfs/dsl_prop.c
@@ -105,8 +105,8 @@ dsl_prop_get_dd(dsl_dir_t *dd, const char *propname,
 		}
 
 		/* Check for a local value. */
-		err = zap_lookup(mos, dd->dd_phys->dd_props_zapobj, propname,
-		    intsz, numints, buf);
+		err = zap_lookup(mos, dsl_dir_phys(dd)->dd_props_zapobj,
+		    propname, intsz, numints, buf);
 		if (err != ENOENT) {
 			if (setpoint != NULL && err == 0)
 				dsl_dir_name(dd, setpoint);
@@ -117,14 +117,14 @@ dsl_prop_get_dd(dsl_dir_t *dd, const char *propname,
 		 * Skip the check for a received value if there is an explicit
 		 * inheritance entry.
 		 */
-		err = zap_contains(mos, dd->dd_phys->dd_props_zapobj,
+		err = zap_contains(mos, dsl_dir_phys(dd)->dd_props_zapobj,
 		    inheritstr);
 		if (err != 0 && err != ENOENT)
 			break;
 
 		if (err == ENOENT) {
 			/* Check for a received value. */
-			err = zap_lookup(mos, dd->dd_phys->dd_props_zapobj,
+			err = zap_lookup(mos, dsl_dir_phys(dd)->dd_props_zapobj,
 			    recvdstr, intsz, numints, buf);
 			if (err != ENOENT) {
 				if (setpoint != NULL && err == 0) {
@@ -169,7 +169,7 @@ dsl_prop_get_ds(dsl_dataset_t *ds, const char *propname,
 	ASSERT(dsl_pool_config_held(ds->ds_dir->dd_pool));
 	inheritable = (prop == ZPROP_INVAL || zfs_prop_inheritable(prop));
 	snapshot = dsl_dataset_is_snapshot(ds);
-	zapobj = ds->ds_phys->ds_props_obj;
+	zapobj = dsl_dataset_phys(ds)->ds_props_obj;
 
 	if (zapobj != 0) {
 		objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset;
@@ -327,7 +327,7 @@ dsl_prop_predict(dsl_dir_t *dd, const char *propname,
 	}
 
 	mos = dd->dd_pool->dp_meta_objset;
-	zapobj = dd->dd_phys->dd_props_zapobj;
+	zapobj = dsl_dir_phys(dd)->dd_props_zapobj;
 	recvdstr = kmem_asprintf("%s%s", propname, ZPROP_RECVD_SUFFIX);
 
 	version = spa_version(dd->dd_pool->dp_spa);
@@ -486,7 +486,8 @@ dsl_prop_changed_notify(dsl_pool_t *dp, uint64_t ddobj,
 		 * If the prop is set here, then this change is not
 		 * being inherited here or below; stop the recursion.
 		 */
-		err = zap_contains(mos, dd->dd_phys->dd_props_zapobj, propname);
+		err = zap_contains(mos, dsl_dir_phys(dd)->dd_props_zapobj,
+		    propname);
 		if (err == 0) {
 			dsl_dir_rele(dd, FTAG);
 			return;
@@ -497,7 +498,7 @@ dsl_prop_changed_notify(dsl_pool_t *dp, uint64_t ddobj,
 	mutex_enter(&dd->dd_lock);
 	for (cbr = list_head(&dd->dd_prop_cbs); cbr;
 	    cbr = list_next(&dd->dd_prop_cbs, cbr)) {
-		uint64_t propobj = cbr->cbr_ds->ds_phys->ds_props_obj;
+		uint64_t propobj = dsl_dataset_phys(cbr->cbr_ds)->ds_props_obj;
 
 		if (strcmp(cbr->cbr_propname, propname) != 0)
 			continue;
@@ -515,7 +516,7 @@ dsl_prop_changed_notify(dsl_pool_t *dp, uint64_t ddobj,
 
 	za = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP);
 	for (zap_cursor_init(&zc, mos,
-	    dd->dd_phys->dd_child_dir_zapobj);
+	    dsl_dir_phys(dd)->dd_child_dir_zapobj);
 	    zap_cursor_retrieve(&zc, za) == 0;
 	    zap_cursor_advance(&zc)) {
 		dsl_prop_changed_notify(dp, za->za_first_integer,
@@ -546,15 +547,15 @@ dsl_prop_set_sync_impl(dsl_dataset_t *ds, const char *propname,
 
 	if (dsl_dataset_is_snapshot(ds)) {
 		ASSERT(version >= SPA_VERSION_SNAP_PROPS);
-		if (ds->ds_phys->ds_props_obj == 0) {
+		if (dsl_dataset_phys(ds)->ds_props_obj == 0) {
 			dmu_buf_will_dirty(ds->ds_dbuf, tx);
-			ds->ds_phys->ds_props_obj =
+			dsl_dataset_phys(ds)->ds_props_obj =
 			    zap_create(mos,
 			    DMU_OT_DSL_PROPS, DMU_OT_NONE, 0, tx);
 		}
-		zapobj = ds->ds_phys->ds_props_obj;
+		zapobj = dsl_dataset_phys(ds)->ds_props_obj;
 	} else {
-		zapobj = ds->ds_dir->dd_phys->dd_props_zapobj;
+		zapobj = dsl_dir_phys(ds->ds_dir)->dd_props_zapobj;
 	}
 
 	if (version < SPA_VERSION_RECVD_PROPS) {
@@ -987,11 +988,11 @@ dsl_prop_get_all_ds(dsl_dataset_t *ds, nvlist_t **nvp,
 
 	ASSERT(dsl_pool_config_held(dp));
 
-	if (ds->ds_phys->ds_props_obj != 0) {
+	if (dsl_dataset_phys(ds)->ds_props_obj != 0) {
 		ASSERT(flags & DSL_PROP_GET_SNAPSHOT);
 		dsl_dataset_name(ds, setpoint);
-		err = dsl_prop_get_all_impl(mos, ds->ds_phys->ds_props_obj,
-		    setpoint, flags, *nvp);
+		err = dsl_prop_get_all_impl(mos,
+		    dsl_dataset_phys(ds)->ds_props_obj, setpoint, flags, *nvp);
 		if (err)
 			goto out;
 	}
@@ -1004,8 +1005,8 @@ dsl_prop_get_all_ds(dsl_dataset_t *ds, nvlist_t **nvp,
 			flags |= DSL_PROP_GET_INHERITING;
 		}
 		dsl_dir_name(dd, setpoint);
-		err = dsl_prop_get_all_impl(mos, dd->dd_phys->dd_props_zapobj,
-		    setpoint, flags, *nvp);
+		err = dsl_prop_get_all_impl(mos,
+		    dsl_dir_phys(dd)->dd_props_zapobj, setpoint, flags, *nvp);
 		if (err)
 			break;
 	}
diff --git a/uts/common/fs/zfs/dsl_scan.c b/uts/common/fs/zfs/dsl_scan.c
index 9dacf88a4eb3..5c9e68fb2ee6 100644
--- a/uts/common/fs/zfs/dsl_scan.c
+++ b/uts/common/fs/zfs/dsl_scan.c
@@ -376,7 +376,7 @@ dsl_scan_ds_maxtxg(dsl_dataset_t *ds)
 {
 	uint64_t smt = ds->ds_dir->dd_pool->dp_scan->scn_phys.scn_max_txg;
 	if (dsl_dataset_is_snapshot(ds))
-		return (MIN(smt, ds->ds_phys->ds_creation_txg));
+		return (MIN(smt, dsl_dataset_phys(ds)->ds_creation_txg));
 	return (smt);
 }
 
@@ -814,11 +814,12 @@ dsl_scan_ds_destroyed(dsl_dataset_t *ds, dmu_tx_t *tx)
 		if (dsl_dataset_is_snapshot(ds)) {
 			/* Note, scn_cur_{min,max}_txg stays the same. */
 			scn->scn_phys.scn_bookmark.zb_objset =
-			    ds->ds_phys->ds_next_snap_obj;
+			    dsl_dataset_phys(ds)->ds_next_snap_obj;
 			zfs_dbgmsg("destroying ds %llu; currently traversing; "
 			    "reset zb_objset to %llu",
 			    (u_longlong_t)ds->ds_object,
-			    (u_longlong_t)ds->ds_phys->ds_next_snap_obj);
+			    (u_longlong_t)dsl_dataset_phys(ds)->
+			    ds_next_snap_obj);
 			scn->scn_phys.scn_flags |= DSF_VISIT_DS_AGAIN;
 		} else {
 			SET_BOOKMARK(&scn->scn_phys.scn_bookmark,
@@ -829,7 +830,7 @@ dsl_scan_ds_destroyed(dsl_dataset_t *ds, dmu_tx_t *tx)
 		}
 	} else if (zap_lookup_int_key(dp->dp_meta_objset,
 	    scn->scn_phys.scn_queue_obj, ds->ds_object, &mintxg) == 0) {
-		ASSERT3U(ds->ds_phys->ds_num_children, <=, 1);
+		ASSERT3U(dsl_dataset_phys(ds)->ds_num_children, <=, 1);
 		VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset,
 		    scn->scn_phys.scn_queue_obj, ds->ds_object, tx));
 		if (dsl_dataset_is_snapshot(ds)) {
@@ -840,11 +841,13 @@ dsl_scan_ds_destroyed(dsl_dataset_t *ds, dmu_tx_t *tx)
 			 */
 			VERIFY(zap_add_int_key(dp->dp_meta_objset,
 			    scn->scn_phys.scn_queue_obj,
-			    ds->ds_phys->ds_next_snap_obj, mintxg, tx) == 0);
+			    dsl_dataset_phys(ds)->ds_next_snap_obj,
+			    mintxg, tx) == 0);
 			zfs_dbgmsg("destroying ds %llu; in queue; "
 			    "replacing with %llu",
 			    (u_longlong_t)ds->ds_object,
-			    (u_longlong_t)ds->ds_phys->ds_next_snap_obj);
+			    (u_longlong_t)dsl_dataset_phys(ds)->
+			    ds_next_snap_obj);
 		} else {
 			zfs_dbgmsg("destroying ds %llu; in queue; removing",
 			    (u_longlong_t)ds->ds_object);
@@ -871,26 +874,26 @@ dsl_scan_ds_snapshotted(dsl_dataset_t *ds, dmu_tx_t *tx)
 	if (scn->scn_phys.scn_state != DSS_SCANNING)
 		return;
 
-	ASSERT(ds->ds_phys->ds_prev_snap_obj != 0);
+	ASSERT(dsl_dataset_phys(ds)->ds_prev_snap_obj != 0);
 
 	if (scn->scn_phys.scn_bookmark.zb_objset == ds->ds_object) {
 		scn->scn_phys.scn_bookmark.zb_objset =
-		    ds->ds_phys->ds_prev_snap_obj;
+		    dsl_dataset_phys(ds)->ds_prev_snap_obj;
 		zfs_dbgmsg("snapshotting ds %llu; currently traversing; "
 		    "reset zb_objset to %llu",
 		    (u_longlong_t)ds->ds_object,
-		    (u_longlong_t)ds->ds_phys->ds_prev_snap_obj);
+		    (u_longlong_t)dsl_dataset_phys(ds)->ds_prev_snap_obj);
 	} else if (zap_lookup_int_key(dp->dp_meta_objset,
 	    scn->scn_phys.scn_queue_obj, ds->ds_object, &mintxg) == 0) {
 		VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset,
 		    scn->scn_phys.scn_queue_obj, ds->ds_object, tx));
 		VERIFY(zap_add_int_key(dp->dp_meta_objset,
 		    scn->scn_phys.scn_queue_obj,
-		    ds->ds_phys->ds_prev_snap_obj, mintxg, tx) == 0);
+		    dsl_dataset_phys(ds)->ds_prev_snap_obj, mintxg, tx) == 0);
 		zfs_dbgmsg("snapshotting ds %llu; in queue; "
 		    "replacing with %llu",
 		    (u_longlong_t)ds->ds_object,
-		    (u_longlong_t)ds->ds_phys->ds_prev_snap_obj);
+		    (u_longlong_t)dsl_dataset_phys(ds)->ds_prev_snap_obj);
 	}
 	dsl_scan_sync_state(scn, tx);
 }
@@ -923,8 +926,8 @@ dsl_scan_ds_clone_swapped(dsl_dataset_t *ds1, dsl_dataset_t *ds2, dmu_tx_t *tx)
 	    ds1->ds_object, &mintxg) == 0) {
 		int err;
 
-		ASSERT3U(mintxg, ==, ds1->ds_phys->ds_prev_snap_txg);
-		ASSERT3U(mintxg, ==, ds2->ds_phys->ds_prev_snap_txg);
+		ASSERT3U(mintxg, ==, dsl_dataset_phys(ds1)->ds_prev_snap_txg);
+		ASSERT3U(mintxg, ==, dsl_dataset_phys(ds2)->ds_prev_snap_txg);
 		VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset,
 		    scn->scn_phys.scn_queue_obj, ds1->ds_object, tx));
 		err = zap_add_int_key(dp->dp_meta_objset,
@@ -942,8 +945,8 @@ dsl_scan_ds_clone_swapped(dsl_dataset_t *ds1, dsl_dataset_t *ds2, dmu_tx_t *tx)
 		    (u_longlong_t)ds2->ds_object);
 	} else if (zap_lookup_int_key(dp->dp_meta_objset,
 	    scn->scn_phys.scn_queue_obj, ds2->ds_object, &mintxg) == 0) {
-		ASSERT3U(mintxg, ==, ds1->ds_phys->ds_prev_snap_txg);
-		ASSERT3U(mintxg, ==, ds2->ds_phys->ds_prev_snap_txg);
+		ASSERT3U(mintxg, ==, dsl_dataset_phys(ds1)->ds_prev_snap_txg);
+		ASSERT3U(mintxg, ==, dsl_dataset_phys(ds2)->ds_prev_snap_txg);
 		VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset,
 		    scn->scn_phys.scn_queue_obj, ds2->ds_object, tx));
 		VERIFY(0 == zap_add_int_key(dp->dp_meta_objset,
@@ -971,17 +974,17 @@ enqueue_clones_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg)
 	int err;
 	dsl_scan_t *scn = dp->dp_scan;
 
-	if (hds->ds_dir->dd_phys->dd_origin_obj != eca->originobj)
+	if (dsl_dir_phys(hds->ds_dir)->dd_origin_obj != eca->originobj)
 		return (0);
 
 	err = dsl_dataset_hold_obj(dp, hds->ds_object, FTAG, &ds);
 	if (err)
 		return (err);
 
-	while (ds->ds_phys->ds_prev_snap_obj != eca->originobj) {
+	while (dsl_dataset_phys(ds)->ds_prev_snap_obj != eca->originobj) {
 		dsl_dataset_t *prev;
 		err = dsl_dataset_hold_obj(dp,
-		    ds->ds_phys->ds_prev_snap_obj, FTAG, &prev);
+		    dsl_dataset_phys(ds)->ds_prev_snap_obj, FTAG, &prev);
 
 		dsl_dataset_rele(ds, FTAG);
 		if (err)
@@ -990,7 +993,7 @@ enqueue_clones_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg)
 	}
 	VERIFY(zap_add_int_key(dp->dp_meta_objset,
 	    scn->scn_phys.scn_queue_obj, ds->ds_object,
-	    ds->ds_phys->ds_prev_snap_txg, eca->tx) == 0);
+	    dsl_dataset_phys(ds)->ds_prev_snap_txg, eca->tx) == 0);
 	dsl_dataset_rele(ds, FTAG);
 	return (0);
 }
@@ -1021,7 +1024,7 @@ dsl_scan_visitds(dsl_scan_t *scn, uint64_t dsobj, dmu_tx_t *tx)
 	 * Iterate over the bps in this ds.
 	 */
 	dmu_buf_will_dirty(ds->ds_dbuf, tx);
-	dsl_scan_visit_rootbp(scn, ds, &ds->ds_phys->ds_bp, tx);
+	dsl_scan_visit_rootbp(scn, ds, &dsl_dataset_phys(ds)->ds_bp, tx);
 
 	char *dsname = kmem_alloc(ZFS_MAXNAMELEN, KM_SLEEP);
 	dsl_dataset_name(ds, dsname);
@@ -1055,14 +1058,15 @@ dsl_scan_visitds(dsl_scan_t *scn, uint64_t dsobj, dmu_tx_t *tx)
 	/*
 	 * Add descendent datasets to work queue.
 	 */
-	if (ds->ds_phys->ds_next_snap_obj != 0) {
+	if (dsl_dataset_phys(ds)->ds_next_snap_obj != 0) {
 		VERIFY(zap_add_int_key(dp->dp_meta_objset,
-		    scn->scn_phys.scn_queue_obj, ds->ds_phys->ds_next_snap_obj,
-		    ds->ds_phys->ds_creation_txg, tx) == 0);
+		    scn->scn_phys.scn_queue_obj,
+		    dsl_dataset_phys(ds)->ds_next_snap_obj,
+		    dsl_dataset_phys(ds)->ds_creation_txg, tx) == 0);
 	}
-	if (ds->ds_phys->ds_num_children > 1) {
+	if (dsl_dataset_phys(ds)->ds_num_children > 1) {
 		boolean_t usenext = B_FALSE;
-		if (ds->ds_phys->ds_next_clones_obj != 0) {
+		if (dsl_dataset_phys(ds)->ds_next_clones_obj != 0) {
 			uint64_t count;
 			/*
 			 * A bug in a previous version of the code could
@@ -1072,17 +1076,17 @@ dsl_scan_visitds(dsl_scan_t *scn, uint64_t dsobj, dmu_tx_t *tx)
 			 * next_clones_obj when its count is correct.
 			 */
 			int err = zap_count(dp->dp_meta_objset,
-			    ds->ds_phys->ds_next_clones_obj, &count);
+			    dsl_dataset_phys(ds)->ds_next_clones_obj, &count);
 			if (err == 0 &&
-			    count == ds->ds_phys->ds_num_children - 1)
+			    count == dsl_dataset_phys(ds)->ds_num_children - 1)
 				usenext = B_TRUE;
 		}
 
 		if (usenext) {
 			VERIFY0(zap_join_key(dp->dp_meta_objset,
-			    ds->ds_phys->ds_next_clones_obj,
+			    dsl_dataset_phys(ds)->ds_next_clones_obj,
 			    scn->scn_phys.scn_queue_obj,
-			    ds->ds_phys->ds_creation_txg, tx));
+			    dsl_dataset_phys(ds)->ds_creation_txg, tx));
 		} else {
 			struct enqueue_clones_arg eca;
 			eca.tx = tx;
@@ -1110,10 +1114,10 @@ enqueue_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg)
 	if (err)
 		return (err);
 
-	while (ds->ds_phys->ds_prev_snap_obj != 0) {
+	while (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) {
 		dsl_dataset_t *prev;
-		err = dsl_dataset_hold_obj(dp, ds->ds_phys->ds_prev_snap_obj,
-		    FTAG, &prev);
+		err = dsl_dataset_hold_obj(dp,
+		    dsl_dataset_phys(ds)->ds_prev_snap_obj, FTAG, &prev);
 		if (err) {
 			dsl_dataset_rele(ds, FTAG);
 			return (err);
@@ -1122,7 +1126,7 @@ enqueue_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg)
 		/*
 		 * If this is a clone, we don't need to worry about it for now.
 		 */
-		if (prev->ds_phys->ds_next_snap_obj != ds->ds_object) {
+		if (dsl_dataset_phys(prev)->ds_next_snap_obj != ds->ds_object) {
 			dsl_dataset_rele(ds, FTAG);
 			dsl_dataset_rele(prev, FTAG);
 			return (0);
@@ -1132,7 +1136,7 @@ enqueue_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg)
 	}
 
 	VERIFY(zap_add_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj,
-	    ds->ds_object, ds->ds_phys->ds_prev_snap_txg, tx) == 0);
+	    ds->ds_object, dsl_dataset_phys(ds)->ds_prev_snap_txg, tx) == 0);
 	dsl_dataset_rele(ds, FTAG);
 	return (0);
 }
@@ -1307,7 +1311,7 @@ dsl_scan_visit(dsl_scan_t *scn, dmu_tx_t *tx)
 		} else {
 			scn->scn_phys.scn_cur_min_txg =
 			    MAX(scn->scn_phys.scn_min_txg,
-			    ds->ds_phys->ds_prev_snap_txg);
+			    dsl_dataset_phys(ds)->ds_prev_snap_txg);
 		}
 		scn->scn_phys.scn_cur_max_txg = dsl_scan_ds_maxtxg(ds);
 		dsl_dataset_rele(ds, FTAG);
@@ -1494,9 +1498,9 @@ dsl_scan_sync(dsl_pool_t *dp, dmu_tx_t *tx)
 	if (err != 0)
 		return;
 	if (!scn->scn_async_destroying && zfs_free_leak_on_eio &&
-	    (dp->dp_free_dir->dd_phys->dd_used_bytes != 0 ||
-	    dp->dp_free_dir->dd_phys->dd_compressed_bytes != 0 ||
-	    dp->dp_free_dir->dd_phys->dd_uncompressed_bytes != 0)) {
+	    (dsl_dir_phys(dp->dp_free_dir)->dd_used_bytes != 0 ||
+	    dsl_dir_phys(dp->dp_free_dir)->dd_compressed_bytes != 0 ||
+	    dsl_dir_phys(dp->dp_free_dir)->dd_uncompressed_bytes != 0)) {
 		/*
 		 * We have finished background destroying, but there is still
 		 * some space left in the dp_free_dir. Transfer this leaked
@@ -1511,19 +1515,19 @@ dsl_scan_sync(dsl_pool_t *dp, dmu_tx_t *tx)
 			rrw_exit(&dp->dp_config_rwlock, FTAG);
 		}
 		dsl_dir_diduse_space(dp->dp_leak_dir, DD_USED_HEAD,
-		    dp->dp_free_dir->dd_phys->dd_used_bytes,
-		    dp->dp_free_dir->dd_phys->dd_compressed_bytes,
-		    dp->dp_free_dir->dd_phys->dd_uncompressed_bytes, tx);
+		    dsl_dir_phys(dp->dp_free_dir)->dd_used_bytes,
+		    dsl_dir_phys(dp->dp_free_dir)->dd_compressed_bytes,
+		    dsl_dir_phys(dp->dp_free_dir)->dd_uncompressed_bytes, tx);
 		dsl_dir_diduse_space(dp->dp_free_dir, DD_USED_HEAD,
-		    -dp->dp_free_dir->dd_phys->dd_used_bytes,
-		    -dp->dp_free_dir->dd_phys->dd_compressed_bytes,
-		    -dp->dp_free_dir->dd_phys->dd_uncompressed_bytes, tx);
+		    -dsl_dir_phys(dp->dp_free_dir)->dd_used_bytes,
+		    -dsl_dir_phys(dp->dp_free_dir)->dd_compressed_bytes,
+		    -dsl_dir_phys(dp->dp_free_dir)->dd_uncompressed_bytes, tx);
 	}
 	if (!scn->scn_async_destroying) {
 		/* finished; verify that space accounting went to zero */
-		ASSERT0(dp->dp_free_dir->dd_phys->dd_used_bytes);
-		ASSERT0(dp->dp_free_dir->dd_phys->dd_compressed_bytes);
-		ASSERT0(dp->dp_free_dir->dd_phys->dd_uncompressed_bytes);
+		ASSERT0(dsl_dir_phys(dp->dp_free_dir)->dd_used_bytes);
+		ASSERT0(dsl_dir_phys(dp->dp_free_dir)->dd_compressed_bytes);
+		ASSERT0(dsl_dir_phys(dp->dp_free_dir)->dd_uncompressed_bytes);
 	}
 
 	if (scn->scn_phys.scn_state != DSS_SCANNING)
diff --git a/uts/common/fs/zfs/dsl_synctask.c b/uts/common/fs/zfs/dsl_synctask.c
index 8e09347678dd..e66e931726f0 100644
--- a/uts/common/fs/zfs/dsl_synctask.c
+++ b/uts/common/fs/zfs/dsl_synctask.c
@@ -163,7 +163,7 @@ dsl_sync_task_sync(dsl_sync_task_t *dst, dmu_tx_t *tx)
 		uint64_t quota = dsl_pool_adjustedsize(dp,
 		    dst->dst_space_check == ZFS_SPACE_CHECK_RESERVED) -
 		    metaslab_class_get_deferred(spa_normal_class(dp->dp_spa));
-		uint64_t used = dp->dp_root_dir->dd_phys->dd_used_bytes;
+		uint64_t used = dsl_dir_phys(dp->dp_root_dir)->dd_used_bytes;
 		/* MOS space is triple-dittoed, so we multiply by 3. */
 		if (dst->dst_space > 0 && used + dst->dst_space * 3 > quota) {
 			dst->dst_error = SET_ERROR(ENOSPC);
diff --git a/uts/common/fs/zfs/dsl_userhold.c b/uts/common/fs/zfs/dsl_userhold.c
index b6878d7ecc44..0985ec91fd50 100644
--- a/uts/common/fs/zfs/dsl_userhold.c
+++ b/uts/common/fs/zfs/dsl_userhold.c
@@ -64,10 +64,10 @@ dsl_dataset_user_hold_check_one(dsl_dataset_t *ds, const char *htag,
 		return (SET_ERROR(E2BIG));
 
 	/* tags must be unique (if ds already exists) */
-	if (ds != NULL && ds->ds_phys->ds_userrefs_obj != 0) {
+	if (ds != NULL && dsl_dataset_phys(ds)->ds_userrefs_obj != 0) {
 		uint64_t value;
 
-		error = zap_lookup(mos, ds->ds_phys->ds_userrefs_obj,
+		error = zap_lookup(mos, dsl_dataset_phys(ds)->ds_userrefs_obj,
 		    htag, 8, 1, &value);
 		if (error == 0)
 			error = SET_ERROR(EEXIST);
@@ -140,16 +140,16 @@ dsl_dataset_user_hold_sync_one_impl(nvlist_t *tmpholds, dsl_dataset_t *ds,
 
 	ASSERT(RRW_WRITE_HELD(&dp->dp_config_rwlock));
 
-	if (ds->ds_phys->ds_userrefs_obj == 0) {
+	if (dsl_dataset_phys(ds)->ds_userrefs_obj == 0) {
 		/*
 		 * This is the first user hold for this dataset.  Create
 		 * the userrefs zap object.
 		 */
 		dmu_buf_will_dirty(ds->ds_dbuf, tx);
-		zapobj = ds->ds_phys->ds_userrefs_obj =
+		zapobj = dsl_dataset_phys(ds)->ds_userrefs_obj =
 		    zap_create(mos, DMU_OT_USERREFS, DMU_OT_NONE, 0, tx);
 	} else {
-		zapobj = ds->ds_phys->ds_userrefs_obj;
+		zapobj = dsl_dataset_phys(ds)->ds_userrefs_obj;
 	}
 	ds->ds_userrefs++;
 
@@ -360,7 +360,7 @@ dsl_dataset_user_release_check_one(dsl_dataset_user_release_arg_t *ddura,
 
 	numholds = 0;
 	mos = ds->ds_dir->dd_pool->dp_meta_objset;
-	zapobj = ds->ds_phys->ds_userrefs_obj;
+	zapobj = dsl_dataset_phys(ds)->ds_userrefs_obj;
 	holds_found = fnvlist_alloc();
 
 	for (nvpair_t *pair = nvlist_next_nvpair(holds, NULL); pair != NULL;
@@ -398,7 +398,8 @@ dsl_dataset_user_release_check_one(dsl_dataset_user_release_arg_t *ddura,
 		numholds++;
 	}
 
-	if (DS_IS_DEFER_DESTROY(ds) && ds->ds_phys->ds_num_children == 1 &&
+	if (DS_IS_DEFER_DESTROY(ds) &&
+	    dsl_dataset_phys(ds)->ds_num_children == 1 &&
 	    ds->ds_userrefs == numholds) {
 		/* we need to destroy the snapshot as well */
 		if (dsl_dataset_long_held(ds)) {
@@ -484,8 +485,8 @@ dsl_dataset_user_release_sync_one(dsl_dataset_t *ds, nvlist_t *holds,
 		error = dsl_pool_user_release(dp, ds->ds_object, holdname, tx);
 		VERIFY(error == 0 || error == ENOENT);
 
-		VERIFY0(zap_remove(mos, ds->ds_phys->ds_userrefs_obj, holdname,
-		    tx));
+		VERIFY0(zap_remove(mos, dsl_dataset_phys(ds)->ds_userrefs_obj,
+		    holdname, tx));
 		ds->ds_userrefs--;
 
 		spa_history_log_internal_ds(ds, "release", tx,
@@ -514,7 +515,7 @@ dsl_dataset_user_release_sync(void *arg, dmu_tx_t *tx)
 		    fnvpair_value_nvlist(pair), tx);
 		if (nvlist_exists(ddura->ddura_todelete, name)) {
 			ASSERT(ds->ds_userrefs == 0 &&
-			    ds->ds_phys->ds_num_children == 1 &&
+			    dsl_dataset_phys(ds)->ds_num_children == 1 &&
 			    DS_IS_DEFER_DESTROY(ds));
 			dsl_destroy_snapshot_sync_impl(ds, B_FALSE, tx);
 		}
@@ -644,13 +645,13 @@ dsl_dataset_get_holds(const char *dsname, nvlist_t *nvl)
 		return (err);
 	}
 
-	if (ds->ds_phys->ds_userrefs_obj != 0) {
+	if (dsl_dataset_phys(ds)->ds_userrefs_obj != 0) {
 		zap_attribute_t *za;
 		zap_cursor_t zc;
 
 		za = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP);
 		for (zap_cursor_init(&zc, ds->ds_dir->dd_pool->dp_meta_objset,
-		    ds->ds_phys->ds_userrefs_obj);
+		    dsl_dataset_phys(ds)->ds_userrefs_obj);
 		    zap_cursor_retrieve(&zc, za) == 0;
 		    zap_cursor_advance(&zc)) {
 			fnvlist_add_uint64(nvl, za->za_name,
diff --git a/uts/common/fs/zfs/sa.c b/uts/common/fs/zfs/sa.c
index 8b3963aed9f5..27d8513541a4 100644
--- a/uts/common/fs/zfs/sa.c
+++ b/uts/common/fs/zfs/sa.c
@@ -1345,7 +1345,7 @@ sa_handle_destroy(sa_handle_t *hdl)
 {
 	mutex_enter(&hdl->sa_lock);
 	(void) dmu_buf_update_user((dmu_buf_t *)hdl->sa_bonus, hdl,
-	    NULL, NULL, NULL);
+	    NULL, NULL);
 
 	if (hdl->sa_bonus_tab) {
 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_bonus_tab);
@@ -1392,8 +1392,7 @@ sa_handle_get_from_db(objset_t *os, dmu_buf_t *db, void *userp,
 
 		error = sa_build_index(handle, SA_BONUS);
 		newhandle = (hdl_type == SA_HDL_SHARED) ?
-		    dmu_buf_set_user_ie(db, handle,
-		    NULL, sa_evict) : NULL;
+		    dmu_buf_set_user_ie(db, handle, sa_evict) : NULL;
 
 		if (newhandle != NULL) {
 			kmem_cache_free(sa_cache, handle);
@@ -1911,7 +1910,7 @@ void
 sa_update_user(sa_handle_t *newhdl, sa_handle_t *oldhdl)
 {
 	(void) dmu_buf_update_user((dmu_buf_t *)newhdl->sa_bonus,
-	    oldhdl, newhdl, NULL, sa_evict);
+	    oldhdl, newhdl, sa_evict);
 	oldhdl->sa_bonus = NULL;
 }
 
diff --git a/uts/common/fs/zfs/spa.c b/uts/common/fs/zfs/spa.c
index a61117812e66..d0408a58023c 100644
--- a/uts/common/fs/zfs/spa.c
+++ b/uts/common/fs/zfs/spa.c
@@ -241,7 +241,8 @@ spa_prop_get_config(spa_t *spa, nvlist_t **nvp)
 		 */
 		if (pool->dp_free_dir != NULL) {
 			spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING, NULL,
-			    pool->dp_free_dir->dd_phys->dd_used_bytes, src);
+			    dsl_dir_phys(pool->dp_free_dir)->dd_used_bytes,
+			    src);
 		} else {
 			spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING,
 			    NULL, 0, src);
@@ -249,7 +250,8 @@ spa_prop_get_config(spa_t *spa, nvlist_t **nvp)
 
 		if (pool->dp_leak_dir != NULL) {
 			spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED, NULL,
-			    pool->dp_leak_dir->dd_phys->dd_used_bytes, src);
+			    dsl_dir_phys(pool->dp_leak_dir)->dd_used_bytes,
+			    src);
 		} else {
 			spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED,
 			    NULL, 0, src);
diff --git a/uts/common/fs/zfs/spa_history.c b/uts/common/fs/zfs/spa_history.c
index ce64f70b28c3..4923e3cba26d 100644
--- a/uts/common/fs/zfs/spa_history.c
+++ b/uts/common/fs/zfs/spa_history.c
@@ -520,7 +520,7 @@ spa_history_log_internal_dd(dsl_dir_t *dd, const char *operation,
 	dsl_dir_name(dd, namebuf);
 	fnvlist_add_string(nvl, ZPOOL_HIST_DSNAME, namebuf);
 	fnvlist_add_uint64(nvl, ZPOOL_HIST_DSID,
-	    dd->dd_phys->dd_head_dataset_obj);
+	    dsl_dir_phys(dd)->dd_head_dataset_obj);
 
 	va_start(adx, fmt);
 	log_internal(nvl, operation, dd->dd_pool->dp_spa, tx, fmt, adx);
diff --git a/uts/common/fs/zfs/sys/dbuf.h b/uts/common/fs/zfs/sys/dbuf.h
index 8ca8753e5e6d..8be8ed6bc85b 100644
--- a/uts/common/fs/zfs/sys/dbuf.h
+++ b/uts/common/fs/zfs/sys/dbuf.h
@@ -228,7 +228,6 @@ typedef struct dmu_buf_impl {
 
 	/* stuff we store for the user (see dmu_buf_set_user) */
 	void *db_user_ptr;
-	void **db_user_data_ptr_ptr;
 	dmu_buf_evict_func_t *db_evict_func;
 
 	uint8_t db_immediate_evict;
diff --git a/uts/common/fs/zfs/sys/dmu.h b/uts/common/fs/zfs/sys/dmu.h
index 93165a95951d..24473daa6d2b 100644
--- a/uts/common/fs/zfs/sys/dmu.h
+++ b/uts/common/fs/zfs/sys/dmu.h
@@ -484,12 +484,6 @@ void dmu_buf_rele_array(dmu_buf_t **, int numbufs, void *tag);
  *
  * user_ptr is for use by the user and can be obtained via dmu_buf_get_user().
  *
- * user_data_ptr_ptr should be NULL, or a pointer to a pointer which
- * will be set to db->db_data when you are allowed to access it.  Note
- * that db->db_data (the pointer) can change when you do dmu_buf_read(),
- * dmu_buf_tryupgrade(), dmu_buf_will_dirty(), or dmu_buf_will_fill().
- * *user_data_ptr_ptr will be set to the new value when it changes.
- *
  * If non-NULL, pageout func will be called when this buffer is being
  * excised from the cache, so that you can clean up the data structure
  * pointed to by user_ptr.
@@ -497,17 +491,16 @@ void dmu_buf_rele_array(dmu_buf_t **, int numbufs, void *tag);
  * dmu_evict_user() will call the pageout func for all buffers in a
  * objset with a given pageout func.
  */
-void *dmu_buf_set_user(dmu_buf_t *db, void *user_ptr, void *user_data_ptr_ptr,
+void *dmu_buf_set_user(dmu_buf_t *db, void *user_ptr,
     dmu_buf_evict_func_t *pageout_func);
 /*
  * set_user_ie is the same as set_user, but request immediate eviction
  * when hold count goes to zero.
  */
 void *dmu_buf_set_user_ie(dmu_buf_t *db, void *user_ptr,
-    void *user_data_ptr_ptr, dmu_buf_evict_func_t *pageout_func);
-void *dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr,
-    void *user_ptr, void *user_data_ptr_ptr,
     dmu_buf_evict_func_t *pageout_func);
+void *dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr,
+    void *user_ptr, dmu_buf_evict_func_t *pageout_func);
 void dmu_evict_user(objset_t *os, dmu_buf_evict_func_t *func);
 
 /*
diff --git a/uts/common/fs/zfs/sys/dsl_dataset.h b/uts/common/fs/zfs/sys/dsl_dataset.h
index ff90f8b439cc..3160a05a8c59 100644
--- a/uts/common/fs/zfs/sys/dsl_dataset.h
+++ b/uts/common/fs/zfs/sys/dsl_dataset.h
@@ -48,7 +48,7 @@ struct dsl_pool;
 
 #define	DS_FLAG_INCONSISTENT	(1ULL<<0)
 #define	DS_IS_INCONSISTENT(ds)	\
-	((ds)->ds_phys->ds_flags & DS_FLAG_INCONSISTENT)
+	(dsl_dataset_phys(ds)->ds_flags & DS_FLAG_INCONSISTENT)
 
 /*
  * Do not allow this dataset to be promoted.
@@ -68,7 +68,7 @@ struct dsl_pool;
  */
 #define	DS_FLAG_DEFER_DESTROY	(1ULL<<3)
 #define	DS_IS_DEFER_DESTROY(ds)	\
-	((ds)->ds_phys->ds_flags & DS_FLAG_DEFER_DESTROY)
+	(dsl_dataset_phys(ds)->ds_flags & DS_FLAG_DEFER_DESTROY)
 
 /*
  * DS_FIELD_* are strings that are used in the "extensified" dataset zap object.
@@ -134,7 +134,6 @@ typedef struct dsl_dataset_phys {
 typedef struct dsl_dataset {
 	/* Immutable: */
 	struct dsl_dir *ds_dir;
-	dsl_dataset_phys_t *ds_phys;
 	dmu_buf_t *ds_dbuf;
 	uint64_t ds_object;
 	uint64_t ds_fsid_guid;
@@ -186,17 +185,26 @@ typedef struct dsl_dataset {
 	char ds_snapname[MAXNAMELEN];
 } dsl_dataset_t;
 
+inline dsl_dataset_phys_t *
+dsl_dataset_phys(dsl_dataset_t *ds)
+{
+	return (ds->ds_dbuf->db_data);
+}
+
 /*
  * The max length of a temporary tag prefix is the number of hex digits
  * required to express UINT64_MAX plus one for the hyphen.
  */
 #define	MAX_TAG_PREFIX_LEN	17
 
-#define	dsl_dataset_is_snapshot(ds) \
-	((ds)->ds_phys->ds_num_children != 0)
+inline boolean_t
+dsl_dataset_is_snapshot(dsl_dataset_t *ds)
+{
+	return (dsl_dataset_phys(ds)->ds_num_children != 0);
+}
 
 #define	DS_UNIQUE_IS_ACCURATE(ds)	\
-	(((ds)->ds_phys->ds_flags & DS_FLAG_UNIQUE_ACCURATE) != 0)
+	((dsl_dataset_phys(ds)->ds_flags & DS_FLAG_UNIQUE_ACCURATE) != 0)
 
 int dsl_dataset_hold(struct dsl_pool *dp, const char *name, void *tag,
     dsl_dataset_t **dsp);
diff --git a/uts/common/fs/zfs/sys/dsl_dir.h b/uts/common/fs/zfs/sys/dsl_dir.h
index a9c4f67515a9..772bfbe6db12 100644
--- a/uts/common/fs/zfs/sys/dsl_dir.h
+++ b/uts/common/fs/zfs/sys/dsl_dir.h
@@ -86,10 +86,11 @@ typedef struct dsl_dir_phys {
 struct dsl_dir {
 	/* These are immutable; no lock needed: */
 	uint64_t dd_object;
-	dsl_dir_phys_t *dd_phys;
-	dmu_buf_t *dd_dbuf;
 	dsl_pool_t *dd_pool;
 
+	/* Stable until user eviction; no lock needed: */
+	dmu_buf_t *dd_dbuf;
+
 	/* protected by lock on pool's dp_dirty_dirs list */
 	txg_node_t dd_dirty_link;
 
@@ -111,6 +112,12 @@ struct dsl_dir {
 	char dd_myname[MAXNAMELEN];
 };
 
+inline dsl_dir_phys_t *
+dsl_dir_phys(dsl_dir_t *dd)
+{
+	return (dd->dd_dbuf->db_data);
+}
+
 void dsl_dir_rele(dsl_dir_t *dd, void *tag);
 int dsl_dir_hold(dsl_pool_t *dp, const char *name, void *tag,
     dsl_dir_t **, const char **tail);
diff --git a/uts/common/fs/zfs/sys/zap_impl.h b/uts/common/fs/zfs/sys/zap_impl.h
index 8b4a8b2b56ed..0d4fc69815fc 100644
--- a/uts/common/fs/zfs/sys/zap_impl.h
+++ b/uts/common/fs/zfs/sys/zap_impl.h
@@ -70,7 +70,7 @@ typedef struct mzap_ent {
 } mzap_ent_t;
 
 #define	MZE_PHYS(zap, mze) \
-	(&(zap)->zap_m.zap_phys->mz_chunk[(mze)->mze_chunkid])
+	(&zap_m_phys(zap)->mz_chunk[(mze)->mze_chunkid])
 
 /*
  * The (fat) zap is stored in one object. It is an array of
@@ -104,7 +104,7 @@ struct zap_leaf;
  * word number (1<<ZAP_EMBEDDED_PTRTBL_SHIFT(zap)).
  */
 #define	ZAP_EMBEDDED_PTRTBL_ENT(zap, idx) \
-	((uint64_t *)(zap)->zap_f.zap_phys) \
+	((uint64_t *)zap_f_phys(zap)) \
 	[(idx) + (1<<ZAP_EMBEDDED_PTRTBL_SHIFT(zap))]
 
 /*
@@ -149,8 +149,6 @@ typedef struct zap {
 	uint64_t zap_salt;
 	union {
 		struct {
-			zap_phys_t *zap_phys;
-
 			/*
 			 * zap_num_entries_mtx protects
 			 * zap_num_entries
@@ -159,7 +157,6 @@ typedef struct zap {
 			int zap_block_shift;
 		} zap_fat;
 		struct {
-			mzap_phys_t *zap_phys;
 			int16_t zap_num_entries;
 			int16_t zap_num_chunks;
 			int16_t zap_alloc_next;
@@ -168,6 +165,18 @@ typedef struct zap {
 	} zap_u;
 } zap_t;
 
+inline zap_phys_t *
+zap_f_phys(zap_t *zap)
+{
+	return (zap->zap_dbuf->db_data);
+}
+
+inline mzap_phys_t *
+zap_m_phys(zap_t *zap)
+{
+	return (zap->zap_dbuf->db_data);
+}
+
 typedef struct zap_name {
 	zap_t *zn_zap;
 	int zn_key_intlen;
diff --git a/uts/common/fs/zfs/sys/zap_leaf.h b/uts/common/fs/zfs/sys/zap_leaf.h
index f6947a72d70e..cd8b74a77a8c 100644
--- a/uts/common/fs/zfs/sys/zap_leaf.h
+++ b/uts/common/fs/zfs/sys/zap_leaf.h
@@ -83,7 +83,7 @@ struct zap_stats;
  */
 #define	ZAP_LEAF_CHUNK(l, idx) \
 	((zap_leaf_chunk_t *) \
-	((l)->l_phys->l_hash + ZAP_LEAF_HASH_NUMENTRIES(l)))[idx]
+	(zap_leaf_phys(l)->l_hash + ZAP_LEAF_HASH_NUMENTRIES(l)))[idx]
 #define	ZAP_LEAF_ENTRY(l, idx) (&ZAP_LEAF_CHUNK(l, idx).l_entry)
 
 typedef enum zap_chunk_type {
@@ -156,9 +156,13 @@ typedef struct zap_leaf {
 	uint64_t l_blkid;		/* 1<<ZAP_BLOCK_SHIFT byte block off */
 	int l_bs;			/* block size shift */
 	dmu_buf_t *l_dbuf;
-	zap_leaf_phys_t *l_phys;
 } zap_leaf_t;
 
+inline zap_leaf_phys_t *
+zap_leaf_phys(zap_leaf_t *l)
+{
+	return (l->l_dbuf->db_data);
+}
 
 typedef struct zap_entry_handle {
 	/* Set by zap_leaf and public to ZAP */
diff --git a/uts/common/fs/zfs/zap.c b/uts/common/fs/zfs/zap.c
index fb79e5a25713..de2d4deb00eb 100644
--- a/uts/common/fs/zfs/zap.c
+++ b/uts/common/fs/zfs/zap.c
@@ -50,10 +50,11 @@
 
 int fzap_default_block_shift = 14; /* 16k blocksize */
 
+extern inline zap_phys_t *zap_f_phys(zap_t *zap);
+
 static void zap_leaf_pageout(dmu_buf_t *db, void *vl);
 static uint64_t zap_allocate_blocks(zap_t *zap, int nblocks);
 
-
 void
 fzap_byteswap(void *vbuf, size_t size)
 {
@@ -80,13 +81,12 @@ fzap_upgrade(zap_t *zap, dmu_tx_t *tx, zap_flags_t flags)
 	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));
 	zap->zap_ismicro = FALSE;
 
-	(void) dmu_buf_update_user(zap->zap_dbuf, zap, zap,
-	    &zap->zap_f.zap_phys, zap_evict);
+	(void) dmu_buf_update_user(zap->zap_dbuf, zap, zap, zap_evict);
 
 	mutex_init(&zap->zap_f.zap_num_entries_mtx, 0, 0, 0);
 	zap->zap_f.zap_block_shift = highbit64(zap->zap_dbuf->db_size) - 1;
 
-	zp = zap->zap_f.zap_phys;
+	zp = zap_f_phys(zap);
 	/*
 	 * explicitly zero it since it might be coming from an
 	 * initialized microzap
@@ -117,7 +117,6 @@ fzap_upgrade(zap_t *zap, dmu_tx_t *tx, zap_flags_t flags)
 
 	l = kmem_zalloc(sizeof (zap_leaf_t), KM_SLEEP);
 	l->l_dbuf = db;
-	l->l_phys = db->db_data;
 
 	zap_leaf_init(l, zp->zap_normflags != 0);
 
@@ -325,10 +324,10 @@ zap_grow_ptrtbl(zap_t *zap, dmu_tx_t *tx)
 	 * If we are within 2 bits of running out, stop growing, since
 	 * this is already an aberrant condition.
 	 */
-	if (zap->zap_f.zap_phys->zap_ptrtbl.zt_shift >= zap_hashbits(zap) - 2)
+	if (zap_f_phys(zap)->zap_ptrtbl.zt_shift >= zap_hashbits(zap) - 2)
 		return (SET_ERROR(ENOSPC));
 
-	if (zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks == 0) {
+	if (zap_f_phys(zap)->zap_ptrtbl.zt_numblks == 0) {
 		/*
 		 * We are outgrowing the "embedded" ptrtbl (the one
 		 * stored in the header block).  Give it its own entire
@@ -338,9 +337,9 @@ zap_grow_ptrtbl(zap_t *zap, dmu_tx_t *tx)
 		dmu_buf_t *db_new;
 		int err;
 
-		ASSERT3U(zap->zap_f.zap_phys->zap_ptrtbl.zt_shift, ==,
+		ASSERT3U(zap_f_phys(zap)->zap_ptrtbl.zt_shift, ==,
 		    ZAP_EMBEDDED_PTRTBL_SHIFT(zap));
-		ASSERT0(zap->zap_f.zap_phys->zap_ptrtbl.zt_blk);
+		ASSERT0(zap_f_phys(zap)->zap_ptrtbl.zt_blk);
 
 		newblk = zap_allocate_blocks(zap, 1);
 		err = dmu_buf_hold(zap->zap_objset, zap->zap_object,
@@ -353,17 +352,17 @@ zap_grow_ptrtbl(zap_t *zap, dmu_tx_t *tx)
 		    db_new->db_data, 1 << ZAP_EMBEDDED_PTRTBL_SHIFT(zap));
 		dmu_buf_rele(db_new, FTAG);
 
-		zap->zap_f.zap_phys->zap_ptrtbl.zt_blk = newblk;
-		zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks = 1;
-		zap->zap_f.zap_phys->zap_ptrtbl.zt_shift++;
+		zap_f_phys(zap)->zap_ptrtbl.zt_blk = newblk;
+		zap_f_phys(zap)->zap_ptrtbl.zt_numblks = 1;
+		zap_f_phys(zap)->zap_ptrtbl.zt_shift++;
 
-		ASSERT3U(1ULL << zap->zap_f.zap_phys->zap_ptrtbl.zt_shift, ==,
-		    zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks <<
+		ASSERT3U(1ULL << zap_f_phys(zap)->zap_ptrtbl.zt_shift, ==,
+		    zap_f_phys(zap)->zap_ptrtbl.zt_numblks <<
 		    (FZAP_BLOCK_SHIFT(zap)-3));
 
 		return (0);
 	} else {
-		return (zap_table_grow(zap, &zap->zap_f.zap_phys->zap_ptrtbl,
+		return (zap_table_grow(zap, &zap_f_phys(zap)->zap_ptrtbl,
 		    zap_ptrtbl_transfer, tx));
 	}
 }
@@ -373,8 +372,8 @@ zap_increment_num_entries(zap_t *zap, int delta, dmu_tx_t *tx)
 {
 	dmu_buf_will_dirty(zap->zap_dbuf, tx);
 	mutex_enter(&zap->zap_f.zap_num_entries_mtx);
-	ASSERT(delta > 0 || zap->zap_f.zap_phys->zap_num_entries >= -delta);
-	zap->zap_f.zap_phys->zap_num_entries += delta;
+	ASSERT(delta > 0 || zap_f_phys(zap)->zap_num_entries >= -delta);
+	zap_f_phys(zap)->zap_num_entries += delta;
 	mutex_exit(&zap->zap_f.zap_num_entries_mtx);
 }
 
@@ -383,8 +382,8 @@ zap_allocate_blocks(zap_t *zap, int nblocks)
 {
 	uint64_t newblk;
 	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));
-	newblk = zap->zap_f.zap_phys->zap_freeblk;
-	zap->zap_f.zap_phys->zap_freeblk += nblocks;
+	newblk = zap_f_phys(zap)->zap_freeblk;
+	zap_f_phys(zap)->zap_freeblk += nblocks;
 	return (newblk);
 }
 
@@ -400,18 +399,17 @@ zap_create_leaf(zap_t *zap, dmu_tx_t *tx)
 	rw_enter(&l->l_rwlock, RW_WRITER);
 	l->l_blkid = zap_allocate_blocks(zap, 1);
 	l->l_dbuf = NULL;
-	l->l_phys = NULL;
 
 	VERIFY(0 == dmu_buf_hold(zap->zap_objset, zap->zap_object,
 	    l->l_blkid << FZAP_BLOCK_SHIFT(zap), NULL, &l->l_dbuf,
 	    DMU_READ_NO_PREFETCH));
-	winner = dmu_buf_set_user(l->l_dbuf, l, &l->l_phys, zap_leaf_pageout);
+	winner = dmu_buf_set_user(l->l_dbuf, l, zap_leaf_pageout);
 	ASSERT(winner == NULL);
 	dmu_buf_will_dirty(l->l_dbuf, tx);
 
 	zap_leaf_init(l, zap->zap_normflags != 0);
 
-	zap->zap_f.zap_phys->zap_num_leafs++;
+	zap_f_phys(zap)->zap_num_leafs++;
 
 	return (l);
 }
@@ -421,7 +419,7 @@ fzap_count(zap_t *zap, uint64_t *count)
 {
 	ASSERT(!zap->zap_ismicro);
 	mutex_enter(&zap->zap_f.zap_num_entries_mtx); /* unnecessary */
-	*count = zap->zap_f.zap_phys->zap_num_entries;
+	*count = zap_f_phys(zap)->zap_num_entries;
 	mutex_exit(&zap->zap_f.zap_num_entries_mtx);
 	return (0);
 }
@@ -460,9 +458,8 @@ zap_open_leaf(uint64_t blkid, dmu_buf_t *db)
 	l->l_blkid = blkid;
 	l->l_bs = highbit64(db->db_size) - 1;
 	l->l_dbuf = db;
-	l->l_phys = NULL;
 
-	winner = dmu_buf_set_user(db, l, &l->l_phys, zap_leaf_pageout);
+	winner = dmu_buf_set_user(db, l, zap_leaf_pageout);
 
 	rw_exit(&l->l_rwlock);
 	if (winner != NULL) {
@@ -476,7 +473,7 @@ zap_open_leaf(uint64_t blkid, dmu_buf_t *db)
 	 * chain.  There should be no chained leafs (as we have removed
 	 * support for them).
 	 */
-	ASSERT0(l->l_phys->l_hdr.lh_pad1);
+	ASSERT0(zap_leaf_phys(l)->l_hdr.lh_pad1);
 
 	/*
 	 * There should be more hash entries than there can be
@@ -486,11 +483,11 @@ zap_open_leaf(uint64_t blkid, dmu_buf_t *db)
 
 	/* The chunks should begin at the end of the hash table */
 	ASSERT3P(&ZAP_LEAF_CHUNK(l, 0), ==,
-	    &l->l_phys->l_hash[ZAP_LEAF_HASH_NUMENTRIES(l)]);
+	    &zap_leaf_phys(l)->l_hash[ZAP_LEAF_HASH_NUMENTRIES(l)]);
 
 	/* The chunks should end at the end of the block */
 	ASSERT3U((uintptr_t)&ZAP_LEAF_CHUNK(l, ZAP_LEAF_NUMCHUNKS(l)) -
-	    (uintptr_t)l->l_phys, ==, l->l_dbuf->db_size);
+	    (uintptr_t)zap_leaf_phys(l), ==, l->l_dbuf->db_size);
 
 	return (l);
 }
@@ -523,16 +520,15 @@ zap_get_leaf_byblk(zap_t *zap, uint64_t blkid, dmu_tx_t *tx, krw_t lt,
 
 	rw_enter(&l->l_rwlock, lt);
 	/*
-	 * Must lock before dirtying, otherwise l->l_phys could change,
+	 * Must lock before dirtying, otherwise zap_leaf_phys(l) could change,
 	 * causing ASSERT below to fail.
 	 */
 	if (lt == RW_WRITER)
 		dmu_buf_will_dirty(db, tx);
 	ASSERT3U(l->l_blkid, ==, blkid);
 	ASSERT3P(l->l_dbuf, ==, db);
-	ASSERT3P(l->l_phys, ==, l->l_dbuf->db_data);
-	ASSERT3U(l->l_phys->l_hdr.lh_block_type, ==, ZBT_LEAF);
-	ASSERT3U(l->l_phys->l_hdr.lh_magic, ==, ZAP_LEAF_MAGIC);
+	ASSERT3U(zap_leaf_phys(l)->l_hdr.lh_block_type, ==, ZBT_LEAF);
+	ASSERT3U(zap_leaf_phys(l)->l_hdr.lh_magic, ==, ZAP_LEAF_MAGIC);
 
 	*lp = l;
 	return (0);
@@ -543,13 +539,13 @@ zap_idx_to_blk(zap_t *zap, uint64_t idx, uint64_t *valp)
 {
 	ASSERT(RW_LOCK_HELD(&zap->zap_rwlock));
 
-	if (zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks == 0) {
+	if (zap_f_phys(zap)->zap_ptrtbl.zt_numblks == 0) {
 		ASSERT3U(idx, <,
-		    (1ULL << zap->zap_f.zap_phys->zap_ptrtbl.zt_shift));
+		    (1ULL << zap_f_phys(zap)->zap_ptrtbl.zt_shift));
 		*valp = ZAP_EMBEDDED_PTRTBL_ENT(zap, idx);
 		return (0);
 	} else {
-		return (zap_table_load(zap, &zap->zap_f.zap_phys->zap_ptrtbl,
+		return (zap_table_load(zap, &zap_f_phys(zap)->zap_ptrtbl,
 		    idx, valp));
 	}
 }
@@ -560,11 +556,11 @@ zap_set_idx_to_blk(zap_t *zap, uint64_t idx, uint64_t blk, dmu_tx_t *tx)
 	ASSERT(tx != NULL);
 	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));
 
-	if (zap->zap_f.zap_phys->zap_ptrtbl.zt_blk == 0) {
+	if (zap_f_phys(zap)->zap_ptrtbl.zt_blk == 0) {
 		ZAP_EMBEDDED_PTRTBL_ENT(zap, idx) = blk;
 		return (0);
 	} else {
-		return (zap_table_store(zap, &zap->zap_f.zap_phys->zap_ptrtbl,
+		return (zap_table_store(zap, &zap_f_phys(zap)->zap_ptrtbl,
 		    idx, blk, tx));
 	}
 }
@@ -576,16 +572,17 @@ zap_deref_leaf(zap_t *zap, uint64_t h, dmu_tx_t *tx, krw_t lt, zap_leaf_t **lp)
 	int err;
 
 	ASSERT(zap->zap_dbuf == NULL ||
-	    zap->zap_f.zap_phys == zap->zap_dbuf->db_data);
-	ASSERT3U(zap->zap_f.zap_phys->zap_magic, ==, ZAP_MAGIC);
-	idx = ZAP_HASH_IDX(h, zap->zap_f.zap_phys->zap_ptrtbl.zt_shift);
+	    zap_f_phys(zap) == zap->zap_dbuf->db_data);
+	ASSERT3U(zap_f_phys(zap)->zap_magic, ==, ZAP_MAGIC);
+	idx = ZAP_HASH_IDX(h, zap_f_phys(zap)->zap_ptrtbl.zt_shift);
 	err = zap_idx_to_blk(zap, idx, &blk);
 	if (err != 0)
 		return (err);
 	err = zap_get_leaf_byblk(zap, blk, tx, lt, lp);
 
-	ASSERT(err || ZAP_HASH_IDX(h, (*lp)->l_phys->l_hdr.lh_prefix_len) ==
-	    (*lp)->l_phys->l_hdr.lh_prefix);
+	ASSERT(err ||
+	    ZAP_HASH_IDX(h, zap_leaf_phys(*lp)->l_hdr.lh_prefix_len) ==
+	    zap_leaf_phys(*lp)->l_hdr.lh_prefix);
 	return (err);
 }
 
@@ -597,16 +594,16 @@ zap_expand_leaf(zap_name_t *zn, zap_leaf_t *l, dmu_tx_t *tx, zap_leaf_t **lp)
 	zap_leaf_t *nl;
 	int prefix_diff, i, err;
 	uint64_t sibling;
-	int old_prefix_len = l->l_phys->l_hdr.lh_prefix_len;
+	int old_prefix_len = zap_leaf_phys(l)->l_hdr.lh_prefix_len;
 
-	ASSERT3U(old_prefix_len, <=, zap->zap_f.zap_phys->zap_ptrtbl.zt_shift);
+	ASSERT3U(old_prefix_len, <=, zap_f_phys(zap)->zap_ptrtbl.zt_shift);
 	ASSERT(RW_LOCK_HELD(&zap->zap_rwlock));
 
 	ASSERT3U(ZAP_HASH_IDX(hash, old_prefix_len), ==,
-	    l->l_phys->l_hdr.lh_prefix);
+	    zap_leaf_phys(l)->l_hdr.lh_prefix);
 
 	if (zap_tryupgradedir(zap, tx) == 0 ||
-	    old_prefix_len == zap->zap_f.zap_phys->zap_ptrtbl.zt_shift) {
+	    old_prefix_len == zap_f_phys(zap)->zap_ptrtbl.zt_shift) {
 		/* We failed to upgrade, or need to grow the pointer table */
 		objset_t *os = zap->zap_objset;
 		uint64_t object = zap->zap_object;
@@ -621,7 +618,7 @@ zap_expand_leaf(zap_name_t *zn, zap_leaf_t *l, dmu_tx_t *tx, zap_leaf_t **lp)
 		ASSERT(!zap->zap_ismicro);
 
 		while (old_prefix_len ==
-		    zap->zap_f.zap_phys->zap_ptrtbl.zt_shift) {
+		    zap_f_phys(zap)->zap_ptrtbl.zt_shift) {
 			err = zap_grow_ptrtbl(zap, tx);
 			if (err)
 				return (err);
@@ -631,18 +628,18 @@ zap_expand_leaf(zap_name_t *zn, zap_leaf_t *l, dmu_tx_t *tx, zap_leaf_t **lp)
 		if (err)
 			return (err);
 
-		if (l->l_phys->l_hdr.lh_prefix_len != old_prefix_len) {
+		if (zap_leaf_phys(l)->l_hdr.lh_prefix_len != old_prefix_len) {
 			/* it split while our locks were down */
 			*lp = l;
 			return (0);
 		}
 	}
 	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));
-	ASSERT3U(old_prefix_len, <, zap->zap_f.zap_phys->zap_ptrtbl.zt_shift);
+	ASSERT3U(old_prefix_len, <, zap_f_phys(zap)->zap_ptrtbl.zt_shift);
 	ASSERT3U(ZAP_HASH_IDX(hash, old_prefix_len), ==,
-	    l->l_phys->l_hdr.lh_prefix);
+	    zap_leaf_phys(l)->l_hdr.lh_prefix);
 
-	prefix_diff = zap->zap_f.zap_phys->zap_ptrtbl.zt_shift -
+	prefix_diff = zap_f_phys(zap)->zap_ptrtbl.zt_shift -
 	    (old_prefix_len + 1);
 	sibling = (ZAP_HASH_IDX(hash, old_prefix_len + 1) | 1) << prefix_diff;
 
@@ -664,7 +661,7 @@ zap_expand_leaf(zap_name_t *zn, zap_leaf_t *l, dmu_tx_t *tx, zap_leaf_t **lp)
 		ASSERT0(err); /* we checked for i/o errors above */
 	}
 
-	if (hash & (1ULL << (64 - l->l_phys->l_hdr.lh_prefix_len))) {
+	if (hash & (1ULL << (64 - zap_leaf_phys(l)->l_hdr.lh_prefix_len))) {
 		/* we want the sibling */
 		zap_put_leaf(l);
 		*lp = nl;
@@ -680,13 +677,13 @@ static void
 zap_put_leaf_maybe_grow_ptrtbl(zap_name_t *zn, zap_leaf_t *l, dmu_tx_t *tx)
 {
 	zap_t *zap = zn->zn_zap;
-	int shift = zap->zap_f.zap_phys->zap_ptrtbl.zt_shift;
-	int leaffull = (l->l_phys->l_hdr.lh_prefix_len == shift &&
-	    l->l_phys->l_hdr.lh_nfree < ZAP_LEAF_LOW_WATER);
+	int shift = zap_f_phys(zap)->zap_ptrtbl.zt_shift;
+	int leaffull = (zap_leaf_phys(l)->l_hdr.lh_prefix_len == shift &&
+	    zap_leaf_phys(l)->l_hdr.lh_nfree < ZAP_LEAF_LOW_WATER);
 
 	zap_put_leaf(l);
 
-	if (leaffull || zap->zap_f.zap_phys->zap_ptrtbl.zt_nextblk) {
+	if (leaffull || zap_f_phys(zap)->zap_ptrtbl.zt_nextblk) {
 		int err;
 
 		/*
@@ -706,7 +703,7 @@ zap_put_leaf_maybe_grow_ptrtbl(zap_name_t *zn, zap_leaf_t *l, dmu_tx_t *tx)
 		}
 
 		/* could have finished growing while our locks were down */
-		if (zap->zap_f.zap_phys->zap_ptrtbl.zt_shift == shift)
+		if (zap_f_phys(zap)->zap_ptrtbl.zt_shift == shift)
 			(void) zap_grow_ptrtbl(zap, tx);
 	}
 }
@@ -937,7 +934,7 @@ fzap_prefetch(zap_name_t *zn)
 	int bs;
 
 	idx = ZAP_HASH_IDX(zn->zn_hash,
-	    zap->zap_f.zap_phys->zap_ptrtbl.zt_shift);
+	    zap_f_phys(zap)->zap_ptrtbl.zt_shift);
 	if (zap_idx_to_blk(zap, idx, &blk) != 0)
 		return;
 	bs = FZAP_BLOCK_SHIFT(zap);
@@ -1169,8 +1166,8 @@ fzap_cursor_retrieve(zap_t *zap, zap_cursor_t *zc, zap_attribute_t *za)
 
 	if (zc->zc_leaf &&
 	    (ZAP_HASH_IDX(zc->zc_hash,
-	    zc->zc_leaf->l_phys->l_hdr.lh_prefix_len) !=
-	    zc->zc_leaf->l_phys->l_hdr.lh_prefix)) {
+	    zap_leaf_phys(zc->zc_leaf)->l_hdr.lh_prefix_len) !=
+	    zap_leaf_phys(zc->zc_leaf)->l_hdr.lh_prefix)) {
 		rw_enter(&zc->zc_leaf->l_rwlock, RW_READER);
 		zap_put_leaf(zc->zc_leaf);
 		zc->zc_leaf = NULL;
@@ -1191,10 +1188,11 @@ again:
 
 	if (err == ENOENT) {
 		uint64_t nocare =
-		    (1ULL << (64 - l->l_phys->l_hdr.lh_prefix_len)) - 1;
+		    (1ULL << (64 - zap_leaf_phys(l)->l_hdr.lh_prefix_len)) - 1;
 		zc->zc_hash = (zc->zc_hash & ~nocare) + nocare + 1;
 		zc->zc_cd = 0;
-		if (l->l_phys->l_hdr.lh_prefix_len == 0 || zc->zc_hash == 0) {
+		if (zap_leaf_phys(l)->l_hdr.lh_prefix_len == 0 ||
+		    zc->zc_hash == 0) {
 			zc->zc_hash = -1ULL;
 		} else {
 			zap_put_leaf(zc->zc_leaf);
@@ -1261,25 +1259,25 @@ fzap_get_stats(zap_t *zap, zap_stats_t *zs)
 	/*
 	 * Set zap_phys_t fields
 	 */
-	zs->zs_num_leafs = zap->zap_f.zap_phys->zap_num_leafs;
-	zs->zs_num_entries = zap->zap_f.zap_phys->zap_num_entries;
-	zs->zs_num_blocks = zap->zap_f.zap_phys->zap_freeblk;
-	zs->zs_block_type = zap->zap_f.zap_phys->zap_block_type;
-	zs->zs_magic = zap->zap_f.zap_phys->zap_magic;
-	zs->zs_salt = zap->zap_f.zap_phys->zap_salt;
+	zs->zs_num_leafs = zap_f_phys(zap)->zap_num_leafs;
+	zs->zs_num_entries = zap_f_phys(zap)->zap_num_entries;
+	zs->zs_num_blocks = zap_f_phys(zap)->zap_freeblk;
+	zs->zs_block_type = zap_f_phys(zap)->zap_block_type;
+	zs->zs_magic = zap_f_phys(zap)->zap_magic;
+	zs->zs_salt = zap_f_phys(zap)->zap_salt;
 
 	/*
 	 * Set zap_ptrtbl fields
 	 */
-	zs->zs_ptrtbl_len = 1ULL << zap->zap_f.zap_phys->zap_ptrtbl.zt_shift;
-	zs->zs_ptrtbl_nextblk = zap->zap_f.zap_phys->zap_ptrtbl.zt_nextblk;
+	zs->zs_ptrtbl_len = 1ULL << zap_f_phys(zap)->zap_ptrtbl.zt_shift;
+	zs->zs_ptrtbl_nextblk = zap_f_phys(zap)->zap_ptrtbl.zt_nextblk;
 	zs->zs_ptrtbl_blks_copied =
-	    zap->zap_f.zap_phys->zap_ptrtbl.zt_blks_copied;
-	zs->zs_ptrtbl_zt_blk = zap->zap_f.zap_phys->zap_ptrtbl.zt_blk;
-	zs->zs_ptrtbl_zt_numblks = zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks;
-	zs->zs_ptrtbl_zt_shift = zap->zap_f.zap_phys->zap_ptrtbl.zt_shift;
+	    zap_f_phys(zap)->zap_ptrtbl.zt_blks_copied;
+	zs->zs_ptrtbl_zt_blk = zap_f_phys(zap)->zap_ptrtbl.zt_blk;
+	zs->zs_ptrtbl_zt_numblks = zap_f_phys(zap)->zap_ptrtbl.zt_numblks;
+	zs->zs_ptrtbl_zt_shift = zap_f_phys(zap)->zap_ptrtbl.zt_shift;
 
-	if (zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks == 0) {
+	if (zap_f_phys(zap)->zap_ptrtbl.zt_numblks == 0) {
 		/* the ptrtbl is entirely in the header block. */
 		zap_stats_ptrtbl(zap, &ZAP_EMBEDDED_PTRTBL_ENT(zap, 0),
 		    1 << ZAP_EMBEDDED_PTRTBL_SHIFT(zap), zs);
@@ -1287,16 +1285,16 @@ fzap_get_stats(zap_t *zap, zap_stats_t *zs)
 		int b;
 
 		dmu_prefetch(zap->zap_objset, zap->zap_object,
-		    zap->zap_f.zap_phys->zap_ptrtbl.zt_blk << bs,
-		    zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks << bs);
+		    zap_f_phys(zap)->zap_ptrtbl.zt_blk << bs,
+		    zap_f_phys(zap)->zap_ptrtbl.zt_numblks << bs);
 
-		for (b = 0; b < zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks;
+		for (b = 0; b < zap_f_phys(zap)->zap_ptrtbl.zt_numblks;
 		    b++) {
 			dmu_buf_t *db;
 			int err;
 
 			err = dmu_buf_hold(zap->zap_objset, zap->zap_object,
-			    (zap->zap_f.zap_phys->zap_ptrtbl.zt_blk + b) << bs,
+			    (zap_f_phys(zap)->zap_ptrtbl.zt_blk + b) << bs,
 			    FTAG, &db, DMU_READ_NO_PREFETCH);
 			if (err == 0) {
 				zap_stats_ptrtbl(zap, db->db_data,
@@ -1333,7 +1331,7 @@ fzap_count_write(zap_name_t *zn, int add, uint64_t *towrite,
 	 *   could extend the table.
 	 */
 	if (add) {
-		if (zap->zap_f.zap_phys->zap_ptrtbl.zt_blk == 0)
+		if (zap_f_phys(zap)->zap_ptrtbl.zt_blk == 0)
 			*towrite += zap->zap_dbuf->db_size;
 		else
 			*towrite += (zap->zap_dbuf->db_size * 3);
diff --git a/uts/common/fs/zfs/zap_leaf.c b/uts/common/fs/zfs/zap_leaf.c
index 78f05d7a7e37..96358f7bd80f 100644
--- a/uts/common/fs/zfs/zap_leaf.c
+++ b/uts/common/fs/zfs/zap_leaf.c
@@ -48,10 +48,12 @@ static uint16_t *zap_leaf_rehash_entry(zap_leaf_t *l, uint16_t entry);
 
 #define	LEAF_HASH(l, h) \
 	((ZAP_LEAF_HASH_NUMENTRIES(l)-1) & \
-	((h) >> (64 - ZAP_LEAF_HASH_SHIFT(l)-(l)->l_phys->l_hdr.lh_prefix_len)))
+	((h) >> \
+	(64 - ZAP_LEAF_HASH_SHIFT(l) - zap_leaf_phys(l)->l_hdr.lh_prefix_len)))
 
-#define	LEAF_HASH_ENTPTR(l, h) (&(l)->l_phys->l_hash[LEAF_HASH(l, h)])
+#define	LEAF_HASH_ENTPTR(l, h) (&zap_leaf_phys(l)->l_hash[LEAF_HASH(l, h)])
 
+extern inline zap_leaf_phys_t *zap_leaf_phys(zap_leaf_t *l);
 
 static void
 zap_memset(void *a, int c, size_t n)
@@ -105,8 +107,11 @@ zap_leaf_byteswap(zap_leaf_phys_t *buf, int size)
 {
 	int i;
 	zap_leaf_t l;
+	dmu_buf_t l_dbuf;
+
+	l_dbuf.db_data = buf;
 	l.l_bs = highbit64(size) - 1;
-	l.l_phys = buf;
+	l.l_dbuf = &l_dbuf;
 
 	buf->l_hdr.lh_block_type =	BSWAP_64(buf->l_hdr.lh_block_type);
 	buf->l_hdr.lh_prefix =		BSWAP_64(buf->l_hdr.lh_prefix);
@@ -158,18 +163,20 @@ zap_leaf_init(zap_leaf_t *l, boolean_t sort)
 	int i;
 
 	l->l_bs = highbit64(l->l_dbuf->db_size) - 1;
-	zap_memset(&l->l_phys->l_hdr, 0, sizeof (struct zap_leaf_header));
-	zap_memset(l->l_phys->l_hash, CHAIN_END, 2*ZAP_LEAF_HASH_NUMENTRIES(l));
+	zap_memset(&zap_leaf_phys(l)->l_hdr, 0,
+	    sizeof (struct zap_leaf_header));
+	zap_memset(zap_leaf_phys(l)->l_hash, CHAIN_END,
+	    2*ZAP_LEAF_HASH_NUMENTRIES(l));
 	for (i = 0; i < ZAP_LEAF_NUMCHUNKS(l); i++) {
 		ZAP_LEAF_CHUNK(l, i).l_free.lf_type = ZAP_CHUNK_FREE;
 		ZAP_LEAF_CHUNK(l, i).l_free.lf_next = i+1;
 	}
 	ZAP_LEAF_CHUNK(l, ZAP_LEAF_NUMCHUNKS(l)-1).l_free.lf_next = CHAIN_END;
-	l->l_phys->l_hdr.lh_block_type = ZBT_LEAF;
-	l->l_phys->l_hdr.lh_magic = ZAP_LEAF_MAGIC;
-	l->l_phys->l_hdr.lh_nfree = ZAP_LEAF_NUMCHUNKS(l);
+	zap_leaf_phys(l)->l_hdr.lh_block_type = ZBT_LEAF;
+	zap_leaf_phys(l)->l_hdr.lh_magic = ZAP_LEAF_MAGIC;
+	zap_leaf_phys(l)->l_hdr.lh_nfree = ZAP_LEAF_NUMCHUNKS(l);
 	if (sort)
-		l->l_phys->l_hdr.lh_flags |= ZLF_ENTRIES_CDSORTED;
+		zap_leaf_phys(l)->l_hdr.lh_flags |= ZLF_ENTRIES_CDSORTED;
 }
 
 /*
@@ -181,15 +188,16 @@ zap_leaf_chunk_alloc(zap_leaf_t *l)
 {
 	int chunk;
 
-	ASSERT(l->l_phys->l_hdr.lh_nfree > 0);
+	ASSERT(zap_leaf_phys(l)->l_hdr.lh_nfree > 0);
 
-	chunk = l->l_phys->l_hdr.lh_freelist;
+	chunk = zap_leaf_phys(l)->l_hdr.lh_freelist;
 	ASSERT3U(chunk, <, ZAP_LEAF_NUMCHUNKS(l));
 	ASSERT3U(ZAP_LEAF_CHUNK(l, chunk).l_free.lf_type, ==, ZAP_CHUNK_FREE);
 
-	l->l_phys->l_hdr.lh_freelist = ZAP_LEAF_CHUNK(l, chunk).l_free.lf_next;
+	zap_leaf_phys(l)->l_hdr.lh_freelist =
+	    ZAP_LEAF_CHUNK(l, chunk).l_free.lf_next;
 
-	l->l_phys->l_hdr.lh_nfree--;
+	zap_leaf_phys(l)->l_hdr.lh_nfree--;
 
 	return (chunk);
 }
@@ -198,16 +206,16 @@ static void
 zap_leaf_chunk_free(zap_leaf_t *l, uint16_t chunk)
 {
 	struct zap_leaf_free *zlf = &ZAP_LEAF_CHUNK(l, chunk).l_free;
-	ASSERT3U(l->l_phys->l_hdr.lh_nfree, <, ZAP_LEAF_NUMCHUNKS(l));
+	ASSERT3U(zap_leaf_phys(l)->l_hdr.lh_nfree, <, ZAP_LEAF_NUMCHUNKS(l));
 	ASSERT3U(chunk, <, ZAP_LEAF_NUMCHUNKS(l));
 	ASSERT(zlf->lf_type != ZAP_CHUNK_FREE);
 
 	zlf->lf_type = ZAP_CHUNK_FREE;
-	zlf->lf_next = l->l_phys->l_hdr.lh_freelist;
+	zlf->lf_next = zap_leaf_phys(l)->l_hdr.lh_freelist;
 	bzero(zlf->lf_pad, sizeof (zlf->lf_pad)); /* help it to compress */
-	l->l_phys->l_hdr.lh_freelist = chunk;
+	zap_leaf_phys(l)->l_hdr.lh_freelist = chunk;
 
-	l->l_phys->l_hdr.lh_nfree++;
+	zap_leaf_phys(l)->l_hdr.lh_nfree++;
 }
 
 /*
@@ -393,7 +401,7 @@ zap_leaf_lookup(zap_leaf_t *l, zap_name_t *zn, zap_entry_handle_t *zeh)
 	uint16_t *chunkp;
 	struct zap_leaf_entry *le;
 
-	ASSERT3U(l->l_phys->l_hdr.lh_magic, ==, ZAP_LEAF_MAGIC);
+	ASSERT3U(zap_leaf_phys(l)->l_hdr.lh_magic, ==, ZAP_LEAF_MAGIC);
 
 again:
 	for (chunkp = LEAF_HASH_ENTPTR(l, zn->zn_hash);
@@ -413,7 +421,7 @@ again:
 		 * lowest-cd match for MT_FIRST.
 		 */
 		ASSERT(zn->zn_matchtype == MT_EXACT ||
-		    (l->l_phys->l_hdr.lh_flags & ZLF_ENTRIES_CDSORTED));
+		    (zap_leaf_phys(l)->l_hdr.lh_flags & ZLF_ENTRIES_CDSORTED));
 		if (zap_leaf_array_match(l, zn, le->le_name_chunk,
 		    le->le_name_numints)) {
 			zeh->zeh_num_integers = le->le_value_numints;
@@ -453,10 +461,10 @@ zap_leaf_lookup_closest(zap_leaf_t *l,
 	uint16_t lh;
 	struct zap_leaf_entry *le;
 
-	ASSERT3U(l->l_phys->l_hdr.lh_magic, ==, ZAP_LEAF_MAGIC);
+	ASSERT3U(zap_leaf_phys(l)->l_hdr.lh_magic, ==, ZAP_LEAF_MAGIC);
 
 	for (lh = LEAF_HASH(l, h); lh <= bestlh; lh++) {
-		for (chunk = l->l_phys->l_hash[lh];
+		for (chunk = zap_leaf_phys(l)->l_hash[lh];
 		    chunk != CHAIN_END; chunk = le->le_next) {
 			le = ZAP_LEAF_ENTRY(l, chunk);
 
@@ -536,7 +544,7 @@ zap_entry_update(zap_entry_handle_t *zeh,
 	delta_chunks = ZAP_LEAF_ARRAY_NCHUNKS(num_integers * integer_size) -
 	    ZAP_LEAF_ARRAY_NCHUNKS(le->le_value_numints * le->le_value_intlen);
 
-	if ((int)l->l_phys->l_hdr.lh_nfree < delta_chunks)
+	if ((int)zap_leaf_phys(l)->l_hdr.lh_nfree < delta_chunks)
 		return (SET_ERROR(EAGAIN));
 
 	zap_leaf_array_free(l, &le->le_value_chunk);
@@ -566,7 +574,7 @@ zap_entry_remove(zap_entry_handle_t *zeh)
 	*zeh->zeh_chunkp = le->le_next;
 	zap_leaf_chunk_free(l, entry_chunk);
 
-	l->l_phys->l_hdr.lh_nentries--;
+	zap_leaf_phys(l)->l_hdr.lh_nentries--;
 }
 
 int
@@ -590,7 +598,7 @@ zap_entry_create(zap_leaf_t *l, zap_name_t *zn, uint32_t cd,
 
 	if (cd == ZAP_NEED_CD) {
 		/* find the lowest unused cd */
-		if (l->l_phys->l_hdr.lh_flags & ZLF_ENTRIES_CDSORTED) {
+		if (zap_leaf_phys(l)->l_hdr.lh_flags & ZLF_ENTRIES_CDSORTED) {
 			cd = 0;
 
 			for (chunk = *LEAF_HASH_ENTPTR(l, h);
@@ -626,7 +634,7 @@ zap_entry_create(zap_leaf_t *l, zap_name_t *zn, uint32_t cd,
 		ASSERT3U(cd, <, zap_maxcd(zn->zn_zap));
 	}
 
-	if (l->l_phys->l_hdr.lh_nfree < numchunks)
+	if (zap_leaf_phys(l)->l_hdr.lh_nfree < numchunks)
 		return (SET_ERROR(EAGAIN));
 
 	/* make the entry */
@@ -647,7 +655,7 @@ zap_entry_create(zap_leaf_t *l, zap_name_t *zn, uint32_t cd,
 	/* XXX if we did the search above, we could just use that */
 	chunkp = zap_leaf_rehash_entry(l, chunk);
 
-	l->l_phys->l_hdr.lh_nentries++;
+	zap_leaf_phys(l)->l_hdr.lh_nentries++;
 
 	zeh->zeh_leaf = l;
 	zeh->zeh_num_integers = num_integers;
@@ -781,8 +789,8 @@ zap_leaf_transfer_entry(zap_leaf_t *l, int entry, zap_leaf_t *nl)
 
 	zap_leaf_chunk_free(l, entry);
 
-	l->l_phys->l_hdr.lh_nentries--;
-	nl->l_phys->l_hdr.lh_nentries++;
+	zap_leaf_phys(l)->l_hdr.lh_nentries--;
+	zap_leaf_phys(nl)->l_hdr.lh_nentries++;
 }
 
 /*
@@ -792,19 +800,22 @@ void
 zap_leaf_split(zap_leaf_t *l, zap_leaf_t *nl, boolean_t sort)
 {
 	int i;
-	int bit = 64 - 1 - l->l_phys->l_hdr.lh_prefix_len;
+	int bit = 64 - 1 - zap_leaf_phys(l)->l_hdr.lh_prefix_len;
 
 	/* set new prefix and prefix_len */
-	l->l_phys->l_hdr.lh_prefix <<= 1;
-	l->l_phys->l_hdr.lh_prefix_len++;
-	nl->l_phys->l_hdr.lh_prefix = l->l_phys->l_hdr.lh_prefix | 1;
-	nl->l_phys->l_hdr.lh_prefix_len = l->l_phys->l_hdr.lh_prefix_len;
+	zap_leaf_phys(l)->l_hdr.lh_prefix <<= 1;
+	zap_leaf_phys(l)->l_hdr.lh_prefix_len++;
+	zap_leaf_phys(nl)->l_hdr.lh_prefix =
+	    zap_leaf_phys(l)->l_hdr.lh_prefix | 1;
+	zap_leaf_phys(nl)->l_hdr.lh_prefix_len =
+	    zap_leaf_phys(l)->l_hdr.lh_prefix_len;
 
 	/* break existing hash chains */
-	zap_memset(l->l_phys->l_hash, CHAIN_END, 2*ZAP_LEAF_HASH_NUMENTRIES(l));
+	zap_memset(zap_leaf_phys(l)->l_hash, CHAIN_END,
+	    2*ZAP_LEAF_HASH_NUMENTRIES(l));
 
 	if (sort)
-		l->l_phys->l_hdr.lh_flags |= ZLF_ENTRIES_CDSORTED;
+		zap_leaf_phys(l)->l_hdr.lh_flags |= ZLF_ENTRIES_CDSORTED;
 
 	/*
 	 * Transfer entries whose hash bit 'bit' is set to nl; rehash
@@ -832,25 +843,25 @@ zap_leaf_stats(zap_t *zap, zap_leaf_t *l, zap_stats_t *zs)
 {
 	int i, n;
 
-	n = zap->zap_f.zap_phys->zap_ptrtbl.zt_shift -
-	    l->l_phys->l_hdr.lh_prefix_len;
+	n = zap_f_phys(zap)->zap_ptrtbl.zt_shift -
+	    zap_leaf_phys(l)->l_hdr.lh_prefix_len;
 	n = MIN(n, ZAP_HISTOGRAM_SIZE-1);
 	zs->zs_leafs_with_2n_pointers[n]++;
 
 
-	n = l->l_phys->l_hdr.lh_nentries/5;
+	n = zap_leaf_phys(l)->l_hdr.lh_nentries/5;
 	n = MIN(n, ZAP_HISTOGRAM_SIZE-1);
 	zs->zs_blocks_with_n5_entries[n]++;
 
 	n = ((1<<FZAP_BLOCK_SHIFT(zap)) -
-	    l->l_phys->l_hdr.lh_nfree * (ZAP_LEAF_ARRAY_BYTES+1))*10 /
+	    zap_leaf_phys(l)->l_hdr.lh_nfree * (ZAP_LEAF_ARRAY_BYTES+1))*10 /
 	    (1<<FZAP_BLOCK_SHIFT(zap));
 	n = MIN(n, ZAP_HISTOGRAM_SIZE-1);
 	zs->zs_blocks_n_tenths_full[n]++;
 
 	for (i = 0; i < ZAP_LEAF_HASH_NUMENTRIES(l); i++) {
 		int nentries = 0;
-		int chunk = l->l_phys->l_hash[i];
+		int chunk = zap_leaf_phys(l)->l_hash[i];
 
 		while (chunk != CHAIN_END) {
 			struct zap_leaf_entry *le =
diff --git a/uts/common/fs/zfs/zap_micro.c b/uts/common/fs/zfs/zap_micro.c
index 59a9f970448a..81ea4f730f97 100644
--- a/uts/common/fs/zfs/zap_micro.c
+++ b/uts/common/fs/zfs/zap_micro.c
@@ -39,6 +39,8 @@
 #include <sys/sunddi.h>
 #endif
 
+extern inline mzap_phys_t *zap_m_phys(zap_t *zap);
+
 static int mzap_upgrade(zap_t **zapp, dmu_tx_t *tx, zap_flags_t flags);
 
 uint64_t
@@ -46,7 +48,7 @@ zap_getflags(zap_t *zap)
 {
 	if (zap->zap_ismicro)
 		return (0);
-	return (zap->zap_u.zap_fat.zap_phys->zap_flags);
+	return (zap_f_phys(zap)->zap_flags);
 }
 
 int
@@ -385,7 +387,7 @@ mzap_open(objset_t *os, uint64_t obj, dmu_buf_t *db)
 	 * it, because zap_lockdir() checks zap_ismicro without the lock
 	 * held.
 	 */
-	winner = dmu_buf_set_user(db, zap, &zap->zap_m.zap_phys, zap_evict);
+	winner = dmu_buf_set_user(db, zap, zap_evict);
 
 	if (winner != NULL) {
 		rw_exit(&zap->zap_rwlock);
@@ -397,15 +399,15 @@ mzap_open(objset_t *os, uint64_t obj, dmu_buf_t *db)
 	}
 
 	if (zap->zap_ismicro) {
-		zap->zap_salt = zap->zap_m.zap_phys->mz_salt;
-		zap->zap_normflags = zap->zap_m.zap_phys->mz_normflags;
+		zap->zap_salt = zap_m_phys(zap)->mz_salt;
+		zap->zap_normflags = zap_m_phys(zap)->mz_normflags;
 		zap->zap_m.zap_num_chunks = db->db_size / MZAP_ENT_LEN - 1;
 		avl_create(&zap->zap_m.zap_avl, mze_compare,
 		    sizeof (mzap_ent_t), offsetof(mzap_ent_t, mze_node));
 
 		for (i = 0; i < zap->zap_m.zap_num_chunks; i++) {
 			mzap_ent_phys_t *mze =
-			    &zap->zap_m.zap_phys->mz_chunk[i];
+			    &zap_m_phys(zap)->mz_chunk[i];
 			if (mze->mze_name[0]) {
 				zap_name_t *zn;
 
@@ -417,8 +419,8 @@ mzap_open(objset_t *os, uint64_t obj, dmu_buf_t *db)
 			}
 		}
 	} else {
-		zap->zap_salt = zap->zap_f.zap_phys->zap_salt;
-		zap->zap_normflags = zap->zap_f.zap_phys->zap_normflags;
+		zap->zap_salt = zap_f_phys(zap)->zap_salt;
+		zap->zap_normflags = zap_f_phys(zap)->zap_normflags;
 
 		ASSERT3U(sizeof (struct zap_leaf_header), ==,
 		    2*ZAP_LEAF_CHUNKSIZE);
@@ -428,7 +430,7 @@ mzap_open(objset_t *os, uint64_t obj, dmu_buf_t *db)
 		 * other members.
 		 */
 		ASSERT3P(&ZAP_EMBEDDED_PTRTBL_ENT(zap, 0), >,
-		    &zap->zap_f.zap_phys->zap_salt);
+		    &zap_f_phys(zap)->zap_salt);
 
 		/*
 		 * The embedded pointer table should end at the end of
@@ -436,7 +438,7 @@ mzap_open(objset_t *os, uint64_t obj, dmu_buf_t *db)
 		 */
 		ASSERT3U((uintptr_t)&ZAP_EMBEDDED_PTRTBL_ENT(zap,
 		    1<<ZAP_EMBEDDED_PTRTBL_SHIFT(zap)) -
-		    (uintptr_t)zap->zap_f.zap_phys, ==,
+		    (uintptr_t)zap_f_phys(zap), ==,
 		    zap->zap_dbuf->db_size);
 	}
 	rw_exit(&zap->zap_rwlock);
@@ -939,7 +941,7 @@ mzap_addent(zap_name_t *zn, uint64_t value)
 
 #ifdef ZFS_DEBUG
 	for (i = 0; i < zap->zap_m.zap_num_chunks; i++) {
-		mzap_ent_phys_t *mze = &zap->zap_m.zap_phys->mz_chunk[i];
+		mzap_ent_phys_t *mze = &zap_m_phys(zap)->mz_chunk[i];
 		ASSERT(strcmp(zn->zn_key_orig, mze->mze_name) != 0);
 	}
 #endif
@@ -950,7 +952,7 @@ mzap_addent(zap_name_t *zn, uint64_t value)
 
 again:
 	for (i = start; i < zap->zap_m.zap_num_chunks; i++) {
-		mzap_ent_phys_t *mze = &zap->zap_m.zap_phys->mz_chunk[i];
+		mzap_ent_phys_t *mze = &zap_m_phys(zap)->mz_chunk[i];
 		if (mze->mze_name[0] == 0) {
 			mze->mze_value = value;
 			mze->mze_cd = cd;
@@ -1151,7 +1153,7 @@ zap_remove_norm(objset_t *os, uint64_t zapobj, const char *name,
 			err = SET_ERROR(ENOENT);
 		} else {
 			zap->zap_m.zap_num_entries--;
-			bzero(&zap->zap_m.zap_phys->mz_chunk[mze->mze_chunkid],
+			bzero(&zap_m_phys(zap)->mz_chunk[mze->mze_chunkid],
 			    sizeof (mzap_ent_phys_t));
 			mze_remove(zap, mze);
 		}
diff --git a/uts/common/fs/zfs/zfs_ioctl.c b/uts/common/fs/zfs/zfs_ioctl.c
index a05e7b2a29b3..20a64666a78e 100644
--- a/uts/common/fs/zfs/zfs_ioctl.c
+++ b/uts/common/fs/zfs/zfs_ioctl.c
@@ -916,7 +916,7 @@ zfs_secpolicy_promote(zfs_cmd_t *zc, nvlist_t *innvl, cred_t *cr)
 		dd = clone->ds_dir;
 
 		error = dsl_dataset_hold_obj(dd->dd_pool,
-		    dd->dd_phys->dd_origin_obj, FTAG, &origin);
+		    dsl_dir_phys(dd)->dd_origin_obj, FTAG, &origin);
 		if (error != 0) {
 			dsl_dataset_rele(clone, FTAG);
 			dsl_pool_rele(dp, FTAG);
@@ -4309,7 +4309,8 @@ zfs_ioc_send(zfs_cmd_t *zc)
 		}
 
 		if (dsl_dir_is_clone(tosnap->ds_dir))
-			zc->zc_fromobj = tosnap->ds_dir->dd_phys->dd_origin_obj;
+			zc->zc_fromobj =
+			    dsl_dir_phys(tosnap->ds_dir)->dd_origin_obj;
 		dsl_dataset_rele(tosnap, FTAG);
 		dsl_pool_rele(dp, FTAG);
 	}
@@ -4873,7 +4874,7 @@ zfs_ioc_next_obj(zfs_cmd_t *zc)
 		return (error);
 
 	error = dmu_object_next(os, &zc->zc_obj, B_FALSE,
-	    os->os_dsl_dataset->ds_phys->ds_prev_snap_txg);
+	    dsl_dataset_phys(os->os_dsl_dataset)->ds_prev_snap_txg);
 
 	dmu_objset_rele(os, FTAG);
 	return (error);

From f0e5c9bcd1a6ace5ffbe4a4d1ed55844614a020e Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Mon, 15 Dec 2014 07:59:33 +0000
Subject: [PATCH 066/380] 5369 arc flags should be an enum 5370 consistent
 arc_buf_hdr_t naming scheme Reviewed by: Matthew Ahrens <mahrens@delphix.com>
 Reviewed by: Alex Reece <alex.reece@delphix.com> Reviewed by: Sebastien Roy
 <sebastien.roy@delphix.com> Reviewed by: Richard Elling
 <richard.elling@richardelling.com> Approved by: Richard Lowe
 <richlowe@richlowe.net> Author: George Wilson <george.wilson@delphix.com>

illumos/illumos-gate@7adb730b589e553bf3b1ccfd9bae2df91c5c1061
---
 uts/common/fs/zfs/arc.c          | 671 +++++++++++++++----------------
 uts/common/fs/zfs/dbuf.c         |  11 +-
 uts/common/fs/zfs/dmu_diff.c     |   2 +-
 uts/common/fs/zfs/dmu_objset.c   |   6 +-
 uts/common/fs/zfs/dmu_send.c     |   6 +-
 uts/common/fs/zfs/dmu_traverse.c |  12 +-
 uts/common/fs/zfs/dsl_scan.c     |   8 +-
 uts/common/fs/zfs/sys/arc.h      |  41 +-
 uts/common/fs/zfs/zil.c          |   4 +-
 uts/common/fs/zfs/zio.c          |   2 +-
 10 files changed, 386 insertions(+), 377 deletions(-)

diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index 57171d7d21be..435096becc75 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -500,7 +500,7 @@ struct arc_buf_hdr {
 
 	arc_buf_hdr_t		*b_hash_next;
 	arc_buf_t		*b_buf;
-	uint32_t		b_flags;
+	arc_flags_t		b_flags;
 	uint32_t		b_datacnt;
 
 	arc_callback_t		*b_acb;
@@ -528,50 +528,26 @@ struct arc_buf_hdr {
 static arc_buf_t *arc_eviction_list;
 static kmutex_t arc_eviction_mtx;
 static arc_buf_hdr_t arc_eviction_hdr;
-static void arc_get_data_buf(arc_buf_t *buf);
-static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock);
-static int arc_evict_needed(arc_buf_contents_t type);
-static void arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes);
-static void arc_buf_watch(arc_buf_t *buf);
-
-static boolean_t l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab);
 
 #define	GHOST_STATE(state)	\
 	((state) == arc_mru_ghost || (state) == arc_mfu_ghost ||	\
 	(state) == arc_l2c_only)
 
-/*
- * Private ARC flags.  These flags are private ARC only flags that will show up
- * in b_flags in the arc_hdr_buf_t.  Some flags are publicly declared, and can
- * be passed in as arc_flags in things like arc_read.  However, these flags
- * should never be passed and should only be set by ARC code.  When adding new
- * public flags, make sure not to smash the private ones.
- */
-
-#define	ARC_IN_HASH_TABLE	(1 << 9)	/* this buffer is hashed */
-#define	ARC_IO_IN_PROGRESS	(1 << 10)	/* I/O in progress for buf */
-#define	ARC_IO_ERROR		(1 << 11)	/* I/O failed for buf */
-#define	ARC_FREED_IN_READ	(1 << 12)	/* buf freed while in read */
-#define	ARC_BUF_AVAILABLE	(1 << 13)	/* block not in active use */
-#define	ARC_INDIRECT		(1 << 14)	/* this is an indirect block */
-#define	ARC_FREE_IN_PROGRESS	(1 << 15)	/* hdr about to be freed */
-#define	ARC_L2_WRITING		(1 << 16)	/* L2ARC write in progress */
-#define	ARC_L2_EVICTED		(1 << 17)	/* evicted during I/O */
-#define	ARC_L2_WRITE_HEAD	(1 << 18)	/* head of write list */
-
-#define	HDR_IN_HASH_TABLE(hdr)	((hdr)->b_flags & ARC_IN_HASH_TABLE)
-#define	HDR_IO_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_IO_IN_PROGRESS)
-#define	HDR_IO_ERROR(hdr)	((hdr)->b_flags & ARC_IO_ERROR)
-#define	HDR_PREFETCH(hdr)	((hdr)->b_flags & ARC_PREFETCH)
-#define	HDR_FREED_IN_READ(hdr)	((hdr)->b_flags & ARC_FREED_IN_READ)
-#define	HDR_BUF_AVAILABLE(hdr)	((hdr)->b_flags & ARC_BUF_AVAILABLE)
-#define	HDR_FREE_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_FREE_IN_PROGRESS)
-#define	HDR_L2CACHE(hdr)	((hdr)->b_flags & ARC_L2CACHE)
-#define	HDR_L2_READING(hdr)	((hdr)->b_flags & ARC_IO_IN_PROGRESS &&	\
-				    (hdr)->b_l2hdr != NULL)
-#define	HDR_L2_WRITING(hdr)	((hdr)->b_flags & ARC_L2_WRITING)
-#define	HDR_L2_EVICTED(hdr)	((hdr)->b_flags & ARC_L2_EVICTED)
-#define	HDR_L2_WRITE_HEAD(hdr)	((hdr)->b_flags & ARC_L2_WRITE_HEAD)
+#define	HDR_IN_HASH_TABLE(hdr)	((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE)
+#define	HDR_IO_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS)
+#define	HDR_IO_ERROR(hdr)	((hdr)->b_flags & ARC_FLAG_IO_ERROR)
+#define	HDR_PREFETCH(hdr)	((hdr)->b_flags & ARC_FLAG_PREFETCH)
+#define	HDR_FREED_IN_READ(hdr)	((hdr)->b_flags & ARC_FLAG_FREED_IN_READ)
+#define	HDR_BUF_AVAILABLE(hdr)	((hdr)->b_flags & ARC_FLAG_BUF_AVAILABLE)
+#define	HDR_FREE_IN_PROGRESS(hdr)	\
+	((hdr)->b_flags & ARC_FLAG_FREE_IN_PROGRESS)
+#define	HDR_L2CACHE(hdr)	((hdr)->b_flags & ARC_FLAG_L2CACHE)
+#define	HDR_L2_READING(hdr)	\
+	((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS &&	\
+	    (hdr)->b_l2hdr != NULL)
+#define	HDR_L2_WRITING(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITING)
+#define	HDR_L2_EVICTED(hdr)	((hdr)->b_flags & ARC_FLAG_L2_EVICTED)
+#define	HDR_L2_WRITE_HEAD(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD)
 
 /*
  * Other sizes
@@ -703,14 +679,20 @@ static kmutex_t l2arc_feed_thr_lock;
 static kcondvar_t l2arc_feed_thr_cv;
 static uint8_t l2arc_thread_exit;
 
-static void l2arc_read_done(zio_t *zio);
+static void arc_get_data_buf(arc_buf_t *);
+static void arc_access(arc_buf_hdr_t *, kmutex_t *);
+static int arc_evict_needed(arc_buf_contents_t);
+static void arc_evict_ghost(arc_state_t *, uint64_t, int64_t);
+static void arc_buf_watch(arc_buf_t *);
+
+static boolean_t l2arc_write_eligible(uint64_t, arc_buf_hdr_t *);
+static void l2arc_read_done(zio_t *);
 static void l2arc_hdr_stat_add(void);
 static void l2arc_hdr_stat_remove(void);
 
-static boolean_t l2arc_compress_buf(l2arc_buf_hdr_t *l2hdr);
-static void l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr,
-    enum zio_compress c);
-static void l2arc_release_cdata_buf(arc_buf_hdr_t *ab);
+static boolean_t l2arc_compress_buf(l2arc_buf_hdr_t *);
+static void l2arc_decompress_zio(zio_t *, arc_buf_hdr_t *, enum zio_compress);
+static void l2arc_release_cdata_buf(arc_buf_hdr_t *);
 
 static uint64_t
 buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
@@ -755,14 +737,14 @@ buf_hash_find(uint64_t spa, const blkptr_t *bp, kmutex_t **lockp)
 	uint64_t birth = BP_PHYSICAL_BIRTH(bp);
 	uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
-	arc_buf_hdr_t *buf;
+	arc_buf_hdr_t *hdr;
 
 	mutex_enter(hash_lock);
-	for (buf = buf_hash_table.ht_table[idx]; buf != NULL;
-	    buf = buf->b_hash_next) {
-		if (BUF_EQUAL(spa, dva, birth, buf)) {
+	for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL;
+	    hdr = hdr->b_hash_next) {
+		if (BUF_EQUAL(spa, dva, birth, hdr)) {
 			*lockp = hash_lock;
-			return (buf);
+			return (hdr);
 		}
 	}
 	mutex_exit(hash_lock);
@@ -777,27 +759,27 @@ buf_hash_find(uint64_t spa, const blkptr_t *bp, kmutex_t **lockp)
  * Otherwise returns NULL.
  */
 static arc_buf_hdr_t *
-buf_hash_insert(arc_buf_hdr_t *buf, kmutex_t **lockp)
+buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp)
 {
-	uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth);
+	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
-	arc_buf_hdr_t *fbuf;
+	arc_buf_hdr_t *fhdr;
 	uint32_t i;
 
-	ASSERT(!DVA_IS_EMPTY(&buf->b_dva));
-	ASSERT(buf->b_birth != 0);
-	ASSERT(!HDR_IN_HASH_TABLE(buf));
+	ASSERT(!DVA_IS_EMPTY(&hdr->b_dva));
+	ASSERT(hdr->b_birth != 0);
+	ASSERT(!HDR_IN_HASH_TABLE(hdr));
 	*lockp = hash_lock;
 	mutex_enter(hash_lock);
-	for (fbuf = buf_hash_table.ht_table[idx], i = 0; fbuf != NULL;
-	    fbuf = fbuf->b_hash_next, i++) {
-		if (BUF_EQUAL(buf->b_spa, &buf->b_dva, buf->b_birth, fbuf))
-			return (fbuf);
+	for (fhdr = buf_hash_table.ht_table[idx], i = 0; fhdr != NULL;
+	    fhdr = fhdr->b_hash_next, i++) {
+		if (BUF_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))
+			return (fhdr);
 	}
 
-	buf->b_hash_next = buf_hash_table.ht_table[idx];
-	buf_hash_table.ht_table[idx] = buf;
-	buf->b_flags |= ARC_IN_HASH_TABLE;
+	hdr->b_hash_next = buf_hash_table.ht_table[idx];
+	buf_hash_table.ht_table[idx] = hdr;
+	hdr->b_flags |= ARC_FLAG_IN_HASH_TABLE;
 
 	/* collect some hash table performance data */
 	if (i > 0) {
@@ -815,22 +797,22 @@ buf_hash_insert(arc_buf_hdr_t *buf, kmutex_t **lockp)
 }
 
 static void
-buf_hash_remove(arc_buf_hdr_t *buf)
+buf_hash_remove(arc_buf_hdr_t *hdr)
 {
-	arc_buf_hdr_t *fbuf, **bufp;
-	uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth);
+	arc_buf_hdr_t *fhdr, **hdrp;
+	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
 
 	ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx)));
-	ASSERT(HDR_IN_HASH_TABLE(buf));
+	ASSERT(HDR_IN_HASH_TABLE(hdr));
 
-	bufp = &buf_hash_table.ht_table[idx];
-	while ((fbuf = *bufp) != buf) {
-		ASSERT(fbuf != NULL);
-		bufp = &fbuf->b_hash_next;
+	hdrp = &buf_hash_table.ht_table[idx];
+	while ((fhdr = *hdrp) != hdr) {
+		ASSERT(fhdr != NULL);
+		hdrp = &fhdr->b_hash_next;
 	}
-	*bufp = buf->b_hash_next;
-	buf->b_hash_next = NULL;
-	buf->b_flags &= ~ARC_IN_HASH_TABLE;
+	*hdrp = hdr->b_hash_next;
+	hdr->b_hash_next = NULL;
+	hdr->b_flags &= ~ARC_FLAG_IN_HASH_TABLE;
 
 	/* collect some hash table performance data */
 	ARCSTAT_BUMPDOWN(arcstat_hash_elements);
@@ -867,12 +849,12 @@ buf_fini(void)
 static int
 hdr_cons(void *vbuf, void *unused, int kmflag)
 {
-	arc_buf_hdr_t *buf = vbuf;
+	arc_buf_hdr_t *hdr = vbuf;
 
-	bzero(buf, sizeof (arc_buf_hdr_t));
-	refcount_create(&buf->b_refcnt);
-	cv_init(&buf->b_cv, NULL, CV_DEFAULT, NULL);
-	mutex_init(&buf->b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
+	bzero(hdr, sizeof (arc_buf_hdr_t));
+	refcount_create(&hdr->b_refcnt);
+	cv_init(&hdr->b_cv, NULL, CV_DEFAULT, NULL);
+	mutex_init(&hdr->b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
 	arc_space_consume(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS);
 
 	return (0);
@@ -899,12 +881,12 @@ buf_cons(void *vbuf, void *unused, int kmflag)
 static void
 hdr_dest(void *vbuf, void *unused)
 {
-	arc_buf_hdr_t *buf = vbuf;
+	arc_buf_hdr_t *hdr = vbuf;
 
-	ASSERT(BUF_EMPTY(buf));
-	refcount_destroy(&buf->b_refcnt);
-	cv_destroy(&buf->b_cv);
-	mutex_destroy(&buf->b_freeze_lock);
+	ASSERT(BUF_EMPTY(hdr));
+	refcount_destroy(&hdr->b_refcnt);
+	cv_destroy(&hdr->b_cv);
+	mutex_destroy(&hdr->b_freeze_lock);
 	arc_space_return(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS);
 }
 
@@ -986,7 +968,7 @@ arc_cksum_verify(arc_buf_t *buf)
 
 	mutex_enter(&buf->b_hdr->b_freeze_lock);
 	if (buf->b_hdr->b_freeze_cksum == NULL ||
-	    (buf->b_hdr->b_flags & ARC_IO_ERROR)) {
+	    (buf->b_hdr->b_flags & ARC_FLAG_IO_ERROR)) {
 		mutex_exit(&buf->b_hdr->b_freeze_lock);
 		return;
 	}
@@ -1077,7 +1059,7 @@ arc_buf_thaw(arc_buf_t *buf)
 	if (zfs_flags & ZFS_DEBUG_MODIFY) {
 		if (buf->b_hdr->b_state != arc_anon)
 			panic("modifying non-anon buffer!");
-		if (buf->b_hdr->b_flags & ARC_IO_IN_PROGRESS)
+		if (buf->b_hdr->b_flags & ARC_FLAG_IO_IN_PROGRESS)
 			panic("modifying buffer while i/o in progress!");
 		arc_cksum_verify(buf);
 	}
@@ -1118,54 +1100,54 @@ arc_buf_freeze(arc_buf_t *buf)
 }
 
 static void
-add_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag)
+add_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
 {
 	ASSERT(MUTEX_HELD(hash_lock));
 
-	if ((refcount_add(&ab->b_refcnt, tag) == 1) &&
-	    (ab->b_state != arc_anon)) {
-		uint64_t delta = ab->b_size * ab->b_datacnt;
-		list_t *list = &ab->b_state->arcs_list[ab->b_type];
-		uint64_t *size = &ab->b_state->arcs_lsize[ab->b_type];
+	if ((refcount_add(&hdr->b_refcnt, tag) == 1) &&
+	    (hdr->b_state != arc_anon)) {
+		uint64_t delta = hdr->b_size * hdr->b_datacnt;
+		list_t *list = &hdr->b_state->arcs_list[hdr->b_type];
+		uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type];
 
-		ASSERT(!MUTEX_HELD(&ab->b_state->arcs_mtx));
-		mutex_enter(&ab->b_state->arcs_mtx);
-		ASSERT(list_link_active(&ab->b_arc_node));
-		list_remove(list, ab);
-		if (GHOST_STATE(ab->b_state)) {
-			ASSERT0(ab->b_datacnt);
-			ASSERT3P(ab->b_buf, ==, NULL);
-			delta = ab->b_size;
+		ASSERT(!MUTEX_HELD(&hdr->b_state->arcs_mtx));
+		mutex_enter(&hdr->b_state->arcs_mtx);
+		ASSERT(list_link_active(&hdr->b_arc_node));
+		list_remove(list, hdr);
+		if (GHOST_STATE(hdr->b_state)) {
+			ASSERT0(hdr->b_datacnt);
+			ASSERT3P(hdr->b_buf, ==, NULL);
+			delta = hdr->b_size;
 		}
 		ASSERT(delta > 0);
 		ASSERT3U(*size, >=, delta);
 		atomic_add_64(size, -delta);
-		mutex_exit(&ab->b_state->arcs_mtx);
+		mutex_exit(&hdr->b_state->arcs_mtx);
 		/* remove the prefetch flag if we get a reference */
-		if (ab->b_flags & ARC_PREFETCH)
-			ab->b_flags &= ~ARC_PREFETCH;
+		if (hdr->b_flags & ARC_FLAG_PREFETCH)
+			hdr->b_flags &= ~ARC_FLAG_PREFETCH;
 	}
 }
 
 static int
-remove_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag)
+remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
 {
 	int cnt;
-	arc_state_t *state = ab->b_state;
+	arc_state_t *state = hdr->b_state;
 
 	ASSERT(state == arc_anon || MUTEX_HELD(hash_lock));
 	ASSERT(!GHOST_STATE(state));
 
-	if (((cnt = refcount_remove(&ab->b_refcnt, tag)) == 0) &&
+	if (((cnt = refcount_remove(&hdr->b_refcnt, tag)) == 0) &&
 	    (state != arc_anon)) {
-		uint64_t *size = &state->arcs_lsize[ab->b_type];
+		uint64_t *size = &state->arcs_lsize[hdr->b_type];
 
 		ASSERT(!MUTEX_HELD(&state->arcs_mtx));
 		mutex_enter(&state->arcs_mtx);
-		ASSERT(!list_link_active(&ab->b_arc_node));
-		list_insert_head(&state->arcs_list[ab->b_type], ab);
-		ASSERT(ab->b_datacnt > 0);
-		atomic_add_64(size, ab->b_size * ab->b_datacnt);
+		ASSERT(!list_link_active(&hdr->b_arc_node));
+		list_insert_head(&state->arcs_list[hdr->b_type], hdr);
+		ASSERT(hdr->b_datacnt > 0);
+		atomic_add_64(size, hdr->b_size * hdr->b_datacnt);
 		mutex_exit(&state->arcs_mtx);
 	}
 	return (cnt);
@@ -1176,19 +1158,20 @@ remove_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag)
  * for the buffer must be held by the caller.
  */
 static void
-arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock)
+arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
+    kmutex_t *hash_lock)
 {
-	arc_state_t *old_state = ab->b_state;
-	int64_t refcnt = refcount_count(&ab->b_refcnt);
+	arc_state_t *old_state = hdr->b_state;
+	int64_t refcnt = refcount_count(&hdr->b_refcnt);
 	uint64_t from_delta, to_delta;
 
 	ASSERT(MUTEX_HELD(hash_lock));
 	ASSERT3P(new_state, !=, old_state);
-	ASSERT(refcnt == 0 || ab->b_datacnt > 0);
-	ASSERT(ab->b_datacnt == 0 || !GHOST_STATE(new_state));
-	ASSERT(ab->b_datacnt <= 1 || old_state != arc_anon);
+	ASSERT(refcnt == 0 || hdr->b_datacnt > 0);
+	ASSERT(hdr->b_datacnt == 0 || !GHOST_STATE(new_state));
+	ASSERT(hdr->b_datacnt <= 1 || old_state != arc_anon);
 
-	from_delta = to_delta = ab->b_datacnt * ab->b_size;
+	from_delta = to_delta = hdr->b_datacnt * hdr->b_size;
 
 	/*
 	 * If this buffer is evictable, transfer it from the
@@ -1197,22 +1180,22 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock)
 	if (refcnt == 0) {
 		if (old_state != arc_anon) {
 			int use_mutex = !MUTEX_HELD(&old_state->arcs_mtx);
-			uint64_t *size = &old_state->arcs_lsize[ab->b_type];
+			uint64_t *size = &old_state->arcs_lsize[hdr->b_type];
 
 			if (use_mutex)
 				mutex_enter(&old_state->arcs_mtx);
 
-			ASSERT(list_link_active(&ab->b_arc_node));
-			list_remove(&old_state->arcs_list[ab->b_type], ab);
+			ASSERT(list_link_active(&hdr->b_arc_node));
+			list_remove(&old_state->arcs_list[hdr->b_type], hdr);
 
 			/*
 			 * If prefetching out of the ghost cache,
 			 * we will have a non-zero datacnt.
 			 */
-			if (GHOST_STATE(old_state) && ab->b_datacnt == 0) {
+			if (GHOST_STATE(old_state) && hdr->b_datacnt == 0) {
 				/* ghost elements have a ghost size */
-				ASSERT(ab->b_buf == NULL);
-				from_delta = ab->b_size;
+				ASSERT(hdr->b_buf == NULL);
+				from_delta = hdr->b_size;
 			}
 			ASSERT3U(*size, >=, from_delta);
 			atomic_add_64(size, -from_delta);
@@ -1222,18 +1205,19 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock)
 		}
 		if (new_state != arc_anon) {
 			int use_mutex = !MUTEX_HELD(&new_state->arcs_mtx);
-			uint64_t *size = &new_state->arcs_lsize[ab->b_type];
+			uint64_t *size = &new_state->arcs_lsize[hdr->b_type];
 
 			if (use_mutex)
 				mutex_enter(&new_state->arcs_mtx);
 
-			list_insert_head(&new_state->arcs_list[ab->b_type], ab);
+			list_insert_head(&new_state->arcs_list[hdr->b_type],
+			    hdr);
 
 			/* ghost elements have a ghost size */
 			if (GHOST_STATE(new_state)) {
-				ASSERT(ab->b_datacnt == 0);
-				ASSERT(ab->b_buf == NULL);
-				to_delta = ab->b_size;
+				ASSERT(hdr->b_datacnt == 0);
+				ASSERT(hdr->b_buf == NULL);
+				to_delta = hdr->b_size;
 			}
 			atomic_add_64(size, to_delta);
 
@@ -1242,9 +1226,9 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock)
 		}
 	}
 
-	ASSERT(!BUF_EMPTY(ab));
-	if (new_state == arc_anon && HDR_IN_HASH_TABLE(ab))
-		buf_hash_remove(ab);
+	ASSERT(!BUF_EMPTY(hdr));
+	if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr))
+		buf_hash_remove(hdr);
 
 	/* adjust state sizes */
 	if (to_delta)
@@ -1253,7 +1237,7 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock)
 		ASSERT3U(old_state->arcs_size, >=, from_delta);
 		atomic_add_64(&old_state->arcs_size, -from_delta);
 	}
-	ab->b_state = new_state;
+	hdr->b_state = new_state;
 
 	/* adjust l2arc hdr stats */
 	if (new_state == arc_l2c_only)
@@ -1455,7 +1439,7 @@ arc_buf_add_ref(arc_buf_t *buf, void* tag)
 	arc_access(hdr, hash_lock);
 	mutex_exit(hash_lock);
 	ARCSTAT_BUMP(arcstat_hits);
-	ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
+	ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_FLAG_PREFETCH),
 	    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
 	    data, metadata, hits);
 }
@@ -1656,7 +1640,7 @@ arc_buf_free(arc_buf_t *buf, void *tag)
 		} else {
 			ASSERT(buf == hdr->b_buf);
 			ASSERT(buf->b_efunc == NULL);
-			hdr->b_flags |= ARC_BUF_AVAILABLE;
+			hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
 		}
 		mutex_exit(hash_lock);
 	} else if (HDR_IO_IN_PROGRESS(hdr)) {
@@ -1707,7 +1691,7 @@ arc_buf_remove_ref(arc_buf_t *buf, void* tag)
 	} else if (no_callback) {
 		ASSERT(hdr->b_buf == buf && buf->b_next == NULL);
 		ASSERT(buf->b_efunc == NULL);
-		hdr->b_flags |= ARC_BUF_AVAILABLE;
+		hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
 	}
 	ASSERT(no_callback || hdr->b_datacnt > 1 ||
 	    refcount_is_zero(&hdr->b_refcnt));
@@ -1782,7 +1766,7 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 {
 	arc_state_t *evicted_state;
 	uint64_t bytes_evicted = 0, skipped = 0, missed = 0;
-	arc_buf_hdr_t *ab, *ab_prev = NULL;
+	arc_buf_hdr_t *hdr, *hdr_prev = NULL;
 	kmutex_t *hash_lock;
 	boolean_t have_lock;
 	void *stolen = NULL;
@@ -1840,24 +1824,24 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 
 	list_t *list = &state->arcs_list[type];
 
-	for (ab = list_tail(list); ab; ab = ab_prev) {
-		ab_prev = list_prev(list, ab);
+	for (hdr = list_tail(list); hdr; hdr = hdr_prev) {
+		hdr_prev = list_prev(list, hdr);
 		/* prefetch buffers have a minimum lifespan */
-		if (HDR_IO_IN_PROGRESS(ab) ||
-		    (spa && ab->b_spa != spa) ||
-		    (ab->b_flags & (ARC_PREFETCH|ARC_INDIRECT) &&
-		    ddi_get_lbolt() - ab->b_arc_access <
+		if (HDR_IO_IN_PROGRESS(hdr) ||
+		    (spa && hdr->b_spa != spa) ||
+		    (hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT) &&
+		    ddi_get_lbolt() - hdr->b_arc_access <
 		    arc_min_prefetch_lifespan)) {
 			skipped++;
 			continue;
 		}
 		/* "lookahead" for better eviction candidate */
-		if (recycle && ab->b_size != bytes &&
-		    ab_prev && ab_prev->b_size == bytes)
+		if (recycle && hdr->b_size != bytes &&
+		    hdr_prev && hdr_prev->b_size == bytes)
 			continue;
 
 		/* ignore markers */
-		if (ab->b_spa == 0)
+		if (hdr->b_spa == 0)
 			continue;
 
 		/*
@@ -1870,34 +1854,34 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 		 * the hot code path, so don't sleep.
 		 */
 		if (!recycle && count++ > arc_evict_iterations) {
-			list_insert_after(list, ab, &marker);
+			list_insert_after(list, hdr, &marker);
 			mutex_exit(&evicted_state->arcs_mtx);
 			mutex_exit(&state->arcs_mtx);
 			kpreempt(KPREEMPT_SYNC);
 			mutex_enter(&state->arcs_mtx);
 			mutex_enter(&evicted_state->arcs_mtx);
-			ab_prev = list_prev(list, &marker);
+			hdr_prev = list_prev(list, &marker);
 			list_remove(list, &marker);
 			count = 0;
 			continue;
 		}
 
-		hash_lock = HDR_LOCK(ab);
+		hash_lock = HDR_LOCK(hdr);
 		have_lock = MUTEX_HELD(hash_lock);
 		if (have_lock || mutex_tryenter(hash_lock)) {
-			ASSERT0(refcount_count(&ab->b_refcnt));
-			ASSERT(ab->b_datacnt > 0);
-			while (ab->b_buf) {
-				arc_buf_t *buf = ab->b_buf;
+			ASSERT0(refcount_count(&hdr->b_refcnt));
+			ASSERT(hdr->b_datacnt > 0);
+			while (hdr->b_buf) {
+				arc_buf_t *buf = hdr->b_buf;
 				if (!mutex_tryenter(&buf->b_evict_lock)) {
 					missed += 1;
 					break;
 				}
 				if (buf->b_data) {
-					bytes_evicted += ab->b_size;
-					if (recycle && ab->b_type == type &&
-					    ab->b_size == bytes &&
-					    !HDR_L2_WRITING(ab)) {
+					bytes_evicted += hdr->b_size;
+					if (recycle && hdr->b_type == type &&
+					    hdr->b_size == bytes &&
+					    !HDR_L2_WRITING(hdr)) {
 						stolen = buf->b_data;
 						recycle = FALSE;
 					}
@@ -1906,7 +1890,7 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 					mutex_enter(&arc_eviction_mtx);
 					arc_buf_destroy(buf,
 					    buf->b_data == stolen, FALSE);
-					ab->b_buf = buf->b_next;
+					hdr->b_buf = buf->b_next;
 					buf->b_hdr = &arc_eviction_hdr;
 					buf->b_next = arc_eviction_list;
 					arc_eviction_list = buf;
@@ -1919,26 +1903,26 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 				}
 			}
 
-			if (ab->b_l2hdr) {
+			if (hdr->b_l2hdr) {
 				ARCSTAT_INCR(arcstat_evict_l2_cached,
-				    ab->b_size);
+				    hdr->b_size);
 			} else {
-				if (l2arc_write_eligible(ab->b_spa, ab)) {
+				if (l2arc_write_eligible(hdr->b_spa, hdr)) {
 					ARCSTAT_INCR(arcstat_evict_l2_eligible,
-					    ab->b_size);
+					    hdr->b_size);
 				} else {
 					ARCSTAT_INCR(
 					    arcstat_evict_l2_ineligible,
-					    ab->b_size);
+					    hdr->b_size);
 				}
 			}
 
-			if (ab->b_datacnt == 0) {
-				arc_change_state(evicted_state, ab, hash_lock);
-				ASSERT(HDR_IN_HASH_TABLE(ab));
-				ab->b_flags |= ARC_IN_HASH_TABLE;
-				ab->b_flags &= ~ARC_BUF_AVAILABLE;
-				DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, ab);
+			if (hdr->b_datacnt == 0) {
+				arc_change_state(evicted_state, hdr, hash_lock);
+				ASSERT(HDR_IN_HASH_TABLE(hdr));
+				hdr->b_flags |= ARC_FLAG_IN_HASH_TABLE;
+				hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE;
+				DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr);
 			}
 			if (!have_lock)
 				mutex_exit(hash_lock);
@@ -1979,7 +1963,7 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 static void
 arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes)
 {
-	arc_buf_hdr_t *ab, *ab_prev;
+	arc_buf_hdr_t *hdr, *hdr_prev;
 	arc_buf_hdr_t marker = { 0 };
 	list_t *list = &state->arcs_list[ARC_BUFC_DATA];
 	kmutex_t *hash_lock;
@@ -1990,18 +1974,18 @@ arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes)
 	ASSERT(GHOST_STATE(state));
 top:
 	mutex_enter(&state->arcs_mtx);
-	for (ab = list_tail(list); ab; ab = ab_prev) {
-		ab_prev = list_prev(list, ab);
-		if (ab->b_type > ARC_BUFC_NUMTYPES)
-			panic("invalid ab=%p", (void *)ab);
-		if (spa && ab->b_spa != spa)
+	for (hdr = list_tail(list); hdr; hdr = hdr_prev) {
+		hdr_prev = list_prev(list, hdr);
+		if (hdr->b_type > ARC_BUFC_NUMTYPES)
+			panic("invalid hdr=%p", (void *)hdr);
+		if (spa && hdr->b_spa != spa)
 			continue;
 
 		/* ignore markers */
-		if (ab->b_spa == 0)
+		if (hdr->b_spa == 0)
 			continue;
 
-		hash_lock = HDR_LOCK(ab);
+		hash_lock = HDR_LOCK(hdr);
 		/* caller may be trying to modify this buffer, skip it */
 		if (MUTEX_HELD(hash_lock))
 			continue;
@@ -2013,35 +1997,35 @@ top:
 		 * before reacquiring the lock.
 		 */
 		if (count++ > arc_evict_iterations) {
-			list_insert_after(list, ab, &marker);
+			list_insert_after(list, hdr, &marker);
 			mutex_exit(&state->arcs_mtx);
 			kpreempt(KPREEMPT_SYNC);
 			mutex_enter(&state->arcs_mtx);
-			ab_prev = list_prev(list, &marker);
+			hdr_prev = list_prev(list, &marker);
 			list_remove(list, &marker);
 			count = 0;
 			continue;
 		}
 		if (mutex_tryenter(hash_lock)) {
-			ASSERT(!HDR_IO_IN_PROGRESS(ab));
-			ASSERT(ab->b_buf == NULL);
+			ASSERT(!HDR_IO_IN_PROGRESS(hdr));
+			ASSERT(hdr->b_buf == NULL);
 			ARCSTAT_BUMP(arcstat_deleted);
-			bytes_deleted += ab->b_size;
+			bytes_deleted += hdr->b_size;
 
-			if (ab->b_l2hdr != NULL) {
+			if (hdr->b_l2hdr != NULL) {
 				/*
 				 * This buffer is cached on the 2nd Level ARC;
 				 * don't destroy the header.
 				 */
-				arc_change_state(arc_l2c_only, ab, hash_lock);
+				arc_change_state(arc_l2c_only, hdr, hash_lock);
 				mutex_exit(hash_lock);
 			} else {
-				arc_change_state(arc_anon, ab, hash_lock);
+				arc_change_state(arc_anon, hdr, hash_lock);
 				mutex_exit(hash_lock);
-				arc_hdr_destroy(ab);
+				arc_hdr_destroy(hdr);
 			}
 
-			DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, ab);
+			DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);
 			if (bytes >= 0 && bytes_deleted >= bytes)
 				break;
 		} else if (bytes < 0) {
@@ -2050,12 +2034,12 @@ top:
 			 * hash lock to become available. Once its
 			 * available, restart from where we left off.
 			 */
-			list_insert_after(list, ab, &marker);
+			list_insert_after(list, hdr, &marker);
 			mutex_exit(&state->arcs_mtx);
 			mutex_enter(hash_lock);
 			mutex_exit(hash_lock);
 			mutex_enter(&state->arcs_mtx);
-			ab_prev = list_prev(list, &marker);
+			hdr_prev = list_prev(list, &marker);
 			list_remove(list, &marker);
 		} else {
 			bufs_skipped += 1;
@@ -2571,7 +2555,8 @@ arc_get_data_buf(arc_buf_t *buf)
 	 * will end up on the mru list; so steal space from there.
 	 */
 	if (state == arc_mfu_ghost)
-		state = buf->b_hdr->b_flags & ARC_PREFETCH ? arc_mru : arc_mfu;
+		state = buf->b_hdr->b_flags & ARC_FLAG_PREFETCH ?
+		    arc_mru : arc_mfu;
 	else if (state == arc_mru_ghost)
 		state = arc_mru;
 
@@ -2626,25 +2611,25 @@ out:
  * NOTE: the hash lock is dropped in this function.
  */
 static void
-arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock)
+arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
 {
 	clock_t now;
 
 	ASSERT(MUTEX_HELD(hash_lock));
 
-	if (buf->b_state == arc_anon) {
+	if (hdr->b_state == arc_anon) {
 		/*
 		 * This buffer is not in the cache, and does not
 		 * appear in our "ghost" list.  Add the new buffer
 		 * to the MRU state.
 		 */
 
-		ASSERT(buf->b_arc_access == 0);
-		buf->b_arc_access = ddi_get_lbolt();
-		DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf);
-		arc_change_state(arc_mru, buf, hash_lock);
+		ASSERT(hdr->b_arc_access == 0);
+		hdr->b_arc_access = ddi_get_lbolt();
+		DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
+		arc_change_state(arc_mru, hdr, hash_lock);
 
-	} else if (buf->b_state == arc_mru) {
+	} else if (hdr->b_state == arc_mru) {
 		now = ddi_get_lbolt();
 
 		/*
@@ -2655,14 +2640,14 @@ arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock)
 		 * - move the buffer to the head of the list if this is
 		 *   another prefetch (to make it less likely to be evicted).
 		 */
-		if ((buf->b_flags & ARC_PREFETCH) != 0) {
-			if (refcount_count(&buf->b_refcnt) == 0) {
-				ASSERT(list_link_active(&buf->b_arc_node));
+		if ((hdr->b_flags & ARC_FLAG_PREFETCH) != 0) {
+			if (refcount_count(&hdr->b_refcnt) == 0) {
+				ASSERT(list_link_active(&hdr->b_arc_node));
 			} else {
-				buf->b_flags &= ~ARC_PREFETCH;
+				hdr->b_flags &= ~ARC_FLAG_PREFETCH;
 				ARCSTAT_BUMP(arcstat_mru_hits);
 			}
-			buf->b_arc_access = now;
+			hdr->b_arc_access = now;
 			return;
 		}
 
@@ -2671,18 +2656,18 @@ arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock)
 		 * but it is still in the cache. Move it to the MFU
 		 * state.
 		 */
-		if (now > buf->b_arc_access + ARC_MINTIME) {
+		if (now > hdr->b_arc_access + ARC_MINTIME) {
 			/*
 			 * More than 125ms have passed since we
 			 * instantiated this buffer.  Move it to the
 			 * most frequently used state.
 			 */
-			buf->b_arc_access = now;
-			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
-			arc_change_state(arc_mfu, buf, hash_lock);
+			hdr->b_arc_access = now;
+			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
+			arc_change_state(arc_mfu, hdr, hash_lock);
 		}
 		ARCSTAT_BUMP(arcstat_mru_hits);
-	} else if (buf->b_state == arc_mru_ghost) {
+	} else if (hdr->b_state == arc_mru_ghost) {
 		arc_state_t	*new_state;
 		/*
 		 * This buffer has been "accessed" recently, but
@@ -2690,21 +2675,21 @@ arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock)
 		 * MFU state.
 		 */
 
-		if (buf->b_flags & ARC_PREFETCH) {
+		if (hdr->b_flags & ARC_FLAG_PREFETCH) {
 			new_state = arc_mru;
-			if (refcount_count(&buf->b_refcnt) > 0)
-				buf->b_flags &= ~ARC_PREFETCH;
-			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf);
+			if (refcount_count(&hdr->b_refcnt) > 0)
+				hdr->b_flags &= ~ARC_FLAG_PREFETCH;
+			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
 		} else {
 			new_state = arc_mfu;
-			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
+			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
 		}
 
-		buf->b_arc_access = ddi_get_lbolt();
-		arc_change_state(new_state, buf, hash_lock);
+		hdr->b_arc_access = ddi_get_lbolt();
+		arc_change_state(new_state, hdr, hash_lock);
 
 		ARCSTAT_BUMP(arcstat_mru_ghost_hits);
-	} else if (buf->b_state == arc_mfu) {
+	} else if (hdr->b_state == arc_mfu) {
 		/*
 		 * This buffer has been accessed more than once and is
 		 * still in the cache.  Keep it in the MFU state.
@@ -2714,13 +2699,13 @@ arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock)
 		 * If it was a prefetch, we will explicitly move it to
 		 * the head of the list now.
 		 */
-		if ((buf->b_flags & ARC_PREFETCH) != 0) {
-			ASSERT(refcount_count(&buf->b_refcnt) == 0);
-			ASSERT(list_link_active(&buf->b_arc_node));
+		if ((hdr->b_flags & ARC_FLAG_PREFETCH) != 0) {
+			ASSERT(refcount_count(&hdr->b_refcnt) == 0);
+			ASSERT(list_link_active(&hdr->b_arc_node));
 		}
 		ARCSTAT_BUMP(arcstat_mfu_hits);
-		buf->b_arc_access = ddi_get_lbolt();
-	} else if (buf->b_state == arc_mfu_ghost) {
+		hdr->b_arc_access = ddi_get_lbolt();
+	} else if (hdr->b_state == arc_mfu_ghost) {
 		arc_state_t	*new_state = arc_mfu;
 		/*
 		 * This buffer has been accessed more than once but has
@@ -2728,28 +2713,28 @@ arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock)
 		 * MFU state.
 		 */
 
-		if (buf->b_flags & ARC_PREFETCH) {
+		if (hdr->b_flags & ARC_FLAG_PREFETCH) {
 			/*
 			 * This is a prefetch access...
 			 * move this block back to the MRU state.
 			 */
-			ASSERT0(refcount_count(&buf->b_refcnt));
+			ASSERT0(refcount_count(&hdr->b_refcnt));
 			new_state = arc_mru;
 		}
 
-		buf->b_arc_access = ddi_get_lbolt();
-		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
-		arc_change_state(new_state, buf, hash_lock);
+		hdr->b_arc_access = ddi_get_lbolt();
+		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
+		arc_change_state(new_state, hdr, hash_lock);
 
 		ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
-	} else if (buf->b_state == arc_l2c_only) {
+	} else if (hdr->b_state == arc_l2c_only) {
 		/*
 		 * This buffer is on the 2nd Level ARC.
 		 */
 
-		buf->b_arc_access = ddi_get_lbolt();
-		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
-		arc_change_state(arc_mfu, buf, hash_lock);
+		hdr->b_arc_access = ddi_get_lbolt();
+		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
+		arc_change_state(arc_mfu, hdr, hash_lock);
 	} else {
 		ASSERT(!"invalid arc state");
 	}
@@ -2817,9 +2802,9 @@ arc_read_done(zio_t *zio)
 		    (found == hdr && HDR_L2_READING(hdr)));
 	}
 
-	hdr->b_flags &= ~ARC_L2_EVICTED;
-	if (l2arc_noprefetch && (hdr->b_flags & ARC_PREFETCH))
-		hdr->b_flags &= ~ARC_L2CACHE;
+	hdr->b_flags &= ~ARC_FLAG_L2_EVICTED;
+	if (l2arc_noprefetch && (hdr->b_flags & ARC_FLAG_PREFETCH))
+		hdr->b_flags &= ~ARC_FLAG_L2CACHE;
 
 	/* byteswap if necessary */
 	callback_list = hdr->b_acb;
@@ -2859,18 +2844,18 @@ arc_read_done(zio_t *zio)
 		}
 	}
 	hdr->b_acb = NULL;
-	hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
+	hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS;
 	ASSERT(!HDR_BUF_AVAILABLE(hdr));
 	if (abuf == buf) {
 		ASSERT(buf->b_efunc == NULL);
 		ASSERT(hdr->b_datacnt == 1);
-		hdr->b_flags |= ARC_BUF_AVAILABLE;
+		hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
 	}
 
 	ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL);
 
 	if (zio->io_error != 0) {
-		hdr->b_flags |= ARC_IO_ERROR;
+		hdr->b_flags |= ARC_FLAG_IO_ERROR;
 		if (hdr->b_state != arc_anon)
 			arc_change_state(arc_anon, hdr, hash_lock);
 		if (HDR_IN_HASH_TABLE(hdr))
@@ -2936,8 +2921,8 @@ arc_read_done(zio_t *zio)
  */
 int
 arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_done_func_t *done,
-    void *private, zio_priority_t priority, int zio_flags, uint32_t *arc_flags,
-    const zbookmark_phys_t *zb)
+    void *private, zio_priority_t priority, int zio_flags,
+    arc_flags_t *arc_flags, const zbookmark_phys_t *zb)
 {
 	arc_buf_hdr_t *hdr = NULL;
 	arc_buf_t *buf = NULL;
@@ -2959,16 +2944,16 @@ top:
 
 	if (hdr != NULL && hdr->b_datacnt > 0) {
 
-		*arc_flags |= ARC_CACHED;
+		*arc_flags |= ARC_FLAG_CACHED;
 
 		if (HDR_IO_IN_PROGRESS(hdr)) {
 
-			if (*arc_flags & ARC_WAIT) {
+			if (*arc_flags & ARC_FLAG_WAIT) {
 				cv_wait(&hdr->b_cv, hash_lock);
 				mutex_exit(hash_lock);
 				goto top;
 			}
-			ASSERT(*arc_flags & ARC_NOWAIT);
+			ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
 
 			if (done) {
 				arc_callback_t	*acb = NULL;
@@ -3006,24 +2991,24 @@ top:
 			ASSERT(buf->b_data);
 			if (HDR_BUF_AVAILABLE(hdr)) {
 				ASSERT(buf->b_efunc == NULL);
-				hdr->b_flags &= ~ARC_BUF_AVAILABLE;
+				hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE;
 			} else {
 				buf = arc_buf_clone(buf);
 			}
 
-		} else if (*arc_flags & ARC_PREFETCH &&
+		} else if (*arc_flags & ARC_FLAG_PREFETCH &&
 		    refcount_count(&hdr->b_refcnt) == 0) {
-			hdr->b_flags |= ARC_PREFETCH;
+			hdr->b_flags |= ARC_FLAG_PREFETCH;
 		}
 		DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
 		arc_access(hdr, hash_lock);
-		if (*arc_flags & ARC_L2CACHE)
-			hdr->b_flags |= ARC_L2CACHE;
-		if (*arc_flags & ARC_L2COMPRESS)
-			hdr->b_flags |= ARC_L2COMPRESS;
+		if (*arc_flags & ARC_FLAG_L2CACHE)
+			hdr->b_flags |= ARC_FLAG_L2CACHE;
+		if (*arc_flags & ARC_FLAG_L2COMPRESS)
+			hdr->b_flags |= ARC_FLAG_L2COMPRESS;
 		mutex_exit(hash_lock);
 		ARCSTAT_BUMP(arcstat_hits);
-		ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
+		ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_FLAG_PREFETCH),
 		    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
 		    data, metadata, hits);
 
@@ -3057,18 +3042,19 @@ top:
 				(void) arc_buf_remove_ref(buf, private);
 				goto top; /* restart the IO request */
 			}
+
 			/* if this is a prefetch, we don't have a reference */
-			if (*arc_flags & ARC_PREFETCH) {
+			if (*arc_flags & ARC_FLAG_PREFETCH) {
 				(void) remove_reference(hdr, hash_lock,
 				    private);
-				hdr->b_flags |= ARC_PREFETCH;
+				hdr->b_flags |= ARC_FLAG_PREFETCH;
 			}
-			if (*arc_flags & ARC_L2CACHE)
-				hdr->b_flags |= ARC_L2CACHE;
-			if (*arc_flags & ARC_L2COMPRESS)
-				hdr->b_flags |= ARC_L2COMPRESS;
+			if (*arc_flags & ARC_FLAG_L2CACHE)
+				hdr->b_flags |= ARC_FLAG_L2CACHE;
+			if (*arc_flags & ARC_FLAG_L2COMPRESS)
+				hdr->b_flags |= ARC_FLAG_L2COMPRESS;
 			if (BP_GET_LEVEL(bp) > 0)
-				hdr->b_flags |= ARC_INDIRECT;
+				hdr->b_flags |= ARC_FLAG_INDIRECT;
 		} else {
 			/* this block is in the ghost cache */
 			ASSERT(GHOST_STATE(hdr->b_state));
@@ -3077,14 +3063,14 @@ top:
 			ASSERT(hdr->b_buf == NULL);
 
 			/* if this is a prefetch, we don't have a reference */
-			if (*arc_flags & ARC_PREFETCH)
-				hdr->b_flags |= ARC_PREFETCH;
+			if (*arc_flags & ARC_FLAG_PREFETCH)
+				hdr->b_flags |= ARC_FLAG_PREFETCH;
 			else
 				add_reference(hdr, hash_lock, private);
-			if (*arc_flags & ARC_L2CACHE)
-				hdr->b_flags |= ARC_L2CACHE;
-			if (*arc_flags & ARC_L2COMPRESS)
-				hdr->b_flags |= ARC_L2COMPRESS;
+			if (*arc_flags & ARC_FLAG_L2CACHE)
+				hdr->b_flags |= ARC_FLAG_L2CACHE;
+			if (*arc_flags & ARC_FLAG_L2COMPRESS)
+				hdr->b_flags |= ARC_FLAG_L2COMPRESS;
 			buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
 			buf->b_hdr = hdr;
 			buf->b_data = NULL;
@@ -3106,7 +3092,7 @@ top:
 
 		ASSERT(hdr->b_acb == NULL);
 		hdr->b_acb = acb;
-		hdr->b_flags |= ARC_IO_IN_PROGRESS;
+		hdr->b_flags |= ARC_FLAG_IO_IN_PROGRESS;
 
 		if (hdr->b_l2hdr != NULL &&
 		    (vd = hdr->b_l2hdr->b_dev->l2ad_vdev) != NULL) {
@@ -3133,7 +3119,7 @@ top:
 		DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp,
 		    uint64_t, size, zbookmark_phys_t *, zb);
 		ARCSTAT_BUMP(arcstat_misses);
-		ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
+		ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_FLAG_PREFETCH),
 		    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
 		    data, metadata, misses);
 
@@ -3195,12 +3181,12 @@ top:
 				    zio_t *, rzio);
 				ARCSTAT_INCR(arcstat_l2_read_bytes, b_asize);
 
-				if (*arc_flags & ARC_NOWAIT) {
+				if (*arc_flags & ARC_FLAG_NOWAIT) {
 					zio_nowait(rzio);
 					return (0);
 				}
 
-				ASSERT(*arc_flags & ARC_WAIT);
+				ASSERT(*arc_flags & ARC_FLAG_WAIT);
 				if (zio_wait(rzio) == 0)
 					return (0);
 
@@ -3226,10 +3212,10 @@ top:
 		rzio = zio_read(pio, spa, bp, buf->b_data, size,
 		    arc_read_done, buf, priority, zio_flags, zb);
 
-		if (*arc_flags & ARC_WAIT)
+		if (*arc_flags & ARC_FLAG_WAIT)
 			return (zio_wait(rzio));
 
-		ASSERT(*arc_flags & ARC_NOWAIT);
+		ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
 		zio_nowait(rzio);
 	}
 	return (0);
@@ -3266,7 +3252,7 @@ arc_freed(spa_t *spa, const blkptr_t *bp)
 	if (HDR_BUF_AVAILABLE(hdr)) {
 		arc_buf_t *buf = hdr->b_buf;
 		add_reference(hdr, hash_lock, FTAG);
-		hdr->b_flags &= ~ARC_BUF_AVAILABLE;
+		hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE;
 		mutex_exit(hash_lock);
 
 		arc_release(buf, FTAG);
@@ -3333,7 +3319,7 @@ arc_clear_callback(arc_buf_t *buf)
 		arc_buf_destroy(buf, FALSE, TRUE);
 	} else {
 		ASSERT(buf == hdr->b_buf);
-		hdr->b_flags |= ARC_BUF_AVAILABLE;
+		hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
 		mutex_exit(&buf->b_evict_lock);
 	}
 
@@ -3439,7 +3425,7 @@ arc_release(arc_buf_t *buf, void *tag)
 		nhdr->b_buf = buf;
 		nhdr->b_state = arc_anon;
 		nhdr->b_arc_access = 0;
-		nhdr->b_flags = flags & ARC_L2_WRITING;
+		nhdr->b_flags = flags & ARC_FLAG_L2_WRITING;
 		nhdr->b_l2hdr = NULL;
 		nhdr->b_datacnt = 1;
 		nhdr->b_freeze_cksum = NULL;
@@ -3523,7 +3509,7 @@ arc_write_ready(zio_t *zio)
 		mutex_exit(&hdr->b_freeze_lock);
 	}
 	arc_cksum_compute(buf, B_FALSE);
-	hdr->b_flags |= ARC_IO_IN_PROGRESS;
+	hdr->b_flags |= ARC_FLAG_IO_IN_PROGRESS;
 }
 
 /*
@@ -3604,13 +3590,13 @@ arc_write_done(zio_t *zio)
 				ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
 			}
 		}
-		hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
+		hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS;
 		/* if it's not anon, we are doing a scrub */
 		if (!exists && hdr->b_state == arc_anon)
 			arc_access(hdr, hash_lock);
 		mutex_exit(hash_lock);
 	} else {
-		hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
+		hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS;
 	}
 
 	ASSERT(!refcount_is_zero(&hdr->b_refcnt));
@@ -3633,12 +3619,12 @@ arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
 	ASSERT(ready != NULL);
 	ASSERT(done != NULL);
 	ASSERT(!HDR_IO_ERROR(hdr));
-	ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0);
+	ASSERT((hdr->b_flags & ARC_FLAG_IO_IN_PROGRESS) == 0);
 	ASSERT(hdr->b_acb == NULL);
 	if (l2arc)
-		hdr->b_flags |= ARC_L2CACHE;
+		hdr->b_flags |= ARC_FLAG_L2CACHE;
 	if (l2arc_compress)
-		hdr->b_flags |= ARC_L2COMPRESS;
+		hdr->b_flags |= ARC_FLAG_L2COMPRESS;
 	callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP);
 	callback->awcb_ready = ready;
 	callback->awcb_physdone = physdone;
@@ -4085,7 +4071,7 @@ arc_fini(void)
  */
 
 static boolean_t
-l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab)
+l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *hdr)
 {
 	/*
 	 * A buffer is *not* eligible for the L2ARC if it:
@@ -4094,8 +4080,8 @@ l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab)
 	 * 3. has an I/O in progress (it may be an incomplete read).
 	 * 4. is flagged not eligible (zfs property).
 	 */
-	if (ab->b_spa != spa_guid || ab->b_l2hdr != NULL ||
-	    HDR_IO_IN_PROGRESS(ab) || !HDR_L2CACHE(ab))
+	if (hdr->b_spa != spa_guid || hdr->b_l2hdr != NULL ||
+	    HDR_IO_IN_PROGRESS(hdr) || !HDR_L2CACHE(hdr))
 		return (B_FALSE);
 
 	return (B_TRUE);
@@ -4256,7 +4242,7 @@ l2arc_write_done(zio_t *zio)
 	l2arc_write_callback_t *cb;
 	l2arc_dev_t *dev;
 	list_t *buflist;
-	arc_buf_hdr_t *head, *ab, *ab_prev;
+	arc_buf_hdr_t *head, *hdr, *hdr_prev;
 	l2arc_buf_hdr_t *abl2;
 	kmutex_t *hash_lock;
 	int64_t bytes_dropped = 0;
@@ -4280,17 +4266,17 @@ l2arc_write_done(zio_t *zio)
 	/*
 	 * All writes completed, or an error was hit.
 	 */
-	for (ab = list_prev(buflist, head); ab; ab = ab_prev) {
-		ab_prev = list_prev(buflist, ab);
-		abl2 = ab->b_l2hdr;
+	for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) {
+		hdr_prev = list_prev(buflist, hdr);
+		abl2 = hdr->b_l2hdr;
 
 		/*
 		 * Release the temporary compressed buffer as soon as possible.
 		 */
 		if (abl2->b_compress != ZIO_COMPRESS_OFF)
-			l2arc_release_cdata_buf(ab);
+			l2arc_release_cdata_buf(hdr);
 
-		hash_lock = HDR_LOCK(ab);
+		hash_lock = HDR_LOCK(hdr);
 		if (!mutex_tryenter(hash_lock)) {
 			/*
 			 * This buffer misses out.  It may be in a stage
@@ -4305,18 +4291,18 @@ l2arc_write_done(zio_t *zio)
 			/*
 			 * Error - drop L2ARC entry.
 			 */
-			list_remove(buflist, ab);
+			list_remove(buflist, hdr);
 			ARCSTAT_INCR(arcstat_l2_asize, -abl2->b_asize);
 			bytes_dropped += abl2->b_asize;
-			ab->b_l2hdr = NULL;
+			hdr->b_l2hdr = NULL;
 			kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
-			ARCSTAT_INCR(arcstat_l2_size, -ab->b_size);
+			ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
 		}
 
 		/*
 		 * Allow ARC to begin reads to this L2ARC entry.
 		 */
-		ab->b_flags &= ~ARC_L2_WRITING;
+		hdr->b_flags &= ~ARC_FLAG_L2_WRITING;
 
 		mutex_exit(hash_lock);
 	}
@@ -4463,7 +4449,7 @@ l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
 {
 	list_t *buflist;
 	l2arc_buf_hdr_t *abl2;
-	arc_buf_hdr_t *ab, *ab_prev;
+	arc_buf_hdr_t *hdr, *hdr_prev;
 	kmutex_t *hash_lock;
 	uint64_t taddr;
 	int64_t bytes_evicted = 0;
@@ -4495,10 +4481,10 @@ l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
 
 top:
 	mutex_enter(&l2arc_buflist_mtx);
-	for (ab = list_tail(buflist); ab; ab = ab_prev) {
-		ab_prev = list_prev(buflist, ab);
+	for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) {
+		hdr_prev = list_prev(buflist, hdr);
 
-		hash_lock = HDR_LOCK(ab);
+		hash_lock = HDR_LOCK(hdr);
 		if (!mutex_tryenter(hash_lock)) {
 			/*
 			 * Missed the hash lock.  Retry.
@@ -4510,19 +4496,19 @@ top:
 			goto top;
 		}
 
-		if (HDR_L2_WRITE_HEAD(ab)) {
+		if (HDR_L2_WRITE_HEAD(hdr)) {
 			/*
 			 * We hit a write head node.  Leave it for
 			 * l2arc_write_done().
 			 */
-			list_remove(buflist, ab);
+			list_remove(buflist, hdr);
 			mutex_exit(hash_lock);
 			continue;
 		}
 
-		if (!all && ab->b_l2hdr != NULL &&
-		    (ab->b_l2hdr->b_daddr > taddr ||
-		    ab->b_l2hdr->b_daddr < dev->l2ad_hand)) {
+		if (!all && hdr->b_l2hdr != NULL &&
+		    (hdr->b_l2hdr->b_daddr > taddr ||
+		    hdr->b_l2hdr->b_daddr < dev->l2ad_hand)) {
 			/*
 			 * We've evicted to the target address,
 			 * or the end of the device.
@@ -4531,7 +4517,7 @@ top:
 			break;
 		}
 
-		if (HDR_FREE_IN_PROGRESS(ab)) {
+		if (HDR_FREE_IN_PROGRESS(hdr)) {
 			/*
 			 * Already on the path to destruction.
 			 */
@@ -4539,44 +4525,44 @@ top:
 			continue;
 		}
 
-		if (ab->b_state == arc_l2c_only) {
-			ASSERT(!HDR_L2_READING(ab));
+		if (hdr->b_state == arc_l2c_only) {
+			ASSERT(!HDR_L2_READING(hdr));
 			/*
 			 * This doesn't exist in the ARC.  Destroy.
 			 * arc_hdr_destroy() will call list_remove()
 			 * and decrement arcstat_l2_size.
 			 */
-			arc_change_state(arc_anon, ab, hash_lock);
-			arc_hdr_destroy(ab);
+			arc_change_state(arc_anon, hdr, hash_lock);
+			arc_hdr_destroy(hdr);
 		} else {
 			/*
 			 * Invalidate issued or about to be issued
 			 * reads, since we may be about to write
 			 * over this location.
 			 */
-			if (HDR_L2_READING(ab)) {
+			if (HDR_L2_READING(hdr)) {
 				ARCSTAT_BUMP(arcstat_l2_evict_reading);
-				ab->b_flags |= ARC_L2_EVICTED;
+				hdr->b_flags |= ARC_FLAG_L2_EVICTED;
 			}
 
 			/*
 			 * Tell ARC this no longer exists in L2ARC.
 			 */
-			if (ab->b_l2hdr != NULL) {
-				abl2 = ab->b_l2hdr;
+			if (hdr->b_l2hdr != NULL) {
+				abl2 = hdr->b_l2hdr;
 				ARCSTAT_INCR(arcstat_l2_asize, -abl2->b_asize);
 				bytes_evicted += abl2->b_asize;
-				ab->b_l2hdr = NULL;
+				hdr->b_l2hdr = NULL;
 				kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
-				ARCSTAT_INCR(arcstat_l2_size, -ab->b_size);
+				ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
 			}
-			list_remove(buflist, ab);
+			list_remove(buflist, hdr);
 
 			/*
 			 * This may have been leftover after a
 			 * failed write.
 			 */
-			ab->b_flags &= ~ARC_L2_WRITING;
+			hdr->b_flags &= ~ARC_FLAG_L2_WRITING;
 		}
 		mutex_exit(hash_lock);
 	}
@@ -4589,7 +4575,7 @@ top:
 /*
  * Find and write ARC buffers to the L2ARC device.
  *
- * An ARC_L2_WRITING flag is set so that the L2ARC buffers are not valid
+ * An ARC_FLAG_L2_WRITING flag is set so that the L2ARC buffers are not valid
  * for reading until they have completed writing.
  * The headroom_boost is an in-out parameter used to maintain headroom boost
  * state between calls to this function.
@@ -4601,7 +4587,7 @@ static uint64_t
 l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
     boolean_t *headroom_boost)
 {
-	arc_buf_hdr_t *ab, *ab_prev, *head;
+	arc_buf_hdr_t *hdr, *hdr_prev, *head;
 	list_t *list;
 	uint64_t write_asize, write_psize, write_sz, headroom,
 	    buf_compress_minsz;
@@ -4622,7 +4608,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 	write_sz = write_asize = write_psize = 0;
 	full = B_FALSE;
 	head = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
-	head->b_flags |= ARC_L2_WRITE_HEAD;
+	head->b_flags |= ARC_FLAG_L2_WRITE_HEAD;
 
 	/*
 	 * We will want to try to compress buffers that are at least 2x the
@@ -4646,25 +4632,25 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 		 * head of the ARC lists rather than the tail.
 		 */
 		if (arc_warm == B_FALSE)
-			ab = list_head(list);
+			hdr = list_head(list);
 		else
-			ab = list_tail(list);
+			hdr = list_tail(list);
 
 		headroom = target_sz * l2arc_headroom;
 		if (do_headroom_boost)
 			headroom = (headroom * l2arc_headroom_boost) / 100;
 
-		for (; ab; ab = ab_prev) {
+		for (; hdr; hdr = hdr_prev) {
 			l2arc_buf_hdr_t *l2hdr;
 			kmutex_t *hash_lock;
 			uint64_t buf_sz;
 
 			if (arc_warm == B_FALSE)
-				ab_prev = list_next(list, ab);
+				hdr_prev = list_next(list, hdr);
 			else
-				ab_prev = list_prev(list, ab);
+				hdr_prev = list_prev(list, hdr);
 
-			hash_lock = HDR_LOCK(ab);
+			hash_lock = HDR_LOCK(hdr);
 			if (!mutex_tryenter(hash_lock)) {
 				/*
 				 * Skip this buffer rather than waiting.
@@ -4672,7 +4658,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 				continue;
 			}
 
-			passed_sz += ab->b_size;
+			passed_sz += hdr->b_size;
 			if (passed_sz > headroom) {
 				/*
 				 * Searched too far.
@@ -4681,12 +4667,12 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 				break;
 			}
 
-			if (!l2arc_write_eligible(guid, ab)) {
+			if (!l2arc_write_eligible(guid, hdr)) {
 				mutex_exit(hash_lock);
 				continue;
 			}
 
-			if ((write_sz + ab->b_size) > target_sz) {
+			if ((write_sz + hdr->b_size) > target_sz) {
 				full = B_TRUE;
 				mutex_exit(hash_lock);
 				break;
@@ -4713,31 +4699,31 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 			 */
 			l2hdr = kmem_zalloc(sizeof (l2arc_buf_hdr_t), KM_SLEEP);
 			l2hdr->b_dev = dev;
-			ab->b_flags |= ARC_L2_WRITING;
+			hdr->b_flags |= ARC_FLAG_L2_WRITING;
 
 			/*
 			 * Temporarily stash the data buffer in b_tmp_cdata.
 			 * The subsequent write step will pick it up from
-			 * there. This is because can't access ab->b_buf
+			 * there. This is because can't access hdr->b_buf
 			 * without holding the hash_lock, which we in turn
 			 * can't access without holding the ARC list locks
 			 * (which we want to avoid during compression/writing).
 			 */
 			l2hdr->b_compress = ZIO_COMPRESS_OFF;
-			l2hdr->b_asize = ab->b_size;
-			l2hdr->b_tmp_cdata = ab->b_buf->b_data;
+			l2hdr->b_asize = hdr->b_size;
+			l2hdr->b_tmp_cdata = hdr->b_buf->b_data;
 
-			buf_sz = ab->b_size;
-			ab->b_l2hdr = l2hdr;
+			buf_sz = hdr->b_size;
+			hdr->b_l2hdr = l2hdr;
 
-			list_insert_head(dev->l2ad_buflist, ab);
+			list_insert_head(dev->l2ad_buflist, hdr);
 
 			/*
 			 * Compute and store the buffer cksum before
 			 * writing.  On debug the cksum is verified first.
 			 */
-			arc_cksum_verify(ab->b_buf);
-			arc_cksum_compute(ab->b_buf, B_TRUE);
+			arc_cksum_verify(hdr->b_buf);
+			arc_cksum_compute(hdr->b_buf, B_TRUE);
 
 			mutex_exit(hash_lock);
 
@@ -4763,21 +4749,22 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 	 * and work backwards, retracing the course of the buffer selector
 	 * loop above.
 	 */
-	for (ab = list_prev(dev->l2ad_buflist, head); ab;
-	    ab = list_prev(dev->l2ad_buflist, ab)) {
+	for (hdr = list_prev(dev->l2ad_buflist, head); hdr;
+	    hdr = list_prev(dev->l2ad_buflist, hdr)) {
 		l2arc_buf_hdr_t *l2hdr;
 		uint64_t buf_sz;
 
 		/*
 		 * We shouldn't need to lock the buffer here, since we flagged
-		 * it as ARC_L2_WRITING in the previous step, but we must take
-		 * care to only access its L2 cache parameters. In particular,
-		 * ab->b_buf may be invalid by now due to ARC eviction.
+		 * it as ARC_FLAG_L2_WRITING in the previous step, but we must
+		 * take care to only access its L2 cache parameters. In
+		 * particular, hdr->b_buf may be invalid by now due to
+		 * ARC eviction.
 		 */
-		l2hdr = ab->b_l2hdr;
+		l2hdr = hdr->b_l2hdr;
 		l2hdr->b_daddr = dev->l2ad_hand;
 
-		if ((ab->b_flags & ARC_L2COMPRESS) &&
+		if ((hdr->b_flags & ARC_FLAG_L2COMPRESS) &&
 		    l2hdr->b_asize >= buf_compress_minsz) {
 			if (l2arc_compress_buf(l2hdr)) {
 				/*
@@ -4981,9 +4968,9 @@ l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr, enum zio_compress c)
  * done, we can dispose of it.
  */
 static void
-l2arc_release_cdata_buf(arc_buf_hdr_t *ab)
+l2arc_release_cdata_buf(arc_buf_hdr_t *hdr)
 {
-	l2arc_buf_hdr_t *l2hdr = ab->b_l2hdr;
+	l2arc_buf_hdr_t *l2hdr = hdr->b_l2hdr;
 
 	if (l2hdr->b_compress == ZIO_COMPRESS_LZ4) {
 		/*
@@ -4991,7 +4978,7 @@ l2arc_release_cdata_buf(arc_buf_hdr_t *ab)
 		 * temporary buffer for it, so now we need to release it.
 		 */
 		ASSERT(l2hdr->b_tmp_cdata != NULL);
-		zio_data_buf_free(l2hdr->b_tmp_cdata, ab->b_size);
+		zio_data_buf_free(l2hdr->b_tmp_cdata, hdr->b_size);
 	}
 	l2hdr->b_tmp_cdata = NULL;
 }
diff --git a/uts/common/fs/zfs/dbuf.c b/uts/common/fs/zfs/dbuf.c
index bcf66bc53d7b..fd539c838d06 100644
--- a/uts/common/fs/zfs/dbuf.c
+++ b/uts/common/fs/zfs/dbuf.c
@@ -507,7 +507,7 @@ dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags)
 {
 	dnode_t *dn;
 	zbookmark_phys_t zb;
-	uint32_t aflags = ARC_NOWAIT;
+	arc_flags_t aflags = ARC_FLAG_NOWAIT;
 
 	DB_DNODE_ENTER(db);
 	dn = DB_DNODE(db);
@@ -560,9 +560,9 @@ dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags)
 	mutex_exit(&db->db_mtx);
 
 	if (DBUF_IS_L2CACHEABLE(db))
-		aflags |= ARC_L2CACHE;
+		aflags |= ARC_FLAG_L2CACHE;
 	if (DBUF_IS_L2COMPRESSIBLE(db))
-		aflags |= ARC_L2COMPRESS;
+		aflags |= ARC_FLAG_L2COMPRESS;
 
 	SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ?
 	    db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET,
@@ -574,7 +574,7 @@ dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags)
 	    dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ,
 	    (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED,
 	    &aflags, &zb);
-	if (aflags & ARC_CACHED)
+	if (aflags & ARC_FLAG_CACHED)
 		*flags |= DB_RF_CACHED;
 }
 
@@ -1863,7 +1863,8 @@ dbuf_prefetch(dnode_t *dn, uint64_t blkid, zio_priority_t prio)
 	if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) {
 		if (bp && !BP_IS_HOLE(bp) && !BP_IS_EMBEDDED(bp)) {
 			dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;
-			uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH;
+			arc_flags_t aflags =
+			    ARC_FLAG_NOWAIT | ARC_FLAG_PREFETCH;
 			zbookmark_phys_t zb;
 
 			SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET,
diff --git a/uts/common/fs/zfs/dmu_diff.c b/uts/common/fs/zfs/dmu_diff.c
index 32e451a77773..91415d0d2dcb 100644
--- a/uts/common/fs/zfs/dmu_diff.c
+++ b/uts/common/fs/zfs/dmu_diff.c
@@ -129,7 +129,7 @@ diff_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 	} else if (zb->zb_level == 0) {
 		dnode_phys_t *blk;
 		arc_buf_t *abuf;
-		uint32_t aflags = ARC_WAIT;
+		arc_flags_t aflags = ARC_FLAG_WAIT;
 		int blksz = BP_GET_LSIZE(bp);
 		int i;
 
diff --git a/uts/common/fs/zfs/dmu_objset.c b/uts/common/fs/zfs/dmu_objset.c
index e39264cc2f93..36ac27a5e0de 100644
--- a/uts/common/fs/zfs/dmu_objset.c
+++ b/uts/common/fs/zfs/dmu_objset.c
@@ -293,15 +293,15 @@ dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
 	os->os_spa = spa;
 	os->os_rootbp = bp;
 	if (!BP_IS_HOLE(os->os_rootbp)) {
-		uint32_t aflags = ARC_WAIT;
+		arc_flags_t aflags = ARC_FLAG_WAIT;
 		zbookmark_phys_t zb;
 		SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET,
 		    ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID);
 
 		if (DMU_OS_IS_L2CACHEABLE(os))
-			aflags |= ARC_L2CACHE;
+			aflags |= ARC_FLAG_L2CACHE;
 		if (DMU_OS_IS_L2COMPRESSIBLE(os))
-			aflags |= ARC_L2COMPRESS;
+			aflags |= ARC_FLAG_L2COMPRESS;
 
 		dprintf_bp(os->os_rootbp, "reading %s", "");
 		err = arc_read(NULL, spa, os->os_rootbp,
diff --git a/uts/common/fs/zfs/dmu_send.c b/uts/common/fs/zfs/dmu_send.c
index dc9f81cd2f28..4bb1290a07c0 100644
--- a/uts/common/fs/zfs/dmu_send.c
+++ b/uts/common/fs/zfs/dmu_send.c
@@ -458,7 +458,7 @@ backup_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 		dnode_phys_t *blk;
 		int i;
 		int blksz = BP_GET_LSIZE(bp);
-		uint32_t aflags = ARC_WAIT;
+		arc_flags_t aflags = ARC_FLAG_WAIT;
 		arc_buf_t *abuf;
 
 		if (arc_read(NULL, spa, bp, arc_getbuf_func, &abuf,
@@ -476,7 +476,7 @@ backup_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 		}
 		(void) arc_buf_remove_ref(abuf, &abuf);
 	} else if (type == DMU_OT_SA) {
-		uint32_t aflags = ARC_WAIT;
+		arc_flags_t aflags = ARC_FLAG_WAIT;
 		arc_buf_t *abuf;
 		int blksz = BP_GET_LSIZE(bp);
 
@@ -493,7 +493,7 @@ backup_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 		err = dump_write_embedded(dsp, zb->zb_object,
 		    zb->zb_blkid * blksz, blksz, bp);
 	} else { /* it's a level-0 block of a regular object */
-		uint32_t aflags = ARC_WAIT;
+		arc_flags_t aflags = ARC_FLAG_WAIT;
 		arc_buf_t *abuf;
 		int blksz = BP_GET_LSIZE(bp);
 		uint64_t offset;
diff --git a/uts/common/fs/zfs/dmu_traverse.c b/uts/common/fs/zfs/dmu_traverse.c
index bffdff65d627..7b1d6befa6ae 100644
--- a/uts/common/fs/zfs/dmu_traverse.c
+++ b/uts/common/fs/zfs/dmu_traverse.c
@@ -178,7 +178,7 @@ static void
 traverse_prefetch_metadata(traverse_data_t *td,
     const blkptr_t *bp, const zbookmark_phys_t *zb)
 {
-	uint32_t flags = ARC_NOWAIT | ARC_PREFETCH;
+	arc_flags_t flags = ARC_FLAG_NOWAIT | ARC_FLAG_PREFETCH;
 
 	if (!(td->td_flags & TRAVERSE_PREFETCH_METADATA))
 		return;
@@ -275,7 +275,7 @@ traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
 	}
 
 	if (BP_GET_LEVEL(bp) > 0) {
-		uint32_t flags = ARC_WAIT;
+		arc_flags_t flags = ARC_FLAG_WAIT;
 		int i;
 		blkptr_t *cbp;
 		int epb = BP_GET_LSIZE(bp) >> SPA_BLKPTRSHIFT;
@@ -303,7 +303,7 @@ traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
 				break;
 		}
 	} else if (BP_GET_TYPE(bp) == DMU_OT_DNODE) {
-		uint32_t flags = ARC_WAIT;
+		arc_flags_t flags = ARC_FLAG_WAIT;
 		int i;
 		int epb = BP_GET_LSIZE(bp) >> DNODE_SHIFT;
 
@@ -326,7 +326,7 @@ traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
 				break;
 		}
 	} else if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) {
-		uint32_t flags = ARC_WAIT;
+		arc_flags_t flags = ARC_FLAG_WAIT;
 		objset_phys_t *osp;
 		dnode_phys_t *dnp;
 
@@ -442,7 +442,7 @@ traverse_prefetcher(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
     const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg)
 {
 	prefetch_data_t *pfd = arg;
-	uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH;
+	arc_flags_t aflags = ARC_FLAG_NOWAIT | ARC_FLAG_PREFETCH;
 
 	ASSERT(pfd->pd_blks_fetched >= 0);
 	if (pfd->pd_cancel)
@@ -533,7 +533,7 @@ traverse_impl(spa_t *spa, dsl_dataset_t *ds, uint64_t objset, blkptr_t *rootbp,
 
 	/* See comment on ZIL traversal in dsl_scan_visitds. */
 	if (ds != NULL && !dsl_dataset_is_snapshot(ds) && !BP_IS_HOLE(rootbp)) {
-		uint32_t flags = ARC_WAIT;
+		arc_flags_t flags = ARC_FLAG_WAIT;
 		objset_phys_t *osp;
 		arc_buf_t *buf;
 
diff --git a/uts/common/fs/zfs/dsl_scan.c b/uts/common/fs/zfs/dsl_scan.c
index 5c9e68fb2ee6..a372b2cdc0ca 100644
--- a/uts/common/fs/zfs/dsl_scan.c
+++ b/uts/common/fs/zfs/dsl_scan.c
@@ -546,7 +546,7 @@ dsl_scan_prefetch(dsl_scan_t *scn, arc_buf_t *buf, blkptr_t *bp,
     uint64_t objset, uint64_t object, uint64_t blkid)
 {
 	zbookmark_phys_t czb;
-	uint32_t flags = ARC_NOWAIT | ARC_PREFETCH;
+	arc_flags_t flags = ARC_FLAG_NOWAIT | ARC_FLAG_PREFETCH;
 
 	if (zfs_no_scrub_prefetch)
 		return;
@@ -611,7 +611,7 @@ dsl_scan_recurse(dsl_scan_t *scn, dsl_dataset_t *ds, dmu_objset_type_t ostype,
 	int err;
 
 	if (BP_GET_LEVEL(bp) > 0) {
-		uint32_t flags = ARC_WAIT;
+		arc_flags_t flags = ARC_FLAG_WAIT;
 		int i;
 		blkptr_t *cbp;
 		int epb = BP_GET_LSIZE(bp) >> SPA_BLKPTRSHIFT;
@@ -638,7 +638,7 @@ dsl_scan_recurse(dsl_scan_t *scn, dsl_dataset_t *ds, dmu_objset_type_t ostype,
 		}
 		(void) arc_buf_remove_ref(buf, &buf);
 	} else if (BP_GET_TYPE(bp) == DMU_OT_DNODE) {
-		uint32_t flags = ARC_WAIT;
+		arc_flags_t flags = ARC_FLAG_WAIT;
 		dnode_phys_t *cdnp;
 		int i, j;
 		int epb = BP_GET_LSIZE(bp) >> DNODE_SHIFT;
@@ -664,7 +664,7 @@ dsl_scan_recurse(dsl_scan_t *scn, dsl_dataset_t *ds, dmu_objset_type_t ostype,
 
 		(void) arc_buf_remove_ref(buf, &buf);
 	} else if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) {
-		uint32_t flags = ARC_WAIT;
+		arc_flags_t flags = ARC_FLAG_WAIT;
 		objset_phys_t *osp;
 		arc_buf_t *buf;
 
diff --git a/uts/common/fs/zfs/sys/arc.h b/uts/common/fs/zfs/sys/arc.h
index 02643bd732dd..83ea6f7fbc7c 100644
--- a/uts/common/fs/zfs/sys/arc.h
+++ b/uts/common/fs/zfs/sys/arc.h
@@ -46,6 +46,36 @@ typedef int arc_evict_func_t(void *private);
 arc_done_func_t arc_bcopy_func;
 arc_done_func_t arc_getbuf_func;
 
+typedef enum arc_flags
+{
+	/*
+	 * Public flags that can be passed into the ARC by external consumers.
+	 */
+	ARC_FLAG_NONE			= 1 << 0,	/* No flags set */
+	ARC_FLAG_WAIT			= 1 << 1,	/* perform sync I/O */
+	ARC_FLAG_NOWAIT			= 1 << 2,	/* perform async I/O */
+	ARC_FLAG_PREFETCH		= 1 << 3,	/* I/O is a prefetch */
+	ARC_FLAG_CACHED			= 1 << 4,	/* I/O was in cache */
+	ARC_FLAG_L2CACHE		= 1 << 5,	/* cache in L2ARC */
+	ARC_FLAG_L2COMPRESS		= 1 << 6,	/* compress in L2ARC */
+
+	/*
+	 * Private ARC flags.  These flags are private ARC only flags that
+	 * will show up in b_flags in the arc_hdr_buf_t. These flags should
+	 * only be set by ARC code.
+	 */
+	ARC_FLAG_IN_HASH_TABLE		= 1 << 7,	/* buffer is hashed */
+	ARC_FLAG_IO_IN_PROGRESS		= 1 << 8,	/* I/O in progress */
+	ARC_FLAG_IO_ERROR		= 1 << 9,	/* I/O failed for buf */
+	ARC_FLAG_FREED_IN_READ		= 1 << 10,	/* freed during read */
+	ARC_FLAG_BUF_AVAILABLE		= 1 << 11,	/* block not in use */
+	ARC_FLAG_INDIRECT		= 1 << 12,	/* indirect block */
+	ARC_FLAG_FREE_IN_PROGRESS	= 1 << 13,	/*  about to be freed */
+	ARC_FLAG_L2_WRITING		= 1 << 14,	/* write in progress */
+	ARC_FLAG_L2_EVICTED		= 1 << 15,	/* evicted during I/O */
+	ARC_FLAG_L2_WRITE_HEAD		= 1 << 16,	/* head of write list */
+} arc_flags_t;
+
 struct arc_buf {
 	arc_buf_hdr_t		*b_hdr;
 	arc_buf_t		*b_next;
@@ -60,15 +90,6 @@ typedef enum arc_buf_contents {
 	ARC_BUFC_METADATA,			/* buffer contains metadata */
 	ARC_BUFC_NUMTYPES
 } arc_buf_contents_t;
-/*
- * These are the flags we pass into calls to the arc
- */
-#define	ARC_WAIT	(1 << 1)	/* perform I/O synchronously */
-#define	ARC_NOWAIT	(1 << 2)	/* perform I/O asynchronously */
-#define	ARC_PREFETCH	(1 << 3)	/* I/O is a prefetch */
-#define	ARC_CACHED	(1 << 4)	/* I/O was already in cache */
-#define	ARC_L2CACHE	(1 << 5)	/* cache in L2ARC */
-#define	ARC_L2COMPRESS	(1 << 6)	/* compress in L2ARC */
 
 /*
  * The following breakdows of arc_size exist for kstat only.
@@ -102,7 +123,7 @@ int arc_referenced(arc_buf_t *buf);
 
 int arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp,
     arc_done_func_t *done, void *private, zio_priority_t priority, int flags,
-    uint32_t *arc_flags, const zbookmark_phys_t *zb);
+    arc_flags_t *arc_flags, const zbookmark_phys_t *zb);
 zio_t *arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
     blkptr_t *bp, arc_buf_t *buf, boolean_t l2arc, boolean_t l2arc_compress,
     const zio_prop_t *zp, arc_done_func_t *ready, arc_done_func_t *physdone,
diff --git a/uts/common/fs/zfs/zil.c b/uts/common/fs/zfs/zil.c
index 6beacadf710d..53944165efb6 100644
--- a/uts/common/fs/zfs/zil.c
+++ b/uts/common/fs/zfs/zil.c
@@ -181,7 +181,7 @@ zil_read_log_block(zilog_t *zilog, const blkptr_t *bp, blkptr_t *nbp, void *dst,
     char **end)
 {
 	enum zio_flag zio_flags = ZIO_FLAG_CANFAIL;
-	uint32_t aflags = ARC_WAIT;
+	arc_flags_t aflags = ARC_FLAG_WAIT;
 	arc_buf_t *abuf = NULL;
 	zbookmark_phys_t zb;
 	int error;
@@ -257,7 +257,7 @@ zil_read_log_data(zilog_t *zilog, const lr_write_t *lr, void *wbuf)
 {
 	enum zio_flag zio_flags = ZIO_FLAG_CANFAIL;
 	const blkptr_t *bp = &lr->lr_blkptr;
-	uint32_t aflags = ARC_WAIT;
+	arc_flags_t aflags = ARC_FLAG_WAIT;
 	arc_buf_t *abuf = NULL;
 	zbookmark_phys_t zb;
 	int error;
diff --git a/uts/common/fs/zfs/zio.c b/uts/common/fs/zfs/zio.c
index b9b31b375d3e..d0a42e9af13b 100644
--- a/uts/common/fs/zfs/zio.c
+++ b/uts/common/fs/zfs/zio.c
@@ -2159,7 +2159,7 @@ zio_ddt_collision(zio_t *zio, ddt_t *ddt, ddt_entry_t *dde)
 
 		if (ddp->ddp_phys_birth != 0) {
 			arc_buf_t *abuf = NULL;
-			uint32_t aflags = ARC_WAIT;
+			arc_flags_t aflags = ARC_FLAG_WAIT;
 			blkptr_t blk = *zio->io_bp;
 			int error;
 

From bc89990d2511b93184a620a140916baf7808d57c Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Thu, 18 Dec 2014 18:59:26 +0000
Subject: [PATCH 067/380] 5422 preserve AVL invariants in dn_dbufs Reviewed by:
 Matthew Ahrens <mahrens@delphix.com> Reviewed by: Paul Dagnelie
 <paul.dagnelie@delphix.com> Reviewed by: Josef 'Jeff' Sipek
 <josef.sipek@nexenta.com> Reviewed by: Albert Lee <trisk@nexenta.com>
 Approved by: Dan McDonald <danmcd@omniti.com> Author: Alex Reece
 <alex@delphix.com>

illumos/illumos-gate@a846f19d279fdfb0e0d63f78ccaf0205a88274d2
---
 uts/common/fs/zfs/dnode.c | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/uts/common/fs/zfs/dnode.c b/uts/common/fs/zfs/dnode.c
index 250dd9f9415e..3303ee709ed7 100644
--- a/uts/common/fs/zfs/dnode.c
+++ b/uts/common/fs/zfs/dnode.c
@@ -79,16 +79,14 @@ dbuf_compare(const void *x1, const void *x2)
 		return (1);
 	}
 
-	if (d1->db_state < d2->db_state) {
+	if (d1->db_state == DB_SEARCH) {
+		ASSERT3S(d2->db_state, !=, DB_SEARCH);
 		return (-1);
-	}
-	if (d1->db_state > d2->db_state) {
+	} else if (d2->db_state == DB_SEARCH) {
+		ASSERT3S(d1->db_state, !=, DB_SEARCH);
 		return (1);
 	}
 
-	ASSERT3S(d1->db_state, !=, DB_SEARCH);
-	ASSERT3S(d2->db_state, !=, DB_SEARCH);
-
 	if ((uintptr_t)d1 < (uintptr_t)d2) {
 		return (-1);
 	}

From 828a3a732c3561407e96d83ed1a7f804823165e0 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Tue, 20 Jan 2015 20:04:51 +0000
Subject: [PATCH 068/380] 5376 arc_kmem_reap_now() should not result in
 clearing arc_no_grow Reviewed by: Christopher Siden
 <christopher.siden@delphix.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Reviewed by: Steven Hartland
 <killing@multiplay.co.uk> Reviewed by: Richard Elling
 <richard.elling@richardelling.com> Approved by: Dan McDonald
 <danmcd@omniti.com> Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@2ec99e3e987d8aa273f1e9ba2b983557d058198c
---
 uts/common/fs/zfs/arc.c | 246 ++++++++++++++++++++++++++--------------
 1 file changed, 162 insertions(+), 84 deletions(-)

diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index 435096becc75..656b4212febd 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -149,13 +149,7 @@ static kmutex_t		arc_reclaim_thr_lock;
 static kcondvar_t	arc_reclaim_thr_cv;	/* used to signal reclaim thr */
 static uint8_t		arc_thread_exit;
 
-#define	ARC_REDUCE_DNLC_PERCENT	3
-uint_t arc_reduce_dnlc_percent = ARC_REDUCE_DNLC_PERCENT;
-
-typedef enum arc_reclaim_strategy {
-	ARC_RECLAIM_AGGR,		/* Aggressive reclaim strategy */
-	ARC_RECLAIM_CONS		/* Conservative reclaim strategy */
-} arc_reclaim_strategy_t;
+uint_t arc_reduce_dnlc_percent = 3;
 
 /*
  * The number of iterations through arc_evict_*() before we
@@ -170,7 +164,19 @@ static int		arc_grow_retry = 60;
 static int		arc_p_min_shift = 4;
 
 /* log2(fraction of arc to reclaim) */
-static int		arc_shrink_shift = 5;
+static int		arc_shrink_shift = 7;
+
+/*
+ * log2(fraction of ARC which must be free to allow growing).
+ * I.e. If there is less than arc_c >> arc_no_grow_shift free memory,
+ * when reading a new block into the ARC, we will evict an equal-sized block
+ * from the ARC.
+ *
+ * This must be less than arc_shrink_shift, so that when we shrink the ARC,
+ * we will still not allow it to grow.
+ */
+int			arc_no_grow_shift = 5;
+
 
 /*
  * minimum lifespan of a prefetch block in clock ticks
@@ -2194,16 +2200,10 @@ arc_flush(spa_t *spa)
 }
 
 void
-arc_shrink(void)
+arc_shrink(int64_t to_free)
 {
 	if (arc_c > arc_c_min) {
-		uint64_t to_free;
 
-#ifdef _KERNEL
-		to_free = MAX(arc_c >> arc_shrink_shift, ptob(needfree));
-#else
-		to_free = arc_c >> arc_shrink_shift;
-#endif
 		if (arc_c > arc_c_min + to_free)
 			atomic_add_64(&arc_c, -to_free);
 		else
@@ -2222,25 +2222,49 @@ arc_shrink(void)
 		arc_adjust();
 }
 
+typedef enum free_memory_reason_t {
+	FMR_UNKNOWN,
+	FMR_NEEDFREE,
+	FMR_LOTSFREE,
+	FMR_SWAPFS_MINFREE,
+	FMR_PAGES_PP_MAXIMUM,
+	FMR_HEAP_ARENA,
+	FMR_ZIO_ARENA,
+} free_memory_reason_t;
+
+int64_t last_free_memory;
+free_memory_reason_t last_free_reason;
+
 /*
- * Determine if the system is under memory pressure and is asking
- * to reclaim memory. A return value of 1 indicates that the system
- * is under memory pressure and that the arc should adjust accordingly.
+ * Additional reserve of pages for pp_reserve.
  */
-static int
-arc_reclaim_needed(void)
+int64_t arc_pages_pp_reserve = 64;
+
+/*
+ * Additional reserve of pages for swapfs.
+ */
+int64_t arc_swapfs_reserve = 64;
+
+/*
+ * Return the amount of memory that can be consumed before reclaim will be
+ * needed.  Positive if there is sufficient free memory, negative indicates
+ * the amount of memory that needs to be freed up.
+ */
+static int64_t
+arc_available_memory(void)
 {
-	uint64_t extra;
+	int64_t lowest = INT64_MAX;
+	int64_t n;
+	free_memory_reason_t r = FMR_UNKNOWN;
 
 #ifdef _KERNEL
-
-	if (needfree)
-		return (1);
-
-	/*
-	 * take 'desfree' extra pages, so we reclaim sooner, rather than later
-	 */
-	extra = desfree;
+	if (needfree > 0) {
+		n = PAGESIZE * (-needfree);
+		if (n < lowest) {
+			lowest = n;
+			r = FMR_NEEDFREE;
+		}
+	}
 
 	/*
 	 * check that we're out of range of the pageout scanner.  It starts to
@@ -2249,8 +2273,11 @@ arc_reclaim_needed(void)
 	 * number of needed free pages.  We add extra pages here to make sure
 	 * the scanner doesn't start up while we're freeing memory.
 	 */
-	if (freemem < lotsfree + needfree + extra)
-		return (1);
+	n = PAGESIZE * (freemem - lotsfree - needfree - desfree);
+	if (n < lowest) {
+		lowest = n;
+		r = FMR_LOTSFREE;
+	}
 
 	/*
 	 * check to make sure that swapfs has enough space so that anon
@@ -2259,8 +2286,13 @@ arc_reclaim_needed(void)
 	 * swap pages.  We also add a bit of extra here just to prevent
 	 * circumstances from getting really dire.
 	 */
-	if (availrmem < swapfs_minfree + swapfs_reserve + extra)
-		return (1);
+	n = PAGESIZE * (availrmem - swapfs_minfree - swapfs_reserve -
+	    desfree - arc_swapfs_reserve);
+	if (n < lowest) {
+		lowest = n;
+		r = FMR_SWAPFS_MINFREE;
+	}
+
 
 	/*
 	 * Check that we have enough availrmem that memory locking (e.g., via
@@ -2269,8 +2301,12 @@ arc_reclaim_needed(void)
 	 * drops below pages_pp_maximum, page locking mechanisms such as
 	 * page_pp_lock() will fail.)
 	 */
-	if (availrmem <= pages_pp_maximum)
-		return (1);
+	n = PAGESIZE * (availrmem - pages_pp_maximum -
+	    arc_pages_pp_reserve);
+	if (n < lowest) {
+		lowest = n;
+		r = FMR_PAGES_PP_MAXIMUM;
+	}
 
 #if defined(__i386)
 	/*
@@ -2284,9 +2320,12 @@ arc_reclaim_needed(void)
 	 * heap is allocated.  (Or, in the calculation, if less than 1/4th is
 	 * free)
 	 */
-	if (vmem_size(heap_arena, VMEM_FREE) <
-	    (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2))
-		return (1);
+	n = vmem_size(heap_arena, VMEM_FREE) -
+	    (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2);
+	if (n < lowest) {
+		lowest = n;
+		r = FMR_HEAP_ARENA;
+	}
 #endif
 
 	/*
@@ -2298,19 +2337,40 @@ arc_reclaim_needed(void)
 	 * to aggressively evict memory from the arc in order to avoid
 	 * memory fragmentation issues.
 	 */
-	if (zio_arena != NULL &&
-	    vmem_size(zio_arena, VMEM_FREE) <
-	    (vmem_size(zio_arena, VMEM_ALLOC) >> 4))
-		return (1);
+	if (zio_arena != NULL) {
+		n = vmem_size(zio_arena, VMEM_FREE) -
+		    (vmem_size(zio_arena, VMEM_ALLOC) >> 4);
+		if (n < lowest) {
+			lowest = n;
+			r = FMR_ZIO_ARENA;
+		}
+	}
 #else
+	/* Every 100 calls, free a small amount */
 	if (spa_get_random(100) == 0)
-		return (1);
+		lowest = -1024;
 #endif
-	return (0);
+
+	last_free_memory = lowest;
+	last_free_reason = r;
+
+	return (lowest);
+}
+
+
+/*
+ * Determine if the system is under memory pressure and is asking
+ * to reclaim memory. A return value of TRUE indicates that the system
+ * is under memory pressure and that the arc should adjust accordingly.
+ */
+static boolean_t
+arc_reclaim_needed(void)
+{
+	return (arc_available_memory() < 0);
 }
 
 static void
-arc_kmem_reap_now(arc_reclaim_strategy_t strat)
+arc_kmem_reap_now(void)
 {
 	size_t			i;
 	kmem_cache_t		*prev_cache = NULL;
@@ -2335,13 +2395,6 @@ arc_kmem_reap_now(arc_reclaim_strategy_t strat)
 #endif
 #endif
 
-	/*
-	 * An aggressive reclamation will shrink the cache size as well as
-	 * reap free buffers from the arc kmem caches.
-	 */
-	if (strat == ARC_RECLAIM_AGGR)
-		arc_shrink();
-
 	for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
 		if (zio_buf_cache[i] != prev_cache) {
 			prev_cache = zio_buf_cache[i];
@@ -2356,47 +2409,57 @@ arc_kmem_reap_now(arc_reclaim_strategy_t strat)
 	kmem_cache_reap_now(hdr_cache);
 	kmem_cache_reap_now(range_seg_cache);
 
-	/*
-	 * Ask the vmem areana to reclaim unused memory from its
-	 * quantum caches.
-	 */
-	if (zio_arena != NULL && strat == ARC_RECLAIM_AGGR)
+	if (zio_arena != NULL) {
+		/*
+		 * Ask the vmem arena to reclaim unused memory from its
+		 * quantum caches.
+		 */
 		vmem_qcache_reap(zio_arena);
+	}
 }
 
 static void
 arc_reclaim_thread(void)
 {
 	clock_t			growtime = 0;
-	arc_reclaim_strategy_t	last_reclaim = ARC_RECLAIM_CONS;
 	callb_cpr_t		cpr;
 
 	CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG);
 
 	mutex_enter(&arc_reclaim_thr_lock);
 	while (arc_thread_exit == 0) {
-		if (arc_reclaim_needed()) {
+		int64_t free_memory = arc_available_memory();
+		if (free_memory < 0) {
 
-			if (arc_no_grow) {
-				if (last_reclaim == ARC_RECLAIM_CONS) {
-					last_reclaim = ARC_RECLAIM_AGGR;
-				} else {
-					last_reclaim = ARC_RECLAIM_CONS;
-				}
-			} else {
-				arc_no_grow = TRUE;
-				last_reclaim = ARC_RECLAIM_AGGR;
-				membar_producer();
-			}
-
-			/* reset the growth delay for every reclaim */
-			growtime = ddi_get_lbolt() + (arc_grow_retry * hz);
-
-			arc_kmem_reap_now(last_reclaim);
+			arc_no_grow = B_TRUE;
 			arc_warm = B_TRUE;
 
-		} else if (arc_no_grow && ddi_get_lbolt() >= growtime) {
-			arc_no_grow = FALSE;
+			/*
+			 * Wait at least zfs_grow_retry (default 60) seconds
+			 * before considering growing.
+			 */
+			growtime = ddi_get_lbolt() + (arc_grow_retry * hz);
+
+			arc_kmem_reap_now();
+
+			/*
+			 * If we are still low on memory, shrink the ARC
+			 * so that we have arc_shrink_min free space.
+			 */
+			free_memory = arc_available_memory();
+
+			int64_t to_free =
+			    (arc_c >> arc_shrink_shift) - free_memory;
+			if (to_free > 0) {
+#ifdef _KERNEL
+				to_free = MAX(to_free, ptob(needfree));
+#endif
+				arc_shrink(to_free);
+			}
+		} else if (free_memory < arc_c >> arc_no_grow_shift) {
+			arc_no_grow = B_TRUE;
+		} else if (ddi_get_lbolt() >= growtime) {
+			arc_no_grow = B_FALSE;
 		}
 
 		arc_adjust();
@@ -3739,6 +3802,15 @@ arc_tempreserve_space(uint64_t reserve, uint64_t txg)
 void
 arc_init(void)
 {
+	/*
+	 * allmem is "all memory that we could possibly use".
+	 */
+#ifdef _KERNEL
+	uint64_t allmem = ptob(physmem - swapfs_minfree);
+#else
+	uint64_t allmem = (physmem * PAGESIZE) / 2;
+#endif
+
 	mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL);
 	cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL);
 
@@ -3746,7 +3818,7 @@ arc_init(void)
 	arc_min_prefetch_lifespan = 1 * hz;
 
 	/* Start out with 1/8 of all memory */
-	arc_c = physmem * PAGESIZE / 8;
+	arc_c = allmem / 8;
 
 #ifdef _KERNEL
 	/*
@@ -3758,21 +3830,21 @@ arc_init(void)
 #endif
 
 	/* set min cache to 1/32 of all memory, or 64MB, whichever is more */
-	arc_c_min = MAX(arc_c / 4, 64<<20);
+	arc_c_min = MAX(allmem / 32, 64 << 20);
 	/* set max to 3/4 of all memory, or all but 1GB, whichever is more */
-	if (arc_c * 8 >= 1<<30)
-		arc_c_max = (arc_c * 8) - (1<<30);
+	if (allmem >= 1 << 30)
+		arc_c_max = allmem - (1 << 30);
 	else
 		arc_c_max = arc_c_min;
-	arc_c_max = MAX(arc_c * 6, arc_c_max);
+	arc_c_max = MAX(allmem * 3 / 4, arc_c_max);
 
 	/*
 	 * Allow the tunables to override our calculations if they are
 	 * reasonable (ie. over 64MB)
 	 */
-	if (zfs_arc_max > 64<<20 && zfs_arc_max < physmem * PAGESIZE)
+	if (zfs_arc_max > 64 << 20 && zfs_arc_max < allmem)
 		arc_c_max = zfs_arc_max;
-	if (zfs_arc_min > 64<<20 && zfs_arc_min <= arc_c_max)
+	if (zfs_arc_min > 64 << 20 && zfs_arc_min <= arc_c_max)
 		arc_c_min = zfs_arc_min;
 
 	arc_c = arc_c_max;
@@ -3800,6 +3872,12 @@ arc_init(void)
 	if (zfs_arc_shrink_shift > 0)
 		arc_shrink_shift = zfs_arc_shrink_shift;
 
+	/*
+	 * Ensure that arc_no_grow_shift is less than arc_shrink_shift.
+	 */
+	if (arc_no_grow_shift >= arc_shrink_shift)
+		arc_no_grow_shift = arc_shrink_shift - 1;
+
 	if (zfs_arc_p_min_shift > 0)
 		arc_p_min_shift = zfs_arc_p_min_shift;
 

From 71702eaf828c0de948f08e2a429ddf9b64a04aba Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Tue, 20 Jan 2015 20:08:50 +0000
Subject: [PATCH 069/380] 5408 managing ZFS cache devices requires lots of RAM
 Reviewed by: Christopher Siden <christopher.siden@delphix.com> Reviewed by:
 George Wilson <george.wilson@delphix.com> Reviewed by: Matthew Ahrens
 <mahrens@delphix.com> Reviewed by: Don Brady <dev.fs.zfs@gmail.com> Reviewed
 by: Josef 'Jeff' Sipek <josef.sipek@nexenta.com> Approved by: Garrett D'Amore
 <garrett@damore.org> Author: Chris Williamson <Chris.Williamson@delphix.com>

illumos/illumos-gate@89c86e32293a30cdd7af530c38b2073fee01411c
---
 uts/common/fs/zfs/arc.c     | 1406 +++++++++++++++++++++--------------
 uts/common/fs/zfs/sys/arc.h |   27 +-
 2 files changed, 873 insertions(+), 560 deletions(-)

diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index 656b4212febd..012dd83cd7c3 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -111,7 +111,7 @@
  * Note that the majority of the performance stats are manipulated
  * with atomic operations.
  *
- * The L2ARC uses the l2arc_buflist_mtx global mutex for the following:
+ * The L2ARC uses the l2ad_mtx on each vdev for the following:
  *
  *	- L2ARC buflist creation
  *	- L2ARC buflist eviction
@@ -314,6 +314,7 @@ typedef struct arc_stats {
 	kstat_named_t arcstat_l2_writes_hdr_miss;
 	kstat_named_t arcstat_l2_evict_lock_retry;
 	kstat_named_t arcstat_l2_evict_reading;
+	kstat_named_t arcstat_l2_evict_l1cached;
 	kstat_named_t arcstat_l2_free_on_write;
 	kstat_named_t arcstat_l2_abort_lowmem;
 	kstat_named_t arcstat_l2_cksum_bad;
@@ -381,6 +382,7 @@ static arc_stats_t arc_stats = {
 	{ "l2_writes_hdr_miss",		KSTAT_DATA_UINT64 },
 	{ "l2_evict_lock_retry",	KSTAT_DATA_UINT64 },
 	{ "l2_evict_reading",		KSTAT_DATA_UINT64 },
+	{ "l2_evict_l1cached",		KSTAT_DATA_UINT64 },
 	{ "l2_free_on_write",		KSTAT_DATA_UINT64 },
 	{ "l2_abort_lowmem",		KSTAT_DATA_UINT64 },
 	{ "l2_cksum_bad",		KSTAT_DATA_UINT64 },
@@ -472,8 +474,6 @@ static int		arc_no_grow;	/* Don't try to grow cache size */
 static uint64_t		arc_tempreserve;
 static uint64_t		arc_loaned_bytes;
 
-typedef struct l2arc_buf_hdr l2arc_buf_hdr_t;
-
 typedef struct arc_callback arc_callback_t;
 
 struct arc_callback {
@@ -494,29 +494,53 @@ struct arc_write_callback {
 	arc_buf_t	*awcb_buf;
 };
 
-struct arc_buf_hdr {
-	/* protected by hash lock */
-	dva_t			b_dva;
-	uint64_t		b_birth;
-	uint64_t		b_cksum0;
-
+/*
+ * ARC buffers are separated into multiple structs as a memory saving measure:
+ *   - Common fields struct, always defined, and embedded within it:
+ *       - L2-only fields, always allocated but undefined when not in L2ARC
+ *       - L1-only fields, only allocated when in L1ARC
+ *
+ *           Buffer in L1                     Buffer only in L2
+ *    +------------------------+          +------------------------+
+ *    | arc_buf_hdr_t          |          | arc_buf_hdr_t          |
+ *    |                        |          |                        |
+ *    |                        |          |                        |
+ *    |                        |          |                        |
+ *    +------------------------+          +------------------------+
+ *    | l2arc_buf_hdr_t        |          | l2arc_buf_hdr_t        |
+ *    | (undefined if L1-only) |          |                        |
+ *    +------------------------+          +------------------------+
+ *    | l1arc_buf_hdr_t        |
+ *    |                        |
+ *    |                        |
+ *    |                        |
+ *    |                        |
+ *    +------------------------+
+ *
+ * Because it's possible for the L2ARC to become extremely large, we can wind
+ * up eating a lot of memory in L2ARC buffer headers, so the size of a header
+ * is minimized by only allocating the fields necessary for an L1-cached buffer
+ * when a header is actually in the L1 cache. The sub-headers (l1arc_buf_hdr and
+ * l2arc_buf_hdr) are embedded rather than allocated separately to save a couple
+ * words in pointers. arc_hdr_realloc() is used to switch a header between
+ * these two allocation states.
+ */
+typedef struct l1arc_buf_hdr {
 	kmutex_t		b_freeze_lock;
-	zio_cksum_t		*b_freeze_cksum;
+#ifdef ZFS_DEBUG
+	/*
+	 * used for debugging wtih kmem_flags - by allocating and freeing
+	 * b_thawed when the buffer is thawed, we get a record of the stack
+	 * trace that thawed it.
+	 */
 	void			*b_thawed;
+#endif
 
-	arc_buf_hdr_t		*b_hash_next;
 	arc_buf_t		*b_buf;
-	arc_flags_t		b_flags;
 	uint32_t		b_datacnt;
-
-	arc_callback_t		*b_acb;
+	/* for waiting on writes to complete */
 	kcondvar_t		b_cv;
 
-	/* immutable */
-	arc_buf_contents_t	b_type;
-	uint64_t		b_size;
-	uint64_t		b_spa;
-
 	/* protected by arc state mutex */
 	arc_state_t		*b_state;
 	list_node_t		b_arc_node;
@@ -527,8 +551,46 @@ struct arc_buf_hdr {
 	/* self protecting */
 	refcount_t		b_refcnt;
 
-	l2arc_buf_hdr_t		*b_l2hdr;
+	arc_callback_t		*b_acb;
+	/* temporary buffer holder for in-flight compressed data */
+	void			*b_tmp_cdata;
+} l1arc_buf_hdr_t;
+
+typedef struct l2arc_dev l2arc_dev_t;
+
+typedef struct l2arc_buf_hdr {
+	/* protected by arc_buf_hdr mutex */
+	l2arc_dev_t		*b_dev;		/* L2ARC device */
+	uint64_t		b_daddr;	/* disk address, offset byte */
+	/* real alloc'd buffer size depending on b_compress applied */
+	int32_t			b_asize;
+
 	list_node_t		b_l2node;
+} l2arc_buf_hdr_t;
+
+struct arc_buf_hdr {
+	/* protected by hash lock */
+	dva_t			b_dva;
+	uint64_t		b_birth;
+	/*
+	 * Even though this checksum is only set/verified when a buffer is in
+	 * the L1 cache, it needs to be in the set of common fields because it
+	 * must be preserved from the time before a buffer is written out to
+	 * L2ARC until after it is read back in.
+	 */
+	zio_cksum_t		*b_freeze_cksum;
+
+	arc_buf_hdr_t		*b_hash_next;
+	arc_flags_t		b_flags;
+
+	/* immutable */
+	int32_t			b_size;
+	uint64_t		b_spa;
+
+	/* L2ARC fields. Undefined when not in L2ARC. */
+	l2arc_buf_hdr_t		b_l2hdr;
+	/* L1ARC fields. Undefined when in l2arc_only state */
+	l1arc_buf_hdr_t		b_l1hdr;
 };
 
 static arc_buf_t *arc_eviction_list;
@@ -545,22 +607,38 @@ static arc_buf_hdr_t arc_eviction_hdr;
 #define	HDR_PREFETCH(hdr)	((hdr)->b_flags & ARC_FLAG_PREFETCH)
 #define	HDR_FREED_IN_READ(hdr)	((hdr)->b_flags & ARC_FLAG_FREED_IN_READ)
 #define	HDR_BUF_AVAILABLE(hdr)	((hdr)->b_flags & ARC_FLAG_BUF_AVAILABLE)
-#define	HDR_FREE_IN_PROGRESS(hdr)	\
-	((hdr)->b_flags & ARC_FLAG_FREE_IN_PROGRESS)
+
 #define	HDR_L2CACHE(hdr)	((hdr)->b_flags & ARC_FLAG_L2CACHE)
+#define	HDR_L2COMPRESS(hdr)	((hdr)->b_flags & ARC_FLAG_L2COMPRESS)
 #define	HDR_L2_READING(hdr)	\
-	((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS &&	\
-	    (hdr)->b_l2hdr != NULL)
+	    (((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) &&	\
+	    ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
 #define	HDR_L2_WRITING(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITING)
 #define	HDR_L2_EVICTED(hdr)	((hdr)->b_flags & ARC_FLAG_L2_EVICTED)
 #define	HDR_L2_WRITE_HEAD(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD)
 
+#define	HDR_ISTYPE_METADATA(hdr)	\
+	    ((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
+#define	HDR_ISTYPE_DATA(hdr)	(!HDR_ISTYPE_METADATA(hdr))
+
+#define	HDR_HAS_L1HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L1HDR)
+#define	HDR_HAS_L2HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)
+
+/* For storing compression mode in b_flags */
+#define	HDR_COMPRESS_OFFSET	24
+#define	HDR_COMPRESS_NBITS	7
+
+#define	HDR_GET_COMPRESS(hdr)	((enum zio_compress)BF32_GET(hdr->b_flags, \
+	    HDR_COMPRESS_OFFSET, HDR_COMPRESS_NBITS))
+#define	HDR_SET_COMPRESS(hdr, cmp) BF32_SET(hdr->b_flags, \
+	    HDR_COMPRESS_OFFSET, HDR_COMPRESS_NBITS, (cmp))
+
 /*
  * Other sizes
  */
 
-#define	HDR_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
-#define	L2HDR_SIZE ((int64_t)sizeof (l2arc_buf_hdr_t))
+#define	HDR_FULL_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
+#define	HDR_L2ONLY_SIZE ((int64_t)offsetof(arc_buf_hdr_t, b_l1hdr))
 
 /*
  * Hash table routines
@@ -624,7 +702,7 @@ boolean_t l2arc_norw = B_TRUE;			/* no reads during writes */
 /*
  * L2ARC Internals
  */
-typedef struct l2arc_dev {
+struct l2arc_dev {
 	vdev_t			*l2ad_vdev;	/* vdev */
 	spa_t			*l2ad_spa;	/* spa */
 	uint64_t		l2ad_hand;	/* next write location */
@@ -633,15 +711,15 @@ typedef struct l2arc_dev {
 	uint64_t		l2ad_evict;	/* last addr eviction reached */
 	boolean_t		l2ad_first;	/* first sweep through */
 	boolean_t		l2ad_writing;	/* currently writing */
-	list_t			*l2ad_buflist;	/* buffer list */
+	kmutex_t		l2ad_mtx;	/* lock for buffer list */
+	list_t			l2ad_buflist;	/* buffer list */
 	list_node_t		l2ad_node;	/* device list node */
-} l2arc_dev_t;
+};
 
 static list_t L2ARC_dev_list;			/* device list */
 static list_t *l2arc_dev_list;			/* device list pointer */
 static kmutex_t l2arc_dev_mtx;			/* device list mutex */
 static l2arc_dev_t *l2arc_dev_last;		/* last device used */
-static kmutex_t l2arc_buflist_mtx;		/* mutex for all buflists */
 static list_t L2ARC_free_on_write;		/* free after write buf list */
 static list_t *l2arc_free_on_write;		/* free after write list ptr */
 static kmutex_t l2arc_free_on_write_mtx;	/* mutex for list */
@@ -661,18 +739,6 @@ typedef struct l2arc_write_callback {
 	arc_buf_hdr_t	*l2wcb_head;		/* head of write buflist */
 } l2arc_write_callback_t;
 
-struct l2arc_buf_hdr {
-	/* protected by arc_buf_hdr  mutex */
-	l2arc_dev_t		*b_dev;		/* L2ARC device */
-	uint64_t		b_daddr;	/* disk address, offset byte */
-	/* compression applied to buffer data */
-	enum zio_compress	b_compress;
-	/* real alloc'd buffer size depending on b_compress applied */
-	int			b_asize;
-	/* temporary buffer holder for in-flight compressed data */
-	void			*b_tmp_cdata;
-};
-
 typedef struct l2arc_data_free {
 	/* protected by l2arc_free_on_write_mtx */
 	void		*l2df_data;
@@ -691,12 +757,13 @@ static int arc_evict_needed(arc_buf_contents_t);
 static void arc_evict_ghost(arc_state_t *, uint64_t, int64_t);
 static void arc_buf_watch(arc_buf_t *);
 
+static arc_buf_contents_t arc_buf_type(arc_buf_hdr_t *);
+static uint32_t arc_bufc_to_flags(arc_buf_contents_t);
+
 static boolean_t l2arc_write_eligible(uint64_t, arc_buf_hdr_t *);
 static void l2arc_read_done(zio_t *);
-static void l2arc_hdr_stat_add(void);
-static void l2arc_hdr_stat_remove(void);
 
-static boolean_t l2arc_compress_buf(l2arc_buf_hdr_t *);
+static boolean_t l2arc_compress_buf(arc_buf_hdr_t *);
 static void l2arc_decompress_zio(zio_t *, arc_buf_hdr_t *, enum zio_compress);
 static void l2arc_release_cdata_buf(arc_buf_hdr_t *);
 
@@ -719,8 +786,7 @@ buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
 
 #define	BUF_EMPTY(buf)						\
 	((buf)->b_dva.dva_word[0] == 0 &&			\
-	(buf)->b_dva.dva_word[1] == 0 &&			\
-	(buf)->b_cksum0 == 0)
+	(buf)->b_dva.dva_word[1] == 0)
 
 #define	BUF_EQUAL(spa, dva, birth, buf)				\
 	((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) &&	\
@@ -733,7 +799,6 @@ buf_discard_identity(arc_buf_hdr_t *hdr)
 	hdr->b_dva.dva_word[0] = 0;
 	hdr->b_dva.dva_word[1] = 0;
 	hdr->b_birth = 0;
-	hdr->b_cksum0 = 0;
 }
 
 static arc_buf_hdr_t *
@@ -763,6 +828,7 @@ buf_hash_find(uint64_t spa, const blkptr_t *bp, kmutex_t **lockp)
  * equal to elem in the hash table, then the already existing element
  * will be returned and the new element will not be inserted.
  * Otherwise returns NULL.
+ * If lockp == NULL, the caller is assumed to already hold the hash lock.
  */
 static arc_buf_hdr_t *
 buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp)
@@ -775,8 +841,14 @@ buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp)
 	ASSERT(!DVA_IS_EMPTY(&hdr->b_dva));
 	ASSERT(hdr->b_birth != 0);
 	ASSERT(!HDR_IN_HASH_TABLE(hdr));
-	*lockp = hash_lock;
-	mutex_enter(hash_lock);
+
+	if (lockp != NULL) {
+		*lockp = hash_lock;
+		mutex_enter(hash_lock);
+	} else {
+		ASSERT(MUTEX_HELD(hash_lock));
+	}
+
 	for (fhdr = buf_hash_table.ht_table[idx], i = 0; fhdr != NULL;
 	    fhdr = fhdr->b_hash_next, i++) {
 		if (BUF_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))
@@ -831,7 +903,8 @@ buf_hash_remove(arc_buf_hdr_t *hdr)
 /*
  * Global data structures and functions for the buf kmem cache.
  */
-static kmem_cache_t *hdr_cache;
+static kmem_cache_t *hdr_full_cache;
+static kmem_cache_t *hdr_l2only_cache;
 static kmem_cache_t *buf_cache;
 
 static void
@@ -843,7 +916,8 @@ buf_fini(void)
 	    (buf_hash_table.ht_mask + 1) * sizeof (void *));
 	for (i = 0; i < BUF_LOCKS; i++)
 		mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock);
-	kmem_cache_destroy(hdr_cache);
+	kmem_cache_destroy(hdr_full_cache);
+	kmem_cache_destroy(hdr_l2only_cache);
 	kmem_cache_destroy(buf_cache);
 }
 
@@ -853,15 +927,27 @@ buf_fini(void)
  */
 /* ARGSUSED */
 static int
-hdr_cons(void *vbuf, void *unused, int kmflag)
+hdr_full_cons(void *vbuf, void *unused, int kmflag)
 {
 	arc_buf_hdr_t *hdr = vbuf;
 
-	bzero(hdr, sizeof (arc_buf_hdr_t));
-	refcount_create(&hdr->b_refcnt);
-	cv_init(&hdr->b_cv, NULL, CV_DEFAULT, NULL);
-	mutex_init(&hdr->b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
-	arc_space_consume(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS);
+	bzero(hdr, HDR_FULL_SIZE);
+	cv_init(&hdr->b_l1hdr.b_cv, NULL, CV_DEFAULT, NULL);
+	refcount_create(&hdr->b_l1hdr.b_refcnt);
+	mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
+	arc_space_consume(HDR_FULL_SIZE, ARC_SPACE_HDRS);
+
+	return (0);
+}
+
+/* ARGSUSED */
+static int
+hdr_l2only_cons(void *vbuf, void *unused, int kmflag)
+{
+	arc_buf_hdr_t *hdr = vbuf;
+
+	bzero(hdr, HDR_L2ONLY_SIZE);
+	arc_space_consume(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
 
 	return (0);
 }
@@ -885,15 +971,25 @@ buf_cons(void *vbuf, void *unused, int kmflag)
  */
 /* ARGSUSED */
 static void
-hdr_dest(void *vbuf, void *unused)
+hdr_full_dest(void *vbuf, void *unused)
 {
 	arc_buf_hdr_t *hdr = vbuf;
 
 	ASSERT(BUF_EMPTY(hdr));
-	refcount_destroy(&hdr->b_refcnt);
-	cv_destroy(&hdr->b_cv);
-	mutex_destroy(&hdr->b_freeze_lock);
-	arc_space_return(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS);
+	cv_destroy(&hdr->b_l1hdr.b_cv);
+	refcount_destroy(&hdr->b_l1hdr.b_refcnt);
+	mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);
+	arc_space_return(HDR_FULL_SIZE, ARC_SPACE_HDRS);
+}
+
+/* ARGSUSED */
+static void
+hdr_l2only_dest(void *vbuf, void *unused)
+{
+	arc_buf_hdr_t *hdr = vbuf;
+
+	ASSERT(BUF_EMPTY(hdr));
+	arc_space_return(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
 }
 
 /* ARGSUSED */
@@ -947,8 +1043,11 @@ retry:
 		goto retry;
 	}
 
-	hdr_cache = kmem_cache_create("arc_buf_hdr_t", sizeof (arc_buf_hdr_t),
-	    0, hdr_cons, hdr_dest, hdr_recl, NULL, NULL, 0);
+	hdr_full_cache = kmem_cache_create("arc_buf_hdr_t_full", HDR_FULL_SIZE,
+	    0, hdr_full_cons, hdr_full_dest, hdr_recl, NULL, NULL, 0);
+	hdr_l2only_cache = kmem_cache_create("arc_buf_hdr_t_l2only",
+	    HDR_L2ONLY_SIZE, 0, hdr_l2only_cons, hdr_l2only_dest, hdr_recl,
+	    NULL, NULL, 0);
 	buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
 	    0, buf_cons, buf_dest, NULL, NULL, NULL, 0);
 
@@ -962,6 +1061,81 @@ retry:
 	}
 }
 
+/*
+ * Transition between the two allocation states for the arc_buf_hdr struct.
+ * The arc_buf_hdr struct can be allocated with (hdr_full_cache) or without
+ * (hdr_l2only_cache) the fields necessary for the L1 cache - the smaller
+ * version is used when a cache buffer is only in the L2ARC in order to reduce
+ * memory usage.
+ */
+static arc_buf_hdr_t *
+arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
+{
+	ASSERT(HDR_HAS_L2HDR(hdr));
+
+	arc_buf_hdr_t *nhdr;
+	l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
+
+	ASSERT((old == hdr_full_cache && new == hdr_l2only_cache) ||
+	    (old == hdr_l2only_cache && new == hdr_full_cache));
+
+	nhdr = kmem_cache_alloc(new, KM_PUSHPAGE);
+
+	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
+	buf_hash_remove(hdr);
+
+	bcopy(hdr, nhdr, HDR_L2ONLY_SIZE);
+	if (new == hdr_full_cache) {
+		nhdr->b_flags |= ARC_FLAG_HAS_L1HDR;
+		/*
+		 * arc_access and arc_change_state need to be aware that a
+		 * header has just come out of L2ARC, so we set its state to
+		 * l2c_only even though it's about to change.
+		 */
+		nhdr->b_l1hdr.b_state = arc_l2c_only;
+	} else {
+		ASSERT(hdr->b_l1hdr.b_buf == NULL);
+		ASSERT0(hdr->b_l1hdr.b_datacnt);
+		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
+		/*
+		 * We might be removing the L1hdr of a buffer which was just
+		 * written out to L2ARC. If such a buffer is compressed then we
+		 * need to free its b_tmp_cdata before destroying the header.
+		 */
+		if (hdr->b_l1hdr.b_tmp_cdata != NULL &&
+		    HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF)
+			l2arc_release_cdata_buf(hdr);
+		nhdr->b_flags &= ~ARC_FLAG_HAS_L1HDR;
+	}
+	/*
+	 * The header has been reallocated so we need to re-insert it into any
+	 * lists it was on.
+	 */
+	(void) buf_hash_insert(nhdr, NULL);
+
+	ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node));
+
+	mutex_enter(&dev->l2ad_mtx);
+
+	/*
+	 * We must place the realloc'ed header back into the list at
+	 * the same spot. Otherwise, if it's placed earlier in the list,
+	 * l2arc_write_buffers() could find it during the function's
+	 * write phase, and try to write it out to the l2arc.
+	 */
+	list_insert_after(&dev->l2ad_buflist, hdr, nhdr);
+	list_remove(&dev->l2ad_buflist, hdr);
+
+	mutex_exit(&dev->l2ad_mtx);
+
+	buf_discard_identity(hdr);
+	hdr->b_freeze_cksum = NULL;
+	kmem_cache_free(old, hdr);
+
+	return (nhdr);
+}
+
+
 #define	ARC_MINTIME	(hz>>4) /* 62 ms */
 
 static void
@@ -972,16 +1146,15 @@ arc_cksum_verify(arc_buf_t *buf)
 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
 		return;
 
-	mutex_enter(&buf->b_hdr->b_freeze_lock);
-	if (buf->b_hdr->b_freeze_cksum == NULL ||
-	    (buf->b_hdr->b_flags & ARC_FLAG_IO_ERROR)) {
-		mutex_exit(&buf->b_hdr->b_freeze_lock);
+	mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
+	if (buf->b_hdr->b_freeze_cksum == NULL || HDR_IO_ERROR(buf->b_hdr)) {
+		mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
 		return;
 	}
 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
 	if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc))
 		panic("buffer modified while frozen!");
-	mutex_exit(&buf->b_hdr->b_freeze_lock);
+	mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
 }
 
 static int
@@ -990,10 +1163,10 @@ arc_cksum_equal(arc_buf_t *buf)
 	zio_cksum_t zc;
 	int equal;
 
-	mutex_enter(&buf->b_hdr->b_freeze_lock);
+	mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
 	equal = ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc);
-	mutex_exit(&buf->b_hdr->b_freeze_lock);
+	mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
 
 	return (equal);
 }
@@ -1004,15 +1177,15 @@ arc_cksum_compute(arc_buf_t *buf, boolean_t force)
 	if (!force && !(zfs_flags & ZFS_DEBUG_MODIFY))
 		return;
 
-	mutex_enter(&buf->b_hdr->b_freeze_lock);
+	mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
 	if (buf->b_hdr->b_freeze_cksum != NULL) {
-		mutex_exit(&buf->b_hdr->b_freeze_lock);
+		mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
 		return;
 	}
 	buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP);
 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size,
 	    buf->b_hdr->b_freeze_cksum);
-	mutex_exit(&buf->b_hdr->b_freeze_lock);
+	mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
 	arc_buf_watch(buf);
 }
 
@@ -1059,30 +1232,58 @@ arc_buf_watch(arc_buf_t *buf)
 #endif
 }
 
+static arc_buf_contents_t
+arc_buf_type(arc_buf_hdr_t *hdr)
+{
+	if (HDR_ISTYPE_METADATA(hdr)) {
+		return (ARC_BUFC_METADATA);
+	} else {
+		return (ARC_BUFC_DATA);
+	}
+}
+
+static uint32_t
+arc_bufc_to_flags(arc_buf_contents_t type)
+{
+	switch (type) {
+	case ARC_BUFC_DATA:
+		/* metadata field is 0 if buffer contains normal data */
+		return (0);
+	case ARC_BUFC_METADATA:
+		return (ARC_FLAG_BUFC_METADATA);
+	default:
+		break;
+	}
+	panic("undefined ARC buffer type!");
+	return ((uint32_t)-1);
+}
+
 void
 arc_buf_thaw(arc_buf_t *buf)
 {
 	if (zfs_flags & ZFS_DEBUG_MODIFY) {
-		if (buf->b_hdr->b_state != arc_anon)
+		if (buf->b_hdr->b_l1hdr.b_state != arc_anon)
 			panic("modifying non-anon buffer!");
-		if (buf->b_hdr->b_flags & ARC_FLAG_IO_IN_PROGRESS)
+		if (HDR_IO_IN_PROGRESS(buf->b_hdr))
 			panic("modifying buffer while i/o in progress!");
 		arc_cksum_verify(buf);
 	}
 
-	mutex_enter(&buf->b_hdr->b_freeze_lock);
+	mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
 	if (buf->b_hdr->b_freeze_cksum != NULL) {
 		kmem_free(buf->b_hdr->b_freeze_cksum, sizeof (zio_cksum_t));
 		buf->b_hdr->b_freeze_cksum = NULL;
 	}
 
+#ifdef ZFS_DEBUG
 	if (zfs_flags & ZFS_DEBUG_MODIFY) {
-		if (buf->b_hdr->b_thawed)
-			kmem_free(buf->b_hdr->b_thawed, 1);
-		buf->b_hdr->b_thawed = kmem_alloc(1, KM_SLEEP);
+		if (buf->b_hdr->b_l1hdr.b_thawed != NULL)
+			kmem_free(buf->b_hdr->b_l1hdr.b_thawed, 1);
+		buf->b_hdr->b_l1hdr.b_thawed = kmem_alloc(1, KM_SLEEP);
 	}
+#endif
 
-	mutex_exit(&buf->b_hdr->b_freeze_lock);
+	mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
 
 	arc_buf_unwatch(buf);
 }
@@ -1099,7 +1300,7 @@ arc_buf_freeze(arc_buf_t *buf)
 	mutex_enter(hash_lock);
 
 	ASSERT(buf->b_hdr->b_freeze_cksum != NULL ||
-	    buf->b_hdr->b_state == arc_anon);
+	    buf->b_hdr->b_l1hdr.b_state == arc_anon);
 	arc_cksum_compute(buf, B_FALSE);
 	mutex_exit(hash_lock);
 
@@ -1108,30 +1309,34 @@ arc_buf_freeze(arc_buf_t *buf)
 static void
 add_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
 {
+	ASSERT(HDR_HAS_L1HDR(hdr));
 	ASSERT(MUTEX_HELD(hash_lock));
+	arc_state_t *state = hdr->b_l1hdr.b_state;
 
-	if ((refcount_add(&hdr->b_refcnt, tag) == 1) &&
-	    (hdr->b_state != arc_anon)) {
-		uint64_t delta = hdr->b_size * hdr->b_datacnt;
-		list_t *list = &hdr->b_state->arcs_list[hdr->b_type];
-		uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type];
+	if ((refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
+	    (state != arc_anon)) {
+		/* We don't use the L2-only state list. */
+		if (state != arc_l2c_only) {
+			uint64_t delta = hdr->b_size * hdr->b_l1hdr.b_datacnt;
+			list_t *list = &state->arcs_list[arc_buf_type(hdr)];
+			uint64_t *size = &state->arcs_lsize[arc_buf_type(hdr)];
 
-		ASSERT(!MUTEX_HELD(&hdr->b_state->arcs_mtx));
-		mutex_enter(&hdr->b_state->arcs_mtx);
-		ASSERT(list_link_active(&hdr->b_arc_node));
-		list_remove(list, hdr);
-		if (GHOST_STATE(hdr->b_state)) {
-			ASSERT0(hdr->b_datacnt);
-			ASSERT3P(hdr->b_buf, ==, NULL);
-			delta = hdr->b_size;
+			ASSERT(!MUTEX_HELD(&state->arcs_mtx));
+			mutex_enter(&state->arcs_mtx);
+			ASSERT(list_link_active(&hdr->b_l1hdr.b_arc_node));
+			list_remove(list, hdr);
+			if (GHOST_STATE(state)) {
+				ASSERT0(hdr->b_l1hdr.b_datacnt);
+				ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
+				delta = hdr->b_size;
+			}
+			ASSERT(delta > 0);
+			ASSERT3U(*size, >=, delta);
+			atomic_add_64(size, -delta);
+			mutex_exit(&state->arcs_mtx);
 		}
-		ASSERT(delta > 0);
-		ASSERT3U(*size, >=, delta);
-		atomic_add_64(size, -delta);
-		mutex_exit(&hdr->b_state->arcs_mtx);
 		/* remove the prefetch flag if we get a reference */
-		if (hdr->b_flags & ARC_FLAG_PREFETCH)
-			hdr->b_flags &= ~ARC_FLAG_PREFETCH;
+		hdr->b_flags &= ~ARC_FLAG_PREFETCH;
 	}
 }
 
@@ -1139,21 +1344,27 @@ static int
 remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
 {
 	int cnt;
-	arc_state_t *state = hdr->b_state;
+	arc_state_t *state = hdr->b_l1hdr.b_state;
 
+	ASSERT(HDR_HAS_L1HDR(hdr));
 	ASSERT(state == arc_anon || MUTEX_HELD(hash_lock));
 	ASSERT(!GHOST_STATE(state));
 
-	if (((cnt = refcount_remove(&hdr->b_refcnt, tag)) == 0) &&
+	/*
+	 * arc_l2c_only counts as a ghost state so we don't need to explicitly
+	 * check to prevent usage of the arc_l2c_only list.
+	 */
+	if (((cnt = refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) &&
 	    (state != arc_anon)) {
-		uint64_t *size = &state->arcs_lsize[hdr->b_type];
+		uint64_t *size = &state->arcs_lsize[arc_buf_type(hdr)];
 
 		ASSERT(!MUTEX_HELD(&state->arcs_mtx));
 		mutex_enter(&state->arcs_mtx);
-		ASSERT(!list_link_active(&hdr->b_arc_node));
-		list_insert_head(&state->arcs_list[hdr->b_type], hdr);
-		ASSERT(hdr->b_datacnt > 0);
-		atomic_add_64(size, hdr->b_size * hdr->b_datacnt);
+		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
+		list_insert_head(&state->arcs_list[arc_buf_type(hdr)], hdr);
+		ASSERT(hdr->b_l1hdr.b_datacnt > 0);
+		atomic_add_64(size, hdr->b_size *
+		    hdr->b_l1hdr.b_datacnt);
 		mutex_exit(&state->arcs_mtx);
 	}
 	return (cnt);
@@ -1167,40 +1378,60 @@ static void
 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
     kmutex_t *hash_lock)
 {
-	arc_state_t *old_state = hdr->b_state;
-	int64_t refcnt = refcount_count(&hdr->b_refcnt);
+	arc_state_t *old_state;
+	int64_t refcnt;
+	uint32_t datacnt;
 	uint64_t from_delta, to_delta;
+	arc_buf_contents_t buftype = arc_buf_type(hdr);
+
+	/*
+	 * We almost always have an L1 hdr here, since we call arc_hdr_realloc()
+	 * in arc_read() when bringing a buffer out of the L2ARC.  However, the
+	 * L1 hdr doesn't always exist when we change state to arc_anon before
+	 * destroying a header, in which case reallocating to add the L1 hdr is
+	 * pointless.
+	 */
+	if (HDR_HAS_L1HDR(hdr)) {
+		old_state = hdr->b_l1hdr.b_state;
+		refcnt = refcount_count(&hdr->b_l1hdr.b_refcnt);
+		datacnt = hdr->b_l1hdr.b_datacnt;
+	} else {
+		old_state = arc_l2c_only;
+		refcnt = 0;
+		datacnt = 0;
+	}
 
 	ASSERT(MUTEX_HELD(hash_lock));
 	ASSERT3P(new_state, !=, old_state);
-	ASSERT(refcnt == 0 || hdr->b_datacnt > 0);
-	ASSERT(hdr->b_datacnt == 0 || !GHOST_STATE(new_state));
-	ASSERT(hdr->b_datacnt <= 1 || old_state != arc_anon);
+	ASSERT(refcnt == 0 || datacnt > 0);
+	ASSERT(!GHOST_STATE(new_state) || datacnt == 0);
+	ASSERT(old_state != arc_anon || datacnt <= 1);
 
-	from_delta = to_delta = hdr->b_datacnt * hdr->b_size;
+	from_delta = to_delta = datacnt * hdr->b_size;
 
 	/*
 	 * If this buffer is evictable, transfer it from the
 	 * old state list to the new state list.
 	 */
 	if (refcnt == 0) {
-		if (old_state != arc_anon) {
+		if (old_state != arc_anon && old_state != arc_l2c_only) {
 			int use_mutex = !MUTEX_HELD(&old_state->arcs_mtx);
-			uint64_t *size = &old_state->arcs_lsize[hdr->b_type];
+			uint64_t *size = &old_state->arcs_lsize[buftype];
 
 			if (use_mutex)
 				mutex_enter(&old_state->arcs_mtx);
 
-			ASSERT(list_link_active(&hdr->b_arc_node));
-			list_remove(&old_state->arcs_list[hdr->b_type], hdr);
+			ASSERT(HDR_HAS_L1HDR(hdr));
+			ASSERT(list_link_active(&hdr->b_l1hdr.b_arc_node));
+			list_remove(&old_state->arcs_list[buftype], hdr);
 
 			/*
 			 * If prefetching out of the ghost cache,
 			 * we will have a non-zero datacnt.
 			 */
-			if (GHOST_STATE(old_state) && hdr->b_datacnt == 0) {
+			if (GHOST_STATE(old_state) && datacnt == 0) {
 				/* ghost elements have a ghost size */
-				ASSERT(hdr->b_buf == NULL);
+				ASSERT(hdr->b_l1hdr.b_buf == NULL);
 				from_delta = hdr->b_size;
 			}
 			ASSERT3U(*size, >=, from_delta);
@@ -1209,20 +1440,26 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
 			if (use_mutex)
 				mutex_exit(&old_state->arcs_mtx);
 		}
-		if (new_state != arc_anon) {
+		if (new_state != arc_anon && new_state != arc_l2c_only) {
 			int use_mutex = !MUTEX_HELD(&new_state->arcs_mtx);
-			uint64_t *size = &new_state->arcs_lsize[hdr->b_type];
+			uint64_t *size = &new_state->arcs_lsize[buftype];
 
+			/*
+			 * An L1 header always exists here, since if we're
+			 * moving to some L1-cached state (i.e. not l2c_only or
+			 * anonymous), we realloc the header to add an L1hdr
+			 * beforehand.
+			 */
+			ASSERT(HDR_HAS_L1HDR(hdr));
 			if (use_mutex)
 				mutex_enter(&new_state->arcs_mtx);
 
-			list_insert_head(&new_state->arcs_list[hdr->b_type],
-			    hdr);
+			list_insert_head(&new_state->arcs_list[buftype], hdr);
 
 			/* ghost elements have a ghost size */
 			if (GHOST_STATE(new_state)) {
-				ASSERT(hdr->b_datacnt == 0);
-				ASSERT(hdr->b_buf == NULL);
+				ASSERT0(datacnt);
+				ASSERT(hdr->b_l1hdr.b_buf == NULL);
 				to_delta = hdr->b_size;
 			}
 			atomic_add_64(size, to_delta);
@@ -1236,20 +1473,22 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
 	if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr))
 		buf_hash_remove(hdr);
 
-	/* adjust state sizes */
-	if (to_delta)
+	/* adjust state sizes (ignore arc_l2c_only) */
+	if (to_delta && new_state != arc_l2c_only)
 		atomic_add_64(&new_state->arcs_size, to_delta);
-	if (from_delta) {
+	if (from_delta && old_state != arc_l2c_only) {
 		ASSERT3U(old_state->arcs_size, >=, from_delta);
 		atomic_add_64(&old_state->arcs_size, -from_delta);
 	}
-	hdr->b_state = new_state;
+	if (HDR_HAS_L1HDR(hdr))
+		hdr->b_l1hdr.b_state = new_state;
 
-	/* adjust l2arc hdr stats */
-	if (new_state == arc_l2c_only)
-		l2arc_hdr_stat_add();
-	else if (old_state == arc_l2c_only)
-		l2arc_hdr_stat_remove();
+	/*
+	 * L2 headers should never be on the L2 state list since they don't
+	 * have L1 headers allocated.
+	 */
+	ASSERT(list_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]) &&
+	    list_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA]));
 }
 
 void
@@ -1305,31 +1544,36 @@ arc_space_return(uint64_t space, arc_space_type_t type)
 }
 
 arc_buf_t *
-arc_buf_alloc(spa_t *spa, int size, void *tag, arc_buf_contents_t type)
+arc_buf_alloc(spa_t *spa, int32_t size, void *tag, arc_buf_contents_t type)
 {
 	arc_buf_hdr_t *hdr;
 	arc_buf_t *buf;
 
 	ASSERT3U(size, >, 0);
-	hdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
+	hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
 	ASSERT(BUF_EMPTY(hdr));
+	ASSERT3P(hdr->b_freeze_cksum, ==, NULL);
 	hdr->b_size = size;
-	hdr->b_type = type;
 	hdr->b_spa = spa_load_guid(spa);
-	hdr->b_state = arc_anon;
-	hdr->b_arc_access = 0;
+
 	buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
 	buf->b_hdr = hdr;
 	buf->b_data = NULL;
 	buf->b_efunc = NULL;
 	buf->b_private = NULL;
 	buf->b_next = NULL;
-	hdr->b_buf = buf;
+
+	hdr->b_flags = arc_bufc_to_flags(type);
+	hdr->b_flags |= ARC_FLAG_HAS_L1HDR;
+
+	hdr->b_l1hdr.b_buf = buf;
+	hdr->b_l1hdr.b_state = arc_anon;
+	hdr->b_l1hdr.b_arc_access = 0;
+	hdr->b_l1hdr.b_datacnt = 1;
+
 	arc_get_data_buf(buf);
-	hdr->b_datacnt = 1;
-	hdr->b_flags = 0;
-	ASSERT(refcount_is_zero(&hdr->b_refcnt));
-	(void) refcount_add(&hdr->b_refcnt, tag);
+	ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
+	(void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
 
 	return (buf);
 }
@@ -1362,8 +1606,9 @@ arc_return_buf(arc_buf_t *buf, void *tag)
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 
 	ASSERT(buf->b_data != NULL);
-	(void) refcount_add(&hdr->b_refcnt, tag);
-	(void) refcount_remove(&hdr->b_refcnt, arc_onloan_tag);
+	ASSERT(HDR_HAS_L1HDR(hdr));
+	(void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
+	(void) refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
 
 	atomic_add_64(&arc_loaned_bytes, -hdr->b_size);
 }
@@ -1372,12 +1617,12 @@ arc_return_buf(arc_buf_t *buf, void *tag)
 void
 arc_loan_inuse_buf(arc_buf_t *buf, void *tag)
 {
-	arc_buf_hdr_t *hdr;
+	arc_buf_hdr_t *hdr = buf->b_hdr;
 
 	ASSERT(buf->b_data != NULL);
-	hdr = buf->b_hdr;
-	(void) refcount_add(&hdr->b_refcnt, arc_onloan_tag);
-	(void) refcount_remove(&hdr->b_refcnt, tag);
+	ASSERT(HDR_HAS_L1HDR(hdr));
+	(void) refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
+	(void) refcount_remove(&hdr->b_l1hdr.b_refcnt, tag);
 	buf->b_efunc = NULL;
 	buf->b_private = NULL;
 
@@ -1391,15 +1636,16 @@ arc_buf_clone(arc_buf_t *from)
 	arc_buf_hdr_t *hdr = from->b_hdr;
 	uint64_t size = hdr->b_size;
 
-	ASSERT(hdr->b_state != arc_anon);
+	ASSERT(HDR_HAS_L1HDR(hdr));
+	ASSERT(hdr->b_l1hdr.b_state != arc_anon);
 
 	buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
 	buf->b_hdr = hdr;
 	buf->b_data = NULL;
 	buf->b_efunc = NULL;
 	buf->b_private = NULL;
-	buf->b_next = hdr->b_buf;
-	hdr->b_buf = buf;
+	buf->b_next = hdr->b_l1hdr.b_buf;
+	hdr->b_l1hdr.b_buf = buf;
 	arc_get_data_buf(buf);
 	bcopy(from->b_data, buf->b_data, size);
 
@@ -1409,11 +1655,11 @@ arc_buf_clone(arc_buf_t *from)
 	 * then track the size and number of duplicates.  These stats will be
 	 * updated as duplicate buffers are created and destroyed.
 	 */
-	if (hdr->b_type == ARC_BUFC_DATA) {
+	if (HDR_ISTYPE_DATA(hdr)) {
 		ARCSTAT_BUMP(arcstat_duplicate_buffers);
 		ARCSTAT_INCR(arcstat_duplicate_buffers_size, size);
 	}
-	hdr->b_datacnt += 1;
+	hdr->b_l1hdr.b_datacnt += 1;
 	return (buf);
 }
 
@@ -1436,17 +1682,20 @@ arc_buf_add_ref(arc_buf_t *buf, void* tag)
 	hash_lock = HDR_LOCK(buf->b_hdr);
 	mutex_enter(hash_lock);
 	hdr = buf->b_hdr;
+	ASSERT(HDR_HAS_L1HDR(hdr));
 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
 	mutex_exit(&buf->b_evict_lock);
 
-	ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
+	ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
+	    hdr->b_l1hdr.b_state == arc_mfu);
+
 	add_reference(hdr, hash_lock, tag);
 	DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
 	arc_access(hdr, hash_lock);
 	mutex_exit(hash_lock);
 	ARCSTAT_BUMP(arcstat_hits);
-	ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_FLAG_PREFETCH),
-	    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
+	ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
+	    demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
 	    data, metadata, hits);
 }
 
@@ -1484,10 +1733,10 @@ arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t remove)
 	arc_buf_t **bufp;
 
 	/* free up data associated with the buf */
-	if (buf->b_data) {
-		arc_state_t *state = buf->b_hdr->b_state;
+	if (buf->b_data != NULL) {
+		arc_state_t *state = buf->b_hdr->b_l1hdr.b_state;
 		uint64_t size = buf->b_hdr->b_size;
-		arc_buf_contents_t type = buf->b_hdr->b_type;
+		arc_buf_contents_t type = arc_buf_type(buf->b_hdr);
 
 		arc_cksum_verify(buf);
 		arc_buf_unwatch(buf);
@@ -1503,11 +1752,12 @@ arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t remove)
 				atomic_add_64(&arc_size, -size);
 			}
 		}
-		if (list_link_active(&buf->b_hdr->b_arc_node)) {
+		if (list_link_active(&buf->b_hdr->b_l1hdr.b_arc_node)) {
 			uint64_t *cnt = &state->arcs_lsize[type];
 
-			ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt));
-			ASSERT(state != arc_anon);
+			ASSERT(refcount_is_zero(
+			    &buf->b_hdr->b_l1hdr.b_refcnt));
+			ASSERT(state != arc_anon && state != arc_l2c_only);
 
 			ASSERT3U(*cnt, >=, size);
 			atomic_add_64(cnt, -size);
@@ -1520,13 +1770,13 @@ arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t remove)
 		 * If we're destroying a duplicate buffer make sure
 		 * that the appropriate statistics are updated.
 		 */
-		if (buf->b_hdr->b_datacnt > 1 &&
-		    buf->b_hdr->b_type == ARC_BUFC_DATA) {
+		if (buf->b_hdr->b_l1hdr.b_datacnt > 1 &&
+		    HDR_ISTYPE_DATA(buf->b_hdr)) {
 			ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
 			ARCSTAT_INCR(arcstat_duplicate_buffers_size, -size);
 		}
-		ASSERT(buf->b_hdr->b_datacnt > 0);
-		buf->b_hdr->b_datacnt -= 1;
+		ASSERT(buf->b_hdr->b_l1hdr.b_datacnt > 0);
+		buf->b_hdr->b_l1hdr.b_datacnt -= 1;
 	}
 
 	/* only remove the buf if requested */
@@ -1534,7 +1784,8 @@ arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t remove)
 		return;
 
 	/* remove the buf from the hdr list */
-	for (bufp = &buf->b_hdr->b_buf; *bufp != buf; bufp = &(*bufp)->b_next)
+	for (bufp = &buf->b_hdr->b_l1hdr.b_buf; *bufp != buf;
+	    bufp = &(*bufp)->b_next)
 		continue;
 	*bufp = buf->b_next;
 	buf->b_next = NULL;
@@ -1549,86 +1800,87 @@ arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t remove)
 static void
 arc_hdr_destroy(arc_buf_hdr_t *hdr)
 {
-	ASSERT(refcount_is_zero(&hdr->b_refcnt));
-	ASSERT3P(hdr->b_state, ==, arc_anon);
+	if (HDR_HAS_L1HDR(hdr)) {
+		ASSERT(hdr->b_l1hdr.b_buf == NULL ||
+		    hdr->b_l1hdr.b_datacnt > 0);
+		ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
+		ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
+	}
 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
-	l2arc_buf_hdr_t *l2hdr = hdr->b_l2hdr;
+	ASSERT(!HDR_IN_HASH_TABLE(hdr));
+
+	if (HDR_HAS_L2HDR(hdr)) {
+		l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
+		boolean_t buflist_held = MUTEX_HELD(&l2hdr->b_dev->l2ad_mtx);
 
-	if (l2hdr != NULL) {
-		boolean_t buflist_held = MUTEX_HELD(&l2arc_buflist_mtx);
-		/*
-		 * To prevent arc_free() and l2arc_evict() from
-		 * attempting to free the same buffer at the same time,
-		 * a FREE_IN_PROGRESS flag is given to arc_free() to
-		 * give it priority.  l2arc_evict() can't destroy this
-		 * header while we are waiting on l2arc_buflist_mtx.
-		 *
-		 * The hdr may be removed from l2ad_buflist before we
-		 * grab l2arc_buflist_mtx, so b_l2hdr is rechecked.
-		 */
 		if (!buflist_held) {
-			mutex_enter(&l2arc_buflist_mtx);
-			l2hdr = hdr->b_l2hdr;
+			mutex_enter(&l2hdr->b_dev->l2ad_mtx);
+			l2hdr = &hdr->b_l2hdr;
 		}
 
-		if (l2hdr != NULL) {
-			list_remove(l2hdr->b_dev->l2ad_buflist, hdr);
-			ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
-			ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
-			vdev_space_update(l2hdr->b_dev->l2ad_vdev,
-			    -l2hdr->b_asize, 0, 0);
-			kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t));
-			if (hdr->b_state == arc_l2c_only)
-				l2arc_hdr_stat_remove();
-			hdr->b_l2hdr = NULL;
-		}
+		list_remove(&l2hdr->b_dev->l2ad_buflist, hdr);
+
+		ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
+		ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
 
 		if (!buflist_held)
-			mutex_exit(&l2arc_buflist_mtx);
+			mutex_exit(&l2hdr->b_dev->l2ad_mtx);
+
+		hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
 	}
 
-	if (!BUF_EMPTY(hdr)) {
-		ASSERT(!HDR_IN_HASH_TABLE(hdr));
+	if (!BUF_EMPTY(hdr))
 		buf_discard_identity(hdr);
-	}
-	while (hdr->b_buf) {
-		arc_buf_t *buf = hdr->b_buf;
 
-		if (buf->b_efunc) {
-			mutex_enter(&arc_eviction_mtx);
-			mutex_enter(&buf->b_evict_lock);
-			ASSERT(buf->b_hdr != NULL);
-			arc_buf_destroy(hdr->b_buf, FALSE, FALSE);
-			hdr->b_buf = buf->b_next;
-			buf->b_hdr = &arc_eviction_hdr;
-			buf->b_next = arc_eviction_list;
-			arc_eviction_list = buf;
-			mutex_exit(&buf->b_evict_lock);
-			mutex_exit(&arc_eviction_mtx);
-		} else {
-			arc_buf_destroy(hdr->b_buf, FALSE, TRUE);
-		}
-	}
 	if (hdr->b_freeze_cksum != NULL) {
 		kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
 		hdr->b_freeze_cksum = NULL;
 	}
-	if (hdr->b_thawed) {
-		kmem_free(hdr->b_thawed, 1);
-		hdr->b_thawed = NULL;
+
+	if (HDR_HAS_L1HDR(hdr)) {
+		while (hdr->b_l1hdr.b_buf) {
+			arc_buf_t *buf = hdr->b_l1hdr.b_buf;
+
+			if (buf->b_efunc != NULL) {
+				mutex_enter(&arc_eviction_mtx);
+				mutex_enter(&buf->b_evict_lock);
+				ASSERT(buf->b_hdr != NULL);
+				arc_buf_destroy(hdr->b_l1hdr.b_buf, FALSE,
+				    FALSE);
+				hdr->b_l1hdr.b_buf = buf->b_next;
+				buf->b_hdr = &arc_eviction_hdr;
+				buf->b_next = arc_eviction_list;
+				arc_eviction_list = buf;
+				mutex_exit(&buf->b_evict_lock);
+				mutex_exit(&arc_eviction_mtx);
+			} else {
+				arc_buf_destroy(hdr->b_l1hdr.b_buf, FALSE,
+				    TRUE);
+			}
+		}
+#ifdef ZFS_DEBUG
+		if (hdr->b_l1hdr.b_thawed != NULL) {
+			kmem_free(hdr->b_l1hdr.b_thawed, 1);
+			hdr->b_l1hdr.b_thawed = NULL;
+		}
+#endif
 	}
 
-	ASSERT(!list_link_active(&hdr->b_arc_node));
 	ASSERT3P(hdr->b_hash_next, ==, NULL);
-	ASSERT3P(hdr->b_acb, ==, NULL);
-	kmem_cache_free(hdr_cache, hdr);
+	if (HDR_HAS_L1HDR(hdr)) {
+		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
+		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
+		kmem_cache_free(hdr_full_cache, hdr);
+	} else {
+		kmem_cache_free(hdr_l2only_cache, hdr);
+	}
 }
 
 void
 arc_buf_free(arc_buf_t *buf, void *tag)
 {
 	arc_buf_hdr_t *hdr = buf->b_hdr;
-	int hashed = hdr->b_state != arc_anon;
+	int hashed = hdr->b_l1hdr.b_state != arc_anon;
 
 	ASSERT(buf->b_efunc == NULL);
 	ASSERT(buf->b_data != NULL);
@@ -1641,10 +1893,10 @@ arc_buf_free(arc_buf_t *buf, void *tag)
 		ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
 
 		(void) remove_reference(hdr, hash_lock, tag);
-		if (hdr->b_datacnt > 1) {
+		if (hdr->b_l1hdr.b_datacnt > 1) {
 			arc_buf_destroy(buf, FALSE, TRUE);
 		} else {
-			ASSERT(buf == hdr->b_buf);
+			ASSERT(buf == hdr->b_l1hdr.b_buf);
 			ASSERT(buf->b_efunc == NULL);
 			hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
 		}
@@ -1658,7 +1910,7 @@ arc_buf_free(arc_buf_t *buf, void *tag)
 		 */
 		mutex_enter(&arc_eviction_mtx);
 		(void) remove_reference(hdr, NULL, tag);
-		ASSERT(refcount_is_zero(&hdr->b_refcnt));
+		ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
 		destroy_hdr = !HDR_IO_IN_PROGRESS(hdr);
 		mutex_exit(&arc_eviction_mtx);
 		if (destroy_hdr)
@@ -1678,34 +1930,35 @@ arc_buf_remove_ref(arc_buf_t *buf, void* tag)
 	kmutex_t *hash_lock = HDR_LOCK(hdr);
 	boolean_t no_callback = (buf->b_efunc == NULL);
 
-	if (hdr->b_state == arc_anon) {
-		ASSERT(hdr->b_datacnt == 1);
+	if (hdr->b_l1hdr.b_state == arc_anon) {
+		ASSERT(hdr->b_l1hdr.b_datacnt == 1);
 		arc_buf_free(buf, tag);
 		return (no_callback);
 	}
 
 	mutex_enter(hash_lock);
 	hdr = buf->b_hdr;
+	ASSERT(hdr->b_l1hdr.b_datacnt > 0);
 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
-	ASSERT(hdr->b_state != arc_anon);
+	ASSERT(hdr->b_l1hdr.b_state != arc_anon);
 	ASSERT(buf->b_data != NULL);
 
 	(void) remove_reference(hdr, hash_lock, tag);
-	if (hdr->b_datacnt > 1) {
+	if (hdr->b_l1hdr.b_datacnt > 1) {
 		if (no_callback)
 			arc_buf_destroy(buf, FALSE, TRUE);
 	} else if (no_callback) {
-		ASSERT(hdr->b_buf == buf && buf->b_next == NULL);
+		ASSERT(hdr->b_l1hdr.b_buf == buf && buf->b_next == NULL);
 		ASSERT(buf->b_efunc == NULL);
 		hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
 	}
-	ASSERT(no_callback || hdr->b_datacnt > 1 ||
-	    refcount_is_zero(&hdr->b_refcnt));
+	ASSERT(no_callback || hdr->b_l1hdr.b_datacnt > 1 ||
+	    refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
 	mutex_exit(hash_lock);
 	return (no_callback);
 }
 
-int
+int32_t
 arc_buf_size(arc_buf_t *buf)
 {
 	return (buf->b_hdr->b_size);
@@ -1746,7 +1999,7 @@ arc_buf_eviction_needed(arc_buf_t *buf)
 		return (B_TRUE);
 	}
 
-	if (hdr->b_datacnt > 1 && hdr->b_type == ARC_BUFC_DATA)
+	if (hdr->b_l1hdr.b_datacnt > 1 && HDR_ISTYPE_DATA(hdr))
 		evict_needed = B_TRUE;
 
 	mutex_exit(&buf->b_evict_lock);
@@ -1783,8 +2036,20 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 
 	evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
 
-	mutex_enter(&state->arcs_mtx);
+	/*
+	 * The ghost list lock must be acquired first in order to prevent
+	 * a 3 party deadlock:
+	 *
+	 *  - arc_evict_ghost acquires arc_*_ghost->arcs_mtx, followed by
+	 *    l2ad_mtx in arc_hdr_realloc
+	 *  - l2arc_write_buffers acquires l2ad_mtx, followed by arc_*->arcs_mtx
+	 *  - arc_evict acquires arc_*_ghost->arcs_mtx, followed by
+	 *    arc_*_ghost->arcs_mtx and forms a deadlock cycle.
+	 *
+	 * This situation is avoided by acquiring the ghost list lock first.
+	 */
 	mutex_enter(&evicted_state->arcs_mtx);
+	mutex_enter(&state->arcs_mtx);
 
 	/*
 	 * Decide which "type" (data vs metadata) to recycle from.
@@ -1800,6 +2065,7 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 		arc_buf_hdr_t *metadata_hdr =
 		    list_tail(&state->arcs_list[ARC_BUFC_METADATA]);
 		arc_buf_contents_t realtype;
+
 		if (data_hdr == NULL) {
 			realtype = ARC_BUFC_METADATA;
 		} else if (metadata_hdr == NULL) {
@@ -1808,13 +2074,13 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 			realtype = ARC_BUFC_METADATA;
 		} else if (arc_meta_used <= arc_meta_min) {
 			realtype = ARC_BUFC_DATA;
+		} else if (HDR_HAS_L1HDR(data_hdr) &&
+		    HDR_HAS_L1HDR(metadata_hdr) &&
+		    data_hdr->b_l1hdr.b_arc_access <
+		    metadata_hdr->b_l1hdr.b_arc_access) {
+			realtype = ARC_BUFC_DATA;
 		} else {
-			if (data_hdr->b_arc_access <
-			    metadata_hdr->b_arc_access) {
-				realtype = ARC_BUFC_DATA;
-			} else {
-				realtype = ARC_BUFC_METADATA;
-			}
+			realtype = ARC_BUFC_METADATA;
 		}
 		if (realtype != type) {
 			/*
@@ -1835,8 +2101,8 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 		/* prefetch buffers have a minimum lifespan */
 		if (HDR_IO_IN_PROGRESS(hdr) ||
 		    (spa && hdr->b_spa != spa) ||
-		    (hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT) &&
-		    ddi_get_lbolt() - hdr->b_arc_access <
+		    ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&
+		    ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <
 		    arc_min_prefetch_lifespan)) {
 			skipped++;
 			continue;
@@ -1861,11 +2127,11 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 		 */
 		if (!recycle && count++ > arc_evict_iterations) {
 			list_insert_after(list, hdr, &marker);
-			mutex_exit(&evicted_state->arcs_mtx);
 			mutex_exit(&state->arcs_mtx);
+			mutex_exit(&evicted_state->arcs_mtx);
 			kpreempt(KPREEMPT_SYNC);
-			mutex_enter(&state->arcs_mtx);
 			mutex_enter(&evicted_state->arcs_mtx);
+			mutex_enter(&state->arcs_mtx);
 			hdr_prev = list_prev(list, &marker);
 			list_remove(list, &marker);
 			count = 0;
@@ -1875,28 +2141,29 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 		hash_lock = HDR_LOCK(hdr);
 		have_lock = MUTEX_HELD(hash_lock);
 		if (have_lock || mutex_tryenter(hash_lock)) {
-			ASSERT0(refcount_count(&hdr->b_refcnt));
-			ASSERT(hdr->b_datacnt > 0);
-			while (hdr->b_buf) {
-				arc_buf_t *buf = hdr->b_buf;
+			ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
+			ASSERT3U(hdr->b_l1hdr.b_datacnt, >, 0);
+			while (hdr->b_l1hdr.b_buf) {
+				arc_buf_t *buf = hdr->b_l1hdr.b_buf;
 				if (!mutex_tryenter(&buf->b_evict_lock)) {
 					missed += 1;
 					break;
 				}
-				if (buf->b_data) {
+				if (buf->b_data != NULL) {
 					bytes_evicted += hdr->b_size;
-					if (recycle && hdr->b_type == type &&
+					if (recycle &&
+					    arc_buf_type(hdr) == type &&
 					    hdr->b_size == bytes &&
 					    !HDR_L2_WRITING(hdr)) {
 						stolen = buf->b_data;
 						recycle = FALSE;
 					}
 				}
-				if (buf->b_efunc) {
+				if (buf->b_efunc != NULL) {
 					mutex_enter(&arc_eviction_mtx);
 					arc_buf_destroy(buf,
 					    buf->b_data == stolen, FALSE);
-					hdr->b_buf = buf->b_next;
+					hdr->b_l1hdr.b_buf = buf->b_next;
 					buf->b_hdr = &arc_eviction_hdr;
 					buf->b_next = arc_eviction_list;
 					arc_eviction_list = buf;
@@ -1909,7 +2176,7 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 				}
 			}
 
-			if (hdr->b_l2hdr) {
+			if (HDR_HAS_L2HDR(hdr)) {
 				ARCSTAT_INCR(arcstat_evict_l2_cached,
 				    hdr->b_size);
 			} else {
@@ -1923,7 +2190,7 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 				}
 			}
 
-			if (hdr->b_datacnt == 0) {
+			if (hdr->b_l1hdr.b_datacnt == 0) {
 				arc_change_state(evicted_state, hdr, hash_lock);
 				ASSERT(HDR_IN_HASH_TABLE(hdr));
 				hdr->b_flags |= ARC_FLAG_IN_HASH_TABLE;
@@ -1939,8 +2206,8 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 		}
 	}
 
-	mutex_exit(&evicted_state->arcs_mtx);
 	mutex_exit(&state->arcs_mtx);
+	mutex_exit(&evicted_state->arcs_mtx);
 
 	if (bytes_evicted < bytes)
 		dprintf("only evicted %lld bytes from %x",
@@ -1982,7 +2249,7 @@ top:
 	mutex_enter(&state->arcs_mtx);
 	for (hdr = list_tail(list); hdr; hdr = hdr_prev) {
 		hdr_prev = list_prev(list, hdr);
-		if (hdr->b_type > ARC_BUFC_NUMTYPES)
+		if (arc_buf_type(hdr) >= ARC_BUFC_NUMTYPES)
 			panic("invalid hdr=%p", (void *)hdr);
 		if (spa && hdr->b_spa != spa)
 			continue;
@@ -2014,16 +2281,23 @@ top:
 		}
 		if (mutex_tryenter(hash_lock)) {
 			ASSERT(!HDR_IO_IN_PROGRESS(hdr));
-			ASSERT(hdr->b_buf == NULL);
+			ASSERT(!HDR_HAS_L1HDR(hdr) ||
+			    hdr->b_l1hdr.b_buf == NULL);
 			ARCSTAT_BUMP(arcstat_deleted);
 			bytes_deleted += hdr->b_size;
 
-			if (hdr->b_l2hdr != NULL) {
+			if (HDR_HAS_L2HDR(hdr)) {
 				/*
 				 * This buffer is cached on the 2nd Level ARC;
 				 * don't destroy the header.
 				 */
 				arc_change_state(arc_l2c_only, hdr, hash_lock);
+				/*
+				 * dropping from L1+L2 cached to L2-only,
+				 * realloc to remove the L1 header.
+				 */
+				hdr = arc_hdr_realloc(hdr, hdr_full_cache,
+				    hdr_l2only_cache);
 				mutex_exit(hash_lock);
 			} else {
 				arc_change_state(arc_anon, hdr, hash_lock);
@@ -2166,27 +2440,27 @@ arc_flush(spa_t *spa)
 {
 	uint64_t guid = 0;
 
-	if (spa)
+	if (spa != NULL)
 		guid = spa_load_guid(spa);
 
 	while (list_head(&arc_mru->arcs_list[ARC_BUFC_DATA])) {
 		(void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_DATA);
-		if (spa)
+		if (spa != NULL)
 			break;
 	}
 	while (list_head(&arc_mru->arcs_list[ARC_BUFC_METADATA])) {
 		(void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_METADATA);
-		if (spa)
+		if (spa != NULL)
 			break;
 	}
 	while (list_head(&arc_mfu->arcs_list[ARC_BUFC_DATA])) {
 		(void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_DATA);
-		if (spa)
+		if (spa != NULL)
 			break;
 	}
 	while (list_head(&arc_mfu->arcs_list[ARC_BUFC_METADATA])) {
 		(void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_METADATA);
-		if (spa)
+		if (spa != NULL)
 			break;
 	}
 
@@ -2406,7 +2680,8 @@ arc_kmem_reap_now(void)
 		}
 	}
 	kmem_cache_reap_now(buf_cache);
-	kmem_cache_reap_now(hdr_cache);
+	kmem_cache_reap_now(hdr_full_cache);
+	kmem_cache_reap_now(hdr_l2only_cache);
 	kmem_cache_reap_now(range_seg_cache);
 
 	if (zio_arena != NULL) {
@@ -2590,9 +2865,9 @@ arc_evict_needed(arc_buf_contents_t type)
 static void
 arc_get_data_buf(arc_buf_t *buf)
 {
-	arc_state_t		*state = buf->b_hdr->b_state;
+	arc_state_t		*state = buf->b_hdr->b_l1hdr.b_state;
 	uint64_t		size = buf->b_hdr->b_size;
-	arc_buf_contents_t	type = buf->b_hdr->b_type;
+	arc_buf_contents_t	type = arc_buf_type(buf->b_hdr);
 
 	arc_adapt(size, state);
 
@@ -2618,8 +2893,7 @@ arc_get_data_buf(arc_buf_t *buf)
 	 * will end up on the mru list; so steal space from there.
 	 */
 	if (state == arc_mfu_ghost)
-		state = buf->b_hdr->b_flags & ARC_FLAG_PREFETCH ?
-		    arc_mru : arc_mfu;
+		state = HDR_PREFETCH(buf->b_hdr) ? arc_mru : arc_mfu;
 	else if (state == arc_mru_ghost)
 		state = arc_mru;
 
@@ -2651,19 +2925,20 @@ out:
 	 * Update the state size.  Note that ghost states have a
 	 * "ghost size" and so don't need to be updated.
 	 */
-	if (!GHOST_STATE(buf->b_hdr->b_state)) {
+	if (!GHOST_STATE(buf->b_hdr->b_l1hdr.b_state)) {
 		arc_buf_hdr_t *hdr = buf->b_hdr;
 
-		atomic_add_64(&hdr->b_state->arcs_size, size);
-		if (list_link_active(&hdr->b_arc_node)) {
-			ASSERT(refcount_is_zero(&hdr->b_refcnt));
-			atomic_add_64(&hdr->b_state->arcs_lsize[type], size);
+		atomic_add_64(&hdr->b_l1hdr.b_state->arcs_size, size);
+		if (list_link_active(&hdr->b_l1hdr.b_arc_node)) {
+			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
+			atomic_add_64(&hdr->b_l1hdr.b_state->arcs_lsize[type],
+			    size);
 		}
 		/*
 		 * If we are growing the cache, and we are adding anonymous
 		 * data, and we have outgrown arc_p, update arc_p
 		 */
-		if (arc_size < arc_c && hdr->b_state == arc_anon &&
+		if (arc_size < arc_c && hdr->b_l1hdr.b_state == arc_anon &&
 		    arc_anon->arcs_size + arc_mru->arcs_size > arc_p)
 			arc_p = MIN(arc_c, arc_p + size);
 	}
@@ -2679,20 +2954,21 @@ arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
 	clock_t now;
 
 	ASSERT(MUTEX_HELD(hash_lock));
+	ASSERT(HDR_HAS_L1HDR(hdr));
 
-	if (hdr->b_state == arc_anon) {
+	if (hdr->b_l1hdr.b_state == arc_anon) {
 		/*
 		 * This buffer is not in the cache, and does not
 		 * appear in our "ghost" list.  Add the new buffer
 		 * to the MRU state.
 		 */
 
-		ASSERT(hdr->b_arc_access == 0);
-		hdr->b_arc_access = ddi_get_lbolt();
+		ASSERT0(hdr->b_l1hdr.b_arc_access);
+		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
 		DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
 		arc_change_state(arc_mru, hdr, hash_lock);
 
-	} else if (hdr->b_state == arc_mru) {
+	} else if (hdr->b_l1hdr.b_state == arc_mru) {
 		now = ddi_get_lbolt();
 
 		/*
@@ -2703,14 +2979,15 @@ arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
 		 * - move the buffer to the head of the list if this is
 		 *   another prefetch (to make it less likely to be evicted).
 		 */
-		if ((hdr->b_flags & ARC_FLAG_PREFETCH) != 0) {
-			if (refcount_count(&hdr->b_refcnt) == 0) {
-				ASSERT(list_link_active(&hdr->b_arc_node));
+		if (HDR_PREFETCH(hdr)) {
+			if (refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
+				ASSERT(list_link_active(
+				    &hdr->b_l1hdr.b_arc_node));
 			} else {
 				hdr->b_flags &= ~ARC_FLAG_PREFETCH;
 				ARCSTAT_BUMP(arcstat_mru_hits);
 			}
-			hdr->b_arc_access = now;
+			hdr->b_l1hdr.b_arc_access = now;
 			return;
 		}
 
@@ -2719,18 +2996,18 @@ arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
 		 * but it is still in the cache. Move it to the MFU
 		 * state.
 		 */
-		if (now > hdr->b_arc_access + ARC_MINTIME) {
+		if (now > hdr->b_l1hdr.b_arc_access + ARC_MINTIME) {
 			/*
 			 * More than 125ms have passed since we
 			 * instantiated this buffer.  Move it to the
 			 * most frequently used state.
 			 */
-			hdr->b_arc_access = now;
+			hdr->b_l1hdr.b_arc_access = now;
 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
 			arc_change_state(arc_mfu, hdr, hash_lock);
 		}
 		ARCSTAT_BUMP(arcstat_mru_hits);
-	} else if (hdr->b_state == arc_mru_ghost) {
+	} else if (hdr->b_l1hdr.b_state == arc_mru_ghost) {
 		arc_state_t	*new_state;
 		/*
 		 * This buffer has been "accessed" recently, but
@@ -2738,9 +3015,9 @@ arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
 		 * MFU state.
 		 */
 
-		if (hdr->b_flags & ARC_FLAG_PREFETCH) {
+		if (HDR_PREFETCH(hdr)) {
 			new_state = arc_mru;
-			if (refcount_count(&hdr->b_refcnt) > 0)
+			if (refcount_count(&hdr->b_l1hdr.b_refcnt) > 0)
 				hdr->b_flags &= ~ARC_FLAG_PREFETCH;
 			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
 		} else {
@@ -2748,11 +3025,11 @@ arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
 		}
 
-		hdr->b_arc_access = ddi_get_lbolt();
+		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
 		arc_change_state(new_state, hdr, hash_lock);
 
 		ARCSTAT_BUMP(arcstat_mru_ghost_hits);
-	} else if (hdr->b_state == arc_mfu) {
+	} else if (hdr->b_l1hdr.b_state == arc_mfu) {
 		/*
 		 * This buffer has been accessed more than once and is
 		 * still in the cache.  Keep it in the MFU state.
@@ -2762,13 +3039,13 @@ arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
 		 * If it was a prefetch, we will explicitly move it to
 		 * the head of the list now.
 		 */
-		if ((hdr->b_flags & ARC_FLAG_PREFETCH) != 0) {
-			ASSERT(refcount_count(&hdr->b_refcnt) == 0);
-			ASSERT(list_link_active(&hdr->b_arc_node));
+		if ((HDR_PREFETCH(hdr)) != 0) {
+			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
+			ASSERT(list_link_active(&hdr->b_l1hdr.b_arc_node));
 		}
 		ARCSTAT_BUMP(arcstat_mfu_hits);
-		hdr->b_arc_access = ddi_get_lbolt();
-	} else if (hdr->b_state == arc_mfu_ghost) {
+		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
+	} else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) {
 		arc_state_t	*new_state = arc_mfu;
 		/*
 		 * This buffer has been accessed more than once but has
@@ -2776,26 +3053,26 @@ arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
 		 * MFU state.
 		 */
 
-		if (hdr->b_flags & ARC_FLAG_PREFETCH) {
+		if (HDR_PREFETCH(hdr)) {
 			/*
 			 * This is a prefetch access...
 			 * move this block back to the MRU state.
 			 */
-			ASSERT0(refcount_count(&hdr->b_refcnt));
+			ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
 			new_state = arc_mru;
 		}
 
-		hdr->b_arc_access = ddi_get_lbolt();
+		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
 		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
 		arc_change_state(new_state, hdr, hash_lock);
 
 		ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
-	} else if (hdr->b_state == arc_l2c_only) {
+	} else if (hdr->b_l1hdr.b_state == arc_l2c_only) {
 		/*
 		 * This buffer is on the 2nd Level ARC.
 		 */
 
-		hdr->b_arc_access = ddi_get_lbolt();
+		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
 		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
 		arc_change_state(arc_mfu, hdr, hash_lock);
 	} else {
@@ -2866,11 +3143,11 @@ arc_read_done(zio_t *zio)
 	}
 
 	hdr->b_flags &= ~ARC_FLAG_L2_EVICTED;
-	if (l2arc_noprefetch && (hdr->b_flags & ARC_FLAG_PREFETCH))
+	if (l2arc_noprefetch && HDR_PREFETCH(hdr))
 		hdr->b_flags &= ~ARC_FLAG_L2CACHE;
 
 	/* byteswap if necessary */
-	callback_list = hdr->b_acb;
+	callback_list = hdr->b_l1hdr.b_acb;
 	ASSERT(callback_list != NULL);
 	if (BP_SHOULD_BYTESWAP(zio->io_bp) && zio->io_error == 0) {
 		dmu_object_byteswap_t bswap =
@@ -2884,7 +3161,8 @@ arc_read_done(zio_t *zio)
 	arc_cksum_compute(buf, B_FALSE);
 	arc_buf_watch(buf);
 
-	if (hash_lock && zio->io_error == 0 && hdr->b_state == arc_anon) {
+	if (hash_lock && zio->io_error == 0 &&
+	    hdr->b_l1hdr.b_state == arc_anon) {
 		/*
 		 * Only call arc_access on anonymous buffers.  This is because
 		 * if we've issued an I/O for an evicted buffer, we've already
@@ -2906,24 +3184,25 @@ arc_read_done(zio_t *zio)
 			abuf = NULL;
 		}
 	}
-	hdr->b_acb = NULL;
+	hdr->b_l1hdr.b_acb = NULL;
 	hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS;
 	ASSERT(!HDR_BUF_AVAILABLE(hdr));
 	if (abuf == buf) {
 		ASSERT(buf->b_efunc == NULL);
-		ASSERT(hdr->b_datacnt == 1);
+		ASSERT(hdr->b_l1hdr.b_datacnt == 1);
 		hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
 	}
 
-	ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL);
+	ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt) ||
+	    callback_list != NULL);
 
 	if (zio->io_error != 0) {
 		hdr->b_flags |= ARC_FLAG_IO_ERROR;
-		if (hdr->b_state != arc_anon)
+		if (hdr->b_l1hdr.b_state != arc_anon)
 			arc_change_state(arc_anon, hdr, hash_lock);
 		if (HDR_IN_HASH_TABLE(hdr))
 			buf_hash_remove(hdr);
-		freeable = refcount_is_zero(&hdr->b_refcnt);
+		freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
 	}
 
 	/*
@@ -2931,9 +3210,9 @@ arc_read_done(zio_t *zio)
 	 * that the hdr (and hence the cv) might be freed before we get to
 	 * the cv_broadcast().
 	 */
-	cv_broadcast(&hdr->b_cv);
+	cv_broadcast(&hdr->b_l1hdr.b_cv);
 
-	if (hash_lock) {
+	if (hash_lock != NULL) {
 		mutex_exit(hash_lock);
 	} else {
 		/*
@@ -2942,8 +3221,8 @@ arc_read_done(zio_t *zio)
 		 * moved to the anonymous state (so that it won't show up
 		 * in the cache).
 		 */
-		ASSERT3P(hdr->b_state, ==, arc_anon);
-		freeable = refcount_is_zero(&hdr->b_refcnt);
+		ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
+		freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
 	}
 
 	/* execute each callback and free its structure */
@@ -3005,14 +3284,14 @@ top:
 		hdr = buf_hash_find(guid, bp, &hash_lock);
 	}
 
-	if (hdr != NULL && hdr->b_datacnt > 0) {
+	if (hdr != NULL && HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_datacnt > 0) {
 
 		*arc_flags |= ARC_FLAG_CACHED;
 
 		if (HDR_IO_IN_PROGRESS(hdr)) {
 
 			if (*arc_flags & ARC_FLAG_WAIT) {
-				cv_wait(&hdr->b_cv, hash_lock);
+				cv_wait(&hdr->b_l1hdr.b_cv, hash_lock);
 				mutex_exit(hash_lock);
 				goto top;
 			}
@@ -3030,8 +3309,8 @@ top:
 					    spa, NULL, NULL, NULL, zio_flags);
 
 				ASSERT(acb->acb_done != NULL);
-				acb->acb_next = hdr->b_acb;
-				hdr->b_acb = acb;
+				acb->acb_next = hdr->b_l1hdr.b_acb;
+				hdr->b_l1hdr.b_acb = acb;
 				add_reference(hdr, hash_lock, private);
 				mutex_exit(hash_lock);
 				return (0);
@@ -3040,7 +3319,8 @@ top:
 			return (0);
 		}
 
-		ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
+		ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
+		    hdr->b_l1hdr.b_state == arc_mfu);
 
 		if (done) {
 			add_reference(hdr, hash_lock, private);
@@ -3049,7 +3329,7 @@ top:
 			 * copy of the data so that we will be guaranteed
 			 * that arc_release() will always succeed.
 			 */
-			buf = hdr->b_buf;
+			buf = hdr->b_l1hdr.b_buf;
 			ASSERT(buf);
 			ASSERT(buf->b_data);
 			if (HDR_BUF_AVAILABLE(hdr)) {
@@ -3060,7 +3340,7 @@ top:
 			}
 
 		} else if (*arc_flags & ARC_FLAG_PREFETCH &&
-		    refcount_count(&hdr->b_refcnt) == 0) {
+		    refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
 			hdr->b_flags |= ARC_FLAG_PREFETCH;
 		}
 		DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
@@ -3071,8 +3351,8 @@ top:
 			hdr->b_flags |= ARC_FLAG_L2COMPRESS;
 		mutex_exit(hash_lock);
 		ARCSTAT_BUMP(arcstat_hits);
-		ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_FLAG_PREFETCH),
-		    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
+		ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
+		    demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
 		    data, metadata, hits);
 
 		if (done)
@@ -3084,7 +3364,7 @@ top:
 		uint64_t addr = 0;
 		boolean_t devw = B_FALSE;
 		enum zio_compress b_compress = ZIO_COMPRESS_OFF;
-		uint64_t b_asize = 0;
+		int32_t b_asize = 0;
 
 		if (hdr == NULL) {
 			/* this block is not in the cache */
@@ -3095,7 +3375,6 @@ top:
 			if (!BP_IS_EMBEDDED(bp)) {
 				hdr->b_dva = *BP_IDENTITY(bp);
 				hdr->b_birth = BP_PHYSICAL_BIRTH(bp);
-				hdr->b_cksum0 = bp->blk_cksum.zc_word[0];
 				exists = buf_hash_insert(hdr, &hash_lock);
 			}
 			if (exists != NULL) {
@@ -3119,11 +3398,20 @@ top:
 			if (BP_GET_LEVEL(bp) > 0)
 				hdr->b_flags |= ARC_FLAG_INDIRECT;
 		} else {
-			/* this block is in the ghost cache */
-			ASSERT(GHOST_STATE(hdr->b_state));
+			/*
+			 * This block is in the ghost cache. If it was L2-only
+			 * (and thus didn't have an L1 hdr), we realloc the
+			 * header to add an L1 hdr.
+			 */
+			if (!HDR_HAS_L1HDR(hdr)) {
+				hdr = arc_hdr_realloc(hdr, hdr_l2only_cache,
+				    hdr_full_cache);
+			}
+
+			ASSERT(GHOST_STATE(hdr->b_l1hdr.b_state));
 			ASSERT(!HDR_IO_IN_PROGRESS(hdr));
-			ASSERT0(refcount_count(&hdr->b_refcnt));
-			ASSERT(hdr->b_buf == NULL);
+			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
+			ASSERT(hdr->b_l1hdr.b_buf == NULL);
 
 			/* if this is a prefetch, we don't have a reference */
 			if (*arc_flags & ARC_FLAG_PREFETCH)
@@ -3140,29 +3428,29 @@ top:
 			buf->b_efunc = NULL;
 			buf->b_private = NULL;
 			buf->b_next = NULL;
-			hdr->b_buf = buf;
-			ASSERT(hdr->b_datacnt == 0);
-			hdr->b_datacnt = 1;
+			hdr->b_l1hdr.b_buf = buf;
+			ASSERT0(hdr->b_l1hdr.b_datacnt);
+			hdr->b_l1hdr.b_datacnt = 1;
 			arc_get_data_buf(buf);
 			arc_access(hdr, hash_lock);
 		}
 
-		ASSERT(!GHOST_STATE(hdr->b_state));
+		ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state));
 
 		acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
 		acb->acb_done = done;
 		acb->acb_private = private;
 
-		ASSERT(hdr->b_acb == NULL);
-		hdr->b_acb = acb;
+		ASSERT(hdr->b_l1hdr.b_acb == NULL);
+		hdr->b_l1hdr.b_acb = acb;
 		hdr->b_flags |= ARC_FLAG_IO_IN_PROGRESS;
 
-		if (hdr->b_l2hdr != NULL &&
-		    (vd = hdr->b_l2hdr->b_dev->l2ad_vdev) != NULL) {
-			devw = hdr->b_l2hdr->b_dev->l2ad_writing;
-			addr = hdr->b_l2hdr->b_daddr;
-			b_compress = hdr->b_l2hdr->b_compress;
-			b_asize = hdr->b_l2hdr->b_asize;
+		if (HDR_HAS_L2HDR(hdr) &&
+		    (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) {
+			devw = hdr->b_l2hdr.b_dev->l2ad_writing;
+			addr = hdr->b_l2hdr.b_daddr;
+			b_compress = HDR_GET_COMPRESS(hdr);
+			b_asize = hdr->b_l2hdr.b_asize;
 			/*
 			 * Lock out device removal.
 			 */
@@ -3182,8 +3470,8 @@ top:
 		DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp,
 		    uint64_t, size, zbookmark_phys_t *, zb);
 		ARCSTAT_BUMP(arcstat_misses);
-		ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_FLAG_PREFETCH),
-		    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
+		ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
+		    demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
 		    data, metadata, misses);
 
 		if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) {
@@ -3196,7 +3484,7 @@ top:
 			 *    also have invalidated the vdev.
 			 * 5. This isn't prefetch and l2arc_noprefetch is set.
 			 */
-			if (hdr->b_l2hdr != NULL &&
+			if (HDR_HAS_L2HDR(hdr) &&
 			    !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) &&
 			    !(l2arc_noprefetch && HDR_PREFETCH(hdr))) {
 				l2arc_read_callback_t *cb;
@@ -3288,8 +3576,9 @@ void
 arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private)
 {
 	ASSERT(buf->b_hdr != NULL);
-	ASSERT(buf->b_hdr->b_state != arc_anon);
-	ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt) || func == NULL);
+	ASSERT(buf->b_hdr->b_l1hdr.b_state != arc_anon);
+	ASSERT(!refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt) ||
+	    func == NULL);
 	ASSERT(buf->b_efunc == NULL);
 	ASSERT(!HDR_BUF_AVAILABLE(buf->b_hdr));
 
@@ -3313,7 +3602,7 @@ arc_freed(spa_t *spa, const blkptr_t *bp)
 	if (hdr == NULL)
 		return;
 	if (HDR_BUF_AVAILABLE(hdr)) {
-		arc_buf_t *buf = hdr->b_buf;
+		arc_buf_t *buf = hdr->b_l1hdr.b_buf;
 		add_reference(hdr, hash_lock, FTAG);
 		hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE;
 		mutex_exit(hash_lock);
@@ -3371,17 +3660,19 @@ arc_clear_callback(arc_buf_t *buf)
 	hdr = buf->b_hdr;
 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
 
-	ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt);
-	ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
+	ASSERT3U(refcount_count(&hdr->b_l1hdr.b_refcnt), <,
+	    hdr->b_l1hdr.b_datacnt);
+	ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
+	    hdr->b_l1hdr.b_state == arc_mfu);
 
 	buf->b_efunc = NULL;
 	buf->b_private = NULL;
 
-	if (hdr->b_datacnt > 1) {
+	if (hdr->b_l1hdr.b_datacnt > 1) {
 		mutex_exit(&buf->b_evict_lock);
 		arc_buf_destroy(buf, FALSE, TRUE);
 	} else {
-		ASSERT(buf == hdr->b_buf);
+		ASSERT(buf == hdr->b_l1hdr.b_buf);
 		hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
 		mutex_exit(&buf->b_evict_lock);
 	}
@@ -3400,10 +3691,7 @@ arc_clear_callback(arc_buf_t *buf)
 void
 arc_release(arc_buf_t *buf, void *tag)
 {
-	arc_buf_hdr_t *hdr;
-	kmutex_t *hash_lock = NULL;
-	l2arc_buf_hdr_t *l2hdr;
-	uint64_t buf_size;
+	arc_buf_hdr_t *hdr = buf->b_hdr;
 
 	/*
 	 * It would be nice to assert that if it's DMU metadata (level >
@@ -3412,56 +3700,87 @@ arc_release(arc_buf_t *buf, void *tag)
 	 */
 
 	mutex_enter(&buf->b_evict_lock);
-	hdr = buf->b_hdr;
+
+	/*
+	 * We don't grab the hash lock prior to this check, because if
+	 * the buffer's header is in the arc_anon state, it won't be
+	 * linked into the hash table.
+	 */
+	if (hdr->b_l1hdr.b_state == arc_anon) {
+		mutex_exit(&buf->b_evict_lock);
+		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
+		ASSERT(!HDR_IN_HASH_TABLE(hdr));
+		ASSERT(!HDR_HAS_L2HDR(hdr));
+		ASSERT(BUF_EMPTY(hdr));
+
+		ASSERT3U(hdr->b_l1hdr.b_datacnt, ==, 1);
+		ASSERT3S(refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1);
+		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
+
+		ASSERT3P(buf->b_efunc, ==, NULL);
+		ASSERT3P(buf->b_private, ==, NULL);
+
+		hdr->b_l1hdr.b_arc_access = 0;
+		arc_buf_thaw(buf);
+
+		return;
+	}
+
+	kmutex_t *hash_lock = HDR_LOCK(hdr);
+	mutex_enter(hash_lock);
+
+	/*
+	 * This assignment is only valid as long as the hash_lock is
+	 * held, we must be careful not to reference state or the
+	 * b_state field after dropping the lock.
+	 */
+	arc_state_t *state = hdr->b_l1hdr.b_state;
+	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
+	ASSERT3P(state, !=, arc_anon);
 
 	/* this buffer is not on any list */
-	ASSERT(refcount_count(&hdr->b_refcnt) > 0);
+	ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) > 0);
 
-	if (hdr->b_state == arc_anon) {
-		/* this buffer is already released */
-		ASSERT(buf->b_efunc == NULL);
-	} else {
-		hash_lock = HDR_LOCK(hdr);
-		mutex_enter(hash_lock);
-		hdr = buf->b_hdr;
-		ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
-	}
+	if (HDR_HAS_L2HDR(hdr)) {
+		ARCSTAT_INCR(arcstat_l2_asize, -hdr->b_l2hdr.b_asize);
+		ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
 
-	l2hdr = hdr->b_l2hdr;
-	if (l2hdr) {
-		mutex_enter(&l2arc_buflist_mtx);
-		hdr->b_l2hdr = NULL;
-		list_remove(l2hdr->b_dev->l2ad_buflist, hdr);
+		mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);
+		list_remove(&hdr->b_l2hdr.b_dev->l2ad_buflist, hdr);
+		mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);
+
+		hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
 	}
-	buf_size = hdr->b_size;
 
 	/*
 	 * Do we have more than one buf?
 	 */
-	if (hdr->b_datacnt > 1) {
+	if (hdr->b_l1hdr.b_datacnt > 1) {
 		arc_buf_hdr_t *nhdr;
 		arc_buf_t **bufp;
 		uint64_t blksz = hdr->b_size;
 		uint64_t spa = hdr->b_spa;
-		arc_buf_contents_t type = hdr->b_type;
+		arc_buf_contents_t type = arc_buf_type(hdr);
 		uint32_t flags = hdr->b_flags;
 
-		ASSERT(hdr->b_buf != buf || buf->b_next != NULL);
+		ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL);
 		/*
 		 * Pull the data off of this hdr and attach it to
 		 * a new anonymous hdr.
 		 */
 		(void) remove_reference(hdr, hash_lock, tag);
-		bufp = &hdr->b_buf;
+		bufp = &hdr->b_l1hdr.b_buf;
 		while (*bufp != buf)
 			bufp = &(*bufp)->b_next;
 		*bufp = buf->b_next;
 		buf->b_next = NULL;
 
-		ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size);
-		atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size);
-		if (refcount_is_zero(&hdr->b_refcnt)) {
-			uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type];
+		ASSERT3P(state, !=, arc_l2c_only);
+		ASSERT3U(state->arcs_size, >=, hdr->b_size);
+		atomic_add_64(&state->arcs_size, -hdr->b_size);
+		if (refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
+			ASSERT3P(state, !=, arc_l2c_only);
+			uint64_t *size = &state->arcs_lsize[type];
 			ASSERT3U(*size, >=, hdr->b_size);
 			atomic_add_64(size, -hdr->b_size);
 		}
@@ -3470,57 +3789,50 @@ arc_release(arc_buf_t *buf, void *tag)
 		 * We're releasing a duplicate user data buffer, update
 		 * our statistics accordingly.
 		 */
-		if (hdr->b_type == ARC_BUFC_DATA) {
+		if (HDR_ISTYPE_DATA(hdr)) {
 			ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
 			ARCSTAT_INCR(arcstat_duplicate_buffers_size,
 			    -hdr->b_size);
 		}
-		hdr->b_datacnt -= 1;
+		hdr->b_l1hdr.b_datacnt -= 1;
 		arc_cksum_verify(buf);
 		arc_buf_unwatch(buf);
 
 		mutex_exit(hash_lock);
 
-		nhdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
+		nhdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
 		nhdr->b_size = blksz;
 		nhdr->b_spa = spa;
-		nhdr->b_type = type;
-		nhdr->b_buf = buf;
-		nhdr->b_state = arc_anon;
-		nhdr->b_arc_access = 0;
+
 		nhdr->b_flags = flags & ARC_FLAG_L2_WRITING;
-		nhdr->b_l2hdr = NULL;
-		nhdr->b_datacnt = 1;
+		nhdr->b_flags |= arc_bufc_to_flags(type);
+		nhdr->b_flags |= ARC_FLAG_HAS_L1HDR;
+
+		nhdr->b_l1hdr.b_buf = buf;
+		nhdr->b_l1hdr.b_datacnt = 1;
+		nhdr->b_l1hdr.b_state = arc_anon;
+		nhdr->b_l1hdr.b_arc_access = 0;
 		nhdr->b_freeze_cksum = NULL;
-		(void) refcount_add(&nhdr->b_refcnt, tag);
+
+		(void) refcount_add(&nhdr->b_l1hdr.b_refcnt, tag);
 		buf->b_hdr = nhdr;
 		mutex_exit(&buf->b_evict_lock);
 		atomic_add_64(&arc_anon->arcs_size, blksz);
 	} else {
 		mutex_exit(&buf->b_evict_lock);
-		ASSERT(refcount_count(&hdr->b_refcnt) == 1);
-		ASSERT(!list_link_active(&hdr->b_arc_node));
+		ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
+		/* protected by hash lock */
+		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
-		if (hdr->b_state != arc_anon)
-			arc_change_state(arc_anon, hdr, hash_lock);
-		hdr->b_arc_access = 0;
-		if (hash_lock)
-			mutex_exit(hash_lock);
+		arc_change_state(arc_anon, hdr, hash_lock);
+		hdr->b_l1hdr.b_arc_access = 0;
+		mutex_exit(hash_lock);
 
 		buf_discard_identity(hdr);
 		arc_buf_thaw(buf);
 	}
 	buf->b_efunc = NULL;
 	buf->b_private = NULL;
-
-	if (l2hdr) {
-		ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
-		vdev_space_update(l2hdr->b_dev->l2ad_vdev,
-		    -l2hdr->b_asize, 0, 0);
-		kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t));
-		ARCSTAT_INCR(arcstat_l2_size, -buf_size);
-		mutex_exit(&l2arc_buflist_mtx);
-	}
 }
 
 int
@@ -3529,7 +3841,8 @@ arc_released(arc_buf_t *buf)
 	int released;
 
 	mutex_enter(&buf->b_evict_lock);
-	released = (buf->b_data != NULL && buf->b_hdr->b_state == arc_anon);
+	released = (buf->b_data != NULL &&
+	    buf->b_hdr->b_l1hdr.b_state == arc_anon);
 	mutex_exit(&buf->b_evict_lock);
 	return (released);
 }
@@ -3541,7 +3854,7 @@ arc_referenced(arc_buf_t *buf)
 	int referenced;
 
 	mutex_enter(&buf->b_evict_lock);
-	referenced = (refcount_count(&buf->b_hdr->b_refcnt));
+	referenced = (refcount_count(&buf->b_hdr->b_l1hdr.b_refcnt));
 	mutex_exit(&buf->b_evict_lock);
 	return (referenced);
 }
@@ -3554,7 +3867,9 @@ arc_write_ready(zio_t *zio)
 	arc_buf_t *buf = callback->awcb_buf;
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 
-	ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt));
+	ASSERT(HDR_HAS_L1HDR(hdr));
+	ASSERT(!refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt));
+	ASSERT(hdr->b_l1hdr.b_datacnt > 0);
 	callback->awcb_ready(zio, buf, callback->awcb_private);
 
 	/*
@@ -3564,12 +3879,12 @@ arc_write_ready(zio_t *zio)
 	 * accounting for any re-write attempt.
 	 */
 	if (HDR_IO_IN_PROGRESS(hdr)) {
-		mutex_enter(&hdr->b_freeze_lock);
+		mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
 		if (hdr->b_freeze_cksum != NULL) {
 			kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
 			hdr->b_freeze_cksum = NULL;
 		}
-		mutex_exit(&hdr->b_freeze_lock);
+		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
 	}
 	arc_cksum_compute(buf, B_FALSE);
 	hdr->b_flags |= ARC_FLAG_IO_IN_PROGRESS;
@@ -3594,7 +3909,7 @@ arc_write_done(zio_t *zio)
 	arc_buf_t *buf = callback->awcb_buf;
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 
-	ASSERT(hdr->b_acb == NULL);
+	ASSERT(hdr->b_l1hdr.b_acb == NULL);
 
 	if (zio->io_error == 0) {
 		if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) {
@@ -3602,7 +3917,6 @@ arc_write_done(zio_t *zio)
 		} else {
 			hdr->b_dva = *BP_IDENTITY(zio->io_bp);
 			hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp);
-			hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0];
 		}
 	} else {
 		ASSERT(BUF_EMPTY(hdr));
@@ -3623,7 +3937,7 @@ arc_write_done(zio_t *zio)
 		arc_cksum_verify(buf);
 
 		exists = buf_hash_insert(hdr, &hash_lock);
-		if (exists) {
+		if (exists != NULL) {
 			/*
 			 * This can only happen if we overwrite for
 			 * sync-to-convergence, because we remove
@@ -3633,7 +3947,8 @@ arc_write_done(zio_t *zio)
 				if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
 					panic("bad overwrite, hdr=%p exists=%p",
 					    (void *)hdr, (void *)exists);
-				ASSERT(refcount_is_zero(&exists->b_refcnt));
+				ASSERT(refcount_is_zero(
+				    &exists->b_l1hdr.b_refcnt));
 				arc_change_state(arc_anon, exists, hash_lock);
 				mutex_exit(hash_lock);
 				arc_hdr_destroy(exists);
@@ -3647,22 +3962,22 @@ arc_write_done(zio_t *zio)
 					    (void *)hdr, (void *)exists);
 			} else {
 				/* Dedup */
-				ASSERT(hdr->b_datacnt == 1);
-				ASSERT(hdr->b_state == arc_anon);
+				ASSERT(hdr->b_l1hdr.b_datacnt == 1);
+				ASSERT(hdr->b_l1hdr.b_state == arc_anon);
 				ASSERT(BP_GET_DEDUP(zio->io_bp));
 				ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
 			}
 		}
 		hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS;
 		/* if it's not anon, we are doing a scrub */
-		if (!exists && hdr->b_state == arc_anon)
+		if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon)
 			arc_access(hdr, hash_lock);
 		mutex_exit(hash_lock);
 	} else {
 		hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS;
 	}
 
-	ASSERT(!refcount_is_zero(&hdr->b_refcnt));
+	ASSERT(!refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
 	callback->awcb_done(zio, buf, callback->awcb_private);
 
 	kmem_free(callback, sizeof (arc_write_callback_t));
@@ -3682,8 +3997,9 @@ arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
 	ASSERT(ready != NULL);
 	ASSERT(done != NULL);
 	ASSERT(!HDR_IO_ERROR(hdr));
-	ASSERT((hdr->b_flags & ARC_FLAG_IO_IN_PROGRESS) == 0);
-	ASSERT(hdr->b_acb == NULL);
+	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
+	ASSERT(hdr->b_l1hdr.b_acb == NULL);
+	ASSERT(hdr->b_l1hdr.b_datacnt > 0);
 	if (l2arc)
 		hdr->b_flags |= ARC_FLAG_L2CACHE;
 	if (l2arc_compress)
@@ -3903,25 +4219,35 @@ arc_init(void)
 	mutex_init(&arc_l2c_only->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
 
 	list_create(&arc_mru->arcs_list[ARC_BUFC_METADATA],
-	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
+	    sizeof (arc_buf_hdr_t),
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
 	list_create(&arc_mru->arcs_list[ARC_BUFC_DATA],
-	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
+	    sizeof (arc_buf_hdr_t),
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
 	list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
-	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
+	    sizeof (arc_buf_hdr_t),
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
 	list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
-	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
+	    sizeof (arc_buf_hdr_t),
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
 	list_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
-	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
+	    sizeof (arc_buf_hdr_t),
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
 	list_create(&arc_mfu->arcs_list[ARC_BUFC_DATA],
-	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
+	    sizeof (arc_buf_hdr_t),
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
 	list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
-	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
+	    sizeof (arc_buf_hdr_t),
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
 	list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
-	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
+	    sizeof (arc_buf_hdr_t),
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
 	list_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
-	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
+	    sizeof (arc_buf_hdr_t),
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
 	list_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
-	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
+	    sizeof (arc_buf_hdr_t),
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
 
 	buf_init();
 
@@ -4000,7 +4326,7 @@ arc_fini(void)
 
 	buf_fini();
 
-	ASSERT(arc_loaned_bytes == 0);
+	ASSERT0(arc_loaned_bytes);
 }
 
 /*
@@ -4158,7 +4484,7 @@ l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *hdr)
 	 * 3. has an I/O in progress (it may be an incomplete read).
 	 * 4. is flagged not eligible (zfs property).
 	 */
-	if (hdr->b_spa != spa_guid || hdr->b_l2hdr != NULL ||
+	if (hdr->b_spa != spa_guid || HDR_HAS_L2HDR(hdr) ||
 	    HDR_IO_IN_PROGRESS(hdr) || !HDR_L2CACHE(hdr))
 		return (B_FALSE);
 
@@ -4211,20 +4537,6 @@ l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote)
 	return (next);
 }
 
-static void
-l2arc_hdr_stat_add(void)
-{
-	ARCSTAT_INCR(arcstat_l2_hdr_size, HDR_SIZE + L2HDR_SIZE);
-	ARCSTAT_INCR(arcstat_hdr_size, -HDR_SIZE);
-}
-
-static void
-l2arc_hdr_stat_remove(void)
-{
-	ARCSTAT_INCR(arcstat_l2_hdr_size, -(HDR_SIZE + L2HDR_SIZE));
-	ARCSTAT_INCR(arcstat_hdr_size, HDR_SIZE);
-}
-
 /*
  * Cycle through L2ARC devices.  This is how L2ARC load balances.
  * If a device is returned, this also returns holding the spa config lock.
@@ -4321,7 +4633,6 @@ l2arc_write_done(zio_t *zio)
 	l2arc_dev_t *dev;
 	list_t *buflist;
 	arc_buf_hdr_t *head, *hdr, *hdr_prev;
-	l2arc_buf_hdr_t *abl2;
 	kmutex_t *hash_lock;
 	int64_t bytes_dropped = 0;
 
@@ -4331,7 +4642,7 @@ l2arc_write_done(zio_t *zio)
 	ASSERT(dev != NULL);
 	head = cb->l2wcb_head;
 	ASSERT(head != NULL);
-	buflist = dev->l2ad_buflist;
+	buflist = &dev->l2ad_buflist;
 	ASSERT(buflist != NULL);
 	DTRACE_PROBE2(l2arc__iodone, zio_t *, zio,
 	    l2arc_write_callback_t *, cb);
@@ -4339,41 +4650,43 @@ l2arc_write_done(zio_t *zio)
 	if (zio->io_error != 0)
 		ARCSTAT_BUMP(arcstat_l2_writes_error);
 
-	mutex_enter(&l2arc_buflist_mtx);
+	mutex_enter(&dev->l2ad_mtx);
 
 	/*
 	 * All writes completed, or an error was hit.
 	 */
 	for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) {
 		hdr_prev = list_prev(buflist, hdr);
-		abl2 = hdr->b_l2hdr;
-
-		/*
-		 * Release the temporary compressed buffer as soon as possible.
-		 */
-		if (abl2->b_compress != ZIO_COMPRESS_OFF)
-			l2arc_release_cdata_buf(hdr);
 
 		hash_lock = HDR_LOCK(hdr);
 		if (!mutex_tryenter(hash_lock)) {
 			/*
 			 * This buffer misses out.  It may be in a stage
-			 * of eviction.  Its ARC_L2_WRITING flag will be
+			 * of eviction.  Its ARC_FLAG_L2_WRITING flag will be
 			 * left set, denying reads to this buffer.
 			 */
 			ARCSTAT_BUMP(arcstat_l2_writes_hdr_miss);
 			continue;
 		}
 
+		/*
+		 * It's possible that this buffer got evicted from the L1 cache
+		 * before we grabbed the vdev + hash locks, in which case
+		 * arc_hdr_realloc freed b_tmp_cdata for us if it was allocated.
+		 * Only free the buffer if we still have an L1 hdr.
+		 */
+		if (HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_tmp_cdata != NULL &&
+		    HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF)
+			l2arc_release_cdata_buf(hdr);
+
 		if (zio->io_error != 0) {
 			/*
 			 * Error - drop L2ARC entry.
 			 */
 			list_remove(buflist, hdr);
-			ARCSTAT_INCR(arcstat_l2_asize, -abl2->b_asize);
-			bytes_dropped += abl2->b_asize;
-			hdr->b_l2hdr = NULL;
-			kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
+			hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
+
+			ARCSTAT_INCR(arcstat_l2_asize, -hdr->b_l2hdr.b_asize);
 			ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
 		}
 
@@ -4387,8 +4700,9 @@ l2arc_write_done(zio_t *zio)
 
 	atomic_inc_64(&l2arc_writes_done);
 	list_remove(buflist, head);
-	kmem_cache_free(hdr_cache, head);
-	mutex_exit(&l2arc_buflist_mtx);
+	ASSERT(!HDR_HAS_L1HDR(head));
+	kmem_cache_free(hdr_l2only_cache, head);
+	mutex_exit(&dev->l2ad_mtx);
 
 	vdev_space_update(dev->l2ad_vdev, -bytes_dropped, 0, 0);
 
@@ -4526,16 +4840,12 @@ static void
 l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
 {
 	list_t *buflist;
-	l2arc_buf_hdr_t *abl2;
 	arc_buf_hdr_t *hdr, *hdr_prev;
 	kmutex_t *hash_lock;
 	uint64_t taddr;
 	int64_t bytes_evicted = 0;
 
-	buflist = dev->l2ad_buflist;
-
-	if (buflist == NULL)
-		return;
+	buflist = &dev->l2ad_buflist;
 
 	if (!all && dev->l2ad_first) {
 		/*
@@ -4558,7 +4868,7 @@ l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
 	    uint64_t, taddr, boolean_t, all);
 
 top:
-	mutex_enter(&l2arc_buflist_mtx);
+	mutex_enter(&dev->l2ad_mtx);
 	for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) {
 		hdr_prev = list_prev(buflist, hdr);
 
@@ -4568,7 +4878,7 @@ top:
 			 * Missed the hash lock.  Retry.
 			 */
 			ARCSTAT_BUMP(arcstat_l2_evict_lock_retry);
-			mutex_exit(&l2arc_buflist_mtx);
+			mutex_exit(&dev->l2ad_mtx);
 			mutex_enter(hash_lock);
 			mutex_exit(hash_lock);
 			goto top;
@@ -4584,9 +4894,9 @@ top:
 			continue;
 		}
 
-		if (!all && hdr->b_l2hdr != NULL &&
-		    (hdr->b_l2hdr->b_daddr > taddr ||
-		    hdr->b_l2hdr->b_daddr < dev->l2ad_hand)) {
+		if (!all && HDR_HAS_L2HDR(hdr) &&
+		    (hdr->b_l2hdr.b_daddr > taddr ||
+		    hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) {
 			/*
 			 * We've evicted to the target address,
 			 * or the end of the device.
@@ -4595,15 +4905,8 @@ top:
 			break;
 		}
 
-		if (HDR_FREE_IN_PROGRESS(hdr)) {
-			/*
-			 * Already on the path to destruction.
-			 */
-			mutex_exit(hash_lock);
-			continue;
-		}
-
-		if (hdr->b_state == arc_l2c_only) {
+		ASSERT(HDR_HAS_L2HDR(hdr));
+		if (!HDR_HAS_L1HDR(hdr)) {
 			ASSERT(!HDR_L2_READING(hdr));
 			/*
 			 * This doesn't exist in the ARC.  Destroy.
@@ -4613,6 +4916,8 @@ top:
 			arc_change_state(arc_anon, hdr, hash_lock);
 			arc_hdr_destroy(hdr);
 		} else {
+			ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only);
+			ARCSTAT_BUMP(arcstat_l2_evict_l1cached);
 			/*
 			 * Invalidate issued or about to be issued
 			 * reads, since we may be about to write
@@ -4623,28 +4928,18 @@ top:
 				hdr->b_flags |= ARC_FLAG_L2_EVICTED;
 			}
 
-			/*
-			 * Tell ARC this no longer exists in L2ARC.
-			 */
-			if (hdr->b_l2hdr != NULL) {
-				abl2 = hdr->b_l2hdr;
-				ARCSTAT_INCR(arcstat_l2_asize, -abl2->b_asize);
-				bytes_evicted += abl2->b_asize;
-				hdr->b_l2hdr = NULL;
-				kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
-				ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
-			}
+			/* Tell ARC this no longer exists in L2ARC. */
+			ARCSTAT_INCR(arcstat_l2_asize, -hdr->b_l2hdr.b_asize);
+			ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
+			hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
 			list_remove(buflist, hdr);
 
-			/*
-			 * This may have been leftover after a
-			 * failed write.
-			 */
+			/* This may have been leftover after a failed write. */
 			hdr->b_flags &= ~ARC_FLAG_L2_WRITING;
 		}
 		mutex_exit(hash_lock);
 	}
-	mutex_exit(&l2arc_buflist_mtx);
+	mutex_exit(&dev->l2ad_mtx);
 
 	vdev_space_update(dev->l2ad_vdev, -bytes_evicted, 0, 0);
 	dev->l2ad_evict = taddr;
@@ -4685,8 +4980,9 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 	pio = NULL;
 	write_sz = write_asize = write_psize = 0;
 	full = B_FALSE;
-	head = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
+	head = kmem_cache_alloc(hdr_l2only_cache, KM_PUSHPAGE);
 	head->b_flags |= ARC_FLAG_L2_WRITE_HEAD;
+	head->b_flags |= ARC_FLAG_HAS_L2HDR;
 
 	/*
 	 * We will want to try to compress buffers that are at least 2x the
@@ -4697,7 +4993,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 	/*
 	 * Copy buffers for L2ARC writing.
 	 */
-	mutex_enter(&l2arc_buflist_mtx);
+	mutex_enter(&dev->l2ad_mtx);
 	for (int try = 0; try <= 3; try++) {
 		uint64_t passed_sz = 0;
 
@@ -4719,7 +5015,6 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 			headroom = (headroom * l2arc_headroom_boost) / 100;
 
 		for (; hdr; hdr = hdr_prev) {
-			l2arc_buf_hdr_t *l2hdr;
 			kmutex_t *hash_lock;
 			uint64_t buf_sz;
 
@@ -4762,7 +5057,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 				 * l2arc_write_done() can find where the
 				 * write buffers begin without searching.
 				 */
-				list_insert_head(dev->l2ad_buflist, head);
+				list_insert_head(&dev->l2ad_buflist, head);
 
 				cb = kmem_alloc(
 				    sizeof (l2arc_write_callback_t), KM_SLEEP);
@@ -4775,33 +5070,31 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 			/*
 			 * Create and add a new L2ARC header.
 			 */
-			l2hdr = kmem_zalloc(sizeof (l2arc_buf_hdr_t), KM_SLEEP);
-			l2hdr->b_dev = dev;
+			hdr->b_l2hdr.b_dev = dev;
 			hdr->b_flags |= ARC_FLAG_L2_WRITING;
-
 			/*
 			 * Temporarily stash the data buffer in b_tmp_cdata.
 			 * The subsequent write step will pick it up from
-			 * there. This is because can't access hdr->b_buf
+			 * there. This is because can't access b_l1hdr.b_buf
 			 * without holding the hash_lock, which we in turn
 			 * can't access without holding the ARC list locks
 			 * (which we want to avoid during compression/writing).
 			 */
-			l2hdr->b_compress = ZIO_COMPRESS_OFF;
-			l2hdr->b_asize = hdr->b_size;
-			l2hdr->b_tmp_cdata = hdr->b_buf->b_data;
+			HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_OFF);
+			hdr->b_l2hdr.b_asize = hdr->b_size;
+			hdr->b_l1hdr.b_tmp_cdata = hdr->b_l1hdr.b_buf->b_data;
 
 			buf_sz = hdr->b_size;
-			hdr->b_l2hdr = l2hdr;
+			hdr->b_flags |= ARC_FLAG_HAS_L2HDR;
 
-			list_insert_head(dev->l2ad_buflist, hdr);
+			list_insert_head(&dev->l2ad_buflist, hdr);
 
 			/*
 			 * Compute and store the buffer cksum before
 			 * writing.  On debug the cksum is verified first.
 			 */
-			arc_cksum_verify(hdr->b_buf);
-			arc_cksum_compute(hdr->b_buf, B_TRUE);
+			arc_cksum_verify(hdr->b_l1hdr.b_buf);
+			arc_cksum_compute(hdr->b_l1hdr.b_buf, B_TRUE);
 
 			mutex_exit(hash_lock);
 
@@ -4817,8 +5110,9 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 	/* No buffers selected for writing? */
 	if (pio == NULL) {
 		ASSERT0(write_sz);
-		mutex_exit(&l2arc_buflist_mtx);
-		kmem_cache_free(hdr_cache, head);
+		mutex_exit(&dev->l2ad_mtx);
+		ASSERT(!HDR_HAS_L1HDR(head));
+		kmem_cache_free(hdr_l2only_cache, head);
 		return (0);
 	}
 
@@ -4827,24 +5121,22 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 	 * and work backwards, retracing the course of the buffer selector
 	 * loop above.
 	 */
-	for (hdr = list_prev(dev->l2ad_buflist, head); hdr;
-	    hdr = list_prev(dev->l2ad_buflist, hdr)) {
-		l2arc_buf_hdr_t *l2hdr;
+	for (hdr = list_prev(&dev->l2ad_buflist, head); hdr;
+	    hdr = list_prev(&dev->l2ad_buflist, hdr)) {
 		uint64_t buf_sz;
 
 		/*
 		 * We shouldn't need to lock the buffer here, since we flagged
 		 * it as ARC_FLAG_L2_WRITING in the previous step, but we must
 		 * take care to only access its L2 cache parameters. In
-		 * particular, hdr->b_buf may be invalid by now due to
+		 * particular, hdr->l1hdr.b_buf may be invalid by now due to
 		 * ARC eviction.
 		 */
-		l2hdr = hdr->b_l2hdr;
-		l2hdr->b_daddr = dev->l2ad_hand;
+		hdr->b_l2hdr.b_daddr = dev->l2ad_hand;
 
-		if ((hdr->b_flags & ARC_FLAG_L2COMPRESS) &&
-		    l2hdr->b_asize >= buf_compress_minsz) {
-			if (l2arc_compress_buf(l2hdr)) {
+		if ((HDR_L2COMPRESS(hdr)) &&
+		    hdr->b_l2hdr.b_asize >= buf_compress_minsz) {
+			if (l2arc_compress_buf(hdr)) {
 				/*
 				 * If compression succeeded, enable headroom
 				 * boost on the next scan cycle.
@@ -4857,8 +5149,8 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 		 * Pick up the buffer data we had previously stashed away
 		 * (and now potentially also compressed).
 		 */
-		buf_data = l2hdr->b_tmp_cdata;
-		buf_sz = l2hdr->b_asize;
+		buf_data = hdr->b_l1hdr.b_tmp_cdata;
+		buf_sz = hdr->b_l2hdr.b_asize;
 
 		/* Compression may have squashed the buffer to zero length. */
 		if (buf_sz != 0) {
@@ -4883,7 +5175,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 		}
 	}
 
-	mutex_exit(&l2arc_buflist_mtx);
+	mutex_exit(&dev->l2ad_mtx);
 
 	ASSERT3U(write_asize, <=, target_sz);
 	ARCSTAT_BUMP(arcstat_l2_writes_sent);
@@ -4911,7 +5203,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 
 /*
  * Compresses an L2ARC buffer.
- * The data to be compressed must be prefilled in l2hdr->b_tmp_cdata and its
+ * The data to be compressed must be prefilled in l1hdr.b_tmp_cdata and its
  * size in l2hdr->b_asize. This routine tries to compress the data and
  * depending on the compression result there are three possible outcomes:
  * *) The buffer was incompressible. The original l2hdr contents were left
@@ -4929,17 +5221,21 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
  * buffer was incompressible).
  */
 static boolean_t
-l2arc_compress_buf(l2arc_buf_hdr_t *l2hdr)
+l2arc_compress_buf(arc_buf_hdr_t *hdr)
 {
 	void *cdata;
 	size_t csize, len, rounded;
+	ASSERT(HDR_HAS_L2HDR(hdr));
+	l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
 
-	ASSERT(l2hdr->b_compress == ZIO_COMPRESS_OFF);
-	ASSERT(l2hdr->b_tmp_cdata != NULL);
+	ASSERT(HDR_HAS_L1HDR(hdr));
+	ASSERT(HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF);
+	ASSERT(hdr->b_l1hdr.b_tmp_cdata != NULL);
 
 	len = l2hdr->b_asize;
 	cdata = zio_data_buf_alloc(len);
-	csize = zio_compress_data(ZIO_COMPRESS_LZ4, l2hdr->b_tmp_cdata,
+	ASSERT3P(cdata, !=, NULL);
+	csize = zio_compress_data(ZIO_COMPRESS_LZ4, hdr->b_l1hdr.b_tmp_cdata,
 	    cdata, l2hdr->b_asize);
 
 	rounded = P2ROUNDUP(csize, (size_t)SPA_MINBLOCKSIZE);
@@ -4951,9 +5247,9 @@ l2arc_compress_buf(l2arc_buf_hdr_t *l2hdr)
 	if (csize == 0) {
 		/* zero block, indicate that there's nothing to write */
 		zio_data_buf_free(cdata, len);
-		l2hdr->b_compress = ZIO_COMPRESS_EMPTY;
+		HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_EMPTY);
 		l2hdr->b_asize = 0;
-		l2hdr->b_tmp_cdata = NULL;
+		hdr->b_l1hdr.b_tmp_cdata = NULL;
 		ARCSTAT_BUMP(arcstat_l2_compress_zeros);
 		return (B_TRUE);
 	} else if (csize > 0 && csize < len) {
@@ -4961,9 +5257,9 @@ l2arc_compress_buf(l2arc_buf_hdr_t *l2hdr)
 		 * Compression succeeded, we'll keep the cdata around for
 		 * writing and release it afterwards.
 		 */
-		l2hdr->b_compress = ZIO_COMPRESS_LZ4;
+		HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_LZ4);
 		l2hdr->b_asize = csize;
-		l2hdr->b_tmp_cdata = cdata;
+		hdr->b_l1hdr.b_tmp_cdata = cdata;
 		ARCSTAT_BUMP(arcstat_l2_compress_successes);
 		return (B_TRUE);
 	} else {
@@ -5008,9 +5304,9 @@ l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr, enum zio_compress c)
 		 * need to fill its io_data after we're done restoring the
 		 * buffer's contents.
 		 */
-		ASSERT(hdr->b_buf != NULL);
-		bzero(hdr->b_buf->b_data, hdr->b_size);
-		zio->io_data = zio->io_orig_data = hdr->b_buf->b_data;
+		ASSERT(hdr->b_l1hdr.b_buf != NULL);
+		bzero(hdr->b_l1hdr.b_buf->b_data, hdr->b_size);
+		zio->io_data = zio->io_orig_data = hdr->b_l1hdr.b_buf->b_data;
 	} else {
 		ASSERT(zio->io_data != NULL);
 		/*
@@ -5048,17 +5344,17 @@ l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr, enum zio_compress c)
 static void
 l2arc_release_cdata_buf(arc_buf_hdr_t *hdr)
 {
-	l2arc_buf_hdr_t *l2hdr = hdr->b_l2hdr;
-
-	if (l2hdr->b_compress == ZIO_COMPRESS_LZ4) {
+	ASSERT(HDR_HAS_L1HDR(hdr));
+	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_EMPTY) {
 		/*
 		 * If the data was compressed, then we've allocated a
 		 * temporary buffer for it, so now we need to release it.
 		 */
-		ASSERT(l2hdr->b_tmp_cdata != NULL);
-		zio_data_buf_free(l2hdr->b_tmp_cdata, hdr->b_size);
+		ASSERT(hdr->b_l1hdr.b_tmp_cdata != NULL);
+		zio_data_buf_free(hdr->b_l1hdr.b_tmp_cdata,
+		    hdr->b_size);
 	}
-	l2hdr->b_tmp_cdata = NULL;
+	hdr->b_l1hdr.b_tmp_cdata = NULL;
 }
 
 /*
@@ -5199,13 +5495,13 @@ l2arc_add_vdev(spa_t *spa, vdev_t *vd)
 	adddev->l2ad_first = B_TRUE;
 	adddev->l2ad_writing = B_FALSE;
 
+	mutex_init(&adddev->l2ad_mtx, NULL, MUTEX_DEFAULT, NULL);
 	/*
 	 * This is a list of all ARC buffers that are still valid on the
 	 * device.
 	 */
-	adddev->l2ad_buflist = kmem_zalloc(sizeof (list_t), KM_SLEEP);
-	list_create(adddev->l2ad_buflist, sizeof (arc_buf_hdr_t),
-	    offsetof(arc_buf_hdr_t, b_l2node));
+	list_create(&adddev->l2ad_buflist, sizeof (arc_buf_hdr_t),
+	    offsetof(arc_buf_hdr_t, b_l2hdr.b_l2node));
 
 	vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand);
 
@@ -5251,8 +5547,8 @@ l2arc_remove_vdev(vdev_t *vd)
 	 * Clear all buflists and ARC references.  L2ARC device flush.
 	 */
 	l2arc_evict(remdev, 0, B_TRUE);
-	list_destroy(remdev->l2ad_buflist);
-	kmem_free(remdev->l2ad_buflist, sizeof (list_t));
+	list_destroy(&remdev->l2ad_buflist);
+	mutex_destroy(&remdev->l2ad_mtx);
 	kmem_free(remdev, sizeof (l2arc_dev_t));
 }
 
@@ -5267,7 +5563,6 @@ l2arc_init(void)
 	mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL);
 	cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL);
 	mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL);
-	mutex_init(&l2arc_buflist_mtx, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL);
 
 	l2arc_dev_list = &L2ARC_dev_list;
@@ -5292,7 +5587,6 @@ l2arc_fini(void)
 	mutex_destroy(&l2arc_feed_thr_lock);
 	cv_destroy(&l2arc_feed_thr_cv);
 	mutex_destroy(&l2arc_dev_mtx);
-	mutex_destroy(&l2arc_buflist_mtx);
 	mutex_destroy(&l2arc_free_on_write_mtx);
 
 	list_destroy(l2arc_dev_list);
diff --git a/uts/common/fs/zfs/sys/arc.h b/uts/common/fs/zfs/sys/arc.h
index 83ea6f7fbc7c..4e9e8e202a7e 100644
--- a/uts/common/fs/zfs/sys/arc.h
+++ b/uts/common/fs/zfs/sys/arc.h
@@ -70,10 +70,29 @@ typedef enum arc_flags
 	ARC_FLAG_FREED_IN_READ		= 1 << 10,	/* freed during read */
 	ARC_FLAG_BUF_AVAILABLE		= 1 << 11,	/* block not in use */
 	ARC_FLAG_INDIRECT		= 1 << 12,	/* indirect block */
-	ARC_FLAG_FREE_IN_PROGRESS	= 1 << 13,	/*  about to be freed */
-	ARC_FLAG_L2_WRITING		= 1 << 14,	/* write in progress */
-	ARC_FLAG_L2_EVICTED		= 1 << 15,	/* evicted during I/O */
-	ARC_FLAG_L2_WRITE_HEAD		= 1 << 16,	/* head of write list */
+	ARC_FLAG_L2_WRITING		= 1 << 13,	/* write in progress */
+	ARC_FLAG_L2_EVICTED		= 1 << 14,	/* evicted during I/O */
+	ARC_FLAG_L2_WRITE_HEAD		= 1 << 15,	/* head of write list */
+	/* indicates that the buffer contains metadata (otherwise, data) */
+	ARC_FLAG_BUFC_METADATA		= 1 << 16,
+
+	/* Flags specifying whether optional hdr struct fields are defined */
+	ARC_FLAG_HAS_L1HDR		= 1 << 17,
+	ARC_FLAG_HAS_L2HDR		= 1 << 18,
+
+	/*
+	 * The arc buffer's compression mode is stored in the top 7 bits of the
+	 * flags field, so these dummy flags are included so that MDB can
+	 * interpret the enum properly.
+	 */
+	ARC_FLAG_COMPRESS_0		= 1 << 24,
+	ARC_FLAG_COMPRESS_1		= 1 << 25,
+	ARC_FLAG_COMPRESS_2		= 1 << 26,
+	ARC_FLAG_COMPRESS_3		= 1 << 27,
+	ARC_FLAG_COMPRESS_4		= 1 << 28,
+	ARC_FLAG_COMPRESS_5		= 1 << 29,
+	ARC_FLAG_COMPRESS_6		= 1 << 30
+
 } arc_flags_t;
 
 struct arc_buf {

From 78ae4cb3787d0bd0ebcc6c4d56d3edf6d9f32043 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Tue, 20 Jan 2015 20:10:03 +0000
Subject: [PATCH 070/380] 5445 Add more visibility via arcstats; specifically
 arc_state_t stats and differentiate between "data" and "metadata" Reviewed
 by: Basil Crow <basil.crow@delphix.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Reviewed by: Matthew Ahrens <mahrens@delphix.com>
 Reviewed by: Bayard Bell <bayard.bell@nexenta.com> Approved by: Robert
 Mustacchi <rm@joyent.com> Author: Prakash Surya <prakash.surya@delphix.com>

illumos/illumos-gate@4076b1bf41cfd9f968a33ed54a7ae76d9e996fe8
---
 uts/common/fs/zfs/arc.c     | 237 +++++++++++++++++++++++++++++++++---
 uts/common/fs/zfs/sys/arc.h |   1 +
 2 files changed, 224 insertions(+), 14 deletions(-)

diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index 012dd83cd7c3..24198579a89d 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -299,9 +299,137 @@ typedef struct arc_stats {
 	kstat_named_t arcstat_c_min;
 	kstat_named_t arcstat_c_max;
 	kstat_named_t arcstat_size;
+	/*
+	 * Number of bytes consumed by internal ARC structures necessary
+	 * for tracking purposes; these structures are not actually
+	 * backed by ARC buffers. This includes arc_buf_hdr_t structures
+	 * (allocated via arc_buf_hdr_t_full and arc_buf_hdr_t_l2only
+	 * caches), and arc_buf_t structures (allocated via arc_buf_t
+	 * cache).
+	 */
 	kstat_named_t arcstat_hdr_size;
+	/*
+	 * Number of bytes consumed by ARC buffers of type equal to
+	 * ARC_BUFC_DATA. This is generally consumed by buffers backing
+	 * on disk user data (e.g. plain file contents).
+	 */
 	kstat_named_t arcstat_data_size;
+	/*
+	 * Number of bytes consumed by ARC buffers of type equal to
+	 * ARC_BUFC_METADATA. This is generally consumed by buffers
+	 * backing on disk data that is used for internal ZFS
+	 * structures (e.g. ZAP, dnode, indirect blocks, etc).
+	 */
+	kstat_named_t arcstat_metadata_size;
+	/*
+	 * Number of bytes consumed by various buffers and structures
+	 * not actually backed with ARC buffers. This includes bonus
+	 * buffers (allocated directly via zio_buf_* functions),
+	 * dmu_buf_impl_t structures (allocated via dmu_buf_impl_t
+	 * cache), and dnode_t structures (allocated via dnode_t cache).
+	 */
 	kstat_named_t arcstat_other_size;
+	/*
+	 * Total number of bytes consumed by ARC buffers residing in the
+	 * arc_anon state. This includes *all* buffers in the arc_anon
+	 * state; e.g. data, metadata, evictable, and unevictable buffers
+	 * are all included in this value.
+	 */
+	kstat_named_t arcstat_anon_size;
+	/*
+	 * Number of bytes consumed by ARC buffers that meet the
+	 * following criteria: backing buffers of type ARC_BUFC_DATA,
+	 * residing in the arc_anon state, and are eligible for eviction
+	 * (e.g. have no outstanding holds on the buffer).
+	 */
+	kstat_named_t arcstat_anon_evictable_data;
+	/*
+	 * Number of bytes consumed by ARC buffers that meet the
+	 * following criteria: backing buffers of type ARC_BUFC_METADATA,
+	 * residing in the arc_anon state, and are eligible for eviction
+	 * (e.g. have no outstanding holds on the buffer).
+	 */
+	kstat_named_t arcstat_anon_evictable_metadata;
+	/*
+	 * Total number of bytes consumed by ARC buffers residing in the
+	 * arc_mru state. This includes *all* buffers in the arc_mru
+	 * state; e.g. data, metadata, evictable, and unevictable buffers
+	 * are all included in this value.
+	 */
+	kstat_named_t arcstat_mru_size;
+	/*
+	 * Number of bytes consumed by ARC buffers that meet the
+	 * following criteria: backing buffers of type ARC_BUFC_DATA,
+	 * residing in the arc_mru state, and are eligible for eviction
+	 * (e.g. have no outstanding holds on the buffer).
+	 */
+	kstat_named_t arcstat_mru_evictable_data;
+	/*
+	 * Number of bytes consumed by ARC buffers that meet the
+	 * following criteria: backing buffers of type ARC_BUFC_METADATA,
+	 * residing in the arc_mru state, and are eligible for eviction
+	 * (e.g. have no outstanding holds on the buffer).
+	 */
+	kstat_named_t arcstat_mru_evictable_metadata;
+	/*
+	 * Total number of bytes that *would have been* consumed by ARC
+	 * buffers in the arc_mru_ghost state. The key thing to note
+	 * here, is the fact that this size doesn't actually indicate
+	 * RAM consumption. The ghost lists only consist of headers and
+	 * don't actually have ARC buffers linked off of these headers.
+	 * Thus, *if* the headers had associated ARC buffers, these
+	 * buffers *would have* consumed this number of bytes.
+	 */
+	kstat_named_t arcstat_mru_ghost_size;
+	/*
+	 * Number of bytes that *would have been* consumed by ARC
+	 * buffers that are eligible for eviction, of type
+	 * ARC_BUFC_DATA, and linked off the arc_mru_ghost state.
+	 */
+	kstat_named_t arcstat_mru_ghost_evictable_data;
+	/*
+	 * Number of bytes that *would have been* consumed by ARC
+	 * buffers that are eligible for eviction, of type
+	 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
+	 */
+	kstat_named_t arcstat_mru_ghost_evictable_metadata;
+	/*
+	 * Total number of bytes consumed by ARC buffers residing in the
+	 * arc_mfu state. This includes *all* buffers in the arc_mfu
+	 * state; e.g. data, metadata, evictable, and unevictable buffers
+	 * are all included in this value.
+	 */
+	kstat_named_t arcstat_mfu_size;
+	/*
+	 * Number of bytes consumed by ARC buffers that are eligible for
+	 * eviction, of type ARC_BUFC_DATA, and reside in the arc_mfu
+	 * state.
+	 */
+	kstat_named_t arcstat_mfu_evictable_data;
+	/*
+	 * Number of bytes consumed by ARC buffers that are eligible for
+	 * eviction, of type ARC_BUFC_METADATA, and reside in the
+	 * arc_mfu state.
+	 */
+	kstat_named_t arcstat_mfu_evictable_metadata;
+	/*
+	 * Total number of bytes that *would have been* consumed by ARC
+	 * buffers in the arc_mfu_ghost state. See the comment above
+	 * arcstat_mru_ghost_size for more details.
+	 */
+	kstat_named_t arcstat_mfu_ghost_size;
+	/*
+	 * Number of bytes that *would have been* consumed by ARC
+	 * buffers that are eligible for eviction, of type
+	 * ARC_BUFC_DATA, and linked off the arc_mfu_ghost state.
+	 */
+	kstat_named_t arcstat_mfu_ghost_evictable_data;
+	/*
+	 * Number of bytes that *would have been* consumed by ARC
+	 * buffers that are eligible for eviction, of type
+	 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
+	 */
+	kstat_named_t arcstat_mfu_ghost_evictable_metadata;
 	kstat_named_t arcstat_l2_hits;
 	kstat_named_t arcstat_l2_misses;
 	kstat_named_t arcstat_l2_feeds;
@@ -369,7 +497,23 @@ static arc_stats_t arc_stats = {
 	{ "size",			KSTAT_DATA_UINT64 },
 	{ "hdr_size",			KSTAT_DATA_UINT64 },
 	{ "data_size",			KSTAT_DATA_UINT64 },
+	{ "metadata_size",		KSTAT_DATA_UINT64 },
 	{ "other_size",			KSTAT_DATA_UINT64 },
+	{ "anon_size",			KSTAT_DATA_UINT64 },
+	{ "anon_evictable_data",	KSTAT_DATA_UINT64 },
+	{ "anon_evictable_metadata",	KSTAT_DATA_UINT64 },
+	{ "mru_size",			KSTAT_DATA_UINT64 },
+	{ "mru_evictable_data",		KSTAT_DATA_UINT64 },
+	{ "mru_evictable_metadata",	KSTAT_DATA_UINT64 },
+	{ "mru_ghost_size",		KSTAT_DATA_UINT64 },
+	{ "mru_ghost_evictable_data",	KSTAT_DATA_UINT64 },
+	{ "mru_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
+	{ "mfu_size",			KSTAT_DATA_UINT64 },
+	{ "mfu_evictable_data",		KSTAT_DATA_UINT64 },
+	{ "mfu_evictable_metadata",	KSTAT_DATA_UINT64 },
+	{ "mfu_ghost_size",		KSTAT_DATA_UINT64 },
+	{ "mfu_ghost_evictable_data",	KSTAT_DATA_UINT64 },
+	{ "mfu_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
 	{ "l2_hits",			KSTAT_DATA_UINT64 },
 	{ "l2_misses",			KSTAT_DATA_UINT64 },
 	{ "l2_feeds",			KSTAT_DATA_UINT64 },
@@ -1500,6 +1644,9 @@ arc_space_consume(uint64_t space, arc_space_type_t type)
 	case ARC_SPACE_DATA:
 		ARCSTAT_INCR(arcstat_data_size, space);
 		break;
+	case ARC_SPACE_META:
+		ARCSTAT_INCR(arcstat_metadata_size, space);
+		break;
 	case ARC_SPACE_OTHER:
 		ARCSTAT_INCR(arcstat_other_size, space);
 		break;
@@ -1511,7 +1658,9 @@ arc_space_consume(uint64_t space, arc_space_type_t type)
 		break;
 	}
 
-	ARCSTAT_INCR(arcstat_meta_used, space);
+	if (type != ARC_SPACE_DATA)
+		ARCSTAT_INCR(arcstat_meta_used, space);
+
 	atomic_add_64(&arc_size, space);
 }
 
@@ -1524,6 +1673,9 @@ arc_space_return(uint64_t space, arc_space_type_t type)
 	case ARC_SPACE_DATA:
 		ARCSTAT_INCR(arcstat_data_size, -space);
 		break;
+	case ARC_SPACE_META:
+		ARCSTAT_INCR(arcstat_metadata_size, -space);
+		break;
 	case ARC_SPACE_OTHER:
 		ARCSTAT_INCR(arcstat_other_size, -space);
 		break;
@@ -1535,10 +1687,13 @@ arc_space_return(uint64_t space, arc_space_type_t type)
 		break;
 	}
 
-	ASSERT(arc_meta_used >= space);
-	if (arc_meta_max < arc_meta_used)
-		arc_meta_max = arc_meta_used;
-	ARCSTAT_INCR(arcstat_meta_used, -space);
+	if (type != ARC_SPACE_DATA) {
+		ASSERT(arc_meta_used >= space);
+		if (arc_meta_max < arc_meta_used)
+			arc_meta_max = arc_meta_used;
+		ARCSTAT_INCR(arcstat_meta_used, -space);
+	}
+
 	ASSERT(arc_size >= space);
 	atomic_add_64(&arc_size, -space);
 }
@@ -1744,12 +1899,11 @@ arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t remove)
 		if (!recycle) {
 			if (type == ARC_BUFC_METADATA) {
 				arc_buf_data_free(buf, zio_buf_free);
-				arc_space_return(size, ARC_SPACE_DATA);
+				arc_space_return(size, ARC_SPACE_META);
 			} else {
 				ASSERT(type == ARC_BUFC_DATA);
 				arc_buf_data_free(buf, zio_data_buf_free);
-				ARCSTAT_INCR(arcstat_data_size, -size);
-				atomic_add_64(&arc_size, -size);
+				arc_space_return(size, ARC_SPACE_DATA);
 			}
 		}
 		if (list_link_active(&buf->b_hdr->b_l1hdr.b_arc_node)) {
@@ -2742,6 +2896,20 @@ arc_reclaim_thread(void)
 		if (arc_eviction_list != NULL)
 			arc_do_user_evicts();
 
+		/*
+		 * This is necessary in order for the mdb ::arc dcmd to
+		 * show up to date information. Since the ::arc command
+		 * does not call the kstat's update function, without
+		 * this call, the command may show stale stats for the
+		 * anon, mru, mru_ghost, mfu, and mfu_ghost lists. Even
+		 * with this change, the data might be up to 1 second
+		 * out of date; but that should suffice. The arc_state_t
+		 * structures can be queried directly if more accurate
+		 * information is needed.
+		 */
+		if (arc_ksp != NULL)
+			arc_ksp->ks_update(arc_ksp, KSTAT_READ);
+
 		/* block until needed, or one second, whichever is shorter */
 		CALLB_CPR_SAFE_BEGIN(&cpr);
 		(void) cv_timedwait(&arc_reclaim_thr_cv,
@@ -2878,12 +3046,11 @@ arc_get_data_buf(arc_buf_t *buf)
 	if (!arc_evict_needed(type)) {
 		if (type == ARC_BUFC_METADATA) {
 			buf->b_data = zio_buf_alloc(size);
-			arc_space_consume(size, ARC_SPACE_DATA);
+			arc_space_consume(size, ARC_SPACE_META);
 		} else {
 			ASSERT(type == ARC_BUFC_DATA);
 			buf->b_data = zio_data_buf_alloc(size);
-			ARCSTAT_INCR(arcstat_data_size, size);
-			atomic_add_64(&arc_size, size);
+			arc_space_consume(size, ARC_SPACE_DATA);
 		}
 		goto out;
 	}
@@ -2910,12 +3077,11 @@ arc_get_data_buf(arc_buf_t *buf)
 	if ((buf->b_data = arc_evict(state, NULL, size, TRUE, type)) == NULL) {
 		if (type == ARC_BUFC_METADATA) {
 			buf->b_data = zio_buf_alloc(size);
-			arc_space_consume(size, ARC_SPACE_DATA);
+			arc_space_consume(size, ARC_SPACE_META);
 		} else {
 			ASSERT(type == ARC_BUFC_DATA);
 			buf->b_data = zio_data_buf_alloc(size);
-			ARCSTAT_INCR(arcstat_data_size, size);
-			atomic_add_64(&arc_size, size);
+			arc_space_consume(size, ARC_SPACE_DATA);
 		}
 		ARCSTAT_BUMP(arcstat_recycle_miss);
 	}
@@ -4115,6 +4281,48 @@ arc_tempreserve_space(uint64_t reserve, uint64_t txg)
 	return (0);
 }
 
+static void
+arc_kstat_update_state(arc_state_t *state, kstat_named_t *size,
+    kstat_named_t *evict_data, kstat_named_t *evict_metadata)
+{
+	size->value.ui64 = state->arcs_size;
+	evict_data->value.ui64 = state->arcs_lsize[ARC_BUFC_DATA];
+	evict_metadata->value.ui64 = state->arcs_lsize[ARC_BUFC_METADATA];
+}
+
+static int
+arc_kstat_update(kstat_t *ksp, int rw)
+{
+	arc_stats_t *as = ksp->ks_data;
+
+	if (rw == KSTAT_WRITE) {
+		return (EACCES);
+	} else {
+		arc_kstat_update_state(arc_anon,
+		    &as->arcstat_anon_size,
+		    &as->arcstat_anon_evictable_data,
+		    &as->arcstat_anon_evictable_metadata);
+		arc_kstat_update_state(arc_mru,
+		    &as->arcstat_mru_size,
+		    &as->arcstat_mru_evictable_data,
+		    &as->arcstat_mru_evictable_metadata);
+		arc_kstat_update_state(arc_mru_ghost,
+		    &as->arcstat_mru_ghost_size,
+		    &as->arcstat_mru_ghost_evictable_data,
+		    &as->arcstat_mru_ghost_evictable_metadata);
+		arc_kstat_update_state(arc_mfu,
+		    &as->arcstat_mfu_size,
+		    &as->arcstat_mfu_evictable_data,
+		    &as->arcstat_mfu_evictable_metadata);
+		arc_kstat_update_state(arc_mfu_ghost,
+		    &as->arcstat_mfu_ghost_size,
+		    &as->arcstat_mfu_ghost_evictable_data,
+		    &as->arcstat_mfu_ghost_evictable_metadata);
+	}
+
+	return (0);
+}
+
 void
 arc_init(void)
 {
@@ -4261,6 +4469,7 @@ arc_init(void)
 
 	if (arc_ksp != NULL) {
 		arc_ksp->ks_data = &arc_stats;
+		arc_ksp->ks_update = arc_kstat_update;
 		kstat_install(arc_ksp);
 	}
 
diff --git a/uts/common/fs/zfs/sys/arc.h b/uts/common/fs/zfs/sys/arc.h
index 4e9e8e202a7e..60112baaba55 100644
--- a/uts/common/fs/zfs/sys/arc.h
+++ b/uts/common/fs/zfs/sys/arc.h
@@ -115,6 +115,7 @@ typedef enum arc_buf_contents {
  */
 typedef enum arc_space_type {
 	ARC_SPACE_DATA,
+	ARC_SPACE_META,
 	ARC_SPACE_HDRS,
 	ARC_SPACE_L2HDRS,
 	ARC_SPACE_OTHER,

From 9a0182dd18de00408d96b47ca6037bcb12afe44b Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Tue, 20 Jan 2015 20:11:30 +0000
Subject: [PATCH 071/380] 5056 ZFS deadlock on db_mtx and dn_holds Reviewed by:
 Will Andrews <willa@spectralogic.com> Reviewed by: Matt Ahrens
 <mahrens@delphix.com> Reviewed by: George Wilson <george.wilson@delphix.com>
 Approved by: Dan McDonald <danmcd@omniti.com> Author: Justin Gibbs
 <justing@spectralogic.com>

illumos/illumos-gate@bc9014e6a81272073b9854d9f65dd59e18d18c35
---
 uts/common/fs/zfs/dbuf.c            | 182 ++++++++++++++++++++--------
 uts/common/fs/zfs/dmu_objset.c      | 111 ++++++++++-------
 uts/common/fs/zfs/dmu_send.c        |   6 +-
 uts/common/fs/zfs/dmu_traverse.c    |   2 +-
 uts/common/fs/zfs/dnode.c           |  85 ++++++++-----
 uts/common/fs/zfs/dnode_sync.c      |  62 ++++------
 uts/common/fs/zfs/dsl_bookmark.c    |   2 +-
 uts/common/fs/zfs/dsl_dataset.c     |  53 ++++----
 uts/common/fs/zfs/dsl_deadlist.c    |   3 +
 uts/common/fs/zfs/dsl_deleg.c       |   2 +-
 uts/common/fs/zfs/dsl_destroy.c     |   8 +-
 uts/common/fs/zfs/dsl_dir.c         |  38 ++++--
 uts/common/fs/zfs/dsl_pool.c        |   3 +
 uts/common/fs/zfs/dsl_prop.c        |  14 +--
 uts/common/fs/zfs/dsl_scan.c        |   8 +-
 uts/common/fs/zfs/dsl_userhold.c    |   2 +-
 uts/common/fs/zfs/sa.c              |  42 ++++---
 uts/common/fs/zfs/spa.c             |  17 +++
 uts/common/fs/zfs/spa_misc.c        |  48 ++++++++
 uts/common/fs/zfs/sys/dbuf.h        |   6 +-
 uts/common/fs/zfs/sys/dmu.h         | 147 +++++++++++++++++-----
 uts/common/fs/zfs/sys/dmu_objset.h  |  14 ++-
 uts/common/fs/zfs/sys/dnode.h       |   4 +-
 uts/common/fs/zfs/sys/dsl_dataset.h |  10 +-
 uts/common/fs/zfs/sys/dsl_dir.h     |   4 +
 uts/common/fs/zfs/sys/sa.h          |   1 -
 uts/common/fs/zfs/sys/sa_impl.h     |   4 +-
 uts/common/fs/zfs/sys/spa.h         |   5 +
 uts/common/fs/zfs/sys/spa_impl.h    |   4 +
 uts/common/fs/zfs/sys/zap_impl.h    |   4 +-
 uts/common/fs/zfs/sys/zap_leaf.h    |   2 +
 uts/common/fs/zfs/zap.c             |  35 +++---
 uts/common/fs/zfs/zap_micro.c       |   9 +-
 uts/common/fs/zfs/zfs_sa.c          |   3 +-
 uts/common/fs/zfs/zil.c             |   2 +-
 35 files changed, 633 insertions(+), 309 deletions(-)

diff --git a/uts/common/fs/zfs/dbuf.c b/uts/common/fs/zfs/dbuf.c
index fd539c838d06..49c659a51b4e 100644
--- a/uts/common/fs/zfs/dbuf.c
+++ b/uts/common/fs/zfs/dbuf.c
@@ -24,6 +24,7 @@
  * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -54,10 +55,16 @@ static void dbuf_destroy(dmu_buf_impl_t *db);
 static boolean_t dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx);
 static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx);
 
+#ifndef __lint
+extern inline void dmu_buf_init_user(dmu_buf_user_t *dbu,
+    dmu_buf_evict_func_t *evict_func, dmu_buf_t **clear_on_evict_dbufp);
+#endif /* ! __lint */
+
 /*
  * Global data structures and functions for the dbuf cache.
  */
 static kmem_cache_t *dbuf_cache;
+static taskq_t *dbu_evict_taskq;
 
 /* ARGSUSED */
 static int
@@ -215,17 +222,72 @@ dbuf_hash_remove(dmu_buf_impl_t *db)
 
 static arc_evict_func_t dbuf_do_evict;
 
+typedef enum {
+	DBVU_EVICTING,
+	DBVU_NOT_EVICTING
+} dbvu_verify_type_t;
+
+static void
+dbuf_verify_user(dmu_buf_impl_t *db, dbvu_verify_type_t verify_type)
+{
+#ifdef ZFS_DEBUG
+	int64_t holds;
+
+	if (db->db_user == NULL)
+		return;
+
+	/* Only data blocks support the attachment of user data. */
+	ASSERT(db->db_level == 0);
+
+	/* Clients must resolve a dbuf before attaching user data. */
+	ASSERT(db->db.db_data != NULL);
+	ASSERT3U(db->db_state, ==, DB_CACHED);
+
+	holds = refcount_count(&db->db_holds);
+	if (verify_type == DBVU_EVICTING) {
+		/*
+		 * Immediate eviction occurs when holds == dirtycnt.
+		 * For normal eviction buffers, holds is zero on
+		 * eviction, except when dbuf_fix_old_data() calls
+		 * dbuf_clear_data().  However, the hold count can grow
+		 * during eviction even though db_mtx is held (see
+		 * dmu_bonus_hold() for an example), so we can only
+		 * test the generic invariant that holds >= dirtycnt.
+		 */
+		ASSERT3U(holds, >=, db->db_dirtycnt);
+	} else {
+		if (db->db_immediate_evict == TRUE)
+			ASSERT3U(holds, >=, db->db_dirtycnt);
+		else
+			ASSERT3U(holds, >, 0);
+	}
+#endif
+}
+
 static void
 dbuf_evict_user(dmu_buf_impl_t *db)
 {
+	dmu_buf_user_t *dbu = db->db_user;
+
 	ASSERT(MUTEX_HELD(&db->db_mtx));
 
-	if (db->db_level != 0 || db->db_evict_func == NULL)
+	if (dbu == NULL)
 		return;
 
-	db->db_evict_func(&db->db, db->db_user_ptr);
-	db->db_user_ptr = NULL;
-	db->db_evict_func = NULL;
+	dbuf_verify_user(db, DBVU_EVICTING);
+	db->db_user = NULL;
+
+#ifdef ZFS_DEBUG
+	if (dbu->dbu_clear_on_evict_dbufp != NULL)
+		*dbu->dbu_clear_on_evict_dbufp = NULL;
+#endif
+
+	/*
+	 * Invoke the callback from a taskq to avoid lock order reversals
+	 * and limit stack depth.
+	 */
+	taskq_dispatch_ent(dbu_evict_taskq, dbu->dbu_evict_func, dbu, 0,
+	    &dbu->dbu_tqent);
 }
 
 boolean_t
@@ -286,6 +348,12 @@ retry:
 
 	for (i = 0; i < DBUF_MUTEXES; i++)
 		mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL);
+
+	/*
+	 * All entries are queued via taskq_dispatch_ent(), so min/maxalloc
+	 * configuration is not required.
+	 */
+	dbu_evict_taskq = taskq_create("dbu_evict", 1, minclsyspri, 0, 0, 0);
 }
 
 void
@@ -298,6 +366,7 @@ dbuf_fini(void)
 		mutex_destroy(&h->hash_mutexes[i]);
 	kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *));
 	kmem_cache_destroy(dbuf_cache);
+	taskq_destroy(dbu_evict_taskq);
 }
 
 /*
@@ -414,22 +483,28 @@ dbuf_verify(dmu_buf_impl_t *db)
 }
 #endif
 
+static void
+dbuf_clear_data(dmu_buf_impl_t *db)
+{
+	ASSERT(MUTEX_HELD(&db->db_mtx));
+	dbuf_evict_user(db);
+	db->db_buf = NULL;
+	db->db.db_data = NULL;
+	if (db->db_state != DB_NOFILL)
+		db->db_state = DB_UNCACHED;
+}
+
 static void
 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf)
 {
 	ASSERT(MUTEX_HELD(&db->db_mtx));
+	ASSERT(buf != NULL);
+
 	db->db_buf = buf;
-	if (buf != NULL) {
-		ASSERT(buf->b_data != NULL);
-		db->db.db_data = buf->b_data;
-		if (!arc_released(buf))
-			arc_set_callback(buf, dbuf_do_evict, db);
-	} else {
-		dbuf_evict_user(db);
-		db->db.db_data = NULL;
-		if (db->db_state != DB_NOFILL)
-			db->db_state = DB_UNCACHED;
-	}
+	ASSERT(buf->b_data != NULL);
+	db->db.db_data = buf->b_data;
+	if (!arc_released(buf))
+		arc_set_callback(buf, dbuf_do_evict, db);
 }
 
 /*
@@ -451,7 +526,7 @@ dbuf_loan_arcbuf(dmu_buf_impl_t *db)
 	} else {
 		abuf = db->db_buf;
 		arc_loan_inuse_buf(abuf, db);
-		dbuf_set_data(db, NULL);
+		dbuf_clear_data(db);
 		mutex_exit(&db->db_mtx);
 	}
 	return (abuf);
@@ -687,7 +762,7 @@ dbuf_noread(dmu_buf_impl_t *db)
 		dbuf_set_data(db, arc_buf_alloc(spa, db->db.db_size, db, type));
 		db->db_state = DB_FILL;
 	} else if (db->db_state == DB_NOFILL) {
-		dbuf_set_data(db, NULL);
+		dbuf_clear_data(db);
 	} else {
 		ASSERT3U(db->db_state, ==, DB_CACHED);
 	}
@@ -743,7 +818,7 @@ dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg)
 		dr->dt.dl.dr_data = arc_buf_alloc(spa, size, db, type);
 		bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size);
 	} else {
-		dbuf_set_data(db, NULL);
+		dbuf_clear_data(db);
 	}
 }
 
@@ -794,7 +869,8 @@ void
 dbuf_free_range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid,
     dmu_tx_t *tx)
 {
-	dmu_buf_impl_t *db, *db_next, db_search;
+	dmu_buf_impl_t db_search;
+	dmu_buf_impl_t *db, *db_next;
 	uint64_t txg = tx->tx_txg;
 	avl_index_t where;
 
@@ -1370,7 +1446,7 @@ dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
 		arc_buf_t *buf = db->db_buf;
 
 		ASSERT(db->db_state == DB_NOFILL || arc_released(buf));
-		dbuf_set_data(db, NULL);
+		dbuf_clear_data(db);
 		VERIFY(arc_buf_remove_ref(buf, db));
 		dbuf_evict(db);
 		return (B_TRUE);
@@ -1710,8 +1786,7 @@ dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid,
 	db->db_parent = parent;
 	db->db_blkptr = blkptr;
 
-	db->db_user_ptr = NULL;
-	db->db_evict_func = NULL;
+	db->db_user = NULL;
 	db->db_immediate_evict = 0;
 	db->db_freed_in_flight = 0;
 
@@ -2114,7 +2189,7 @@ dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag)
 			/*
 			 * This dbuf has anonymous data associated with it.
 			 */
-			dbuf_set_data(db, NULL);
+			dbuf_clear_data(db);
 			VERIFY(arc_buf_remove_ref(buf, db));
 			dbuf_evict(db);
 		} else {
@@ -2147,7 +2222,8 @@ dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag)
 				} else {
 					dbuf_clear(db);
 				}
-			} else if (arc_buf_eviction_needed(db->db_buf)) {
+			} else if (db->db_objset->os_evicting ||
+			    arc_buf_eviction_needed(db->db_buf)) {
 				dbuf_clear(db);
 			} else {
 				mutex_exit(&db->db_mtx);
@@ -2166,51 +2242,57 @@ dbuf_refcount(dmu_buf_impl_t *db)
 }
 
 void *
-dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr,
-    dmu_buf_evict_func_t *evict_func)
+dmu_buf_replace_user(dmu_buf_t *db_fake, dmu_buf_user_t *old_user,
+    dmu_buf_user_t *new_user)
 {
-	return (dmu_buf_update_user(db_fake, NULL, user_ptr, evict_func));
+	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
+
+	mutex_enter(&db->db_mtx);
+	dbuf_verify_user(db, DBVU_NOT_EVICTING);
+	if (db->db_user == old_user)
+		db->db_user = new_user;
+	else
+		old_user = db->db_user;
+	dbuf_verify_user(db, DBVU_NOT_EVICTING);
+	mutex_exit(&db->db_mtx);
+
+	return (old_user);
 }
 
 void *
-dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr,
-    dmu_buf_evict_func_t *evict_func)
+dmu_buf_set_user(dmu_buf_t *db_fake, dmu_buf_user_t *user)
+{
+	return (dmu_buf_replace_user(db_fake, NULL, user));
+}
+
+void *
+dmu_buf_set_user_ie(dmu_buf_t *db_fake, dmu_buf_user_t *user)
 {
 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
 
 	db->db_immediate_evict = TRUE;
-	return (dmu_buf_update_user(db_fake, NULL, user_ptr, evict_func));
+	return (dmu_buf_set_user(db_fake, user));
 }
 
 void *
-dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr,
-    dmu_buf_evict_func_t *evict_func)
+dmu_buf_remove_user(dmu_buf_t *db_fake, dmu_buf_user_t *user)
 {
-	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
-	ASSERT(db->db_level == 0);
-
-	ASSERT((user_ptr == NULL) == (evict_func == NULL));
-
-	mutex_enter(&db->db_mtx);
-
-	if (db->db_user_ptr == old_user_ptr) {
-		db->db_user_ptr = user_ptr;
-		db->db_evict_func = evict_func;
-	} else {
-		old_user_ptr = db->db_user_ptr;
-	}
-
-	mutex_exit(&db->db_mtx);
-	return (old_user_ptr);
+	return (dmu_buf_replace_user(db_fake, user, NULL));
 }
 
 void *
 dmu_buf_get_user(dmu_buf_t *db_fake)
 {
 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
-	ASSERT(!refcount_is_zero(&db->db_holds));
 
-	return (db->db_user_ptr);
+	dbuf_verify_user(db, DBVU_NOT_EVICTING);
+	return (db->db_user);
+}
+
+void
+dmu_buf_user_evict_wait()
+{
+	taskq_wait(dbu_evict_taskq);
 }
 
 boolean_t
diff --git a/uts/common/fs/zfs/dmu_objset.c b/uts/common/fs/zfs/dmu_objset.c
index 36ac27a5e0de..8593e2c94f6c 100644
--- a/uts/common/fs/zfs/dmu_objset.c
+++ b/uts/common/fs/zfs/dmu_objset.c
@@ -23,6 +23,7 @@
  * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 /* Portions Copyright 2010 Robert Milkowski */
@@ -355,7 +356,7 @@ dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
 			    zfs_prop_to_name(ZFS_PROP_SECONDARYCACHE),
 			    secondary_cache_changed_cb, os);
 		}
-		if (!dsl_dataset_is_snapshot(ds)) {
+		if (!ds->ds_is_snapshot) {
 			if (err == 0) {
 				err = dsl_prop_register(ds,
 				    zfs_prop_to_name(ZFS_PROP_CHECKSUM),
@@ -417,7 +418,7 @@ dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
 		os->os_secondary_cache = ZFS_CACHE_ALL;
 	}
 
-	if (ds == NULL || !dsl_dataset_is_snapshot(ds))
+	if (ds == NULL || !ds->ds_is_snapshot)
 		os->os_zil_header = os->os_phys->os_zil_header;
 	os->os_zil = zil_alloc(os, &os->os_zil_header);
 
@@ -436,16 +437,13 @@ dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
 	mutex_init(&os->os_obj_lock, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&os->os_user_ptr_lock, NULL, MUTEX_DEFAULT, NULL);
 
-	DMU_META_DNODE(os) = dnode_special_open(os,
-	    &os->os_phys->os_meta_dnode, DMU_META_DNODE_OBJECT,
-	    &os->os_meta_dnode);
+	dnode_special_open(os, &os->os_phys->os_meta_dnode,
+	    DMU_META_DNODE_OBJECT, &os->os_meta_dnode);
 	if (arc_buf_size(os->os_phys_buf) >= sizeof (objset_phys_t)) {
-		DMU_USERUSED_DNODE(os) = dnode_special_open(os,
-		    &os->os_phys->os_userused_dnode, DMU_USERUSED_OBJECT,
-		    &os->os_userused_dnode);
-		DMU_GROUPUSED_DNODE(os) = dnode_special_open(os,
-		    &os->os_phys->os_groupused_dnode, DMU_GROUPUSED_OBJECT,
-		    &os->os_groupused_dnode);
+		dnode_special_open(os, &os->os_phys->os_userused_dnode,
+		    DMU_USERUSED_OBJECT, &os->os_userused_dnode);
+		dnode_special_open(os, &os->os_phys->os_groupused_dnode,
+		    DMU_GROUPUSED_OBJECT, &os->os_groupused_dnode);
 	}
 
 	*osp = os;
@@ -533,7 +531,7 @@ dmu_objset_own(const char *name, dmu_objset_type_t type,
 	} else if (type != DMU_OST_ANY && type != (*osp)->os_phys->os_type) {
 		dsl_dataset_disown(ds, tag);
 		return (SET_ERROR(EINVAL));
-	} else if (!readonly && dsl_dataset_is_snapshot(ds)) {
+	} else if (!readonly && ds->ds_is_snapshot) {
 		dsl_dataset_disown(ds, tag);
 		return (SET_ERROR(EROFS));
 	}
@@ -589,41 +587,53 @@ dmu_objset_disown(objset_t *os, void *tag)
 void
 dmu_objset_evict_dbufs(objset_t *os)
 {
+	dnode_t dn_marker;
 	dnode_t *dn;
 
 	mutex_enter(&os->os_lock);
+	dn = list_head(&os->os_dnodes);
+	while (dn != NULL) {
+		/*
+		 * Skip dnodes without holds.  We have to do this dance
+		 * because dnode_add_ref() only works if there is already a
+		 * hold.  If the dnode has no holds, then it has no dbufs.
+		 */
+		if (dnode_add_ref(dn, FTAG)) {
+			list_insert_after(&os->os_dnodes, dn, &dn_marker);
+			mutex_exit(&os->os_lock);
 
-	/* process the mdn last, since the other dnodes have holds on it */
-	list_remove(&os->os_dnodes, DMU_META_DNODE(os));
-	list_insert_tail(&os->os_dnodes, DMU_META_DNODE(os));
+			dnode_evict_dbufs(dn);
+			dnode_rele(dn, FTAG);
 
-	/*
-	 * Find the first dnode with holds.  We have to do this dance
-	 * because dnode_add_ref() only works if you already have a
-	 * hold.  If there are no holds then it has no dbufs so OK to
-	 * skip.
-	 */
-	for (dn = list_head(&os->os_dnodes);
-	    dn && !dnode_add_ref(dn, FTAG);
-	    dn = list_next(&os->os_dnodes, dn))
-		continue;
-
-	while (dn) {
-		dnode_t *next_dn = dn;
-
-		do {
-			next_dn = list_next(&os->os_dnodes, next_dn);
-		} while (next_dn && !dnode_add_ref(next_dn, FTAG));
-
-		mutex_exit(&os->os_lock);
-		dnode_evict_dbufs(dn);
-		dnode_rele(dn, FTAG);
-		mutex_enter(&os->os_lock);
-		dn = next_dn;
+			mutex_enter(&os->os_lock);
+			dn = list_next(&os->os_dnodes, &dn_marker);
+			list_remove(&os->os_dnodes, &dn_marker);
+		} else {
+			dn = list_next(&os->os_dnodes, dn);
+		}
 	}
 	mutex_exit(&os->os_lock);
+
+	if (DMU_USERUSED_DNODE(os) != NULL) {
+		dnode_evict_dbufs(DMU_GROUPUSED_DNODE(os));
+		dnode_evict_dbufs(DMU_USERUSED_DNODE(os));
+	}
+	dnode_evict_dbufs(DMU_META_DNODE(os));
 }
 
+/*
+ * Objset eviction processing is split into into two pieces.
+ * The first marks the objset as evicting, evicts any dbufs that
+ * have a refcount of zero, and then queues up the objset for the
+ * second phase of eviction.  Once os->os_dnodes has been cleared by
+ * dnode_buf_pageout()->dnode_destroy(), the second phase is executed.
+ * The second phase closes the special dnodes, dequeues the objset from
+ * the list of those undergoing eviction, and finally frees the objset.
+ *
+ * NOTE: Due to asynchronous eviction processing (invocation of
+ *       dnode_buf_pageout()), it is possible for the meta dnode for the
+ *       objset to have no holds even though os->os_dnodes is not empty.
+ */
 void
 dmu_objset_evict(objset_t *os)
 {
@@ -633,7 +643,7 @@ dmu_objset_evict(objset_t *os)
 		ASSERT(!dmu_objset_is_dirty(os, t));
 
 	if (ds) {
-		if (!dsl_dataset_is_snapshot(ds)) {
+		if (!ds->ds_is_snapshot) {
 			VERIFY0(dsl_prop_unregister(ds,
 			    zfs_prop_to_name(ZFS_PROP_CHECKSUM),
 			    checksum_changed_cb, os));
@@ -670,8 +680,24 @@ dmu_objset_evict(objset_t *os)
 	if (os->os_sa)
 		sa_tear_down(os);
 
+	os->os_evicting = B_TRUE;
 	dmu_objset_evict_dbufs(os);
 
+	mutex_enter(&os->os_lock);
+	spa_evicting_os_register(os->os_spa, os);
+	if (list_is_empty(&os->os_dnodes)) {
+		mutex_exit(&os->os_lock);
+		dmu_objset_evict_done(os);
+	} else {
+		mutex_exit(&os->os_lock);
+	}
+}
+
+void
+dmu_objset_evict_done(objset_t *os)
+{
+	ASSERT3P(list_head(&os->os_dnodes), ==, NULL);
+
 	dnode_special_close(&os->os_meta_dnode);
 	if (DMU_USERUSED_DNODE(os)) {
 		dnode_special_close(&os->os_userused_dnode);
@@ -679,8 +705,6 @@ dmu_objset_evict(objset_t *os)
 	}
 	zil_free(os->os_zil);
 
-	ASSERT3P(list_head(&os->os_dnodes), ==, NULL);
-
 	VERIFY(arc_buf_remove_ref(os->os_phys_buf, &os->os_phys_buf));
 
 	/*
@@ -695,6 +719,7 @@ dmu_objset_evict(objset_t *os)
 	mutex_destroy(&os->os_lock);
 	mutex_destroy(&os->os_obj_lock);
 	mutex_destroy(&os->os_user_ptr_lock);
+	spa_evicting_os_deregister(os->os_spa, os);
 	kmem_free(os, sizeof (objset_t));
 }
 
@@ -903,7 +928,7 @@ dmu_objset_clone_check(void *arg, dmu_tx_t *tx)
 	}
 
 	/* You can only clone snapshots, not the head datasets. */
-	if (!dsl_dataset_is_snapshot(origin)) {
+	if (!origin->ds_is_snapshot) {
 		dsl_dataset_rele(origin, FTAG);
 		return (SET_ERROR(EINVAL));
 	}
@@ -1467,7 +1492,7 @@ int
 dmu_objset_is_snapshot(objset_t *os)
 {
 	if (os->os_dsl_dataset != NULL)
-		return (dsl_dataset_is_snapshot(os->os_dsl_dataset));
+		return (os->os_dsl_dataset->ds_is_snapshot);
 	else
 		return (B_FALSE);
 }
diff --git a/uts/common/fs/zfs/dmu_send.c b/uts/common/fs/zfs/dmu_send.c
index 4bb1290a07c0..3c156054877b 100644
--- a/uts/common/fs/zfs/dmu_send.c
+++ b/uts/common/fs/zfs/dmu_send.c
@@ -611,7 +611,7 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *ds,
 		fromtxg = fromzb->zbm_creation_txg;
 	}
 	dsl_dataset_name(ds, drr->drr_u.drr_begin.drr_toname);
-	if (!dsl_dataset_is_snapshot(ds)) {
+	if (!ds->ds_is_snapshot) {
 		(void) strlcat(drr->drr_u.drr_begin.drr_toname, "@--head--",
 		    sizeof (drr->drr_u.drr_begin.drr_toname));
 	}
@@ -818,7 +818,7 @@ dmu_send_estimate(dsl_dataset_t *ds, dsl_dataset_t *fromds, uint64_t *sizep)
 	ASSERT(dsl_pool_config_held(dp));
 
 	/* tosnap must be a snapshot */
-	if (!dsl_dataset_is_snapshot(ds))
+	if (!ds->ds_is_snapshot)
 		return (SET_ERROR(EINVAL));
 
 	/*
@@ -1065,7 +1065,7 @@ dmu_recv_begin_check(void *arg, dmu_tx_t *tx)
 				dsl_dataset_rele(ds, FTAG);
 				return (error);
 			}
-			if (!dsl_dataset_is_snapshot(origin)) {
+			if (!origin->ds_is_snapshot) {
 				dsl_dataset_rele(origin, FTAG);
 				dsl_dataset_rele(ds, FTAG);
 				return (SET_ERROR(EINVAL));
diff --git a/uts/common/fs/zfs/dmu_traverse.c b/uts/common/fs/zfs/dmu_traverse.c
index 7b1d6befa6ae..14eef2e7516c 100644
--- a/uts/common/fs/zfs/dmu_traverse.c
+++ b/uts/common/fs/zfs/dmu_traverse.c
@@ -532,7 +532,7 @@ traverse_impl(spa_t *spa, dsl_dataset_t *ds, uint64_t objset, blkptr_t *rootbp,
 	cv_init(&pd.pd_cv, NULL, CV_DEFAULT, NULL);
 
 	/* See comment on ZIL traversal in dsl_scan_visitds. */
-	if (ds != NULL && !dsl_dataset_is_snapshot(ds) && !BP_IS_HOLE(rootbp)) {
+	if (ds != NULL && !ds->ds_is_snapshot && !BP_IS_HOLE(rootbp)) {
 		arc_flags_t flags = ARC_FLAG_WAIT;
 		objset_phys_t *osp;
 		arc_buf_t *buf;
diff --git a/uts/common/fs/zfs/dnode.c b/uts/common/fs/zfs/dnode.c
index 3303ee709ed7..4e376fc4d920 100644
--- a/uts/common/fs/zfs/dnode.c
+++ b/uts/common/fs/zfs/dnode.c
@@ -21,6 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -402,8 +403,9 @@ static dnode_t *
 dnode_create(objset_t *os, dnode_phys_t *dnp, dmu_buf_impl_t *db,
     uint64_t object, dnode_handle_t *dnh)
 {
-	dnode_t *dn = kmem_cache_alloc(dnode_cache, KM_SLEEP);
+	dnode_t *dn;
 
+	dn = kmem_cache_alloc(dnode_cache, KM_SLEEP);
 	ASSERT(!POINTER_IS_VALID(dn->dn_objset));
 	dn->dn_moved = 0;
 
@@ -440,13 +442,31 @@ dnode_create(objset_t *os, dnode_phys_t *dnp, dmu_buf_impl_t *db,
 	ASSERT(DMU_OT_IS_VALID(dn->dn_phys->dn_type));
 
 	mutex_enter(&os->os_lock);
-	list_insert_head(&os->os_dnodes, dn);
-	membar_producer();
+	if (dnh->dnh_dnode != NULL) {
+		/* Lost the allocation race. */
+		mutex_exit(&os->os_lock);
+		kmem_cache_free(dnode_cache, dn);
+		return (dnh->dnh_dnode);
+	}
+
 	/*
-	 * Everything else must be valid before assigning dn_objset makes the
-	 * dnode eligible for dnode_move().
+	 * Exclude special dnodes from os_dnodes so an empty os_dnodes
+	 * signifies that the special dnodes have no references from
+	 * their children (the entries in os_dnodes).  This allows
+	 * dnode_destroy() to easily determine if the last child has
+	 * been removed and then complete eviction of the objset.
+	 */
+	if (!DMU_OBJECT_IS_SPECIAL(object))
+		list_insert_head(&os->os_dnodes, dn);
+	membar_producer();
+
+	/*
+	 * Everything else must be valid before assigning dn_objset
+	 * makes the dnode eligible for dnode_move().
 	 */
 	dn->dn_objset = os;
+
+	dnh->dnh_dnode = dn;
 	mutex_exit(&os->os_lock);
 
 	arc_space_consume(sizeof (dnode_t), ARC_SPACE_OTHER);
@@ -460,12 +480,18 @@ static void
 dnode_destroy(dnode_t *dn)
 {
 	objset_t *os = dn->dn_objset;
+	boolean_t complete_os_eviction = B_FALSE;
 
 	ASSERT((dn->dn_id_flags & DN_ID_NEW_EXIST) == 0);
 
 	mutex_enter(&os->os_lock);
 	POINTER_INVALIDATE(&dn->dn_objset);
-	list_remove(&os->os_dnodes, dn);
+	if (!DMU_OBJECT_IS_SPECIAL(dn->dn_object)) {
+		list_remove(&os->os_dnodes, dn);
+		complete_os_eviction =
+		    list_is_empty(&os->os_dnodes) &&
+		    list_link_active(&os->os_evicting_node);
+	}
 	mutex_exit(&os->os_lock);
 
 	/* the dnode can no longer move, so we can release the handle */
@@ -500,6 +526,9 @@ dnode_destroy(dnode_t *dn)
 	dmu_zfetch_rele(&dn->dn_zfetch);
 	kmem_cache_free(dnode_cache, dn);
 	arc_space_return(sizeof (dnode_t), ARC_SPACE_OTHER);
+
+	if (complete_os_eviction)
+		dmu_objset_evict_done(os);
 }
 
 void
@@ -966,33 +995,32 @@ dnode_special_close(dnode_handle_t *dnh)
 	 */
 	while (refcount_count(&dn->dn_holds) > 0)
 		delay(1);
+	ASSERT(dn->dn_dbuf == NULL ||
+	    dmu_buf_get_user(&dn->dn_dbuf->db) == NULL);
 	zrl_add(&dnh->dnh_zrlock);
 	dnode_destroy(dn); /* implicit zrl_remove() */
 	zrl_destroy(&dnh->dnh_zrlock);
 	dnh->dnh_dnode = NULL;
 }
 
-dnode_t *
+void
 dnode_special_open(objset_t *os, dnode_phys_t *dnp, uint64_t object,
     dnode_handle_t *dnh)
 {
-	dnode_t *dn = dnode_create(os, dnp, NULL, object, dnh);
-	dnh->dnh_dnode = dn;
+	dnode_t *dn;
+
+	dn = dnode_create(os, dnp, NULL, object, dnh);
 	zrl_init(&dnh->dnh_zrlock);
 	DNODE_VERIFY(dn);
-	return (dn);
 }
 
 static void
-dnode_buf_pageout(dmu_buf_t *db, void *arg)
+dnode_buf_pageout(void *dbu)
 {
-	dnode_children_t *children_dnodes = arg;
+	dnode_children_t *children_dnodes = dbu;
 	int i;
-	int epb = db->db_size >> DNODE_SHIFT;
 
-	ASSERT(epb == children_dnodes->dnc_count);
-
-	for (i = 0; i < epb; i++) {
+	for (i = 0; i < children_dnodes->dnc_count; i++) {
 		dnode_handle_t *dnh = &children_dnodes->dnc_children[i];
 		dnode_t *dn;
 
@@ -1022,7 +1050,7 @@ dnode_buf_pageout(dmu_buf_t *db, void *arg)
 		dnh->dnh_dnode = NULL;
 	}
 	kmem_free(children_dnodes, sizeof (dnode_children_t) +
-	    epb * sizeof (dnode_handle_t));
+	    children_dnodes->dnc_count * sizeof (dnode_handle_t));
 }
 
 /*
@@ -1106,16 +1134,17 @@ dnode_hold_impl(objset_t *os, uint64_t object, int flag,
 	if (children_dnodes == NULL) {
 		int i;
 		dnode_children_t *winner;
-		children_dnodes = kmem_alloc(sizeof (dnode_children_t) +
+		children_dnodes = kmem_zalloc(sizeof (dnode_children_t) +
 		    epb * sizeof (dnode_handle_t), KM_SLEEP);
 		children_dnodes->dnc_count = epb;
 		dnh = &children_dnodes->dnc_children[0];
 		for (i = 0; i < epb; i++) {
 			zrl_init(&dnh[i].dnh_zrlock);
-			dnh[i].dnh_dnode = NULL;
 		}
-		if (winner = dmu_buf_set_user(&db->db, children_dnodes,
-		    dnode_buf_pageout)) {
+		dmu_buf_init_user(&children_dnodes->dnc_dbu,
+		    dnode_buf_pageout, NULL);
+		winner = dmu_buf_set_user(&db->db, &children_dnodes->dnc_dbu);
+		if (winner != NULL) {
 
 			for (i = 0; i < epb; i++) {
 				zrl_destroy(&dnh[i].dnh_zrlock);
@@ -1130,17 +1159,11 @@ dnode_hold_impl(objset_t *os, uint64_t object, int flag,
 
 	dnh = &children_dnodes->dnc_children[idx];
 	zrl_add(&dnh->dnh_zrlock);
-	if ((dn = dnh->dnh_dnode) == NULL) {
+	dn = dnh->dnh_dnode;
+	if (dn == NULL) {
 		dnode_phys_t *phys = (dnode_phys_t *)db->db.db_data+idx;
-		dnode_t *winner;
 
 		dn = dnode_create(os, phys, db, object, dnh);
-		winner = atomic_cas_ptr(&dnh->dnh_dnode, NULL, dn);
-		if (winner != NULL) {
-			zrl_add(&dnh->dnh_zrlock);
-			dnode_destroy(dn); /* implicit zrl_remove() */
-			dn = winner;
-		}
 	}
 
 	mutex_enter(&dn->dn_mtx);
@@ -1154,10 +1177,10 @@ dnode_hold_impl(objset_t *os, uint64_t object, int flag,
 		dbuf_rele(db, FTAG);
 		return (type == DMU_OT_NONE ? ENOENT : EEXIST);
 	}
-	mutex_exit(&dn->dn_mtx);
-
 	if (refcount_add(&dn->dn_holds, tag) == 1)
 		dbuf_add_ref(db, dnh);
+	mutex_exit(&dn->dn_mtx);
+
 	/* Now we can rely on the hold to prevent the dnode from moving. */
 	zrl_remove(&dnh->dnh_zrlock);
 
diff --git a/uts/common/fs/zfs/dnode_sync.c b/uts/common/fs/zfs/dnode_sync.c
index 63bfc94f9aee..99ecb9e3ff73 100644
--- a/uts/common/fs/zfs/dnode_sync.c
+++ b/uts/common/fs/zfs/dnode_sync.c
@@ -22,6 +22,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -396,49 +397,37 @@ dnode_sync_free_range(void *arg, uint64_t blkid, uint64_t nblks)
 void
 dnode_evict_dbufs(dnode_t *dn)
 {
-	int progress;
-	int pass = 0;
+	dmu_buf_impl_t db_marker;
+	dmu_buf_impl_t *db, *db_next;
 
-	do {
-		dmu_buf_impl_t *db, *db_next;
-		int evicting = FALSE;
+	mutex_enter(&dn->dn_dbufs_mtx);
+	for (db = avl_first(&dn->dn_dbufs); db != NULL; db = db_next) {
 
-		progress = FALSE;
-		mutex_enter(&dn->dn_dbufs_mtx);
-		for (db = avl_first(&dn->dn_dbufs); db != NULL; db = db_next) {
-			db_next = AVL_NEXT(&dn->dn_dbufs, db);
 #ifdef	DEBUG
-			DB_DNODE_ENTER(db);
-			ASSERT3P(DB_DNODE(db), ==, dn);
-			DB_DNODE_EXIT(db);
+		DB_DNODE_ENTER(db);
+		ASSERT3P(DB_DNODE(db), ==, dn);
+		DB_DNODE_EXIT(db);
 #endif	/* DEBUG */
 
-			mutex_enter(&db->db_mtx);
-			if (db->db_state == DB_EVICTING) {
-				progress = TRUE;
-				evicting = TRUE;
-				mutex_exit(&db->db_mtx);
-			} else if (refcount_is_zero(&db->db_holds)) {
-				progress = TRUE;
-				dbuf_clear(db); /* exits db_mtx for us */
-			} else {
-				mutex_exit(&db->db_mtx);
-			}
+		mutex_enter(&db->db_mtx);
+		if (db->db_state != DB_EVICTING &&
+		    refcount_is_zero(&db->db_holds)) {
+			db_marker.db_level = db->db_level;
+			db_marker.db_blkid = db->db_blkid;
+			db_marker.db_state = DB_SEARCH;
+			avl_insert_here(&dn->dn_dbufs, &db_marker, db,
+			    AVL_BEFORE);
 
+			dbuf_clear(db);
+
+			db_next = AVL_NEXT(&dn->dn_dbufs, &db_marker);
+			avl_remove(&dn->dn_dbufs, &db_marker);
+		} else {
+			mutex_exit(&db->db_mtx);
+			db_next = AVL_NEXT(&dn->dn_dbufs, db);
 		}
-		/*
-		 * NB: we need to drop dn_dbufs_mtx between passes so
-		 * that any DB_EVICTING dbufs can make progress.
-		 * Ideally, we would have some cv we could wait on, but
-		 * since we don't, just wait a bit to give the other
-		 * thread a chance to run.
-		 */
-		mutex_exit(&dn->dn_dbufs_mtx);
-		if (evicting)
-			delay(1);
-		pass++;
-		ASSERT(pass < 100); /* sanity check */
-	} while (progress);
+	}
+	mutex_exit(&dn->dn_dbufs_mtx);
 
 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
 	if (dn->dn_bonus && refcount_is_zero(&dn->dn_bonus->db_holds)) {
@@ -497,7 +486,6 @@ dnode_sync_free(dnode_t *dn, dmu_tx_t *tx)
 	dnode_undirty_dbufs(&dn->dn_dirty_records[txgoff]);
 	dnode_evict_dbufs(dn);
 	ASSERT(avl_is_empty(&dn->dn_dbufs));
-	ASSERT3P(dn->dn_bonus, ==, NULL);
 
 	/*
 	 * XXX - It would be nice to assert this, but we may still
diff --git a/uts/common/fs/zfs/dsl_bookmark.c b/uts/common/fs/zfs/dsl_bookmark.c
index 5fb7f96600dc..bc08d1c10e8e 100644
--- a/uts/common/fs/zfs/dsl_bookmark.c
+++ b/uts/common/fs/zfs/dsl_bookmark.c
@@ -120,7 +120,7 @@ dsl_bookmark_create_check_impl(dsl_dataset_t *snapds, const char *bookmark_name,
 	int error;
 	zfs_bookmark_phys_t bmark_phys;
 
-	if (!dsl_dataset_is_snapshot(snapds))
+	if (!snapds->ds_is_snapshot)
 		return (SET_ERROR(EINVAL));
 
 	error = dsl_bookmark_hold_ds(dp, bookmark_name,
diff --git a/uts/common/fs/zfs/dsl_dataset.c b/uts/common/fs/zfs/dsl_dataset.c
index 5baf5c3c0a11..d3ca43af47d1 100644
--- a/uts/common/fs/zfs/dsl_dataset.c
+++ b/uts/common/fs/zfs/dsl_dataset.c
@@ -23,6 +23,7 @@
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2014, Joyent, Inc. All rights reserved.
  * Copyright (c) 2014 RackTop Systems.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #include <sys/dmu_objset.h>
@@ -71,7 +72,6 @@ int zfs_max_recordsize = 1 * 1024 * 1024;
 #define	DS_REF_MAX	(1ULL << 62)
 
 extern inline dsl_dataset_phys_t *dsl_dataset_phys(dsl_dataset_t *ds);
-extern inline boolean_t dsl_dataset_is_snapshot(dsl_dataset_t *ds);
 
 /*
  * Figure out how much of this delta should be propogated to the dsl_dir
@@ -155,7 +155,7 @@ dsl_dataset_block_kill(dsl_dataset_t *ds, const blkptr_t *bp, dmu_tx_t *tx,
 	}
 	ASSERT3P(tx->tx_pool, ==, ds->ds_dir->dd_pool);
 
-	ASSERT(!dsl_dataset_is_snapshot(ds));
+	ASSERT(!ds->ds_is_snapshot);
 	dmu_buf_will_dirty(ds->ds_dbuf, tx);
 
 	if (bp->blk_birth > dsl_dataset_phys(ds)->ds_prev_snap_txg) {
@@ -253,14 +253,15 @@ dsl_dataset_block_freeable(dsl_dataset_t *ds, const blkptr_t *bp,
 	return (B_TRUE);
 }
 
-/* ARGSUSED */
 static void
-dsl_dataset_evict(dmu_buf_t *db, void *dsv)
+dsl_dataset_evict(void *dbu)
 {
-	dsl_dataset_t *ds = dsv;
+	dsl_dataset_t *ds = dbu;
 
 	ASSERT(ds->ds_owner == NULL);
 
+	ds->ds_dbuf = NULL;
+
 	unique_remove(ds->ds_fsid_guid);
 
 	if (ds->ds_objset != NULL)
@@ -272,10 +273,10 @@ dsl_dataset_evict(dmu_buf_t *db, void *dsv)
 	}
 
 	bplist_destroy(&ds->ds_pending_deadlist);
-	if (dsl_dataset_phys(ds)->ds_deadlist_obj != 0)
+	if (ds->ds_deadlist.dl_os != NULL)
 		dsl_deadlist_close(&ds->ds_deadlist);
 	if (ds->ds_dir)
-		dsl_dir_rele(ds->ds_dir, ds);
+		dsl_dir_async_rele(ds->ds_dir, ds);
 
 	ASSERT(!list_link_active(&ds->ds_synced_link));
 
@@ -389,6 +390,7 @@ dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
 		ds = kmem_zalloc(sizeof (dsl_dataset_t), KM_SLEEP);
 		ds->ds_dbuf = dbuf;
 		ds->ds_object = dsobj;
+		ds->ds_is_snapshot = dsl_dataset_phys(ds)->ds_num_children != 0;
 
 		mutex_init(&ds->ds_lock, NULL, MUTEX_DEFAULT, NULL);
 		mutex_init(&ds->ds_opening_lock, NULL, MUTEX_DEFAULT, NULL);
@@ -427,7 +429,7 @@ dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
 			return (err);
 		}
 
-		if (!dsl_dataset_is_snapshot(ds)) {
+		if (!ds->ds_is_snapshot) {
 			ds->ds_snapname[0] = '\0';
 			if (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) {
 				err = dsl_dataset_hold_obj(dp,
@@ -454,7 +456,7 @@ dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
 			}
 		}
 
-		if (err == 0 && !dsl_dataset_is_snapshot(ds)) {
+		if (err == 0 && !ds->ds_is_snapshot) {
 			err = dsl_prop_get_int_ds(ds,
 			    zfs_prop_to_name(ZFS_PROP_REFRESERVATION),
 			    &ds->ds_reserved);
@@ -467,8 +469,11 @@ dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
 			ds->ds_reserved = ds->ds_quota = 0;
 		}
 
-		if (err != 0 || (winner = dmu_buf_set_user_ie(dbuf, ds,
-		    dsl_dataset_evict)) != NULL) {
+		dmu_buf_init_user(&ds->ds_dbu, dsl_dataset_evict, &ds->ds_dbuf);
+		if (err == 0)
+			winner = dmu_buf_set_user_ie(dbuf, &ds->ds_dbu);
+
+		if (err != 0 || winner != NULL) {
 			bplist_destroy(&ds->ds_pending_deadlist);
 			dsl_deadlist_close(&ds->ds_deadlist);
 			if (ds->ds_prev)
@@ -848,7 +853,7 @@ dsl_dataset_recalc_head_uniq(dsl_dataset_t *ds)
 	uint64_t mrs_used;
 	uint64_t dlused, dlcomp, dluncomp;
 
-	ASSERT(!dsl_dataset_is_snapshot(ds));
+	ASSERT(!ds->ds_is_snapshot);
 
 	if (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0)
 		mrs_used = dsl_dataset_phys(ds->ds_prev)->ds_referenced_bytes;
@@ -1591,7 +1596,7 @@ dsl_dataset_stats(dsl_dataset_t *ds, nvlist_t *nv)
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_LOGICALREFERENCED,
 	    dsl_dataset_phys(ds)->ds_uncompressed_bytes);
 
-	if (dsl_dataset_is_snapshot(ds)) {
+	if (ds->ds_is_snapshot) {
 		dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_COMPRESSRATIO, ratio);
 		dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_USED,
 		    dsl_dataset_phys(ds)->ds_unique_bytes);
@@ -1659,7 +1664,7 @@ dsl_dataset_fast_stat(dsl_dataset_t *ds, dmu_objset_stats_t *stat)
 	    dsl_dataset_phys(ds)->ds_flags & DS_FLAG_INCONSISTENT;
 	stat->dds_guid = dsl_dataset_phys(ds)->ds_guid;
 	stat->dds_origin[0] = '\0';
-	if (dsl_dataset_is_snapshot(ds)) {
+	if (ds->ds_is_snapshot) {
 		stat->dds_is_snapshot = B_TRUE;
 		stat->dds_num_clones =
 		    dsl_dataset_phys(ds)->ds_num_children - 1;
@@ -1919,7 +1924,7 @@ dsl_dataset_rollback_check(void *arg, dmu_tx_t *tx)
 		return (error);
 
 	/* must not be a snapshot */
-	if (dsl_dataset_is_snapshot(ds)) {
+	if (ds->ds_is_snapshot) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(EINVAL));
 	}
@@ -2491,7 +2496,7 @@ promote_hold(dsl_dataset_promote_arg_t *ddpa, dsl_pool_t *dp, void *tag)
 		return (error);
 	dd = ddpa->ddpa_clone->ds_dir;
 
-	if (dsl_dataset_is_snapshot(ddpa->ddpa_clone) ||
+	if (ddpa->ddpa_clone->ds_is_snapshot ||
 	    !dsl_dir_is_clone(dd)) {
 		dsl_dataset_rele(ddpa->ddpa_clone, tag);
 		return (SET_ERROR(EINVAL));
@@ -2583,8 +2588,8 @@ dsl_dataset_clone_swap_check_impl(dsl_dataset_t *clone,
 	int64_t unused_refres_delta;
 
 	/* they should both be heads */
-	if (dsl_dataset_is_snapshot(clone) ||
-	    dsl_dataset_is_snapshot(origin_head))
+	if (clone->ds_is_snapshot ||
+	    origin_head->ds_is_snapshot)
 		return (SET_ERROR(EINVAL));
 
 	/* if we are not forcing, the branch point should be just before them */
@@ -2863,7 +2868,7 @@ dsl_dataset_set_refquota_check(void *arg, dmu_tx_t *tx)
 	if (error != 0)
 		return (error);
 
-	if (dsl_dataset_is_snapshot(ds)) {
+	if (ds->ds_is_snapshot) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(EINVAL));
 	}
@@ -2946,7 +2951,7 @@ dsl_dataset_set_refreservation_check(void *arg, dmu_tx_t *tx)
 	if (error != 0)
 		return (error);
 
-	if (dsl_dataset_is_snapshot(ds)) {
+	if (ds->ds_is_snapshot) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(EINVAL));
 	}
@@ -3160,8 +3165,8 @@ dsl_dataset_space_wouldfree(dsl_dataset_t *firstsnap,
 	uint64_t snapobj;
 	dsl_pool_t *dp = firstsnap->ds_dir->dd_pool;
 
-	ASSERT(dsl_dataset_is_snapshot(firstsnap));
-	ASSERT(dsl_dataset_is_snapshot(lastsnap));
+	ASSERT(firstsnap->ds_is_snapshot);
+	ASSERT(lastsnap->ds_is_snapshot);
 
 	/*
 	 * Check that the snapshots are in the same dsl_dir, and firstsnap
@@ -3286,12 +3291,12 @@ dsl_dataset_is_before(dsl_dataset_t *later, dsl_dataset_t *earlier,
 	boolean_t ret;
 
 	ASSERT(dsl_pool_config_held(dp));
-	ASSERT(dsl_dataset_is_snapshot(earlier) || earlier_txg != 0);
+	ASSERT(earlier->ds_is_snapshot || earlier_txg != 0);
 
 	if (earlier_txg == 0)
 		earlier_txg = dsl_dataset_phys(earlier)->ds_creation_txg;
 
-	if (dsl_dataset_is_snapshot(later) &&
+	if (later->ds_is_snapshot &&
 	    earlier_txg >= dsl_dataset_phys(later)->ds_creation_txg)
 		return (B_FALSE);
 
diff --git a/uts/common/fs/zfs/dsl_deadlist.c b/uts/common/fs/zfs/dsl_deadlist.c
index 4ac562bfdba3..d26c6cd35797 100644
--- a/uts/common/fs/zfs/dsl_deadlist.c
+++ b/uts/common/fs/zfs/dsl_deadlist.c
@@ -21,6 +21,7 @@
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012 by Delphix. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #include <sys/dsl_dataset.h>
@@ -119,6 +120,8 @@ dsl_deadlist_close(dsl_deadlist_t *dl)
 	void *cookie = NULL;
 	dsl_deadlist_entry_t *dle;
 
+	dl->dl_os = NULL;
+
 	if (dl->dl_oldfmt) {
 		dl->dl_oldfmt = B_FALSE;
 		bpobj_close(&dl->dl_bpobj);
diff --git a/uts/common/fs/zfs/dsl_deleg.c b/uts/common/fs/zfs/dsl_deleg.c
index dc405ebe58b5..32fe03803f21 100644
--- a/uts/common/fs/zfs/dsl_deleg.c
+++ b/uts/common/fs/zfs/dsl_deleg.c
@@ -561,7 +561,7 @@ dsl_deleg_access_impl(dsl_dataset_t *ds, const char *perm, cred_t *cr)
 	    SPA_VERSION_DELEGATED_PERMS)
 		return (SET_ERROR(EPERM));
 
-	if (dsl_dataset_is_snapshot(ds)) {
+	if (ds->ds_is_snapshot) {
 		/*
 		 * Snapshots are treated as descendents only,
 		 * local permissions do not apply.
diff --git a/uts/common/fs/zfs/dsl_destroy.c b/uts/common/fs/zfs/dsl_destroy.c
index a776e144f1d2..7f9046957f9c 100644
--- a/uts/common/fs/zfs/dsl_destroy.c
+++ b/uts/common/fs/zfs/dsl_destroy.c
@@ -51,7 +51,7 @@ typedef struct dmu_snapshots_destroy_arg {
 int
 dsl_destroy_snapshot_check_impl(dsl_dataset_t *ds, boolean_t defer)
 {
-	if (!dsl_dataset_is_snapshot(ds))
+	if (!ds->ds_is_snapshot)
 		return (SET_ERROR(EINVAL));
 
 	if (dsl_dataset_long_held(ds))
@@ -354,7 +354,7 @@ dsl_destroy_snapshot_sync_impl(dsl_dataset_t *ds, boolean_t defer, dmu_tx_t *tx)
 	dsl_dataset_remove_clones_key(ds,
 	    dsl_dataset_phys(ds)->ds_creation_txg, tx);
 
-	if (dsl_dataset_is_snapshot(ds_next)) {
+	if (ds_next->ds_is_snapshot) {
 		dsl_dataset_t *ds_nextnext;
 
 		/*
@@ -604,8 +604,8 @@ dsl_destroy_head_check_impl(dsl_dataset_t *ds, int expected_holds)
 	uint64_t count;
 	objset_t *mos;
 
-	ASSERT(!dsl_dataset_is_snapshot(ds));
-	if (dsl_dataset_is_snapshot(ds))
+	ASSERT(!ds->ds_is_snapshot);
+	if (ds->ds_is_snapshot)
 		return (SET_ERROR(EINVAL));
 
 	if (refcount_count(&ds->ds_longholds) != expected_holds)
diff --git a/uts/common/fs/zfs/dsl_dir.c b/uts/common/fs/zfs/dsl_dir.c
index f49584168344..ee69cc8dbffe 100644
--- a/uts/common/fs/zfs/dsl_dir.c
+++ b/uts/common/fs/zfs/dsl_dir.c
@@ -23,6 +23,7 @@
  * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 Martin Matuska. All rights reserved.
  * Copyright (c) 2014 Joyent, Inc. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #include <sys/dmu.h>
@@ -125,14 +126,15 @@ extern inline dsl_dir_phys_t *dsl_dir_phys(dsl_dir_t *dd);
 
 static uint64_t dsl_dir_space_towrite(dsl_dir_t *dd);
 
-/* ARGSUSED */
 static void
-dsl_dir_evict(dmu_buf_t *db, void *arg)
+dsl_dir_evict(void *dbu)
 {
-	dsl_dir_t *dd = arg;
+	dsl_dir_t *dd = dbu;
 	dsl_pool_t *dp = dd->dd_pool;
 	int t;
 
+	dd->dd_dbuf = NULL;
+
 	for (t = 0; t < TXG_SIZE; t++) {
 		ASSERT(!txg_list_member(&dp->dp_dirty_dirs, dd, t));
 		ASSERT(dd->dd_tempreserved[t] == 0);
@@ -140,9 +142,9 @@ dsl_dir_evict(dmu_buf_t *db, void *arg)
 	}
 
 	if (dd->dd_parent)
-		dsl_dir_rele(dd->dd_parent, dd);
+		dsl_dir_async_rele(dd->dd_parent, dd);
 
-	spa_close(dd->dd_pool->dp_spa, dd);
+	spa_async_close(dd->dd_pool->dp_spa, dd);
 
 	/*
 	 * The props callback list should have been cleaned up by
@@ -238,8 +240,9 @@ dsl_dir_hold_obj(dsl_pool_t *dp, uint64_t ddobj,
 			dmu_buf_rele(origin_bonus, FTAG);
 		}
 
-		winner = dmu_buf_set_user_ie(dbuf, dd, dsl_dir_evict);
-		if (winner) {
+		dmu_buf_init_user(&dd->dd_dbu, dsl_dir_evict, &dd->dd_dbuf);
+		winner = dmu_buf_set_user_ie(dbuf, &dd->dd_dbu);
+		if (winner != NULL) {
 			if (dd->dd_parent)
 				dsl_dir_rele(dd->dd_parent, dd);
 			mutex_destroy(&dd->dd_lock);
@@ -283,6 +286,21 @@ dsl_dir_rele(dsl_dir_t *dd, void *tag)
 	dmu_buf_rele(dd->dd_dbuf, tag);
 }
 
+/*
+ * Remove a reference to the given dsl dir that is being asynchronously
+ * released.  Async releases occur from a taskq performing eviction of
+ * dsl datasets and dirs.  This process is identical to a normal release
+ * with the exception of using the async API for releasing the reference on
+ * the spa.
+ */
+void
+dsl_dir_async_rele(dsl_dir_t *dd, void *tag)
+{
+	dprintf_dd(dd, "%s\n", "");
+	spa_async_close(dd->dd_pool->dp_spa, tag);
+	dmu_buf_rele(dd->dd_dbuf, tag);
+}
+
 /* buf must be long enough (MAXNAMELEN + strlen(MOS_DIR_NAME) + 1 should do) */
 void
 dsl_dir_name(dsl_dir_t *dd, char *buf)
@@ -413,7 +431,7 @@ dsl_dir_hold(dsl_pool_t *dp, const char *name, void *tag,
 	}
 
 	while (next != NULL) {
-		dsl_dir_t *child_ds;
+		dsl_dir_t *child_dd;
 		err = getcomponent(next, buf, &nextnext);
 		if (err != 0)
 			break;
@@ -432,11 +450,11 @@ dsl_dir_hold(dsl_pool_t *dp, const char *name, void *tag,
 			break;
 		}
 
-		err = dsl_dir_hold_obj(dp, ddobj, buf, tag, &child_ds);
+		err = dsl_dir_hold_obj(dp, ddobj, buf, tag, &child_dd);
 		if (err != 0)
 			break;
 		dsl_dir_rele(dd, tag);
-		dd = child_ds;
+		dd = child_dd;
 		next = nextnext;
 	}
 
diff --git a/uts/common/fs/zfs/dsl_pool.c b/uts/common/fs/zfs/dsl_pool.c
index 39a797c27958..30b7b4c8f024 100644
--- a/uts/common/fs/zfs/dsl_pool.c
+++ b/uts/common/fs/zfs/dsl_pool.c
@@ -22,6 +22,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #include <sys/dsl_pool.h>
@@ -317,6 +318,8 @@ dsl_pool_close(dsl_pool_t *dp)
 	arc_flush(dp->dp_spa);
 	txg_fini(dp);
 	dsl_scan_fini(dp);
+	dmu_buf_user_evict_wait();
+
 	rrw_destroy(&dp->dp_config_rwlock);
 	mutex_destroy(&dp->dp_lock);
 	taskq_destroy(dp->dp_vnrele_taskq);
diff --git a/uts/common/fs/zfs/dsl_prop.c b/uts/common/fs/zfs/dsl_prop.c
index 4e93e43f9698..e795d735c6f0 100644
--- a/uts/common/fs/zfs/dsl_prop.c
+++ b/uts/common/fs/zfs/dsl_prop.c
@@ -163,19 +163,17 @@ dsl_prop_get_ds(dsl_dataset_t *ds, const char *propname,
 {
 	zfs_prop_t prop = zfs_name_to_prop(propname);
 	boolean_t inheritable;
-	boolean_t snapshot;
 	uint64_t zapobj;
 
 	ASSERT(dsl_pool_config_held(ds->ds_dir->dd_pool));
 	inheritable = (prop == ZPROP_INVAL || zfs_prop_inheritable(prop));
-	snapshot = dsl_dataset_is_snapshot(ds);
 	zapobj = dsl_dataset_phys(ds)->ds_props_obj;
 
 	if (zapobj != 0) {
 		objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset;
 		int err;
 
-		ASSERT(snapshot);
+		ASSERT(ds->ds_is_snapshot);
 
 		/* Check for a local value. */
 		err = zap_lookup(mos, zapobj, propname, intsz, numints, buf);
@@ -215,7 +213,7 @@ dsl_prop_get_ds(dsl_dataset_t *ds, const char *propname,
 	}
 
 	return (dsl_prop_get_dd(ds->ds_dir, propname,
-	    intsz, numints, buf, setpoint, snapshot));
+	    intsz, numints, buf, setpoint, ds->ds_is_snapshot));
 }
 
 /*
@@ -545,7 +543,7 @@ dsl_prop_set_sync_impl(dsl_dataset_t *ds, const char *propname,
 
 	isint = (dodefault(propname, 8, 1, &intval) == 0);
 
-	if (dsl_dataset_is_snapshot(ds)) {
+	if (ds->ds_is_snapshot) {
 		ASSERT(version >= SPA_VERSION_SNAP_PROPS);
 		if (dsl_dataset_phys(ds)->ds_props_obj == 0) {
 			dmu_buf_will_dirty(ds->ds_dbuf, tx);
@@ -642,7 +640,7 @@ dsl_prop_set_sync_impl(dsl_dataset_t *ds, const char *propname,
 	if (isint) {
 		VERIFY0(dsl_prop_get_int_ds(ds, propname, &intval));
 
-		if (dsl_dataset_is_snapshot(ds)) {
+		if (ds->ds_is_snapshot) {
 			dsl_prop_cb_record_t *cbr;
 			/*
 			 * It's a snapshot; nothing can inherit this
@@ -760,7 +758,7 @@ dsl_props_set_check(void *arg, dmu_tx_t *tx)
 		}
 	}
 
-	if (dsl_dataset_is_snapshot(ds) && version < SPA_VERSION_SNAP_PROPS) {
+	if (ds->ds_is_snapshot && version < SPA_VERSION_SNAP_PROPS) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(ENOTSUP));
 	}
@@ -983,7 +981,7 @@ dsl_prop_get_all_ds(dsl_dataset_t *ds, nvlist_t **nvp,
 
 	VERIFY(nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP) == 0);
 
-	if (dsl_dataset_is_snapshot(ds))
+	if (ds->ds_is_snapshot)
 		flags |= DSL_PROP_GET_SNAPSHOT;
 
 	ASSERT(dsl_pool_config_held(dp));
diff --git a/uts/common/fs/zfs/dsl_scan.c b/uts/common/fs/zfs/dsl_scan.c
index a372b2cdc0ca..0894ebd45451 100644
--- a/uts/common/fs/zfs/dsl_scan.c
+++ b/uts/common/fs/zfs/dsl_scan.c
@@ -375,7 +375,7 @@ static uint64_t
 dsl_scan_ds_maxtxg(dsl_dataset_t *ds)
 {
 	uint64_t smt = ds->ds_dir->dd_pool->dp_scan->scn_phys.scn_max_txg;
-	if (dsl_dataset_is_snapshot(ds))
+	if (ds->ds_is_snapshot)
 		return (MIN(smt, dsl_dataset_phys(ds)->ds_creation_txg));
 	return (smt);
 }
@@ -811,7 +811,7 @@ dsl_scan_ds_destroyed(dsl_dataset_t *ds, dmu_tx_t *tx)
 		return;
 
 	if (scn->scn_phys.scn_bookmark.zb_objset == ds->ds_object) {
-		if (dsl_dataset_is_snapshot(ds)) {
+		if (ds->ds_is_snapshot) {
 			/* Note, scn_cur_{min,max}_txg stays the same. */
 			scn->scn_phys.scn_bookmark.zb_objset =
 			    dsl_dataset_phys(ds)->ds_next_snap_obj;
@@ -833,7 +833,7 @@ dsl_scan_ds_destroyed(dsl_dataset_t *ds, dmu_tx_t *tx)
 		ASSERT3U(dsl_dataset_phys(ds)->ds_num_children, <=, 1);
 		VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset,
 		    scn->scn_phys.scn_queue_obj, ds->ds_object, tx));
-		if (dsl_dataset_is_snapshot(ds)) {
+		if (ds->ds_is_snapshot) {
 			/*
 			 * We keep the same mintxg; it could be >
 			 * ds_creation_txg if the previous snapshot was
@@ -1017,7 +1017,7 @@ dsl_scan_visitds(dsl_scan_t *scn, uint64_t dsobj, dmu_tx_t *tx)
 	 * ZIL here, rather than in scan_recurse(), because the regular
 	 * snapshot block-sharing rules don't apply to it.
 	 */
-	if (DSL_SCAN_IS_SCRUB_RESILVER(scn) && !dsl_dataset_is_snapshot(ds))
+	if (DSL_SCAN_IS_SCRUB_RESILVER(scn) && !ds->ds_is_snapshot)
 		dsl_scan_zil(dp, &os->os_zil_header);
 
 	/*
diff --git a/uts/common/fs/zfs/dsl_userhold.c b/uts/common/fs/zfs/dsl_userhold.c
index 0985ec91fd50..a67db5f19e30 100644
--- a/uts/common/fs/zfs/dsl_userhold.c
+++ b/uts/common/fs/zfs/dsl_userhold.c
@@ -352,7 +352,7 @@ dsl_dataset_user_release_check_one(dsl_dataset_user_release_arg_t *ddura,
 	objset_t *mos;
 	int numholds;
 
-	if (!dsl_dataset_is_snapshot(ds))
+	if (!ds->ds_is_snapshot)
 		return (SET_ERROR(EINVAL));
 
 	if (nvlist_empty(holds))
diff --git a/uts/common/fs/zfs/sa.c b/uts/common/fs/zfs/sa.c
index 27d8513541a4..2e3156c00a4d 100644
--- a/uts/common/fs/zfs/sa.c
+++ b/uts/common/fs/zfs/sa.c
@@ -23,6 +23,7 @@
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  * Portions Copyright 2011 iXsystems, Inc
  * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -210,6 +211,7 @@ sa_cache_constructor(void *buf, void *unused, int kmflag)
 {
 	sa_handle_t *hdl = buf;
 
+	hdl->sa_dbu.dbu_evict_func = NULL;
 	hdl->sa_bonus_tab = NULL;
 	hdl->sa_spill_tab = NULL;
 	hdl->sa_os = NULL;
@@ -225,6 +227,7 @@ static void
 sa_cache_destructor(void *buf, void *unused)
 {
 	sa_handle_t *hdl = buf;
+	hdl->sa_dbu.dbu_evict_func = NULL;
 	mutex_destroy(&hdl->sa_lock);
 }
 
@@ -1301,10 +1304,10 @@ sa_build_index(sa_handle_t *hdl, sa_buf_type_t buftype)
 }
 
 /*ARGSUSED*/
-void
-sa_evict(dmu_buf_t *db, void *sap)
+static void
+sa_evict(void *dbu)
 {
-	panic("evicting sa dbuf %p\n", (void *)db);
+	panic("evicting sa dbuf\n");
 }
 
 static void
@@ -1343,9 +1346,10 @@ sa_idx_tab_hold(objset_t *os, sa_idx_tab_t *idx_tab)
 void
 sa_handle_destroy(sa_handle_t *hdl)
 {
+	dmu_buf_t *db = hdl->sa_bonus;
+
 	mutex_enter(&hdl->sa_lock);
-	(void) dmu_buf_update_user((dmu_buf_t *)hdl->sa_bonus, hdl,
-	    NULL, NULL);
+	(void) dmu_buf_remove_user(db, &hdl->sa_dbu);
 
 	if (hdl->sa_bonus_tab) {
 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_bonus_tab);
@@ -1371,7 +1375,7 @@ sa_handle_get_from_db(objset_t *os, dmu_buf_t *db, void *userp,
 {
 	int error = 0;
 	dmu_object_info_t doi;
-	sa_handle_t *handle;
+	sa_handle_t *handle = NULL;
 
 #ifdef ZFS_DEBUG
 	dmu_object_info_from_db(db, &doi);
@@ -1381,9 +1385,12 @@ sa_handle_get_from_db(objset_t *os, dmu_buf_t *db, void *userp,
 	/* find handle, if it exists */
 	/* if one doesn't exist then create a new one, and initialize it */
 
-	handle = (hdl_type == SA_HDL_SHARED) ? dmu_buf_get_user(db) : NULL;
+	if (hdl_type == SA_HDL_SHARED)
+		handle = dmu_buf_get_user(db);
+
 	if (handle == NULL) {
-		sa_handle_t *newhandle;
+		sa_handle_t *winner = NULL;
+
 		handle = kmem_cache_alloc(sa_cache, KM_SLEEP);
 		handle->sa_userp = userp;
 		handle->sa_bonus = db;
@@ -1391,12 +1398,15 @@ sa_handle_get_from_db(objset_t *os, dmu_buf_t *db, void *userp,
 		handle->sa_spill = NULL;
 
 		error = sa_build_index(handle, SA_BONUS);
-		newhandle = (hdl_type == SA_HDL_SHARED) ?
-		    dmu_buf_set_user_ie(db, handle, sa_evict) : NULL;
 
-		if (newhandle != NULL) {
+		if (hdl_type == SA_HDL_SHARED) {
+			dmu_buf_init_user(&handle->sa_dbu, sa_evict, NULL);
+			winner = dmu_buf_set_user_ie(db, &handle->sa_dbu);
+		}
+
+		if (winner != NULL) {
 			kmem_cache_free(sa_cache, handle);
-			handle = newhandle;
+			handle = winner;
 		}
 	}
 	*handlepp = handle;
@@ -1906,14 +1916,6 @@ sa_object_size(sa_handle_t *hdl, uint32_t *blksize, u_longlong_t *nblocks)
 	    blksize, nblocks);
 }
 
-void
-sa_update_user(sa_handle_t *newhdl, sa_handle_t *oldhdl)
-{
-	(void) dmu_buf_update_user((dmu_buf_t *)newhdl->sa_bonus,
-	    oldhdl, newhdl, sa_evict);
-	oldhdl->sa_bonus = NULL;
-}
-
 void
 sa_set_userp(sa_handle_t *hdl, void *ptr)
 {
diff --git a/uts/common/fs/zfs/spa.c b/uts/common/fs/zfs/spa.c
index d0408a58023c..7947d25f48d1 100644
--- a/uts/common/fs/zfs/spa.c
+++ b/uts/common/fs/zfs/spa.c
@@ -23,6 +23,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013, 2014, Nexenta Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 /*
@@ -1088,6 +1089,8 @@ spa_activate(spa_t *spa, int mode)
 
 	list_create(&spa->spa_config_dirty_list, sizeof (vdev_t),
 	    offsetof(vdev_t, vdev_config_dirty_node));
+	list_create(&spa->spa_evicting_os_list, sizeof (objset_t),
+	    offsetof(objset_t, os_evicting_node));
 	list_create(&spa->spa_state_dirty_list, sizeof (vdev_t),
 	    offsetof(vdev_t, vdev_state_dirty_node));
 
@@ -1114,9 +1117,12 @@ spa_deactivate(spa_t *spa)
 	ASSERT(spa->spa_async_zio_root == NULL);
 	ASSERT(spa->spa_state != POOL_STATE_UNINITIALIZED);
 
+	spa_evicting_os_wait(spa);
+
 	txg_list_destroy(&spa->spa_vdev_txg_list);
 
 	list_destroy(&spa->spa_config_dirty_list);
+	list_destroy(&spa->spa_evicting_os_list);
 	list_destroy(&spa->spa_state_dirty_list);
 
 	for (int t = 0; t < ZIO_TYPES; t++) {
@@ -2107,6 +2113,11 @@ spa_load(spa_t *spa, spa_load_state_t state, spa_import_type_t type,
 		    mosconfig, &ereport);
 	}
 
+	/*
+	 * Don't count references from objsets that are already closed
+	 * and are making their way through the eviction process.
+	 */
+	spa_evicting_os_wait(spa);
 	spa->spa_minref = refcount_count(&spa->spa_refcount);
 	if (error) {
 		if (error != EEXIST) {
@@ -3675,6 +3686,11 @@ spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props,
 
 	spa_history_log_version(spa, "create");
 
+	/*
+	 * Don't count references from objsets that are already closed
+	 * and are making their way through the eviction process.
+	 */
+	spa_evicting_os_wait(spa);
 	spa->spa_minref = refcount_count(&spa->spa_refcount);
 
 	mutex_exit(&spa_namespace_lock);
@@ -4207,6 +4223,7 @@ spa_export_common(char *pool, int new_state, nvlist_t **oldconfig,
 		 * have to force it to sync before checking spa_refcnt.
 		 */
 		txg_wait_synced(spa->spa_dsl_pool, 0);
+		spa_evicting_os_wait(spa);
 
 		/*
 		 * A pool cannot be exported or destroyed if there are active
diff --git a/uts/common/fs/zfs/spa_misc.c b/uts/common/fs/zfs/spa_misc.c
index 1729ba0503be..56ad27cf9ae7 100644
--- a/uts/common/fs/zfs/spa_misc.c
+++ b/uts/common/fs/zfs/spa_misc.c
@@ -22,6 +22,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -551,6 +552,7 @@ spa_add(const char *name, nvlist_t *config, const char *altroot)
 	mutex_init(&spa->spa_async_lock, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&spa->spa_errlist_lock, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&spa->spa_errlog_lock, NULL, MUTEX_DEFAULT, NULL);
+	mutex_init(&spa->spa_evicting_os_lock, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&spa->spa_history_lock, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&spa->spa_proc_lock, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&spa->spa_props_lock, NULL, MUTEX_DEFAULT, NULL);
@@ -560,6 +562,7 @@ spa_add(const char *name, nvlist_t *config, const char *altroot)
 	mutex_init(&spa->spa_iokstat_lock, NULL, MUTEX_DEFAULT, NULL);
 
 	cv_init(&spa->spa_async_cv, NULL, CV_DEFAULT, NULL);
+	cv_init(&spa->spa_evicting_os_cv, NULL, CV_DEFAULT, NULL);
 	cv_init(&spa->spa_proc_cv, NULL, CV_DEFAULT, NULL);
 	cv_init(&spa->spa_scrub_io_cv, NULL, CV_DEFAULT, NULL);
 	cv_init(&spa->spa_suspend_cv, NULL, CV_DEFAULT, NULL);
@@ -669,6 +672,7 @@ spa_remove(spa_t *spa)
 
 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
 	ASSERT(spa->spa_state == POOL_STATE_UNINITIALIZED);
+	ASSERT3U(refcount_count(&spa->spa_refcount), ==, 0);
 
 	nvlist_free(spa->spa_config_splitting);
 
@@ -710,6 +714,7 @@ spa_remove(spa_t *spa)
 		bplist_destroy(&spa->spa_free_bplist[t]);
 
 	cv_destroy(&spa->spa_async_cv);
+	cv_destroy(&spa->spa_evicting_os_cv);
 	cv_destroy(&spa->spa_proc_cv);
 	cv_destroy(&spa->spa_scrub_io_cv);
 	cv_destroy(&spa->spa_suspend_cv);
@@ -717,6 +722,7 @@ spa_remove(spa_t *spa)
 	mutex_destroy(&spa->spa_async_lock);
 	mutex_destroy(&spa->spa_errlist_lock);
 	mutex_destroy(&spa->spa_errlog_lock);
+	mutex_destroy(&spa->spa_evicting_os_lock);
 	mutex_destroy(&spa->spa_history_lock);
 	mutex_destroy(&spa->spa_proc_lock);
 	mutex_destroy(&spa->spa_props_lock);
@@ -773,6 +779,20 @@ spa_close(spa_t *spa, void *tag)
 	(void) refcount_remove(&spa->spa_refcount, tag);
 }
 
+/*
+ * Remove a reference to the given spa_t held by a dsl dir that is
+ * being asynchronously released.  Async releases occur from a taskq
+ * performing eviction of dsl datasets and dirs.  The namespace lock
+ * isn't held and the hold by the object being evicted may contribute to
+ * spa_minref (e.g. dataset or directory released during pool export),
+ * so the asserts in spa_close() do not apply.
+ */
+void
+spa_async_close(spa_t *spa, void *tag)
+{
+	(void) refcount_remove(&spa->spa_refcount, tag);
+}
+
 /*
  * Check to see if the spa refcount is zero.  Must be called with
  * spa_namespace_lock held.  We really compare against spa_minref, which is the
@@ -1669,6 +1689,34 @@ spa_log_class(spa_t *spa)
 	return (spa->spa_log_class);
 }
 
+void
+spa_evicting_os_register(spa_t *spa, objset_t *os)
+{
+	mutex_enter(&spa->spa_evicting_os_lock);
+	list_insert_head(&spa->spa_evicting_os_list, os);
+	mutex_exit(&spa->spa_evicting_os_lock);
+}
+
+void
+spa_evicting_os_deregister(spa_t *spa, objset_t *os)
+{
+	mutex_enter(&spa->spa_evicting_os_lock);
+	list_remove(&spa->spa_evicting_os_list, os);
+	cv_broadcast(&spa->spa_evicting_os_cv);
+	mutex_exit(&spa->spa_evicting_os_lock);
+}
+
+void
+spa_evicting_os_wait(spa_t *spa)
+{
+	mutex_enter(&spa->spa_evicting_os_lock);
+	while (!list_is_empty(&spa->spa_evicting_os_list))
+		cv_wait(&spa->spa_evicting_os_cv, &spa->spa_evicting_os_lock);
+	mutex_exit(&spa->spa_evicting_os_lock);
+
+	dmu_buf_user_evict_wait();
+}
+
 int
 spa_max_replication(spa_t *spa)
 {
diff --git a/uts/common/fs/zfs/sys/dbuf.h b/uts/common/fs/zfs/sys/dbuf.h
index 8be8ed6bc85b..6da5dccc05f7 100644
--- a/uts/common/fs/zfs/sys/dbuf.h
+++ b/uts/common/fs/zfs/sys/dbuf.h
@@ -22,6 +22,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #ifndef	_SYS_DBUF_H
@@ -226,9 +227,8 @@ typedef struct dmu_buf_impl {
 
 	/* Data which is unique to data (leaf) blocks: */
 
-	/* stuff we store for the user (see dmu_buf_set_user) */
-	void *db_user_ptr;
-	dmu_buf_evict_func_t *db_evict_func;
+	/* User callback information. */
+	dmu_buf_user_t *db_user;
 
 	uint8_t db_immediate_evict;
 	uint8_t db_freed_in_flight;
diff --git a/uts/common/fs/zfs/sys/dmu.h b/uts/common/fs/zfs/sys/dmu.h
index 24473daa6d2b..9b8b9ddc2016 100644
--- a/uts/common/fs/zfs/sys/dmu.h
+++ b/uts/common/fs/zfs/sys/dmu.h
@@ -26,6 +26,7 @@
  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
  * Copyright 2013 DEY Storage Systems, Inc.
  * Copyright 2014 HybridCluster. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 /* Portions Copyright 2010 Robert Milkowski */
@@ -41,11 +42,9 @@
  * dmu_spa.h.
  */
 
+#include <sys/zfs_context.h>
 #include <sys/inttypes.h>
-#include <sys/types.h>
-#include <sys/param.h>
 #include <sys/cred.h>
-#include <sys/time.h>
 #include <sys/fs/zfs.h>
 
 #ifdef	__cplusplus
@@ -291,8 +290,6 @@ typedef struct dmu_buf {
 	void *db_data;			/* data in buffer */
 } dmu_buf_t;
 
-typedef void dmu_buf_evict_func_t(struct dmu_buf *db, void *user_ptr);
-
 /*
  * The names of zap entries in the DIRECTORY_OBJECT of the MOS.
  */
@@ -478,36 +475,126 @@ int dmu_buf_hold_array_by_bonus(dmu_buf_t *db, uint64_t offset,
     uint64_t length, int read, void *tag, int *numbufsp, dmu_buf_t ***dbpp);
 void dmu_buf_rele_array(dmu_buf_t **, int numbufs, void *tag);
 
-/*
- * Returns NULL on success, or the existing user ptr if it's already
- * been set.
- *
- * user_ptr is for use by the user and can be obtained via dmu_buf_get_user().
- *
- * If non-NULL, pageout func will be called when this buffer is being
- * excised from the cache, so that you can clean up the data structure
- * pointed to by user_ptr.
- *
- * dmu_evict_user() will call the pageout func for all buffers in a
- * objset with a given pageout func.
- */
-void *dmu_buf_set_user(dmu_buf_t *db, void *user_ptr,
-    dmu_buf_evict_func_t *pageout_func);
-/*
- * set_user_ie is the same as set_user, but request immediate eviction
- * when hold count goes to zero.
- */
-void *dmu_buf_set_user_ie(dmu_buf_t *db, void *user_ptr,
-    dmu_buf_evict_func_t *pageout_func);
-void *dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr,
-    void *user_ptr, dmu_buf_evict_func_t *pageout_func);
-void dmu_evict_user(objset_t *os, dmu_buf_evict_func_t *func);
+typedef void dmu_buf_evict_func_t(void *user_ptr);
 
 /*
- * Returns the user_ptr set with dmu_buf_set_user(), or NULL if not set.
+ * A DMU buffer user object may be associated with a dbuf for the
+ * duration of its lifetime.  This allows the user of a dbuf (client)
+ * to attach private data to a dbuf (e.g. in-core only data such as a
+ * dnode_children_t, zap_t, or zap_leaf_t) and be optionally notified
+ * when that dbuf has been evicted.  Clients typically respond to the
+ * eviction notification by freeing their private data, thus ensuring
+ * the same lifetime for both dbuf and private data.
+ *
+ * The mapping from a dmu_buf_user_t to any client private data is the
+ * client's responsibility.  All current consumers of the API with private
+ * data embed a dmu_buf_user_t as the first member of the structure for
+ * their private data.  This allows conversions between the two types
+ * with a simple cast.  Since the DMU buf user API never needs access
+ * to the private data, other strategies can be employed if necessary
+ * or convenient for the client (e.g. using container_of() to do the
+ * conversion for private data that cannot have the dmu_buf_user_t as
+ * its first member).
+ *
+ * Eviction callbacks are executed without the dbuf mutex held or any
+ * other type of mechanism to guarantee that the dbuf is still available.
+ * For this reason, users must assume the dbuf has already been freed
+ * and not reference the dbuf from the callback context.
+ *
+ * Users requesting "immediate eviction" are notified as soon as the dbuf
+ * is only referenced by dirty records (dirties == holds).  Otherwise the
+ * notification occurs after eviction processing for the dbuf begins.
+ */
+typedef struct dmu_buf_user {
+	/*
+	 * Asynchronous user eviction callback state.
+	 */
+	taskq_ent_t	dbu_tqent;
+
+	/* This instance's eviction function pointer. */
+	dmu_buf_evict_func_t *dbu_evict_func;
+#ifdef ZFS_DEBUG
+	/*
+	 * Pointer to user's dbuf pointer.  NULL for clients that do
+	 * not associate a dbuf with their user data.
+	 *
+	 * The dbuf pointer is cleared upon eviction so as to catch
+	 * use-after-evict bugs in clients.
+	 */
+	dmu_buf_t **dbu_clear_on_evict_dbufp;
+#endif
+} dmu_buf_user_t;
+
+/*
+ * Initialize the given dmu_buf_user_t instance with the eviction function
+ * evict_func, to be called when the user is evicted.
+ *
+ * NOTE: This function should only be called once on a given dmu_buf_user_t.
+ *       To allow enforcement of this, dbu must already be zeroed on entry.
+ */
+#ifdef __lint
+/* Very ugly, but it beats issuing suppression directives in many Makefiles. */
+extern void
+dmu_buf_init_user(dmu_buf_user_t *dbu, dmu_buf_evict_func_t *evict_func,
+    dmu_buf_t **clear_on_evict_dbufp);
+#else /* __lint */
+inline void
+dmu_buf_init_user(dmu_buf_user_t *dbu, dmu_buf_evict_func_t *evict_func,
+    dmu_buf_t **clear_on_evict_dbufp)
+{
+	ASSERT(dbu->dbu_evict_func == NULL);
+	ASSERT(evict_func != NULL);
+	dbu->dbu_evict_func = evict_func;
+#ifdef ZFS_DEBUG
+	dbu->dbu_clear_on_evict_dbufp = clear_on_evict_dbufp;
+#endif
+}
+#endif /* __lint */
+
+/*
+ * Attach user data to a dbuf and mark it for normal (when the dbuf's
+ * data is cleared or its reference count goes to zero) eviction processing.
+ *
+ * Returns NULL on success, or the existing user if another user currently
+ * owns the buffer.
+ */
+void *dmu_buf_set_user(dmu_buf_t *db, dmu_buf_user_t *user);
+
+/*
+ * Attach user data to a dbuf and mark it for immediate (its dirty and
+ * reference counts are equal) eviction processing.
+ *
+ * Returns NULL on success, or the existing user if another user currently
+ * owns the buffer.
+ */
+void *dmu_buf_set_user_ie(dmu_buf_t *db, dmu_buf_user_t *user);
+
+/*
+ * Replace the current user of a dbuf.
+ *
+ * If given the current user of a dbuf, replaces the dbuf's user with
+ * "new_user" and returns the user data pointer that was replaced.
+ * Otherwise returns the current, and unmodified, dbuf user pointer.
+ */
+void *dmu_buf_replace_user(dmu_buf_t *db,
+    dmu_buf_user_t *old_user, dmu_buf_user_t *new_user);
+
+/*
+ * Remove the specified user data for a DMU buffer.
+ *
+ * Returns the user that was removed on success, or the current user if
+ * another user currently owns the buffer.
+ */
+void *dmu_buf_remove_user(dmu_buf_t *db, dmu_buf_user_t *user);
+
+/*
+ * Returns the user data (dmu_buf_user_t *) associated with this dbuf.
  */
 void *dmu_buf_get_user(dmu_buf_t *db);
 
+/* Block until any in-progress dmu buf user evictions complete. */
+void dmu_buf_user_evict_wait(void);
+
 /*
  * Returns the blkptr associated with this dbuf, or NULL if not set.
  */
diff --git a/uts/common/fs/zfs/sys/dmu_objset.h b/uts/common/fs/zfs/sys/dmu_objset.h
index 804f0c182b6f..e00847e6cbba 100644
--- a/uts/common/fs/zfs/sys/dmu_objset.h
+++ b/uts/common/fs/zfs/sys/dmu_objset.h
@@ -22,6 +22,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 /* Portions Copyright 2010 Robert Milkowski */
@@ -74,22 +75,25 @@ struct objset {
 	arc_buf_t *os_phys_buf;
 	objset_phys_t *os_phys;
 	/*
-	 * The following "special" dnodes have no parent and are exempt from
-	 * dnode_move(), but they root their descendents in this objset using
-	 * handles anyway, so that all access to dnodes from dbufs consistently
-	 * uses handles.
+	 * The following "special" dnodes have no parent, are exempt
+	 * from dnode_move(), and are not recorded in os_dnodes, but they
+	 * root their descendents in this objset using handles anyway, so
+	 * that all access to dnodes from dbufs consistently uses handles.
 	 */
 	dnode_handle_t os_meta_dnode;
 	dnode_handle_t os_userused_dnode;
 	dnode_handle_t os_groupused_dnode;
 	zilog_t *os_zil;
 
+	list_node_t os_evicting_node;
+
 	/* can change, under dsl_dir's locks: */
 	enum zio_checksum os_checksum;
 	enum zio_compress os_compress;
 	uint8_t os_copies;
 	enum zio_checksum os_dedup_checksum;
 	boolean_t os_dedup_verify;
+	boolean_t os_evicting;
 	zfs_logbias_op_t os_logbias;
 	zfs_cache_type_t os_primary_cache;
 	zfs_cache_type_t os_secondary_cache;
@@ -170,6 +174,8 @@ int dmu_objset_userspace_upgrade(objset_t *os);
 boolean_t dmu_objset_userspace_present(objset_t *os);
 int dmu_fsname(const char *snapname, char *buf);
 
+void dmu_objset_evict_done(objset_t *os);
+
 void dmu_objset_init(void);
 void dmu_objset_fini(void);
 
diff --git a/uts/common/fs/zfs/sys/dnode.h b/uts/common/fs/zfs/sys/dnode.h
index 5668af1fef08..406954a65323 100644
--- a/uts/common/fs/zfs/sys/dnode.h
+++ b/uts/common/fs/zfs/sys/dnode.h
@@ -21,6 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #ifndef	_SYS_DNODE_H
@@ -255,6 +256,7 @@ typedef struct dnode_handle {
 } dnode_handle_t;
 
 typedef struct dnode_children {
+	dmu_buf_user_t dnc_dbu;		/* User evict data */
 	size_t dnc_count;		/* number of children */
 	dnode_handle_t dnc_children[];	/* sized dynamically */
 } dnode_children_t;
@@ -265,7 +267,7 @@ typedef struct free_range {
 	uint64_t fr_nblks;
 } free_range_t;
 
-dnode_t *dnode_special_open(struct objset *dd, dnode_phys_t *dnp,
+void dnode_special_open(struct objset *dd, dnode_phys_t *dnp,
     uint64_t object, dnode_handle_t *dnh);
 void dnode_special_close(dnode_handle_t *dnh);
 
diff --git a/uts/common/fs/zfs/sys/dsl_dataset.h b/uts/common/fs/zfs/sys/dsl_dataset.h
index 3160a05a8c59..f0c78c971bef 100644
--- a/uts/common/fs/zfs/sys/dsl_dataset.h
+++ b/uts/common/fs/zfs/sys/dsl_dataset.h
@@ -23,6 +23,7 @@
  * Copyright (c) 2013 by Delphix. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #ifndef	_SYS_DSL_DATASET_H
@@ -132,11 +133,14 @@ typedef struct dsl_dataset_phys {
 } dsl_dataset_phys_t;
 
 typedef struct dsl_dataset {
+	dmu_buf_user_t ds_dbu;
+
 	/* Immutable: */
 	struct dsl_dir *ds_dir;
 	dmu_buf_t *ds_dbuf;
 	uint64_t ds_object;
 	uint64_t ds_fsid_guid;
+	boolean_t ds_is_snapshot;
 
 	/* only used in syncing context, only valid for non-snapshots: */
 	struct dsl_dataset *ds_prev;
@@ -197,12 +201,6 @@ dsl_dataset_phys(dsl_dataset_t *ds)
  */
 #define	MAX_TAG_PREFIX_LEN	17
 
-inline boolean_t
-dsl_dataset_is_snapshot(dsl_dataset_t *ds)
-{
-	return (dsl_dataset_phys(ds)->ds_num_children != 0);
-}
-
 #define	DS_UNIQUE_IS_ACCURATE(ds)	\
 	((dsl_dataset_phys(ds)->ds_flags & DS_FLAG_UNIQUE_ACCURATE) != 0)
 
diff --git a/uts/common/fs/zfs/sys/dsl_dir.h b/uts/common/fs/zfs/sys/dsl_dir.h
index 772bfbe6db12..f50014d95a81 100644
--- a/uts/common/fs/zfs/sys/dsl_dir.h
+++ b/uts/common/fs/zfs/sys/dsl_dir.h
@@ -22,6 +22,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2013 by Delphix. All rights reserved.
  * Copyright (c) 2014, Joyent, Inc. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #ifndef	_SYS_DSL_DIR_H
@@ -84,6 +85,8 @@ typedef struct dsl_dir_phys {
 } dsl_dir_phys_t;
 
 struct dsl_dir {
+	dmu_buf_user_t dd_dbu;
+
 	/* These are immutable; no lock needed: */
 	uint64_t dd_object;
 	dsl_pool_t *dd_pool;
@@ -119,6 +122,7 @@ dsl_dir_phys(dsl_dir_t *dd)
 }
 
 void dsl_dir_rele(dsl_dir_t *dd, void *tag);
+void dsl_dir_async_rele(dsl_dir_t *dd, void *tag);
 int dsl_dir_hold(dsl_pool_t *dp, const char *name, void *tag,
     dsl_dir_t **, const char **tail);
 int dsl_dir_hold_obj(dsl_pool_t *dp, uint64_t ddobj,
diff --git a/uts/common/fs/zfs/sys/sa.h b/uts/common/fs/zfs/sys/sa.h
index bc89fa07d222..a60eb3772cd4 100644
--- a/uts/common/fs/zfs/sys/sa.h
+++ b/uts/common/fs/zfs/sys/sa.h
@@ -133,7 +133,6 @@ int sa_update_from_cb(sa_handle_t *, sa_attr_type_t,
     uint32_t buflen, sa_data_locator_t *, void *userdata, dmu_tx_t *);
 void sa_object_info(sa_handle_t *, dmu_object_info_t *);
 void sa_object_size(sa_handle_t *, uint32_t *, u_longlong_t *);
-void sa_update_user(sa_handle_t *, sa_handle_t *);
 void *sa_get_userdata(sa_handle_t *);
 void sa_set_userp(sa_handle_t *, void *);
 dmu_buf_t *sa_get_db(sa_handle_t *);
diff --git a/uts/common/fs/zfs/sys/sa_impl.h b/uts/common/fs/zfs/sys/sa_impl.h
index 6b9af2ef4f89..50874c6bf083 100644
--- a/uts/common/fs/zfs/sys/sa_impl.h
+++ b/uts/common/fs/zfs/sys/sa_impl.h
@@ -21,6 +21,7 @@
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #ifndef	_SYS_SA_IMPL_H
@@ -210,11 +211,12 @@ typedef enum sa_data_op {
  */
 
 struct sa_handle {
+	dmu_buf_user_t	sa_dbu;
 	kmutex_t	sa_lock;
 	dmu_buf_t	*sa_bonus;
 	dmu_buf_t	*sa_spill;
 	objset_t	*sa_os;
-	void 		*sa_userp;
+	void		*sa_userp;
 	sa_idx_tab_t	*sa_bonus_tab;	 /* idx of bonus */
 	sa_idx_tab_t	*sa_spill_tab; /* only present if spill activated */
 };
diff --git a/uts/common/fs/zfs/sys/spa.h b/uts/common/fs/zfs/sys/spa.h
index cf307a2ec977..4dd6ce69af91 100644
--- a/uts/common/fs/zfs/sys/spa.h
+++ b/uts/common/fs/zfs/sys/spa.h
@@ -22,6 +22,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #ifndef _SYS_SPA_H
@@ -682,6 +683,7 @@ extern spa_t *spa_next(spa_t *prev);
 /* Refcount functions */
 extern void spa_open_ref(spa_t *spa, void *tag);
 extern void spa_close(spa_t *spa, void *tag);
+extern void spa_async_close(spa_t *spa, void *tag);
 extern boolean_t spa_refcount_zero(spa_t *spa);
 
 #define	SCL_NONE	0x00
@@ -754,6 +756,9 @@ extern uint64_t spa_version(spa_t *spa);
 extern boolean_t spa_deflate(spa_t *spa);
 extern metaslab_class_t *spa_normal_class(spa_t *spa);
 extern metaslab_class_t *spa_log_class(spa_t *spa);
+extern void spa_evicting_os_register(spa_t *, objset_t *os);
+extern void spa_evicting_os_deregister(spa_t *, objset_t *os);
+extern void spa_evicting_os_wait(spa_t *spa);
 extern int spa_max_replication(spa_t *spa);
 extern int spa_prev_software_version(spa_t *spa);
 extern int spa_busy(void);
diff --git a/uts/common/fs/zfs/sys/spa_impl.h b/uts/common/fs/zfs/sys/spa_impl.h
index 48b28eb5a139..9bbb240e0a11 100644
--- a/uts/common/fs/zfs/sys/spa_impl.h
+++ b/uts/common/fs/zfs/sys/spa_impl.h
@@ -22,6 +22,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #ifndef _SYS_SPA_IMPL_H
@@ -144,6 +145,9 @@ struct spa {
 	uint64_t	spa_claim_max_txg;	/* highest claimed birth txg */
 	timespec_t	spa_loaded_ts;		/* 1st successful open time */
 	objset_t	*spa_meta_objset;	/* copy of dp->dp_meta_objset */
+	kmutex_t	spa_evicting_os_lock;	/* Evicting objset list lock */
+	list_t		spa_evicting_os_list;	/* Objsets being evicted. */
+	kcondvar_t	spa_evicting_os_cv;	/* Objset Eviction Completion */
 	txg_list_t	spa_vdev_txg_list;	/* per-txg dirty vdev list */
 	vdev_t		*spa_root_vdev;		/* top-level vdev container */
 	uint64_t	spa_config_guid;	/* config pool guid */
diff --git a/uts/common/fs/zfs/sys/zap_impl.h b/uts/common/fs/zfs/sys/zap_impl.h
index 0d4fc69815fc..acc795cbc44e 100644
--- a/uts/common/fs/zfs/sys/zap_impl.h
+++ b/uts/common/fs/zfs/sys/zap_impl.h
@@ -21,6 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #ifndef	_SYS_ZAP_IMPL_H
@@ -140,6 +141,7 @@ typedef struct zap_phys {
 typedef struct zap_table_phys zap_table_phys_t;
 
 typedef struct zap {
+	dmu_buf_user_t zap_dbu;
 	objset_t *zap_objset;
 	uint64_t zap_object;
 	struct dmu_buf *zap_dbuf;
@@ -196,7 +198,7 @@ boolean_t zap_match(zap_name_t *zn, const char *matchname);
 int zap_lockdir(objset_t *os, uint64_t obj, dmu_tx_t *tx,
     krw_t lti, boolean_t fatreader, boolean_t adding, zap_t **zapp);
 void zap_unlockdir(zap_t *zap);
-void zap_evict(dmu_buf_t *db, void *vmzap);
+void zap_evict(void *dbu);
 zap_name_t *zap_name_alloc(zap_t *zap, const char *key, matchtype_t mt);
 void zap_name_free(zap_name_t *zn);
 int zap_hashbits(zap_t *zap);
diff --git a/uts/common/fs/zfs/sys/zap_leaf.h b/uts/common/fs/zfs/sys/zap_leaf.h
index cd8b74a77a8c..76b3ecc72557 100644
--- a/uts/common/fs/zfs/sys/zap_leaf.h
+++ b/uts/common/fs/zfs/sys/zap_leaf.h
@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #ifndef	_SYS_ZAP_LEAF_H
@@ -152,6 +153,7 @@ typedef union zap_leaf_chunk {
 } zap_leaf_chunk_t;
 
 typedef struct zap_leaf {
+	dmu_buf_user_t l_dbu;
 	krwlock_t l_rwlock;
 	uint64_t l_blkid;		/* 1<<ZAP_BLOCK_SHIFT byte block off */
 	int l_bs;			/* block size shift */
diff --git a/uts/common/fs/zfs/zap.c b/uts/common/fs/zfs/zap.c
index de2d4deb00eb..20f349651000 100644
--- a/uts/common/fs/zfs/zap.c
+++ b/uts/common/fs/zfs/zap.c
@@ -21,6 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 /*
@@ -52,7 +53,6 @@ int fzap_default_block_shift = 14; /* 16k blocksize */
 
 extern inline zap_phys_t *zap_f_phys(zap_t *zap);
 
-static void zap_leaf_pageout(dmu_buf_t *db, void *vl);
 static uint64_t zap_allocate_blocks(zap_t *zap, int nblocks);
 
 void
@@ -81,7 +81,7 @@ fzap_upgrade(zap_t *zap, dmu_tx_t *tx, zap_flags_t flags)
 	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));
 	zap->zap_ismicro = FALSE;
 
-	(void) dmu_buf_update_user(zap->zap_dbuf, zap, zap, zap_evict);
+	zap->zap_dbu.dbu_evict_func = zap_evict;
 
 	mutex_init(&zap->zap_f.zap_num_entries_mtx, 0, 0, 0);
 	zap->zap_f.zap_block_shift = highbit64(zap->zap_dbuf->db_size) - 1;
@@ -387,11 +387,20 @@ zap_allocate_blocks(zap_t *zap, int nblocks)
 	return (newblk);
 }
 
+static void
+zap_leaf_pageout(void *dbu)
+{
+	zap_leaf_t *l = dbu;
+
+	rw_destroy(&l->l_rwlock);
+	kmem_free(l, sizeof (zap_leaf_t));
+}
+
 static zap_leaf_t *
 zap_create_leaf(zap_t *zap, dmu_tx_t *tx)
 {
 	void *winner;
-	zap_leaf_t *l = kmem_alloc(sizeof (zap_leaf_t), KM_SLEEP);
+	zap_leaf_t *l = kmem_zalloc(sizeof (zap_leaf_t), KM_SLEEP);
 
 	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));
 
@@ -403,7 +412,8 @@ zap_create_leaf(zap_t *zap, dmu_tx_t *tx)
 	VERIFY(0 == dmu_buf_hold(zap->zap_objset, zap->zap_object,
 	    l->l_blkid << FZAP_BLOCK_SHIFT(zap), NULL, &l->l_dbuf,
 	    DMU_READ_NO_PREFETCH));
-	winner = dmu_buf_set_user(l->l_dbuf, l, zap_leaf_pageout);
+	dmu_buf_init_user(&l->l_dbu, zap_leaf_pageout, &l->l_dbuf);
+	winner = dmu_buf_set_user(l->l_dbuf, &l->l_dbu);
 	ASSERT(winner == NULL);
 	dmu_buf_will_dirty(l->l_dbuf, tx);
 
@@ -435,16 +445,6 @@ zap_put_leaf(zap_leaf_t *l)
 	dmu_buf_rele(l->l_dbuf, NULL);
 }
 
-_NOTE(ARGSUSED(0))
-static void
-zap_leaf_pageout(dmu_buf_t *db, void *vl)
-{
-	zap_leaf_t *l = vl;
-
-	rw_destroy(&l->l_rwlock);
-	kmem_free(l, sizeof (zap_leaf_t));
-}
-
 static zap_leaf_t *
 zap_open_leaf(uint64_t blkid, dmu_buf_t *db)
 {
@@ -452,19 +452,20 @@ zap_open_leaf(uint64_t blkid, dmu_buf_t *db)
 
 	ASSERT(blkid != 0);
 
-	l = kmem_alloc(sizeof (zap_leaf_t), KM_SLEEP);
+	l = kmem_zalloc(sizeof (zap_leaf_t), KM_SLEEP);
 	rw_init(&l->l_rwlock, 0, 0, 0);
 	rw_enter(&l->l_rwlock, RW_WRITER);
 	l->l_blkid = blkid;
 	l->l_bs = highbit64(db->db_size) - 1;
 	l->l_dbuf = db;
 
-	winner = dmu_buf_set_user(db, l, zap_leaf_pageout);
+	dmu_buf_init_user(&l->l_dbu, zap_leaf_pageout, &l->l_dbuf);
+	winner = dmu_buf_set_user(db, &l->l_dbu);
 
 	rw_exit(&l->l_rwlock);
 	if (winner != NULL) {
 		/* someone else set it first */
-		zap_leaf_pageout(NULL, l);
+		zap_leaf_pageout(&l->l_dbu);
 		l = winner;
 	}
 
diff --git a/uts/common/fs/zfs/zap_micro.c b/uts/common/fs/zfs/zap_micro.c
index 81ea4f730f97..3742471f78e6 100644
--- a/uts/common/fs/zfs/zap_micro.c
+++ b/uts/common/fs/zfs/zap_micro.c
@@ -21,6 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #include <sys/zio.h>
@@ -387,7 +388,8 @@ mzap_open(objset_t *os, uint64_t obj, dmu_buf_t *db)
 	 * it, because zap_lockdir() checks zap_ismicro without the lock
 	 * held.
 	 */
-	winner = dmu_buf_set_user(db, zap, zap_evict);
+	dmu_buf_init_user(&zap->zap_dbu, zap_evict, &zap->zap_dbuf);
+	winner = dmu_buf_set_user(db, &zap->zap_dbu);
 
 	if (winner != NULL) {
 		rw_exit(&zap->zap_rwlock);
@@ -679,11 +681,10 @@ zap_destroy(objset_t *os, uint64_t zapobj, dmu_tx_t *tx)
 	return (dmu_object_free(os, zapobj, tx));
 }
 
-_NOTE(ARGSUSED(0))
 void
-zap_evict(dmu_buf_t *db, void *vzap)
+zap_evict(void *dbu)
 {
-	zap_t *zap = vzap;
+	zap_t *zap = dbu;
 
 	rw_destroy(&zap->zap_rwlock);
 
diff --git a/uts/common/fs/zfs/zfs_sa.c b/uts/common/fs/zfs/zfs_sa.c
index ed5f27647510..3a472aa11a45 100644
--- a/uts/common/fs/zfs/zfs_sa.c
+++ b/uts/common/fs/zfs/zfs_sa.c
@@ -22,8 +22,7 @@
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
-#include <sys/types.h>
-#include <sys/param.h>
+#include <sys/zfs_context.h>
 #include <sys/vnode.h>
 #include <sys/sa.h>
 #include <sys/zfs_acl.h>
diff --git a/uts/common/fs/zfs/zil.c b/uts/common/fs/zfs/zil.c
index 53944165efb6..f48828dc9067 100644
--- a/uts/common/fs/zfs/zil.c
+++ b/uts/common/fs/zfs/zil.c
@@ -474,7 +474,7 @@ zilog_dirty(zilog_t *zilog, uint64_t txg)
 	dsl_pool_t *dp = zilog->zl_dmu_pool;
 	dsl_dataset_t *ds = dmu_objset_ds(zilog->zl_os);
 
-	if (dsl_dataset_is_snapshot(ds))
+	if (ds->ds_is_snapshot)
 		panic("dirtying snapshot!");
 
 	if (txg_list_add(&dp->dp_dirty_zilogs, zilog, txg)) {

From 799d0fe46cce0554b90c74bee1738ba066dada2f Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Tue, 20 Jan 2015 20:14:50 +0000
Subject: [PATCH 072/380] 5313 Allow I/Os to be aggregated across ZIO priority
 classes Reviewed by: Andriy Gapon <avg@FreeBSD.org> Reviewed by: Will Andrews
 <willa@SpectraLogic.com> Reviewed by: Matt Ahrens <mahrens@delphix.com>
 Reviewed by: George Wilson <george@delphix.com> Approved by: Robert Mustacchi
 <rm@joyent.com> Author: Justin T. Gibbs <justing@spectralogic.com>

illumos/illumos-gate@fe319232d24f4ae183730a5a24a09423d8ab4429
---
 uts/common/fs/zfs/sys/vdev_impl.h |  2 +
 uts/common/fs/zfs/sys/zio.h       |  1 +
 uts/common/fs/zfs/vdev_queue.c    | 78 +++++++++++++++++++------------
 3 files changed, 51 insertions(+), 30 deletions(-)

diff --git a/uts/common/fs/zfs/sys/vdev_impl.h b/uts/common/fs/zfs/sys/vdev_impl.h
index 6d9bcb17d00f..3af0e2a552a7 100644
--- a/uts/common/fs/zfs/sys/vdev_impl.h
+++ b/uts/common/fs/zfs/sys/vdev_impl.h
@@ -113,6 +113,8 @@ struct vdev_queue {
 	vdev_t		*vq_vdev;
 	vdev_queue_class_t vq_class[ZIO_PRIORITY_NUM_QUEUEABLE];
 	avl_tree_t	vq_active_tree;
+	avl_tree_t	vq_read_offset_tree;
+	avl_tree_t	vq_write_offset_tree;
 	uint64_t	vq_last_offset;
 	hrtime_t	vq_io_complete_ts; /* time last i/o completed */
 	kmutex_t	vq_lock;
diff --git a/uts/common/fs/zfs/sys/zio.h b/uts/common/fs/zfs/sys/zio.h
index f6cf259bf71a..71bb6b96a01c 100644
--- a/uts/common/fs/zfs/sys/zio.h
+++ b/uts/common/fs/zfs/sys/zio.h
@@ -419,6 +419,7 @@ struct zio {
 	uint64_t	io_offset;
 	hrtime_t	io_timestamp;
 	avl_node_t	io_queue_node;
+	avl_node_t	io_offset_node;
 
 	/* Internal pipeline state */
 	enum zio_flag	io_flags;
diff --git a/uts/common/fs/zfs/vdev_queue.c b/uts/common/fs/zfs/vdev_queue.c
index f47c4cd1e26d..79d6e13b3be4 100644
--- a/uts/common/fs/zfs/vdev_queue.c
+++ b/uts/common/fs/zfs/vdev_queue.c
@@ -185,6 +185,22 @@ vdev_queue_offset_compare(const void *x1, const void *x2)
 	return (0);
 }
 
+static inline avl_tree_t *
+vdev_queue_class_tree(vdev_queue_t *vq, zio_priority_t p)
+{
+	return (&vq->vq_class[p].vqc_queued_tree);
+}
+
+static inline avl_tree_t *
+vdev_queue_type_tree(vdev_queue_t *vq, zio_type_t t)
+{
+	ASSERT(t == ZIO_TYPE_READ || t == ZIO_TYPE_WRITE);
+	if (t == ZIO_TYPE_READ)
+		return (&vq->vq_read_offset_tree);
+	else
+		return (&vq->vq_write_offset_tree);
+}
+
 int
 vdev_queue_timestamp_compare(const void *x1, const void *x2)
 {
@@ -214,19 +230,27 @@ vdev_queue_init(vdev_t *vd)
 
 	avl_create(&vq->vq_active_tree, vdev_queue_offset_compare,
 	    sizeof (zio_t), offsetof(struct zio, io_queue_node));
+	avl_create(vdev_queue_type_tree(vq, ZIO_TYPE_READ),
+	    vdev_queue_offset_compare, sizeof (zio_t),
+	    offsetof(struct zio, io_offset_node));
+	avl_create(vdev_queue_type_tree(vq, ZIO_TYPE_WRITE),
+	    vdev_queue_offset_compare, sizeof (zio_t),
+	    offsetof(struct zio, io_offset_node));
 
 	for (zio_priority_t p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++) {
+		int (*compfn) (const void *, const void *);
+
 		/*
-		 * The synchronous i/o queues are FIFO rather than LBA ordered.
-		 * This provides more consistent latency for these i/os, and
-		 * they tend to not be tightly clustered anyway so there is
-		 * little to no throughput loss.
+		 * The synchronous i/o queues are dispatched in FIFO rather
+		 * than LBA order.  This provides more consistent latency for
+		 * these i/os.
 		 */
-		boolean_t fifo = (p == ZIO_PRIORITY_SYNC_READ ||
-		    p == ZIO_PRIORITY_SYNC_WRITE);
-		avl_create(&vq->vq_class[p].vqc_queued_tree,
-		    fifo ? vdev_queue_timestamp_compare :
-		    vdev_queue_offset_compare,
+		if (p == ZIO_PRIORITY_SYNC_READ || p == ZIO_PRIORITY_SYNC_WRITE)
+			compfn = vdev_queue_timestamp_compare;
+		else
+			compfn = vdev_queue_offset_compare;
+
+		avl_create(vdev_queue_class_tree(vq, p), compfn,
 		    sizeof (zio_t), offsetof(struct zio, io_queue_node));
 	}
 }
@@ -237,8 +261,10 @@ vdev_queue_fini(vdev_t *vd)
 	vdev_queue_t *vq = &vd->vdev_queue;
 
 	for (zio_priority_t p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++)
-		avl_destroy(&vq->vq_class[p].vqc_queued_tree);
+		avl_destroy(vdev_queue_class_tree(vq, p));
 	avl_destroy(&vq->vq_active_tree);
+	avl_destroy(vdev_queue_type_tree(vq, ZIO_TYPE_READ));
+	avl_destroy(vdev_queue_type_tree(vq, ZIO_TYPE_WRITE));
 
 	mutex_destroy(&vq->vq_lock);
 }
@@ -248,7 +274,8 @@ vdev_queue_io_add(vdev_queue_t *vq, zio_t *zio)
 {
 	spa_t *spa = zio->io_spa;
 	ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);
-	avl_add(&vq->vq_class[zio->io_priority].vqc_queued_tree, zio);
+	avl_add(vdev_queue_class_tree(vq, zio->io_priority), zio);
+	avl_add(vdev_queue_type_tree(vq, zio->io_type), zio);
 
 	mutex_enter(&spa->spa_iokstat_lock);
 	spa->spa_queue_stats[zio->io_priority].spa_queued++;
@@ -262,7 +289,8 @@ vdev_queue_io_remove(vdev_queue_t *vq, zio_t *zio)
 {
 	spa_t *spa = zio->io_spa;
 	ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);
-	avl_remove(&vq->vq_class[zio->io_priority].vqc_queued_tree, zio);
+	avl_remove(vdev_queue_class_tree(vq, zio->io_priority), zio);
+	avl_remove(vdev_queue_type_tree(vq, zio->io_type), zio);
 
 	mutex_enter(&spa->spa_iokstat_lock);
 	ASSERT3U(spa->spa_queue_stats[zio->io_priority].spa_queued, >, 0);
@@ -423,7 +451,7 @@ vdev_queue_class_to_issue(vdev_queue_t *vq)
 
 	/* find a queue that has not reached its minimum # outstanding i/os */
 	for (p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++) {
-		if (avl_numnodes(&vq->vq_class[p].vqc_queued_tree) > 0 &&
+		if (avl_numnodes(vdev_queue_class_tree(vq, p)) > 0 &&
 		    vq->vq_class[p].vqc_active <
 		    vdev_queue_class_min_active(p))
 			return (p);
@@ -434,7 +462,7 @@ vdev_queue_class_to_issue(vdev_queue_t *vq)
 	 * maximum # outstanding i/os.
 	 */
 	for (p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++) {
-		if (avl_numnodes(&vq->vq_class[p].vqc_queued_tree) > 0 &&
+		if (avl_numnodes(vdev_queue_class_tree(vq, p)) > 0 &&
 		    vq->vq_class[p].vqc_active <
 		    vdev_queue_class_max_active(spa, p))
 			return (p);
@@ -460,22 +488,12 @@ vdev_queue_aggregate(vdev_queue_t *vq, zio_t *zio)
 	uint64_t maxgap = 0;
 	uint64_t size;
 	boolean_t stretch = B_FALSE;
-	vdev_queue_class_t *vqc = &vq->vq_class[zio->io_priority];
-	avl_tree_t *t = &vqc->vqc_queued_tree;
+	avl_tree_t *t = vdev_queue_type_tree(vq, zio->io_type);
 	enum zio_flag flags = zio->io_flags & ZIO_FLAG_AGG_INHERIT;
 
 	if (zio->io_flags & ZIO_FLAG_DONT_AGGREGATE)
 		return (NULL);
 
-	/*
-	 * The synchronous i/o queues are not sorted by LBA, so we can't
-	 * find adjacent i/os.  These i/os tend to not be tightly clustered,
-	 * or too large to aggregate, so this has little impact on performance.
-	 */
-	if (zio->io_priority == ZIO_PRIORITY_SYNC_READ ||
-	    zio->io_priority == ZIO_PRIORITY_SYNC_WRITE)
-		return (NULL);
-
 	first = last = zio;
 
 	if (zio->io_type == ZIO_TYPE_READ)
@@ -607,7 +625,7 @@ vdev_queue_io_to_issue(vdev_queue_t *vq)
 	zio_t *zio, *aio;
 	zio_priority_t p;
 	avl_index_t idx;
-	vdev_queue_class_t *vqc;
+	avl_tree_t *tree;
 	zio_t search;
 
 again:
@@ -626,13 +644,13 @@ again:
 	 *
 	 * For FIFO queues (sync), issue the i/o with the lowest timestamp.
 	 */
-	vqc = &vq->vq_class[p];
+	tree = vdev_queue_class_tree(vq, p);
 	search.io_timestamp = 0;
 	search.io_offset = vq->vq_last_offset + 1;
-	VERIFY3P(avl_find(&vqc->vqc_queued_tree, &search, &idx), ==, NULL);
-	zio = avl_nearest(&vqc->vqc_queued_tree, idx, AVL_AFTER);
+	VERIFY3P(avl_find(tree, &search, &idx), ==, NULL);
+	zio = avl_nearest(tree, idx, AVL_AFTER);
 	if (zio == NULL)
-		zio = avl_first(&vqc->vqc_queued_tree);
+		zio = avl_first(tree);
 	ASSERT3U(zio->io_priority, ==, p);
 
 	aio = vdev_queue_aggregate(vq, zio);

From 6ee177cd18971881cb1e39a5dc631bd49932f786 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Tue, 20 Jan 2015 20:17:29 +0000
Subject: [PATCH 073/380] 5497 lock contention on arcs_mtx Reviewed by: George
 Wilson <george.wilson@delphix.com> Reviewed by: Matthew Ahrens
 <mahrens@delphix.com> Reviewed by: Richard Elling
 <richard.elling@richardelling.com> Approved by: Dan McDonald
 <danmcd@omniti.com> Author: Prakash Surya <prakash.surya@delphix.com>

illumos/illumos-gate@244781f10dcd82684fd8163c016540667842f203
---
 uts/common/fs/zfs/arc.c           | 1826 +++++++++++++++++++----------
 uts/common/fs/zfs/dsl_pool.c      |    9 +-
 uts/common/fs/zfs/multilist.c     |  366 ++++++
 uts/common/fs/zfs/sys/arc.h       |    8 +-
 uts/common/fs/zfs/sys/multilist.h |  106 ++
 uts/common/fs/zfs/zio_inject.c    |    6 +-
 6 files changed, 1679 insertions(+), 642 deletions(-)
 create mode 100644 uts/common/fs/zfs/multilist.c
 create mode 100644 uts/common/fs/zfs/sys/multilist.h

diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index 24198579a89d..a227c424708c 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -129,6 +129,7 @@
 #include <sys/vdev.h>
 #include <sys/vdev_impl.h>
 #include <sys/dsl_pool.h>
+#include <sys/multilist.h>
 #ifdef _KERNEL
 #include <sys/vmsystm.h>
 #include <vm/anon.h>
@@ -145,21 +146,39 @@ boolean_t arc_watch = B_FALSE;
 int arc_procfd;
 #endif
 
-static kmutex_t		arc_reclaim_thr_lock;
-static kcondvar_t	arc_reclaim_thr_cv;	/* used to signal reclaim thr */
-static uint8_t		arc_thread_exit;
+static kmutex_t		arc_reclaim_lock;
+static kcondvar_t	arc_reclaim_thread_cv;
+static boolean_t	arc_reclaim_thread_exit;
+static kcondvar_t	arc_reclaim_waiters_cv;
+
+static kmutex_t		arc_user_evicts_lock;
+static kcondvar_t	arc_user_evicts_cv;
+static boolean_t	arc_user_evicts_thread_exit;
 
 uint_t arc_reduce_dnlc_percent = 3;
 
 /*
- * The number of iterations through arc_evict_*() before we
- * drop & reacquire the lock.
+ * The number of headers to evict in arc_evict_state_impl() before
+ * dropping the sublist lock and evicting from another sublist. A lower
+ * value means we're more likely to evict the "correct" header (i.e. the
+ * oldest header in the arc state), but comes with higher overhead
+ * (i.e. more invocations of arc_evict_state_impl()).
  */
-int arc_evict_iterations = 100;
+int zfs_arc_evict_batch_limit = 10;
+
+/*
+ * The number of sublists used for each of the arc state lists. If this
+ * is not set to a suitable value by the user, it will be configured to
+ * the number of CPUs on the system in arc_init().
+ */
+int zfs_arc_num_sublists_per_state = 0;
 
 /* number of seconds before growing cache again */
 static int		arc_grow_retry = 60;
 
+/* shift of arc_c for calculating overflow limit in arc_get_data_buf */
+int		zfs_arc_overflow_shift = 8;
+
 /* shift of arc_c for calculating both min and max arc_p */
 static int		arc_p_min_shift = 4;
 
@@ -242,10 +261,19 @@ int zfs_arc_average_blocksize = 8 * 1024; /* 8KB */
  */
 
 typedef struct arc_state {
-	list_t	arcs_list[ARC_BUFC_NUMTYPES];	/* list of evictable buffers */
-	uint64_t arcs_lsize[ARC_BUFC_NUMTYPES];	/* amount of evictable data */
-	uint64_t arcs_size;	/* total amount of data in this state */
-	kmutex_t arcs_mtx;
+	/*
+	 * list of evictable buffers
+	 */
+	multilist_t arcs_list[ARC_BUFC_NUMTYPES];
+	/*
+	 * total amount of evictable data in this state
+	 */
+	uint64_t arcs_lsize[ARC_BUFC_NUMTYPES];
+	/*
+	 * total amount of data in this state; this includes: evictable,
+	 * non-evictable, ARC_BUFC_DATA, and ARC_BUFC_METADATA.
+	 */
+	uint64_t arcs_size;
 } arc_state_t;
 
 /* The 6 states: */
@@ -272,7 +300,6 @@ typedef struct arc_stats {
 	kstat_named_t arcstat_mfu_hits;
 	kstat_named_t arcstat_mfu_ghost_hits;
 	kstat_named_t arcstat_deleted;
-	kstat_named_t arcstat_recycle_miss;
 	/*
 	 * Number of buffers that could not be evicted because the hash lock
 	 * was held by another thread.  The lock may not necessarily be held
@@ -286,9 +313,15 @@ typedef struct arc_stats {
 	 * not from the spa we're trying to evict from.
 	 */
 	kstat_named_t arcstat_evict_skip;
+	/*
+	 * Number of times arc_evict_state() was unable to evict enough
+	 * buffers to reach it's target amount.
+	 */
+	kstat_named_t arcstat_evict_not_enough;
 	kstat_named_t arcstat_evict_l2_cached;
 	kstat_named_t arcstat_evict_l2_eligible;
 	kstat_named_t arcstat_evict_l2_ineligible;
+	kstat_named_t arcstat_evict_l2_skip;
 	kstat_named_t arcstat_hash_elements;
 	kstat_named_t arcstat_hash_elements_max;
 	kstat_named_t arcstat_hash_collisions;
@@ -439,11 +472,12 @@ typedef struct arc_stats {
 	kstat_named_t arcstat_l2_writes_sent;
 	kstat_named_t arcstat_l2_writes_done;
 	kstat_named_t arcstat_l2_writes_error;
-	kstat_named_t arcstat_l2_writes_hdr_miss;
+	kstat_named_t arcstat_l2_writes_lock_retry;
 	kstat_named_t arcstat_l2_evict_lock_retry;
 	kstat_named_t arcstat_l2_evict_reading;
 	kstat_named_t arcstat_l2_evict_l1cached;
 	kstat_named_t arcstat_l2_free_on_write;
+	kstat_named_t arcstat_l2_cdata_free_on_write;
 	kstat_named_t arcstat_l2_abort_lowmem;
 	kstat_named_t arcstat_l2_cksum_bad;
 	kstat_named_t arcstat_l2_io_error;
@@ -479,12 +513,13 @@ static arc_stats_t arc_stats = {
 	{ "mfu_hits",			KSTAT_DATA_UINT64 },
 	{ "mfu_ghost_hits",		KSTAT_DATA_UINT64 },
 	{ "deleted",			KSTAT_DATA_UINT64 },
-	{ "recycle_miss",		KSTAT_DATA_UINT64 },
 	{ "mutex_miss",			KSTAT_DATA_UINT64 },
 	{ "evict_skip",			KSTAT_DATA_UINT64 },
+	{ "evict_not_enough",		KSTAT_DATA_UINT64 },
 	{ "evict_l2_cached",		KSTAT_DATA_UINT64 },
 	{ "evict_l2_eligible",		KSTAT_DATA_UINT64 },
 	{ "evict_l2_ineligible",	KSTAT_DATA_UINT64 },
+	{ "evict_l2_skip",		KSTAT_DATA_UINT64 },
 	{ "hash_elements",		KSTAT_DATA_UINT64 },
 	{ "hash_elements_max",		KSTAT_DATA_UINT64 },
 	{ "hash_collisions",		KSTAT_DATA_UINT64 },
@@ -523,11 +558,12 @@ static arc_stats_t arc_stats = {
 	{ "l2_writes_sent",		KSTAT_DATA_UINT64 },
 	{ "l2_writes_done",		KSTAT_DATA_UINT64 },
 	{ "l2_writes_error",		KSTAT_DATA_UINT64 },
-	{ "l2_writes_hdr_miss",		KSTAT_DATA_UINT64 },
+	{ "l2_writes_lock_retry",	KSTAT_DATA_UINT64 },
 	{ "l2_evict_lock_retry",	KSTAT_DATA_UINT64 },
 	{ "l2_evict_reading",		KSTAT_DATA_UINT64 },
 	{ "l2_evict_l1cached",		KSTAT_DATA_UINT64 },
 	{ "l2_free_on_write",		KSTAT_DATA_UINT64 },
+	{ "l2_cdata_free_on_write",	KSTAT_DATA_UINT64 },
 	{ "l2_abort_lowmem",		KSTAT_DATA_UINT64 },
 	{ "l2_cksum_bad",		KSTAT_DATA_UINT64 },
 	{ "l2_io_error",		KSTAT_DATA_UINT64 },
@@ -687,7 +723,7 @@ typedef struct l1arc_buf_hdr {
 
 	/* protected by arc state mutex */
 	arc_state_t		*b_state;
-	list_node_t		b_arc_node;
+	multilist_node_t	b_arc_node;
 
 	/* updated atomically */
 	clock_t			b_arc_access;
@@ -738,7 +774,6 @@ struct arc_buf_hdr {
 };
 
 static arc_buf_t *arc_eviction_list;
-static kmutex_t arc_eviction_mtx;
 static arc_buf_hdr_t arc_eviction_hdr;
 
 #define	GHOST_STATE(state)	\
@@ -897,8 +932,7 @@ static uint8_t l2arc_thread_exit;
 
 static void arc_get_data_buf(arc_buf_t *);
 static void arc_access(arc_buf_hdr_t *, kmutex_t *);
-static int arc_evict_needed(arc_buf_contents_t);
-static void arc_evict_ghost(arc_state_t *, uint64_t, int64_t);
+static boolean_t arc_is_overflowing();
 static void arc_buf_watch(arc_buf_t *);
 
 static arc_buf_contents_t arc_buf_type(arc_buf_hdr_t *);
@@ -1079,6 +1113,7 @@ hdr_full_cons(void *vbuf, void *unused, int kmflag)
 	cv_init(&hdr->b_l1hdr.b_cv, NULL, CV_DEFAULT, NULL);
 	refcount_create(&hdr->b_l1hdr.b_refcnt);
 	mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
+	multilist_link_init(&hdr->b_l1hdr.b_arc_node);
 	arc_space_consume(HDR_FULL_SIZE, ARC_SPACE_HDRS);
 
 	return (0);
@@ -1123,6 +1158,7 @@ hdr_full_dest(void *vbuf, void *unused)
 	cv_destroy(&hdr->b_l1hdr.b_cv);
 	refcount_destroy(&hdr->b_l1hdr.b_refcnt);
 	mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);
+	ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
 	arc_space_return(HDR_FULL_SIZE, ARC_SPACE_HDRS);
 }
 
@@ -1159,7 +1195,7 @@ hdr_recl(void *unused)
 	 * which is after we do arc_fini().
 	 */
 	if (!arc_dead)
-		cv_signal(&arc_reclaim_thr_cv);
+		cv_signal(&arc_reclaim_thread_cv);
 }
 
 static void
@@ -1237,18 +1273,31 @@ arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
 		 * l2c_only even though it's about to change.
 		 */
 		nhdr->b_l1hdr.b_state = arc_l2c_only;
+
+		/* Verify previous threads set to NULL before freeing */
+		ASSERT3P(nhdr->b_l1hdr.b_tmp_cdata, ==, NULL);
 	} else {
 		ASSERT(hdr->b_l1hdr.b_buf == NULL);
 		ASSERT0(hdr->b_l1hdr.b_datacnt);
-		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
+
 		/*
-		 * We might be removing the L1hdr of a buffer which was just
-		 * written out to L2ARC. If such a buffer is compressed then we
-		 * need to free its b_tmp_cdata before destroying the header.
+		 * If we've reached here, We must have been called from
+		 * arc_evict_hdr(), as such we should have already been
+		 * removed from any ghost list we were previously on
+		 * (which protects us from racing with arc_evict_state),
+		 * thus no locking is needed during this check.
 		 */
-		if (hdr->b_l1hdr.b_tmp_cdata != NULL &&
-		    HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF)
-			l2arc_release_cdata_buf(hdr);
+		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
+
+		/*
+		 * A buffer must not be moved into the arc_l2c_only
+		 * state if it's not finished being written out to the
+		 * l2arc device. Otherwise, the b_l1hdr.b_tmp_cdata field
+		 * might try to be accessed, even though it was removed.
+		 */
+		VERIFY(!HDR_L2_WRITING(hdr));
+		VERIFY3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
+
 		nhdr->b_flags &= ~ARC_FLAG_HAS_L1HDR;
 	}
 	/*
@@ -1461,14 +1510,13 @@ add_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
 	    (state != arc_anon)) {
 		/* We don't use the L2-only state list. */
 		if (state != arc_l2c_only) {
+			arc_buf_contents_t type = arc_buf_type(hdr);
 			uint64_t delta = hdr->b_size * hdr->b_l1hdr.b_datacnt;
-			list_t *list = &state->arcs_list[arc_buf_type(hdr)];
-			uint64_t *size = &state->arcs_lsize[arc_buf_type(hdr)];
+			multilist_t *list = &state->arcs_list[type];
+			uint64_t *size = &state->arcs_lsize[type];
+
+			multilist_remove(list, hdr);
 
-			ASSERT(!MUTEX_HELD(&state->arcs_mtx));
-			mutex_enter(&state->arcs_mtx);
-			ASSERT(list_link_active(&hdr->b_l1hdr.b_arc_node));
-			list_remove(list, hdr);
 			if (GHOST_STATE(state)) {
 				ASSERT0(hdr->b_l1hdr.b_datacnt);
 				ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
@@ -1477,7 +1525,6 @@ add_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
 			ASSERT(delta > 0);
 			ASSERT3U(*size, >=, delta);
 			atomic_add_64(size, -delta);
-			mutex_exit(&state->arcs_mtx);
 		}
 		/* remove the prefetch flag if we get a reference */
 		hdr->b_flags &= ~ARC_FLAG_PREFETCH;
@@ -1500,22 +1547,21 @@ remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
 	 */
 	if (((cnt = refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) &&
 	    (state != arc_anon)) {
-		uint64_t *size = &state->arcs_lsize[arc_buf_type(hdr)];
+		arc_buf_contents_t type = arc_buf_type(hdr);
+		multilist_t *list = &state->arcs_list[type];
+		uint64_t *size = &state->arcs_lsize[type];
+
+		multilist_insert(list, hdr);
 
-		ASSERT(!MUTEX_HELD(&state->arcs_mtx));
-		mutex_enter(&state->arcs_mtx);
-		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
-		list_insert_head(&state->arcs_list[arc_buf_type(hdr)], hdr);
 		ASSERT(hdr->b_l1hdr.b_datacnt > 0);
 		atomic_add_64(size, hdr->b_size *
 		    hdr->b_l1hdr.b_datacnt);
-		mutex_exit(&state->arcs_mtx);
 	}
 	return (cnt);
 }
 
 /*
- * Move the supplied buffer to the indicated state.  The mutex
+ * Move the supplied buffer to the indicated state. The hash lock
  * for the buffer must be held by the caller.
  */
 static void
@@ -1559,15 +1605,10 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
 	 */
 	if (refcnt == 0) {
 		if (old_state != arc_anon && old_state != arc_l2c_only) {
-			int use_mutex = !MUTEX_HELD(&old_state->arcs_mtx);
 			uint64_t *size = &old_state->arcs_lsize[buftype];
 
-			if (use_mutex)
-				mutex_enter(&old_state->arcs_mtx);
-
 			ASSERT(HDR_HAS_L1HDR(hdr));
-			ASSERT(list_link_active(&hdr->b_l1hdr.b_arc_node));
-			list_remove(&old_state->arcs_list[buftype], hdr);
+			multilist_remove(&old_state->arcs_list[buftype], hdr);
 
 			/*
 			 * If prefetching out of the ghost cache,
@@ -1580,12 +1621,8 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
 			}
 			ASSERT3U(*size, >=, from_delta);
 			atomic_add_64(size, -from_delta);
-
-			if (use_mutex)
-				mutex_exit(&old_state->arcs_mtx);
 		}
 		if (new_state != arc_anon && new_state != arc_l2c_only) {
-			int use_mutex = !MUTEX_HELD(&new_state->arcs_mtx);
 			uint64_t *size = &new_state->arcs_lsize[buftype];
 
 			/*
@@ -1595,10 +1632,7 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
 			 * beforehand.
 			 */
 			ASSERT(HDR_HAS_L1HDR(hdr));
-			if (use_mutex)
-				mutex_enter(&new_state->arcs_mtx);
-
-			list_insert_head(&new_state->arcs_list[buftype], hdr);
+			multilist_insert(&new_state->arcs_list[buftype], hdr);
 
 			/* ghost elements have a ghost size */
 			if (GHOST_STATE(new_state)) {
@@ -1607,9 +1641,6 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
 				to_delta = hdr->b_size;
 			}
 			atomic_add_64(size, to_delta);
-
-			if (use_mutex)
-				mutex_exit(&new_state->arcs_mtx);
 		}
 	}
 
@@ -1631,8 +1662,8 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
 	 * L2 headers should never be on the L2 state list since they don't
 	 * have L1 headers allocated.
 	 */
-	ASSERT(list_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]) &&
-	    list_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA]));
+	ASSERT(multilist_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]) &&
+	    multilist_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA]));
 }
 
 void
@@ -1725,6 +1756,7 @@ arc_buf_alloc(spa_t *spa, int32_t size, void *tag, arc_buf_contents_t type)
 	hdr->b_l1hdr.b_state = arc_anon;
 	hdr->b_l1hdr.b_arc_access = 0;
 	hdr->b_l1hdr.b_datacnt = 1;
+	hdr->b_l1hdr.b_tmp_cdata = NULL;
 
 	arc_get_data_buf(buf);
 	ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
@@ -1854,6 +1886,21 @@ arc_buf_add_ref(arc_buf_t *buf, void* tag)
 	    data, metadata, hits);
 }
 
+static void
+arc_buf_free_on_write(void *data, size_t size,
+    void (*free_func)(void *, size_t))
+{
+	l2arc_data_free_t *df;
+
+	df = kmem_alloc(sizeof (*df), KM_SLEEP);
+	df->l2df_data = data;
+	df->l2df_size = size;
+	df->l2df_func = free_func;
+	mutex_enter(&l2arc_free_on_write_mtx);
+	list_insert_head(l2arc_free_on_write, df);
+	mutex_exit(&l2arc_free_on_write_mtx);
+}
+
 /*
  * Free the arc data buffer.  If it is an l2arc write in progress,
  * the buffer is placed on l2arc_free_on_write to be freed later.
@@ -1864,26 +1911,74 @@ arc_buf_data_free(arc_buf_t *buf, void (*free_func)(void *, size_t))
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 
 	if (HDR_L2_WRITING(hdr)) {
-		l2arc_data_free_t *df;
-		df = kmem_alloc(sizeof (l2arc_data_free_t), KM_SLEEP);
-		df->l2df_data = buf->b_data;
-		df->l2df_size = hdr->b_size;
-		df->l2df_func = free_func;
-		mutex_enter(&l2arc_free_on_write_mtx);
-		list_insert_head(l2arc_free_on_write, df);
-		mutex_exit(&l2arc_free_on_write_mtx);
+		arc_buf_free_on_write(buf->b_data, hdr->b_size, free_func);
 		ARCSTAT_BUMP(arcstat_l2_free_on_write);
 	} else {
 		free_func(buf->b_data, hdr->b_size);
 	}
 }
 
+static void
+arc_buf_l2_cdata_free(arc_buf_hdr_t *hdr)
+{
+	ASSERT(HDR_HAS_L2HDR(hdr));
+	ASSERT(MUTEX_HELD(&hdr->b_l2hdr.b_dev->l2ad_mtx));
+
+	/*
+	 * The b_tmp_cdata field is linked off of the b_l1hdr, so if
+	 * that doesn't exist, the header is in the arc_l2c_only state,
+	 * and there isn't anything to free (it's already been freed).
+	 */
+	if (!HDR_HAS_L1HDR(hdr))
+		return;
+
+	/*
+	 * The header isn't being written to the l2arc device, thus it
+	 * shouldn't have a b_tmp_cdata to free.
+	 */
+	if (!HDR_L2_WRITING(hdr)) {
+		ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
+		return;
+	}
+
+	/*
+	 * The header does not have compression enabled. This can be due
+	 * to the buffer not being compressible, or because we're
+	 * freeing the buffer before the second phase of
+	 * l2arc_write_buffer() has started (which does the compression
+	 * step). In either case, b_tmp_cdata does not point to a
+	 * separately compressed buffer, so there's nothing to free (it
+	 * points to the same buffer as the arc_buf_t's b_data field).
+	 */
+	if (HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF) {
+		hdr->b_l1hdr.b_tmp_cdata = NULL;
+		return;
+	}
+
+	/*
+	 * There's nothing to free since the buffer was all zero's and
+	 * compressed to a zero length buffer.
+	 */
+	if (HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_EMPTY) {
+		ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
+		return;
+	}
+
+	ASSERT(L2ARC_IS_VALID_COMPRESS(HDR_GET_COMPRESS(hdr)));
+
+	arc_buf_free_on_write(hdr->b_l1hdr.b_tmp_cdata,
+	    hdr->b_size, zio_data_buf_free);
+
+	ARCSTAT_BUMP(arcstat_l2_cdata_free_on_write);
+	hdr->b_l1hdr.b_tmp_cdata = NULL;
+}
+
 /*
  * Free up buf->b_data and if 'remove' is set, then pull the
  * arc_buf_t off of the the arc_buf_hdr_t's list and free it.
  */
 static void
-arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t remove)
+arc_buf_destroy(arc_buf_t *buf, boolean_t remove)
 {
 	arc_buf_t **bufp;
 
@@ -1896,17 +1991,17 @@ arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t remove)
 		arc_cksum_verify(buf);
 		arc_buf_unwatch(buf);
 
-		if (!recycle) {
-			if (type == ARC_BUFC_METADATA) {
-				arc_buf_data_free(buf, zio_buf_free);
-				arc_space_return(size, ARC_SPACE_META);
-			} else {
-				ASSERT(type == ARC_BUFC_DATA);
-				arc_buf_data_free(buf, zio_data_buf_free);
-				arc_space_return(size, ARC_SPACE_DATA);
-			}
+		if (type == ARC_BUFC_METADATA) {
+			arc_buf_data_free(buf, zio_buf_free);
+			arc_space_return(size, ARC_SPACE_META);
+		} else {
+			ASSERT(type == ARC_BUFC_DATA);
+			arc_buf_data_free(buf, zio_data_buf_free);
+			arc_space_return(size, ARC_SPACE_DATA);
 		}
-		if (list_link_active(&buf->b_hdr->b_l1hdr.b_arc_node)) {
+
+		/* protected by hash lock, if in the hash table */
+		if (multilist_link_active(&buf->b_hdr->b_l1hdr.b_arc_node)) {
 			uint64_t *cnt = &state->arcs_lsize[type];
 
 			ASSERT(refcount_is_zero(
@@ -1974,6 +2069,12 @@ arc_hdr_destroy(arc_buf_hdr_t *hdr)
 
 		list_remove(&l2hdr->b_dev->l2ad_buflist, hdr);
 
+		/*
+		 * We don't want to leak the b_tmp_cdata buffer that was
+		 * allocated in l2arc_write_buffers()
+		 */
+		arc_buf_l2_cdata_free(hdr);
+
 		ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
 		ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
 
@@ -1996,20 +2097,19 @@ arc_hdr_destroy(arc_buf_hdr_t *hdr)
 			arc_buf_t *buf = hdr->b_l1hdr.b_buf;
 
 			if (buf->b_efunc != NULL) {
-				mutex_enter(&arc_eviction_mtx);
+				mutex_enter(&arc_user_evicts_lock);
 				mutex_enter(&buf->b_evict_lock);
 				ASSERT(buf->b_hdr != NULL);
-				arc_buf_destroy(hdr->b_l1hdr.b_buf, FALSE,
-				    FALSE);
+				arc_buf_destroy(hdr->b_l1hdr.b_buf, FALSE);
 				hdr->b_l1hdr.b_buf = buf->b_next;
 				buf->b_hdr = &arc_eviction_hdr;
 				buf->b_next = arc_eviction_list;
 				arc_eviction_list = buf;
 				mutex_exit(&buf->b_evict_lock);
-				mutex_exit(&arc_eviction_mtx);
+				cv_signal(&arc_user_evicts_cv);
+				mutex_exit(&arc_user_evicts_lock);
 			} else {
-				arc_buf_destroy(hdr->b_l1hdr.b_buf, FALSE,
-				    TRUE);
+				arc_buf_destroy(hdr->b_l1hdr.b_buf, TRUE);
 			}
 		}
 #ifdef ZFS_DEBUG
@@ -2022,7 +2122,7 @@ arc_hdr_destroy(arc_buf_hdr_t *hdr)
 
 	ASSERT3P(hdr->b_hash_next, ==, NULL);
 	if (HDR_HAS_L1HDR(hdr)) {
-		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
+		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
 		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
 		kmem_cache_free(hdr_full_cache, hdr);
 	} else {
@@ -2048,7 +2148,7 @@ arc_buf_free(arc_buf_t *buf, void *tag)
 
 		(void) remove_reference(hdr, hash_lock, tag);
 		if (hdr->b_l1hdr.b_datacnt > 1) {
-			arc_buf_destroy(buf, FALSE, TRUE);
+			arc_buf_destroy(buf, TRUE);
 		} else {
 			ASSERT(buf == hdr->b_l1hdr.b_buf);
 			ASSERT(buf->b_efunc == NULL);
@@ -2062,16 +2162,16 @@ arc_buf_free(arc_buf_t *buf, void *tag)
 		 * this buffer unless the write completes before we finish
 		 * decrementing the reference count.
 		 */
-		mutex_enter(&arc_eviction_mtx);
+		mutex_enter(&arc_user_evicts_lock);
 		(void) remove_reference(hdr, NULL, tag);
 		ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
 		destroy_hdr = !HDR_IO_IN_PROGRESS(hdr);
-		mutex_exit(&arc_eviction_mtx);
+		mutex_exit(&arc_user_evicts_lock);
 		if (destroy_hdr)
 			arc_hdr_destroy(hdr);
 	} else {
 		if (remove_reference(hdr, NULL, tag) > 0)
-			arc_buf_destroy(buf, FALSE, TRUE);
+			arc_buf_destroy(buf, TRUE);
 		else
 			arc_hdr_destroy(hdr);
 	}
@@ -2100,7 +2200,7 @@ arc_buf_remove_ref(arc_buf_t *buf, void* tag)
 	(void) remove_reference(hdr, hash_lock, tag);
 	if (hdr->b_l1hdr.b_datacnt > 1) {
 		if (no_callback)
-			arc_buf_destroy(buf, FALSE, TRUE);
+			arc_buf_destroy(buf, TRUE);
 	} else if (no_callback) {
 		ASSERT(hdr->b_l1hdr.b_buf == buf && buf->b_next == NULL);
 		ASSERT(buf->b_efunc == NULL);
@@ -2161,418 +2261,675 @@ arc_buf_eviction_needed(arc_buf_t *buf)
 }
 
 /*
- * Evict buffers from list until we've removed the specified number of
- * bytes.  Move the removed buffers to the appropriate evict state.
- * If the recycle flag is set, then attempt to "recycle" a buffer:
- * - look for a buffer to evict that is `bytes' long.
- * - return the data block from this buffer rather than freeing it.
- * This flag is used by callers that are trying to make space for a
- * new buffer in a full arc cache.
+ * Evict the arc_buf_hdr that is provided as a parameter. The resultant
+ * state of the header is dependent on it's state prior to entering this
+ * function. The following transitions are possible:
  *
- * This function makes a "best effort".  It skips over any buffers
- * it can't get a hash_lock on, and so may not catch all candidates.
- * It may also return without evicting as much space as requested.
+ *    - arc_mru -> arc_mru_ghost
+ *    - arc_mfu -> arc_mfu_ghost
+ *    - arc_mru_ghost -> arc_l2c_only
+ *    - arc_mru_ghost -> deleted
+ *    - arc_mfu_ghost -> arc_l2c_only
+ *    - arc_mfu_ghost -> deleted
  */
-static void *
-arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
-    arc_buf_contents_t type)
+static int64_t
+arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
 {
-	arc_state_t *evicted_state;
-	uint64_t bytes_evicted = 0, skipped = 0, missed = 0;
-	arc_buf_hdr_t *hdr, *hdr_prev = NULL;
-	kmutex_t *hash_lock;
-	boolean_t have_lock;
-	void *stolen = NULL;
-	arc_buf_hdr_t marker = { 0 };
-	int count = 0;
+	arc_state_t *evicted_state, *state;
+	int64_t bytes_evicted = 0;
+
+	ASSERT(MUTEX_HELD(hash_lock));
+	ASSERT(HDR_HAS_L1HDR(hdr));
+
+	state = hdr->b_l1hdr.b_state;
+	if (GHOST_STATE(state)) {
+		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
+		ASSERT(hdr->b_l1hdr.b_buf == NULL);
+
+		/*
+		 * l2arc_write_buffers() relies on a header's L1 portion
+		 * (i.e. it's b_tmp_cdata field) during it's write phase.
+		 * Thus, we cannot push a header onto the arc_l2c_only
+		 * state (removing it's L1 piece) until the header is
+		 * done being written to the l2arc.
+		 */
+		if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) {
+			ARCSTAT_BUMP(arcstat_evict_l2_skip);
+			return (bytes_evicted);
+		}
+
+		ARCSTAT_BUMP(arcstat_deleted);
+		bytes_evicted += hdr->b_size;
+
+		DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);
+
+		if (HDR_HAS_L2HDR(hdr)) {
+			/*
+			 * This buffer is cached on the 2nd Level ARC;
+			 * don't destroy the header.
+			 */
+			arc_change_state(arc_l2c_only, hdr, hash_lock);
+			/*
+			 * dropping from L1+L2 cached to L2-only,
+			 * realloc to remove the L1 header.
+			 */
+			hdr = arc_hdr_realloc(hdr, hdr_full_cache,
+			    hdr_l2only_cache);
+		} else {
+			arc_change_state(arc_anon, hdr, hash_lock);
+			arc_hdr_destroy(hdr);
+		}
+		return (bytes_evicted);
+	}
 
 	ASSERT(state == arc_mru || state == arc_mfu);
-
 	evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
 
-	/*
-	 * The ghost list lock must be acquired first in order to prevent
-	 * a 3 party deadlock:
-	 *
-	 *  - arc_evict_ghost acquires arc_*_ghost->arcs_mtx, followed by
-	 *    l2ad_mtx in arc_hdr_realloc
-	 *  - l2arc_write_buffers acquires l2ad_mtx, followed by arc_*->arcs_mtx
-	 *  - arc_evict acquires arc_*_ghost->arcs_mtx, followed by
-	 *    arc_*_ghost->arcs_mtx and forms a deadlock cycle.
-	 *
-	 * This situation is avoided by acquiring the ghost list lock first.
-	 */
-	mutex_enter(&evicted_state->arcs_mtx);
-	mutex_enter(&state->arcs_mtx);
+	/* prefetch buffers have a minimum lifespan */
+	if (HDR_IO_IN_PROGRESS(hdr) ||
+	    ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&
+	    ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <
+	    arc_min_prefetch_lifespan)) {
+		ARCSTAT_BUMP(arcstat_evict_skip);
+		return (bytes_evicted);
+	}
 
-	/*
-	 * Decide which "type" (data vs metadata) to recycle from.
-	 *
-	 * If we are over the metadata limit, recycle from metadata.
-	 * If we are under the metadata minimum, recycle from data.
-	 * Otherwise, recycle from whichever type has the oldest (least
-	 * recently accessed) header.
-	 */
-	if (recycle) {
-		arc_buf_hdr_t *data_hdr =
-		    list_tail(&state->arcs_list[ARC_BUFC_DATA]);
-		arc_buf_hdr_t *metadata_hdr =
-		    list_tail(&state->arcs_list[ARC_BUFC_METADATA]);
-		arc_buf_contents_t realtype;
-
-		if (data_hdr == NULL) {
-			realtype = ARC_BUFC_METADATA;
-		} else if (metadata_hdr == NULL) {
-			realtype = ARC_BUFC_DATA;
-		} else if (arc_meta_used >= arc_meta_limit) {
-			realtype = ARC_BUFC_METADATA;
-		} else if (arc_meta_used <= arc_meta_min) {
-			realtype = ARC_BUFC_DATA;
-		} else if (HDR_HAS_L1HDR(data_hdr) &&
-		    HDR_HAS_L1HDR(metadata_hdr) &&
-		    data_hdr->b_l1hdr.b_arc_access <
-		    metadata_hdr->b_l1hdr.b_arc_access) {
-			realtype = ARC_BUFC_DATA;
-		} else {
-			realtype = ARC_BUFC_METADATA;
+	ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
+	ASSERT3U(hdr->b_l1hdr.b_datacnt, >, 0);
+	while (hdr->b_l1hdr.b_buf) {
+		arc_buf_t *buf = hdr->b_l1hdr.b_buf;
+		if (!mutex_tryenter(&buf->b_evict_lock)) {
+			ARCSTAT_BUMP(arcstat_mutex_miss);
+			break;
 		}
-		if (realtype != type) {
-			/*
-			 * If we want to evict from a different list,
-			 * we can not recycle, because DATA vs METADATA
-			 * buffers are segregated into different kmem
-			 * caches (and vmem arenas).
-			 */
-			type = realtype;
-			recycle = B_FALSE;
+		if (buf->b_data != NULL)
+			bytes_evicted += hdr->b_size;
+		if (buf->b_efunc != NULL) {
+			mutex_enter(&arc_user_evicts_lock);
+			arc_buf_destroy(buf, FALSE);
+			hdr->b_l1hdr.b_buf = buf->b_next;
+			buf->b_hdr = &arc_eviction_hdr;
+			buf->b_next = arc_eviction_list;
+			arc_eviction_list = buf;
+			cv_signal(&arc_user_evicts_cv);
+			mutex_exit(&arc_user_evicts_lock);
+			mutex_exit(&buf->b_evict_lock);
+		} else {
+			mutex_exit(&buf->b_evict_lock);
+			arc_buf_destroy(buf, TRUE);
 		}
 	}
 
-	list_t *list = &state->arcs_list[type];
+	if (HDR_HAS_L2HDR(hdr)) {
+		ARCSTAT_INCR(arcstat_evict_l2_cached, hdr->b_size);
+	} else {
+		if (l2arc_write_eligible(hdr->b_spa, hdr))
+			ARCSTAT_INCR(arcstat_evict_l2_eligible, hdr->b_size);
+		else
+			ARCSTAT_INCR(arcstat_evict_l2_ineligible, hdr->b_size);
+	}
 
-	for (hdr = list_tail(list); hdr; hdr = hdr_prev) {
-		hdr_prev = list_prev(list, hdr);
-		/* prefetch buffers have a minimum lifespan */
-		if (HDR_IO_IN_PROGRESS(hdr) ||
-		    (spa && hdr->b_spa != spa) ||
-		    ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&
-		    ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <
-		    arc_min_prefetch_lifespan)) {
-			skipped++;
-			continue;
-		}
-		/* "lookahead" for better eviction candidate */
-		if (recycle && hdr->b_size != bytes &&
-		    hdr_prev && hdr_prev->b_size == bytes)
-			continue;
+	if (hdr->b_l1hdr.b_datacnt == 0) {
+		arc_change_state(evicted_state, hdr, hash_lock);
+		ASSERT(HDR_IN_HASH_TABLE(hdr));
+		hdr->b_flags |= ARC_FLAG_IN_HASH_TABLE;
+		hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE;
+		DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr);
+	}
 
-		/* ignore markers */
+	return (bytes_evicted);
+}
+
+static uint64_t
+arc_evict_state_impl(multilist_t *ml, int idx, arc_buf_hdr_t *marker,
+    uint64_t spa, int64_t bytes)
+{
+	multilist_sublist_t *mls;
+	uint64_t bytes_evicted = 0;
+	arc_buf_hdr_t *hdr;
+	kmutex_t *hash_lock;
+	int evict_count = 0;
+
+	ASSERT3P(marker, !=, NULL);
+	IMPLY(bytes < 0, bytes == ARC_EVICT_ALL);
+
+	mls = multilist_sublist_lock(ml, idx);
+
+	for (hdr = multilist_sublist_prev(mls, marker); hdr != NULL;
+	    hdr = multilist_sublist_prev(mls, marker)) {
+		if ((bytes != ARC_EVICT_ALL && bytes_evicted >= bytes) ||
+		    (evict_count >= zfs_arc_evict_batch_limit))
+			break;
+
+		/*
+		 * To keep our iteration location, move the marker
+		 * forward. Since we're not holding hdr's hash lock, we
+		 * must be very careful and not remove 'hdr' from the
+		 * sublist. Otherwise, other consumers might mistake the
+		 * 'hdr' as not being on a sublist when they call the
+		 * multilist_link_active() function (they all rely on
+		 * the hash lock protecting concurrent insertions and
+		 * removals). multilist_sublist_move_forward() was
+		 * specifically implemented to ensure this is the case
+		 * (only 'marker' will be removed and re-inserted).
+		 */
+		multilist_sublist_move_forward(mls, marker);
+
+		/*
+		 * The only case where the b_spa field should ever be
+		 * zero, is the marker headers inserted by
+		 * arc_evict_state(). It's possible for multiple threads
+		 * to be calling arc_evict_state() concurrently (e.g.
+		 * dsl_pool_close() and zio_inject_fault()), so we must
+		 * skip any markers we see from these other threads.
+		 */
 		if (hdr->b_spa == 0)
 			continue;
 
-		/*
-		 * It may take a long time to evict all the bufs requested.
-		 * To avoid blocking all arc activity, periodically drop
-		 * the arcs_mtx and give other threads a chance to run
-		 * before reacquiring the lock.
-		 *
-		 * If we are looking for a buffer to recycle, we are in
-		 * the hot code path, so don't sleep.
-		 */
-		if (!recycle && count++ > arc_evict_iterations) {
-			list_insert_after(list, hdr, &marker);
-			mutex_exit(&state->arcs_mtx);
-			mutex_exit(&evicted_state->arcs_mtx);
-			kpreempt(KPREEMPT_SYNC);
-			mutex_enter(&evicted_state->arcs_mtx);
-			mutex_enter(&state->arcs_mtx);
-			hdr_prev = list_prev(list, &marker);
-			list_remove(list, &marker);
-			count = 0;
+		/* we're only interested in evicting buffers of a certain spa */
+		if (spa != 0 && hdr->b_spa != spa) {
+			ARCSTAT_BUMP(arcstat_evict_skip);
 			continue;
 		}
 
 		hash_lock = HDR_LOCK(hdr);
-		have_lock = MUTEX_HELD(hash_lock);
-		if (have_lock || mutex_tryenter(hash_lock)) {
-			ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
-			ASSERT3U(hdr->b_l1hdr.b_datacnt, >, 0);
-			while (hdr->b_l1hdr.b_buf) {
-				arc_buf_t *buf = hdr->b_l1hdr.b_buf;
-				if (!mutex_tryenter(&buf->b_evict_lock)) {
-					missed += 1;
-					break;
-				}
-				if (buf->b_data != NULL) {
-					bytes_evicted += hdr->b_size;
-					if (recycle &&
-					    arc_buf_type(hdr) == type &&
-					    hdr->b_size == bytes &&
-					    !HDR_L2_WRITING(hdr)) {
-						stolen = buf->b_data;
-						recycle = FALSE;
-					}
-				}
-				if (buf->b_efunc != NULL) {
-					mutex_enter(&arc_eviction_mtx);
-					arc_buf_destroy(buf,
-					    buf->b_data == stolen, FALSE);
-					hdr->b_l1hdr.b_buf = buf->b_next;
-					buf->b_hdr = &arc_eviction_hdr;
-					buf->b_next = arc_eviction_list;
-					arc_eviction_list = buf;
-					mutex_exit(&arc_eviction_mtx);
-					mutex_exit(&buf->b_evict_lock);
-				} else {
-					mutex_exit(&buf->b_evict_lock);
-					arc_buf_destroy(buf,
-					    buf->b_data == stolen, TRUE);
-				}
-			}
 
-			if (HDR_HAS_L2HDR(hdr)) {
-				ARCSTAT_INCR(arcstat_evict_l2_cached,
-				    hdr->b_size);
-			} else {
-				if (l2arc_write_eligible(hdr->b_spa, hdr)) {
-					ARCSTAT_INCR(arcstat_evict_l2_eligible,
-					    hdr->b_size);
-				} else {
-					ARCSTAT_INCR(
-					    arcstat_evict_l2_ineligible,
-					    hdr->b_size);
-				}
-			}
+		/*
+		 * We aren't calling this function from any code path
+		 * that would already be holding a hash lock, so we're
+		 * asserting on this assumption to be defensive in case
+		 * this ever changes. Without this check, it would be
+		 * possible to incorrectly increment arcstat_mutex_miss
+		 * below (e.g. if the code changed such that we called
+		 * this function with a hash lock held).
+		 */
+		ASSERT(!MUTEX_HELD(hash_lock));
 
-			if (hdr->b_l1hdr.b_datacnt == 0) {
-				arc_change_state(evicted_state, hdr, hash_lock);
-				ASSERT(HDR_IN_HASH_TABLE(hdr));
-				hdr->b_flags |= ARC_FLAG_IN_HASH_TABLE;
-				hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE;
-				DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr);
-			}
-			if (!have_lock)
-				mutex_exit(hash_lock);
-			if (bytes >= 0 && bytes_evicted >= bytes)
-				break;
+		if (mutex_tryenter(hash_lock)) {
+			uint64_t evicted = arc_evict_hdr(hdr, hash_lock);
+			mutex_exit(hash_lock);
+
+			bytes_evicted += evicted;
+
+			/*
+			 * If evicted is zero, arc_evict_hdr() must have
+			 * decided to skip this header, don't increment
+			 * evict_count in this case.
+			 */
+			if (evicted != 0)
+				evict_count++;
+
+			/*
+			 * If arc_size isn't overflowing, signal any
+			 * threads that might happen to be waiting.
+			 *
+			 * For each header evicted, we wake up a single
+			 * thread. If we used cv_broadcast, we could
+			 * wake up "too many" threads causing arc_size
+			 * to significantly overflow arc_c; since
+			 * arc_get_data_buf() doesn't check for overflow
+			 * when it's woken up (it doesn't because it's
+			 * possible for the ARC to be overflowing while
+			 * full of un-evictable buffers, and the
+			 * function should proceed in this case).
+			 *
+			 * If threads are left sleeping, due to not
+			 * using cv_broadcast, they will be woken up
+			 * just before arc_reclaim_thread() sleeps.
+			 */
+			mutex_enter(&arc_reclaim_lock);
+			if (!arc_is_overflowing())
+				cv_signal(&arc_reclaim_waiters_cv);
+			mutex_exit(&arc_reclaim_lock);
 		} else {
-			missed += 1;
+			ARCSTAT_BUMP(arcstat_mutex_miss);
 		}
 	}
 
-	mutex_exit(&state->arcs_mtx);
-	mutex_exit(&evicted_state->arcs_mtx);
+	multilist_sublist_unlock(mls);
 
-	if (bytes_evicted < bytes)
-		dprintf("only evicted %lld bytes from %x",
-		    (longlong_t)bytes_evicted, state);
-
-	if (skipped)
-		ARCSTAT_INCR(arcstat_evict_skip, skipped);
-
-	if (missed)
-		ARCSTAT_INCR(arcstat_mutex_miss, missed);
-
-	/*
-	 * Note: we have just evicted some data into the ghost state,
-	 * potentially putting the ghost size over the desired size.  Rather
-	 * that evicting from the ghost list in this hot code path, leave
-	 * this chore to the arc_reclaim_thread().
-	 */
-
-	return (stolen);
+	return (bytes_evicted);
 }
 
 /*
- * Remove buffers from list until we've removed the specified number of
- * bytes.  Destroy the buffers that are removed.
+ * Evict buffers from the given arc state, until we've removed the
+ * specified number of bytes. Move the removed buffers to the
+ * appropriate evict state.
+ *
+ * This function makes a "best effort". It skips over any buffers
+ * it can't get a hash_lock on, and so, may not catch all candidates.
+ * It may also return without evicting as much space as requested.
+ *
+ * If bytes is specified using the special value ARC_EVICT_ALL, this
+ * will evict all available (i.e. unlocked and evictable) buffers from
+ * the given arc state; which is used by arc_flush().
  */
-static void
-arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes)
+static uint64_t
+arc_evict_state(arc_state_t *state, uint64_t spa, int64_t bytes,
+    arc_buf_contents_t type)
 {
-	arc_buf_hdr_t *hdr, *hdr_prev;
-	arc_buf_hdr_t marker = { 0 };
-	list_t *list = &state->arcs_list[ARC_BUFC_DATA];
-	kmutex_t *hash_lock;
-	uint64_t bytes_deleted = 0;
-	uint64_t bufs_skipped = 0;
-	int count = 0;
+	uint64_t total_evicted = 0;
+	multilist_t *ml = &state->arcs_list[type];
+	int num_sublists;
+	arc_buf_hdr_t **markers;
 
-	ASSERT(GHOST_STATE(state));
-top:
-	mutex_enter(&state->arcs_mtx);
-	for (hdr = list_tail(list); hdr; hdr = hdr_prev) {
-		hdr_prev = list_prev(list, hdr);
-		if (arc_buf_type(hdr) >= ARC_BUFC_NUMTYPES)
-			panic("invalid hdr=%p", (void *)hdr);
-		if (spa && hdr->b_spa != spa)
-			continue;
+	IMPLY(bytes < 0, bytes == ARC_EVICT_ALL);
 
-		/* ignore markers */
-		if (hdr->b_spa == 0)
-			continue;
+	num_sublists = multilist_get_num_sublists(ml);
 
-		hash_lock = HDR_LOCK(hdr);
-		/* caller may be trying to modify this buffer, skip it */
-		if (MUTEX_HELD(hash_lock))
-			continue;
+	/*
+	 * If we've tried to evict from each sublist, made some
+	 * progress, but still have not hit the target number of bytes
+	 * to evict, we want to keep trying. The markers allow us to
+	 * pick up where we left off for each individual sublist, rather
+	 * than starting from the tail each time.
+	 */
+	markers = kmem_zalloc(sizeof (*markers) * num_sublists, KM_SLEEP);
+	for (int i = 0; i < num_sublists; i++) {
+		markers[i] = kmem_cache_alloc(hdr_full_cache, KM_SLEEP);
 
 		/*
-		 * It may take a long time to evict all the bufs requested.
-		 * To avoid blocking all arc activity, periodically drop
-		 * the arcs_mtx and give other threads a chance to run
-		 * before reacquiring the lock.
+		 * A b_spa of 0 is used to indicate that this header is
+		 * a marker. This fact is used in arc_adjust_type() and
+		 * arc_evict_state_impl().
 		 */
-		if (count++ > arc_evict_iterations) {
-			list_insert_after(list, hdr, &marker);
-			mutex_exit(&state->arcs_mtx);
-			kpreempt(KPREEMPT_SYNC);
-			mutex_enter(&state->arcs_mtx);
-			hdr_prev = list_prev(list, &marker);
-			list_remove(list, &marker);
-			count = 0;
-			continue;
-		}
-		if (mutex_tryenter(hash_lock)) {
-			ASSERT(!HDR_IO_IN_PROGRESS(hdr));
-			ASSERT(!HDR_HAS_L1HDR(hdr) ||
-			    hdr->b_l1hdr.b_buf == NULL);
-			ARCSTAT_BUMP(arcstat_deleted);
-			bytes_deleted += hdr->b_size;
+		markers[i]->b_spa = 0;
 
-			if (HDR_HAS_L2HDR(hdr)) {
-				/*
-				 * This buffer is cached on the 2nd Level ARC;
-				 * don't destroy the header.
-				 */
-				arc_change_state(arc_l2c_only, hdr, hash_lock);
-				/*
-				 * dropping from L1+L2 cached to L2-only,
-				 * realloc to remove the L1 header.
-				 */
-				hdr = arc_hdr_realloc(hdr, hdr_full_cache,
-				    hdr_l2only_cache);
-				mutex_exit(hash_lock);
-			} else {
-				arc_change_state(arc_anon, hdr, hash_lock);
-				mutex_exit(hash_lock);
-				arc_hdr_destroy(hdr);
+		multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
+		multilist_sublist_insert_tail(mls, markers[i]);
+		multilist_sublist_unlock(mls);
+	}
+
+	/*
+	 * While we haven't hit our target number of bytes to evict, or
+	 * we're evicting all available buffers.
+	 */
+	while (total_evicted < bytes || bytes == ARC_EVICT_ALL) {
+		/*
+		 * Start eviction using a randomly selected sublist,
+		 * this is to try and evenly balance eviction across all
+		 * sublists. Always starting at the same sublist
+		 * (e.g. index 0) would cause evictions to favor certain
+		 * sublists over others.
+		 */
+		int sublist_idx = multilist_get_random_index(ml);
+		uint64_t scan_evicted = 0;
+
+		for (int i = 0; i < num_sublists; i++) {
+			uint64_t bytes_remaining;
+			uint64_t bytes_evicted;
+
+			if (bytes == ARC_EVICT_ALL)
+				bytes_remaining = ARC_EVICT_ALL;
+			else if (total_evicted < bytes)
+				bytes_remaining = bytes - total_evicted;
+			else
+				break;
+
+			bytes_evicted = arc_evict_state_impl(ml, sublist_idx,
+			    markers[sublist_idx], spa, bytes_remaining);
+
+			scan_evicted += bytes_evicted;
+			total_evicted += bytes_evicted;
+
+			/* we've reached the end, wrap to the beginning */
+			if (++sublist_idx >= num_sublists)
+				sublist_idx = 0;
+		}
+
+		/*
+		 * If we didn't evict anything during this scan, we have
+		 * no reason to believe we'll evict more during another
+		 * scan, so break the loop.
+		 */
+		if (scan_evicted == 0) {
+			/* This isn't possible, let's make that obvious */
+			ASSERT3S(bytes, !=, 0);
+
+			/*
+			 * When bytes is ARC_EVICT_ALL, the only way to
+			 * break the loop is when scan_evicted is zero.
+			 * In that case, we actually have evicted enough,
+			 * so we don't want to increment the kstat.
+			 */
+			if (bytes != ARC_EVICT_ALL) {
+				ASSERT3S(total_evicted, <, bytes);
+				ARCSTAT_BUMP(arcstat_evict_not_enough);
 			}
 
-			DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);
-			if (bytes >= 0 && bytes_deleted >= bytes)
-				break;
-		} else if (bytes < 0) {
-			/*
-			 * Insert a list marker and then wait for the
-			 * hash lock to become available. Once its
-			 * available, restart from where we left off.
-			 */
-			list_insert_after(list, hdr, &marker);
-			mutex_exit(&state->arcs_mtx);
-			mutex_enter(hash_lock);
-			mutex_exit(hash_lock);
-			mutex_enter(&state->arcs_mtx);
-			hdr_prev = list_prev(list, &marker);
-			list_remove(list, &marker);
-		} else {
-			bufs_skipped += 1;
+			break;
 		}
-
-	}
-	mutex_exit(&state->arcs_mtx);
-
-	if (list == &state->arcs_list[ARC_BUFC_DATA] &&
-	    (bytes < 0 || bytes_deleted < bytes)) {
-		list = &state->arcs_list[ARC_BUFC_METADATA];
-		goto top;
 	}
 
-	if (bufs_skipped) {
-		ARCSTAT_INCR(arcstat_mutex_miss, bufs_skipped);
-		ASSERT(bytes >= 0);
-	}
+	for (int i = 0; i < num_sublists; i++) {
+		multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
+		multilist_sublist_remove(mls, markers[i]);
+		multilist_sublist_unlock(mls);
 
-	if (bytes_deleted < bytes)
-		dprintf("only deleted %lld bytes from %p",
-		    (longlong_t)bytes_deleted, state);
+		kmem_cache_free(hdr_full_cache, markers[i]);
+	}
+	kmem_free(markers, sizeof (*markers) * num_sublists);
+
+	return (total_evicted);
 }
 
-static void
+/*
+ * Flush all "evictable" data of the given type from the arc state
+ * specified. This will not evict any "active" buffers (i.e. referenced).
+ *
+ * When 'retry' is set to FALSE, the function will make a single pass
+ * over the state and evict any buffers that it can. Since it doesn't
+ * continually retry the eviction, it might end up leaving some buffers
+ * in the ARC due to lock misses.
+ *
+ * When 'retry' is set to TRUE, the function will continually retry the
+ * eviction until *all* evictable buffers have been removed from the
+ * state. As a result, if concurrent insertions into the state are
+ * allowed (e.g. if the ARC isn't shutting down), this function might
+ * wind up in an infinite loop, continually trying to evict buffers.
+ */
+static uint64_t
+arc_flush_state(arc_state_t *state, uint64_t spa, arc_buf_contents_t type,
+    boolean_t retry)
+{
+	uint64_t evicted = 0;
+
+	while (state->arcs_lsize[type] != 0) {
+		evicted += arc_evict_state(state, spa, ARC_EVICT_ALL, type);
+
+		if (!retry)
+			break;
+	}
+
+	return (evicted);
+}
+
+/*
+ * Evict the specified number of bytes from the state specified,
+ * restricting eviction to the spa and type given. This function
+ * prevents us from trying to evict more from a state's list than
+ * is "evictable", and to skip evicting altogether when passed a
+ * negative value for "bytes". In contrast, arc_evict_state() will
+ * evict everything it can, when passed a negative value for "bytes".
+ */
+static uint64_t
+arc_adjust_impl(arc_state_t *state, uint64_t spa, int64_t bytes,
+    arc_buf_contents_t type)
+{
+	int64_t delta;
+
+	if (bytes > 0 && state->arcs_lsize[type] > 0) {
+		delta = MIN(state->arcs_lsize[type], bytes);
+		return (arc_evict_state(state, spa, delta, type));
+	}
+
+	return (0);
+}
+
+/*
+ * Evict metadata buffers from the cache, such that arc_meta_used is
+ * capped by the arc_meta_limit tunable.
+ */
+static uint64_t
+arc_adjust_meta(void)
+{
+	uint64_t total_evicted = 0;
+	int64_t target;
+
+	/*
+	 * If we're over the meta limit, we want to evict enough
+	 * metadata to get back under the meta limit. We don't want to
+	 * evict so much that we drop the MRU below arc_p, though. If
+	 * we're over the meta limit more than we're over arc_p, we
+	 * evict some from the MRU here, and some from the MFU below.
+	 */
+	target = MIN((int64_t)(arc_meta_used - arc_meta_limit),
+	    (int64_t)(arc_anon->arcs_size + arc_mru->arcs_size - arc_p));
+
+	total_evicted += arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
+
+	/*
+	 * Similar to the above, we want to evict enough bytes to get us
+	 * below the meta limit, but not so much as to drop us below the
+	 * space alloted to the MFU (which is defined as arc_c - arc_p).
+	 */
+	target = MIN((int64_t)(arc_meta_used - arc_meta_limit),
+	    (int64_t)(arc_mfu->arcs_size - (arc_c - arc_p)));
+
+	total_evicted += arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
+
+	return (total_evicted);
+}
+
+/*
+ * Return the type of the oldest buffer in the given arc state
+ *
+ * This function will select a random sublist of type ARC_BUFC_DATA and
+ * a random sublist of type ARC_BUFC_METADATA. The tail of each sublist
+ * is compared, and the type which contains the "older" buffer will be
+ * returned.
+ */
+static arc_buf_contents_t
+arc_adjust_type(arc_state_t *state)
+{
+	multilist_t *data_ml = &state->arcs_list[ARC_BUFC_DATA];
+	multilist_t *meta_ml = &state->arcs_list[ARC_BUFC_METADATA];
+	int data_idx = multilist_get_random_index(data_ml);
+	int meta_idx = multilist_get_random_index(meta_ml);
+	multilist_sublist_t *data_mls;
+	multilist_sublist_t *meta_mls;
+	arc_buf_contents_t type;
+	arc_buf_hdr_t *data_hdr;
+	arc_buf_hdr_t *meta_hdr;
+
+	/*
+	 * We keep the sublist lock until we're finished, to prevent
+	 * the headers from being destroyed via arc_evict_state().
+	 */
+	data_mls = multilist_sublist_lock(data_ml, data_idx);
+	meta_mls = multilist_sublist_lock(meta_ml, meta_idx);
+
+	/*
+	 * These two loops are to ensure we skip any markers that
+	 * might be at the tail of the lists due to arc_evict_state().
+	 */
+
+	for (data_hdr = multilist_sublist_tail(data_mls); data_hdr != NULL;
+	    data_hdr = multilist_sublist_prev(data_mls, data_hdr)) {
+		if (data_hdr->b_spa != 0)
+			break;
+	}
+
+	for (meta_hdr = multilist_sublist_tail(meta_mls); meta_hdr != NULL;
+	    meta_hdr = multilist_sublist_prev(meta_mls, meta_hdr)) {
+		if (meta_hdr->b_spa != 0)
+			break;
+	}
+
+	if (data_hdr == NULL && meta_hdr == NULL) {
+		type = ARC_BUFC_DATA;
+	} else if (data_hdr == NULL) {
+		ASSERT3P(meta_hdr, !=, NULL);
+		type = ARC_BUFC_METADATA;
+	} else if (meta_hdr == NULL) {
+		ASSERT3P(data_hdr, !=, NULL);
+		type = ARC_BUFC_DATA;
+	} else {
+		ASSERT3P(data_hdr, !=, NULL);
+		ASSERT3P(meta_hdr, !=, NULL);
+
+		/* The headers can't be on the sublist without an L1 header */
+		ASSERT(HDR_HAS_L1HDR(data_hdr));
+		ASSERT(HDR_HAS_L1HDR(meta_hdr));
+
+		if (data_hdr->b_l1hdr.b_arc_access <
+		    meta_hdr->b_l1hdr.b_arc_access) {
+			type = ARC_BUFC_DATA;
+		} else {
+			type = ARC_BUFC_METADATA;
+		}
+	}
+
+	multilist_sublist_unlock(meta_mls);
+	multilist_sublist_unlock(data_mls);
+
+	return (type);
+}
+
+/*
+ * Evict buffers from the cache, such that arc_size is capped by arc_c.
+ */
+static uint64_t
 arc_adjust(void)
 {
-	int64_t adjustment, delta;
+	uint64_t total_evicted = 0;
+	uint64_t bytes;
+	int64_t target;
+
+	/*
+	 * If we're over arc_meta_limit, we want to correct that before
+	 * potentially evicting data buffers below.
+	 */
+	total_evicted += arc_adjust_meta();
 
 	/*
 	 * Adjust MRU size
+	 *
+	 * If we're over the target cache size, we want to evict enough
+	 * from the list to get back to our target size. We don't want
+	 * to evict too much from the MRU, such that it drops below
+	 * arc_p. So, if we're over our target cache size more than
+	 * the MRU is over arc_p, we'll evict enough to get back to
+	 * arc_p here, and then evict more from the MFU below.
 	 */
-
-	adjustment = MIN((int64_t)(arc_size - arc_c),
+	target = MIN((int64_t)(arc_size - arc_c),
 	    (int64_t)(arc_anon->arcs_size + arc_mru->arcs_size + arc_meta_used -
 	    arc_p));
 
-	if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_DATA] > 0) {
-		delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_DATA], adjustment);
-		(void) arc_evict(arc_mru, NULL, delta, FALSE, ARC_BUFC_DATA);
-		adjustment -= delta;
-	}
+	/*
+	 * If we're below arc_meta_min, always prefer to evict data.
+	 * Otherwise, try to satisfy the requested number of bytes to
+	 * evict from the type which contains older buffers; in an
+	 * effort to keep newer buffers in the cache regardless of their
+	 * type. If we cannot satisfy the number of bytes from this
+	 * type, spill over into the next type.
+	 */
+	if (arc_adjust_type(arc_mru) == ARC_BUFC_METADATA &&
+	    arc_meta_used > arc_meta_min) {
+		bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
+		total_evicted += bytes;
 
-	if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_METADATA] > 0) {
-		delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_METADATA], adjustment);
-		(void) arc_evict(arc_mru, NULL, delta, FALSE,
-		    ARC_BUFC_METADATA);
+		/*
+		 * If we couldn't evict our target number of bytes from
+		 * metadata, we try to get the rest from data.
+		 */
+		target -= bytes;
+
+		total_evicted +=
+		    arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA);
+	} else {
+		bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA);
+		total_evicted += bytes;
+
+		/*
+		 * If we couldn't evict our target number of bytes from
+		 * data, we try to get the rest from metadata.
+		 */
+		target -= bytes;
+
+		total_evicted +=
+		    arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
 	}
 
 	/*
 	 * Adjust MFU size
+	 *
+	 * Now that we've tried to evict enough from the MRU to get its
+	 * size back to arc_p, if we're still above the target cache
+	 * size, we evict the rest from the MFU.
 	 */
+	target = arc_size - arc_c;
 
-	adjustment = arc_size - arc_c;
+	if (arc_adjust_type(arc_mru) == ARC_BUFC_METADATA &&
+	    arc_meta_used > arc_meta_min) {
+		bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
+		total_evicted += bytes;
 
-	if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_DATA] > 0) {
-		delta = MIN(adjustment, arc_mfu->arcs_lsize[ARC_BUFC_DATA]);
-		(void) arc_evict(arc_mfu, NULL, delta, FALSE, ARC_BUFC_DATA);
-		adjustment -= delta;
-	}
+		/*
+		 * If we couldn't evict our target number of bytes from
+		 * metadata, we try to get the rest from data.
+		 */
+		target -= bytes;
 
-	if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_METADATA] > 0) {
-		int64_t delta = MIN(adjustment,
-		    arc_mfu->arcs_lsize[ARC_BUFC_METADATA]);
-		(void) arc_evict(arc_mfu, NULL, delta, FALSE,
-		    ARC_BUFC_METADATA);
+		total_evicted +=
+		    arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA);
+	} else {
+		bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA);
+		total_evicted += bytes;
+
+		/*
+		 * If we couldn't evict our target number of bytes from
+		 * data, we try to get the rest from data.
+		 */
+		target -= bytes;
+
+		total_evicted +=
+		    arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
 	}
 
 	/*
 	 * Adjust ghost lists
+	 *
+	 * In addition to the above, the ARC also defines target values
+	 * for the ghost lists. The sum of the mru list and mru ghost
+	 * list should never exceed the target size of the cache, and
+	 * the sum of the mru list, mfu list, mru ghost list, and mfu
+	 * ghost list should never exceed twice the target size of the
+	 * cache. The following logic enforces these limits on the ghost
+	 * caches, and evicts from them as needed.
 	 */
+	target = arc_mru->arcs_size + arc_mru_ghost->arcs_size - arc_c;
 
-	adjustment = arc_mru->arcs_size + arc_mru_ghost->arcs_size - arc_c;
+	bytes = arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_DATA);
+	total_evicted += bytes;
 
-	if (adjustment > 0 && arc_mru_ghost->arcs_size > 0) {
-		delta = MIN(arc_mru_ghost->arcs_size, adjustment);
-		arc_evict_ghost(arc_mru_ghost, NULL, delta);
-	}
+	target -= bytes;
 
-	adjustment =
-	    arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size - arc_c;
+	total_evicted +=
+	    arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_METADATA);
 
-	if (adjustment > 0 && arc_mfu_ghost->arcs_size > 0) {
-		delta = MIN(arc_mfu_ghost->arcs_size, adjustment);
-		arc_evict_ghost(arc_mfu_ghost, NULL, delta);
-	}
+	/*
+	 * We assume the sum of the mru list and mfu list is less than
+	 * or equal to arc_c (we enforced this above), which means we
+	 * can use the simpler of the two equations below:
+	 *
+	 *	mru + mfu + mru ghost + mfu ghost <= 2 * arc_c
+	 *		    mru ghost + mfu ghost <= arc_c
+	 */
+	target = arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size - arc_c;
+
+	bytes = arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_DATA);
+	total_evicted += bytes;
+
+	target -= bytes;
+
+	total_evicted +=
+	    arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_METADATA);
+
+	return (total_evicted);
 }
 
 static void
 arc_do_user_evicts(void)
 {
-	mutex_enter(&arc_eviction_mtx);
+	mutex_enter(&arc_user_evicts_lock);
 	while (arc_eviction_list != NULL) {
 		arc_buf_t *buf = arc_eviction_list;
 		arc_eviction_list = buf->b_next;
 		mutex_enter(&buf->b_evict_lock);
 		buf->b_hdr = NULL;
 		mutex_exit(&buf->b_evict_lock);
-		mutex_exit(&arc_eviction_mtx);
+		mutex_exit(&arc_user_evicts_lock);
 
 		if (buf->b_efunc != NULL)
 			VERIFY0(buf->b_efunc(buf->b_private));
@@ -2580,50 +2937,39 @@ arc_do_user_evicts(void)
 		buf->b_efunc = NULL;
 		buf->b_private = NULL;
 		kmem_cache_free(buf_cache, buf);
-		mutex_enter(&arc_eviction_mtx);
+		mutex_enter(&arc_user_evicts_lock);
 	}
-	mutex_exit(&arc_eviction_mtx);
+	mutex_exit(&arc_user_evicts_lock);
 }
 
-/*
- * Flush all *evictable* data from the cache for the given spa.
- * NOTE: this will not touch "active" (i.e. referenced) data.
- */
 void
-arc_flush(spa_t *spa)
+arc_flush(spa_t *spa, boolean_t retry)
 {
 	uint64_t guid = 0;
 
+	/*
+	 * If retry is TRUE, a spa must not be specified since we have
+	 * no good way to determine if all of a spa's buffers have been
+	 * evicted from an arc state.
+	 */
+	ASSERT(!retry || spa == 0);
+
 	if (spa != NULL)
 		guid = spa_load_guid(spa);
 
-	while (list_head(&arc_mru->arcs_list[ARC_BUFC_DATA])) {
-		(void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_DATA);
-		if (spa != NULL)
-			break;
-	}
-	while (list_head(&arc_mru->arcs_list[ARC_BUFC_METADATA])) {
-		(void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_METADATA);
-		if (spa != NULL)
-			break;
-	}
-	while (list_head(&arc_mfu->arcs_list[ARC_BUFC_DATA])) {
-		(void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_DATA);
-		if (spa != NULL)
-			break;
-	}
-	while (list_head(&arc_mfu->arcs_list[ARC_BUFC_METADATA])) {
-		(void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_METADATA);
-		if (spa != NULL)
-			break;
-	}
+	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_DATA, retry);
+	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_METADATA, retry);
 
-	arc_evict_ghost(arc_mru_ghost, guid, -1);
-	arc_evict_ghost(arc_mfu_ghost, guid, -1);
+	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_DATA, retry);
+	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_METADATA, retry);
+
+	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_DATA, retry);
+	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_METADATA, retry);
+
+	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_DATA, retry);
+	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_METADATA, retry);
 
-	mutex_enter(&arc_reclaim_thr_lock);
 	arc_do_user_evicts();
-	mutex_exit(&arc_reclaim_thr_lock);
 	ASSERT(spa || arc_eviction_list == NULL);
 }
 
@@ -2647,7 +2993,7 @@ arc_shrink(int64_t to_free)
 	}
 
 	if (arc_size > arc_c)
-		arc_adjust();
+		(void) arc_adjust();
 }
 
 typedef enum free_memory_reason_t {
@@ -2847,17 +3193,37 @@ arc_kmem_reap_now(void)
 	}
 }
 
+/*
+ * Threads can block in arc_get_data_buf() waiting for this thread to evict
+ * enough data and signal them to proceed. When this happens, the threads in
+ * arc_get_data_buf() are sleeping while holding the hash lock for their
+ * particular arc header. Thus, we must be careful to never sleep on a
+ * hash lock in this thread. This is to prevent the following deadlock:
+ *
+ *  - Thread A sleeps on CV in arc_get_data_buf() holding hash lock "L",
+ *    waiting for the reclaim thread to signal it.
+ *
+ *  - arc_reclaim_thread() tries to acquire hash lock "L" using mutex_enter,
+ *    fails, and goes to sleep forever.
+ *
+ * This possible deadlock is avoided by always acquiring a hash lock
+ * using mutex_tryenter() from arc_reclaim_thread().
+ */
 static void
 arc_reclaim_thread(void)
 {
 	clock_t			growtime = 0;
 	callb_cpr_t		cpr;
 
-	CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG);
+	CALLB_CPR_INIT(&cpr, &arc_reclaim_lock, callb_generic_cpr, FTAG);
 
-	mutex_enter(&arc_reclaim_thr_lock);
-	while (arc_thread_exit == 0) {
+	mutex_enter(&arc_reclaim_lock);
+	while (!arc_reclaim_thread_exit) {
 		int64_t free_memory = arc_available_memory();
+		uint64_t evicted = 0;
+
+		mutex_exit(&arc_reclaim_lock);
+
 		if (free_memory < 0) {
 
 			arc_no_grow = B_TRUE;
@@ -2891,10 +3257,57 @@ arc_reclaim_thread(void)
 			arc_no_grow = B_FALSE;
 		}
 
-		arc_adjust();
+		evicted = arc_adjust();
 
-		if (arc_eviction_list != NULL)
-			arc_do_user_evicts();
+		mutex_enter(&arc_reclaim_lock);
+
+		/*
+		 * If evicted is zero, we couldn't evict anything via
+		 * arc_adjust(). This could be due to hash lock
+		 * collisions, but more likely due to the majority of
+		 * arc buffers being unevictable. Therefore, even if
+		 * arc_size is above arc_c, another pass is unlikely to
+		 * be helpful and could potentially cause us to enter an
+		 * infinite loop.
+		 */
+		if (arc_size <= arc_c || evicted == 0) {
+			/*
+			 * We're either no longer overflowing, or we
+			 * can't evict anything more, so we should wake
+			 * up any threads before we go to sleep.
+			 */
+			cv_broadcast(&arc_reclaim_waiters_cv);
+
+			/*
+			 * Block until signaled, or after one second (we
+			 * might need to perform arc_kmem_reap_now()
+			 * even if we aren't being signalled)
+			 */
+			CALLB_CPR_SAFE_BEGIN(&cpr);
+			(void) cv_timedwait(&arc_reclaim_thread_cv,
+			    &arc_reclaim_lock, ddi_get_lbolt() + hz);
+			CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_lock);
+		}
+	}
+
+	arc_reclaim_thread_exit = FALSE;
+	cv_broadcast(&arc_reclaim_thread_cv);
+	CALLB_CPR_EXIT(&cpr);		/* drops arc_reclaim_lock */
+	thread_exit();
+}
+
+static void
+arc_user_evicts_thread(void)
+{
+	callb_cpr_t cpr;
+
+	CALLB_CPR_INIT(&cpr, &arc_user_evicts_lock, callb_generic_cpr, FTAG);
+
+	mutex_enter(&arc_user_evicts_lock);
+	while (!arc_user_evicts_thread_exit) {
+		mutex_exit(&arc_user_evicts_lock);
+
+		arc_do_user_evicts();
 
 		/*
 		 * This is necessary in order for the mdb ::arc dcmd to
@@ -2910,16 +3323,21 @@ arc_reclaim_thread(void)
 		if (arc_ksp != NULL)
 			arc_ksp->ks_update(arc_ksp, KSTAT_READ);
 
-		/* block until needed, or one second, whichever is shorter */
+		mutex_enter(&arc_user_evicts_lock);
+
+		/*
+		 * Block until signaled, or after one second (we need to
+		 * call the arc's kstat update function regularly).
+		 */
 		CALLB_CPR_SAFE_BEGIN(&cpr);
-		(void) cv_timedwait(&arc_reclaim_thr_cv,
-		    &arc_reclaim_thr_lock, (ddi_get_lbolt() + hz));
-		CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_thr_lock);
+		(void) cv_timedwait(&arc_user_evicts_cv,
+		    &arc_user_evicts_lock, ddi_get_lbolt() + hz);
+		CALLB_CPR_SAFE_END(&cpr, &arc_user_evicts_lock);
 	}
 
-	arc_thread_exit = 0;
-	cv_broadcast(&arc_reclaim_thr_cv);
-	CALLB_CPR_EXIT(&cpr);		/* drops arc_reclaim_thr_lock */
+	arc_user_evicts_thread_exit = FALSE;
+	cv_broadcast(&arc_user_evicts_cv);
+	CALLB_CPR_EXIT(&cpr);		/* drops arc_user_evicts_lock */
 	thread_exit();
 }
 
@@ -2965,7 +3383,7 @@ arc_adapt(int bytes, arc_state_t *state)
 	ASSERT((int64_t)arc_p >= 0);
 
 	if (arc_reclaim_needed()) {
-		cv_signal(&arc_reclaim_thr_cv);
+		cv_signal(&arc_reclaim_thread_cv);
 		return;
 	}
 
@@ -2992,43 +3410,25 @@ arc_adapt(int bytes, arc_state_t *state)
 }
 
 /*
- * Check if the cache has reached its limits and eviction is required
- * prior to insert.
+ * Check if arc_size has grown past our upper threshold, determined by
+ * zfs_arc_overflow_shift.
  */
-static int
-arc_evict_needed(arc_buf_contents_t type)
+static boolean_t
+arc_is_overflowing(void)
 {
-	if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit)
-		return (1);
+	/* Always allow at least one block of overflow */
+	uint64_t overflow = MAX(SPA_MAXBLOCKSIZE,
+	    arc_c >> zfs_arc_overflow_shift);
 
-	if (arc_reclaim_needed())
-		return (1);
-
-	return (arc_size > arc_c);
+	return (arc_size >= arc_c + overflow);
 }
 
 /*
- * The buffer, supplied as the first argument, needs a data block.
- * So, if we are at cache max, determine which cache should be victimized.
- * We have the following cases:
- *
- * 1. Insert for MRU, p > sizeof(arc_anon + arc_mru) ->
- * In this situation if we're out of space, but the resident size of the MFU is
- * under the limit, victimize the MFU cache to satisfy this insertion request.
- *
- * 2. Insert for MRU, p <= sizeof(arc_anon + arc_mru) ->
- * Here, we've used up all of the available space for the MRU, so we need to
- * evict from our own cache instead.  Evict from the set of resident MRU
- * entries.
- *
- * 3. Insert for MFU (c - p) > sizeof(arc_mfu) ->
- * c minus p represents the MFU space in the cache, since p is the size of the
- * cache that is dedicated to the MRU.  In this situation there's still space on
- * the MFU side, so the MRU side needs to be victimized.
- *
- * 4. Insert for MFU (c - p) < sizeof(arc_mfu) ->
- * MFU's resident set is consuming more space than it has been allotted.  In
- * this situation, we must victimize our own cache, the MFU, for this insertion.
+ * The buffer, supplied as the first argument, needs a data block. If we
+ * are hitting the hard limit for the cache size, we must sleep, waiting
+ * for the eviction thread to catch up. If we're past the target size
+ * but below the hard limit, we'll only signal the reclaim thread and
+ * continue on.
  */
 static void
 arc_get_data_buf(arc_buf_t *buf)
@@ -3040,53 +3440,50 @@ arc_get_data_buf(arc_buf_t *buf)
 	arc_adapt(size, state);
 
 	/*
-	 * We have not yet reached cache maximum size,
-	 * just allocate a new buffer.
+	 * If arc_size is currently overflowing, and has grown past our
+	 * upper limit, we must be adding data faster than the evict
+	 * thread can evict. Thus, to ensure we don't compound the
+	 * problem by adding more data and forcing arc_size to grow even
+	 * further past it's target size, we halt and wait for the
+	 * eviction thread to catch up.
+	 *
+	 * It's also possible that the reclaim thread is unable to evict
+	 * enough buffers to get arc_size below the overflow limit (e.g.
+	 * due to buffers being un-evictable, or hash lock collisions).
+	 * In this case, we want to proceed regardless if we're
+	 * overflowing; thus we don't use a while loop here.
 	 */
-	if (!arc_evict_needed(type)) {
-		if (type == ARC_BUFC_METADATA) {
-			buf->b_data = zio_buf_alloc(size);
-			arc_space_consume(size, ARC_SPACE_META);
-		} else {
-			ASSERT(type == ARC_BUFC_DATA);
-			buf->b_data = zio_data_buf_alloc(size);
-			arc_space_consume(size, ARC_SPACE_DATA);
+	if (arc_is_overflowing()) {
+		mutex_enter(&arc_reclaim_lock);
+
+		/*
+		 * Now that we've acquired the lock, we may no longer be
+		 * over the overflow limit, lets check.
+		 *
+		 * We're ignoring the case of spurious wake ups. If that
+		 * were to happen, it'd let this thread consume an ARC
+		 * buffer before it should have (i.e. before we're under
+		 * the overflow limit and were signalled by the reclaim
+		 * thread). As long as that is a rare occurrence, it
+		 * shouldn't cause any harm.
+		 */
+		if (arc_is_overflowing()) {
+			cv_signal(&arc_reclaim_thread_cv);
+			cv_wait(&arc_reclaim_waiters_cv, &arc_reclaim_lock);
 		}
-		goto out;
+
+		mutex_exit(&arc_reclaim_lock);
 	}
 
-	/*
-	 * If we are prefetching from the mfu ghost list, this buffer
-	 * will end up on the mru list; so steal space from there.
-	 */
-	if (state == arc_mfu_ghost)
-		state = HDR_PREFETCH(buf->b_hdr) ? arc_mru : arc_mfu;
-	else if (state == arc_mru_ghost)
-		state = arc_mru;
-
-	if (state == arc_mru || state == arc_anon) {
-		uint64_t mru_used = arc_anon->arcs_size + arc_mru->arcs_size;
-		state = (arc_mfu->arcs_lsize[type] >= size &&
-		    arc_p > mru_used) ? arc_mfu : arc_mru;
+	if (type == ARC_BUFC_METADATA) {
+		buf->b_data = zio_buf_alloc(size);
+		arc_space_consume(size, ARC_SPACE_META);
 	} else {
-		/* MFU cases */
-		uint64_t mfu_space = arc_c - arc_p;
-		state =  (arc_mru->arcs_lsize[type] >= size &&
-		    mfu_space > arc_mfu->arcs_size) ? arc_mru : arc_mfu;
+		ASSERT(type == ARC_BUFC_DATA);
+		buf->b_data = zio_data_buf_alloc(size);
+		arc_space_consume(size, ARC_SPACE_DATA);
 	}
-	if ((buf->b_data = arc_evict(state, NULL, size, TRUE, type)) == NULL) {
-		if (type == ARC_BUFC_METADATA) {
-			buf->b_data = zio_buf_alloc(size);
-			arc_space_consume(size, ARC_SPACE_META);
-		} else {
-			ASSERT(type == ARC_BUFC_DATA);
-			buf->b_data = zio_data_buf_alloc(size);
-			arc_space_consume(size, ARC_SPACE_DATA);
-		}
-		ARCSTAT_BUMP(arcstat_recycle_miss);
-	}
-	ASSERT(buf->b_data != NULL);
-out:
+
 	/*
 	 * Update the state size.  Note that ghost states have a
 	 * "ghost size" and so don't need to be updated.
@@ -3095,7 +3492,17 @@ out:
 		arc_buf_hdr_t *hdr = buf->b_hdr;
 
 		atomic_add_64(&hdr->b_l1hdr.b_state->arcs_size, size);
-		if (list_link_active(&hdr->b_l1hdr.b_arc_node)) {
+
+		/*
+		 * If this is reached via arc_read, the link is
+		 * protected by the hash lock. If reached via
+		 * arc_buf_alloc, the header should not be accessed by
+		 * any other thread. And, if reached via arc_read_done,
+		 * the hash lock will protect it if it's found in the
+		 * hash table; otherwise no other thread should be
+		 * trying to [add|remove]_reference it.
+		 */
+		if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
 			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
 			atomic_add_64(&hdr->b_l1hdr.b_state->arcs_lsize[type],
 			    size);
@@ -3147,7 +3554,8 @@ arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
 		 */
 		if (HDR_PREFETCH(hdr)) {
 			if (refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
-				ASSERT(list_link_active(
+				/* link protected by hash lock */
+				ASSERT(multilist_link_active(
 				    &hdr->b_l1hdr.b_arc_node));
 			} else {
 				hdr->b_flags &= ~ARC_FLAG_PREFETCH;
@@ -3207,7 +3615,8 @@ arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
 		 */
 		if ((HDR_PREFETCH(hdr)) != 0) {
 			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
-			ASSERT(list_link_active(&hdr->b_l1hdr.b_arc_node));
+			/* link protected by hash_lock */
+			ASSERT(multilist_link_active(&hdr->b_l1hdr.b_arc_node));
 		}
 		ARCSTAT_BUMP(arcstat_mfu_hits);
 		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
@@ -3577,7 +3986,7 @@ top:
 			ASSERT(GHOST_STATE(hdr->b_l1hdr.b_state));
 			ASSERT(!HDR_IO_IN_PROGRESS(hdr));
 			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
-			ASSERT(hdr->b_l1hdr.b_buf == NULL);
+			ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
 
 			/* if this is a prefetch, we don't have a reference */
 			if (*arc_flags & ARC_FLAG_PREFETCH)
@@ -3836,7 +4245,7 @@ arc_clear_callback(arc_buf_t *buf)
 
 	if (hdr->b_l1hdr.b_datacnt > 1) {
 		mutex_exit(&buf->b_evict_lock);
-		arc_buf_destroy(buf, FALSE, TRUE);
+		arc_buf_destroy(buf, TRUE);
 	} else {
 		ASSERT(buf == hdr->b_l1hdr.b_buf);
 		hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
@@ -3867,6 +4276,8 @@ arc_release(arc_buf_t *buf, void *tag)
 
 	mutex_enter(&buf->b_evict_lock);
 
+	ASSERT(HDR_HAS_L1HDR(hdr));
+
 	/*
 	 * We don't grab the hash lock prior to this check, because if
 	 * the buffer's header is in the arc_anon state, it won't be
@@ -3913,6 +4324,13 @@ arc_release(arc_buf_t *buf, void *tag)
 
 		mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);
 		list_remove(&hdr->b_l2hdr.b_dev->l2ad_buflist, hdr);
+
+		/*
+		 * We don't want to leak the b_tmp_cdata buffer that was
+		 * allocated in l2arc_write_buffers()
+		 */
+		arc_buf_l2_cdata_free(hdr);
+
 		mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);
 
 		hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
@@ -3978,6 +4396,7 @@ arc_release(arc_buf_t *buf, void *tag)
 		nhdr->b_l1hdr.b_datacnt = 1;
 		nhdr->b_l1hdr.b_state = arc_anon;
 		nhdr->b_l1hdr.b_arc_access = 0;
+		nhdr->b_l1hdr.b_tmp_cdata = NULL;
 		nhdr->b_freeze_cksum = NULL;
 
 		(void) refcount_add(&nhdr->b_l1hdr.b_refcnt, tag);
@@ -3987,8 +4406,8 @@ arc_release(arc_buf_t *buf, void *tag)
 	} else {
 		mutex_exit(&buf->b_evict_lock);
 		ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
-		/* protected by hash lock */
-		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
+		/* protected by hash lock, or hdr is on arc_anon */
+		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
 		arc_change_state(arc_anon, hdr, hash_lock);
 		hdr->b_l1hdr.b_arc_access = 0;
@@ -4323,6 +4742,41 @@ arc_kstat_update(kstat_t *ksp, int rw)
 	return (0);
 }
 
+/*
+ * This function *must* return indices evenly distributed between all
+ * sublists of the multilist. This is needed due to how the ARC eviction
+ * code is laid out; arc_evict_state() assumes ARC buffers are evenly
+ * distributed between all sublists and uses this assumption when
+ * deciding which sublist to evict from and how much to evict from it.
+ */
+unsigned int
+arc_state_multilist_index_func(multilist_t *ml, void *obj)
+{
+	arc_buf_hdr_t *hdr = obj;
+
+	/*
+	 * We rely on b_dva to generate evenly distributed index
+	 * numbers using buf_hash below. So, as an added precaution,
+	 * let's make sure we never add empty buffers to the arc lists.
+	 */
+	ASSERT(!BUF_EMPTY(hdr));
+
+	/*
+	 * The assumption here, is the hash value for a given
+	 * arc_buf_hdr_t will remain constant throughout it's lifetime
+	 * (i.e. it's b_spa, b_dva, and b_birth fields don't change).
+	 * Thus, we don't need to store the header's sublist index
+	 * on insertion, as this index can be recalculated on removal.
+	 *
+	 * Also, the low order bits of the hash value are thought to be
+	 * distributed evenly. Otherwise, in the case that the multilist
+	 * has a power of two number of sublists, each sublists' usage
+	 * would not be evenly distributed.
+	 */
+	return (buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) %
+	    multilist_get_num_sublists(ml));
+}
+
 void
 arc_init(void)
 {
@@ -4335,8 +4789,12 @@ arc_init(void)
 	uint64_t allmem = (physmem * PAGESIZE) / 2;
 #endif
 
-	mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL);
-	cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL);
+	mutex_init(&arc_reclaim_lock, NULL, MUTEX_DEFAULT, NULL);
+	cv_init(&arc_reclaim_thread_cv, NULL, CV_DEFAULT, NULL);
+	cv_init(&arc_reclaim_waiters_cv, NULL, CV_DEFAULT, NULL);
+
+	mutex_init(&arc_user_evicts_lock, NULL, MUTEX_DEFAULT, NULL);
+	cv_init(&arc_user_evicts_cv, NULL, CV_DEFAULT, NULL);
 
 	/* Convert seconds to clock ticks */
 	arc_min_prefetch_lifespan = 1 * hz;
@@ -4405,6 +4863,9 @@ arc_init(void)
 	if (zfs_arc_p_min_shift > 0)
 		arc_p_min_shift = zfs_arc_p_min_shift;
 
+	if (zfs_arc_num_sublists_per_state < 1)
+		zfs_arc_num_sublists_per_state = MAX(boot_ncpus, 1);
+
 	/* if kmem_flags are set, lets try to use less memory */
 	if (kmem_debugging())
 		arc_c = arc_c / 2;
@@ -4419,49 +4880,52 @@ arc_init(void)
 	arc_l2c_only = &ARC_l2c_only;
 	arc_size = 0;
 
-	mutex_init(&arc_anon->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
-	mutex_init(&arc_mru->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
-	mutex_init(&arc_mru_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
-	mutex_init(&arc_mfu->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
-	mutex_init(&arc_mfu_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
-	mutex_init(&arc_l2c_only->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
-
-	list_create(&arc_mru->arcs_list[ARC_BUFC_METADATA],
+	multilist_create(&arc_mru->arcs_list[ARC_BUFC_METADATA],
 	    sizeof (arc_buf_hdr_t),
-	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
-	list_create(&arc_mru->arcs_list[ARC_BUFC_DATA],
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
+	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
+	multilist_create(&arc_mru->arcs_list[ARC_BUFC_DATA],
 	    sizeof (arc_buf_hdr_t),
-	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
-	list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
+	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
+	multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
 	    sizeof (arc_buf_hdr_t),
-	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
-	list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
+	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
+	multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
 	    sizeof (arc_buf_hdr_t),
-	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
-	list_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
+	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
+	multilist_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
 	    sizeof (arc_buf_hdr_t),
-	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
-	list_create(&arc_mfu->arcs_list[ARC_BUFC_DATA],
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
+	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
+	multilist_create(&arc_mfu->arcs_list[ARC_BUFC_DATA],
 	    sizeof (arc_buf_hdr_t),
-	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
-	list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
+	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
+	multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
 	    sizeof (arc_buf_hdr_t),
-	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
-	list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
+	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
+	multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
 	    sizeof (arc_buf_hdr_t),
-	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
-	list_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
+	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
+	multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
 	    sizeof (arc_buf_hdr_t),
-	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
-	list_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
+	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
+	multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
 	    sizeof (arc_buf_hdr_t),
-	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
+	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
 
 	buf_init();
 
-	arc_thread_exit = 0;
+	arc_reclaim_thread_exit = FALSE;
+	arc_user_evicts_thread_exit = FALSE;
 	arc_eviction_list = NULL;
-	mutex_init(&arc_eviction_mtx, NULL, MUTEX_DEFAULT, NULL);
 	bzero(&arc_eviction_hdr, sizeof (arc_buf_hdr_t));
 
 	arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED,
@@ -4476,6 +4940,9 @@ arc_init(void)
 	(void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0,
 	    TS_RUN, minclsyspri);
 
+	(void) thread_create(NULL, 0, arc_user_evicts_thread, NULL, 0, &p0,
+	    TS_RUN, minclsyspri);
+
 	arc_dead = FALSE;
 	arc_warm = B_FALSE;
 
@@ -4498,13 +4965,32 @@ arc_init(void)
 void
 arc_fini(void)
 {
-	mutex_enter(&arc_reclaim_thr_lock);
-	arc_thread_exit = 1;
-	while (arc_thread_exit != 0)
-		cv_wait(&arc_reclaim_thr_cv, &arc_reclaim_thr_lock);
-	mutex_exit(&arc_reclaim_thr_lock);
+	mutex_enter(&arc_reclaim_lock);
+	arc_reclaim_thread_exit = TRUE;
+	/*
+	 * The reclaim thread will set arc_reclaim_thread_exit back to
+	 * FALSE when it is finished exiting; we're waiting for that.
+	 */
+	while (arc_reclaim_thread_exit) {
+		cv_signal(&arc_reclaim_thread_cv);
+		cv_wait(&arc_reclaim_thread_cv, &arc_reclaim_lock);
+	}
+	mutex_exit(&arc_reclaim_lock);
 
-	arc_flush(NULL);
+	mutex_enter(&arc_user_evicts_lock);
+	arc_user_evicts_thread_exit = TRUE;
+	/*
+	 * The user evicts thread will set arc_user_evicts_thread_exit
+	 * to FALSE when it is finished exiting; we're waiting for that.
+	 */
+	while (arc_user_evicts_thread_exit) {
+		cv_signal(&arc_user_evicts_cv);
+		cv_wait(&arc_user_evicts_cv, &arc_user_evicts_lock);
+	}
+	mutex_exit(&arc_user_evicts_lock);
+
+	/* Use TRUE to ensure *all* buffers are evicted */
+	arc_flush(NULL, TRUE);
 
 	arc_dead = TRUE;
 
@@ -4513,25 +4999,21 @@ arc_fini(void)
 		arc_ksp = NULL;
 	}
 
-	mutex_destroy(&arc_eviction_mtx);
-	mutex_destroy(&arc_reclaim_thr_lock);
-	cv_destroy(&arc_reclaim_thr_cv);
+	mutex_destroy(&arc_reclaim_lock);
+	cv_destroy(&arc_reclaim_thread_cv);
+	cv_destroy(&arc_reclaim_waiters_cv);
 
-	list_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
-	list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
-	list_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
-	list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
-	list_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
-	list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
-	list_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
-	list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
+	mutex_destroy(&arc_user_evicts_lock);
+	cv_destroy(&arc_user_evicts_cv);
 
-	mutex_destroy(&arc_anon->arcs_mtx);
-	mutex_destroy(&arc_mru->arcs_mtx);
-	mutex_destroy(&arc_mru_ghost->arcs_mtx);
-	mutex_destroy(&arc_mfu->arcs_mtx);
-	mutex_destroy(&arc_mfu_ghost->arcs_mtx);
-	mutex_destroy(&arc_l2c_only->arcs_mtx);
+	multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
+	multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
+	multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
+	multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
+	multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
+	multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
+	multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
+	multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
 
 	buf_fini();
 
@@ -4859,34 +5341,62 @@ l2arc_write_done(zio_t *zio)
 	if (zio->io_error != 0)
 		ARCSTAT_BUMP(arcstat_l2_writes_error);
 
-	mutex_enter(&dev->l2ad_mtx);
-
 	/*
 	 * All writes completed, or an error was hit.
 	 */
+top:
+	mutex_enter(&dev->l2ad_mtx);
 	for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) {
 		hdr_prev = list_prev(buflist, hdr);
 
 		hash_lock = HDR_LOCK(hdr);
+
+		/*
+		 * We cannot use mutex_enter or else we can deadlock
+		 * with l2arc_write_buffers (due to swapping the order
+		 * the hash lock and l2ad_mtx are taken).
+		 */
 		if (!mutex_tryenter(hash_lock)) {
 			/*
-			 * This buffer misses out.  It may be in a stage
-			 * of eviction.  Its ARC_FLAG_L2_WRITING flag will be
-			 * left set, denying reads to this buffer.
+			 * Missed the hash lock. We must retry so we
+			 * don't leave the ARC_FLAG_L2_WRITING bit set.
 			 */
-			ARCSTAT_BUMP(arcstat_l2_writes_hdr_miss);
-			continue;
+			ARCSTAT_BUMP(arcstat_l2_writes_lock_retry);
+
+			/*
+			 * We don't want to rescan the headers we've
+			 * already marked as having been written out, so
+			 * we reinsert the head node so we can pick up
+			 * where we left off.
+			 */
+			list_remove(buflist, head);
+			list_insert_after(buflist, hdr, head);
+
+			mutex_exit(&dev->l2ad_mtx);
+
+			/*
+			 * We wait for the hash lock to become available
+			 * to try and prevent busy waiting, and increase
+			 * the chance we'll be able to acquire the lock
+			 * the next time around.
+			 */
+			mutex_enter(hash_lock);
+			mutex_exit(hash_lock);
+			goto top;
 		}
 
 		/*
-		 * It's possible that this buffer got evicted from the L1 cache
-		 * before we grabbed the vdev + hash locks, in which case
-		 * arc_hdr_realloc freed b_tmp_cdata for us if it was allocated.
-		 * Only free the buffer if we still have an L1 hdr.
+		 * We could not have been moved into the arc_l2c_only
+		 * state while in-flight due to our ARC_FLAG_L2_WRITING
+		 * bit being set. Let's just ensure that's being enforced.
 		 */
-		if (HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_tmp_cdata != NULL &&
-		    HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF)
-			l2arc_release_cdata_buf(hdr);
+		ASSERT(HDR_HAS_L1HDR(hdr));
+
+		/*
+		 * We may have allocated a buffer for L2ARC compression,
+		 * we must release it to avoid leaking this data.
+		 */
+		l2arc_release_cdata_buf(hdr);
 
 		if (zio->io_error != 0) {
 			/*
@@ -4900,7 +5410,8 @@ l2arc_write_done(zio_t *zio)
 		}
 
 		/*
-		 * Allow ARC to begin reads to this L2ARC entry.
+		 * Allow ARC to begin reads and ghost list evictions to
+		 * this L2ARC entry.
 		 */
 		hdr->b_flags &= ~ARC_FLAG_L2_WRITING;
 
@@ -5008,35 +5519,37 @@ l2arc_read_done(zio_t *zio)
  * the data lists.  This function returns a locked list, and also returns
  * the lock pointer.
  */
-static list_t *
-l2arc_list_locked(int list_num, kmutex_t **lock)
+static multilist_sublist_t *
+l2arc_sublist_lock(int list_num)
 {
-	list_t *list = NULL;
+	multilist_t *ml = NULL;
+	unsigned int idx;
 
 	ASSERT(list_num >= 0 && list_num <= 3);
 
 	switch (list_num) {
 	case 0:
-		list = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
-		*lock = &arc_mfu->arcs_mtx;
+		ml = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
 		break;
 	case 1:
-		list = &arc_mru->arcs_list[ARC_BUFC_METADATA];
-		*lock = &arc_mru->arcs_mtx;
+		ml = &arc_mru->arcs_list[ARC_BUFC_METADATA];
 		break;
 	case 2:
-		list = &arc_mfu->arcs_list[ARC_BUFC_DATA];
-		*lock = &arc_mfu->arcs_mtx;
+		ml = &arc_mfu->arcs_list[ARC_BUFC_DATA];
 		break;
 	case 3:
-		list = &arc_mru->arcs_list[ARC_BUFC_DATA];
-		*lock = &arc_mru->arcs_mtx;
+		ml = &arc_mru->arcs_list[ARC_BUFC_DATA];
 		break;
 	}
 
-	ASSERT(!(MUTEX_HELD(*lock)));
-	mutex_enter(*lock);
-	return (list);
+	/*
+	 * Return a randomly-selected sublist. This is acceptable
+	 * because the caller feeds only a little bit of data for each
+	 * call (8MB). Subsequent calls will result in different
+	 * sublists being selected.
+	 */
+	idx = multilist_get_random_index(ml);
+	return (multilist_sublist_lock(ml, idx));
 }
 
 /*
@@ -5082,6 +5595,12 @@ top:
 		hdr_prev = list_prev(buflist, hdr);
 
 		hash_lock = HDR_LOCK(hdr);
+
+		/*
+		 * We cannot use mutex_enter or else we can deadlock
+		 * with l2arc_write_buffers (due to swapping the order
+		 * the hash lock and l2ad_mtx are taken).
+		 */
 		if (!mutex_tryenter(hash_lock)) {
 			/*
 			 * Missed the hash lock.  Retry.
@@ -5143,8 +5662,9 @@ top:
 			hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
 			list_remove(buflist, hdr);
 
-			/* This may have been leftover after a failed write. */
-			hdr->b_flags &= ~ARC_FLAG_L2_WRITING;
+			/* Ensure this header has finished being written */
+			ASSERT(!HDR_L2_WRITING(hdr));
+			ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
 		}
 		mutex_exit(hash_lock);
 	}
@@ -5170,11 +5690,9 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
     boolean_t *headroom_boost)
 {
 	arc_buf_hdr_t *hdr, *hdr_prev, *head;
-	list_t *list;
 	uint64_t write_asize, write_psize, write_sz, headroom,
 	    buf_compress_minsz;
 	void *buf_data;
-	kmutex_t *list_lock;
 	boolean_t full;
 	l2arc_write_callback_t *cb;
 	zio_t *pio, *wzio;
@@ -5202,12 +5720,10 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 	/*
 	 * Copy buffers for L2ARC writing.
 	 */
-	mutex_enter(&dev->l2ad_mtx);
 	for (int try = 0; try <= 3; try++) {
+		multilist_sublist_t *mls = l2arc_sublist_lock(try);
 		uint64_t passed_sz = 0;
 
-		list = l2arc_list_locked(try, &list_lock);
-
 		/*
 		 * L2ARC fast warmup.
 		 *
@@ -5215,9 +5731,9 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 		 * head of the ARC lists rather than the tail.
 		 */
 		if (arc_warm == B_FALSE)
-			hdr = list_head(list);
+			hdr = multilist_sublist_head(mls);
 		else
-			hdr = list_tail(list);
+			hdr = multilist_sublist_tail(mls);
 
 		headroom = target_sz * l2arc_headroom;
 		if (do_headroom_boost)
@@ -5228,9 +5744,9 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 			uint64_t buf_sz;
 
 			if (arc_warm == B_FALSE)
-				hdr_prev = list_next(list, hdr);
+				hdr_prev = multilist_sublist_next(mls, hdr);
 			else
-				hdr_prev = list_prev(list, hdr);
+				hdr_prev = multilist_sublist_prev(mls, hdr);
 
 			hash_lock = HDR_LOCK(hdr);
 			if (!mutex_tryenter(hash_lock)) {
@@ -5266,7 +5782,9 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 				 * l2arc_write_done() can find where the
 				 * write buffers begin without searching.
 				 */
+				mutex_enter(&dev->l2ad_mtx);
 				list_insert_head(&dev->l2ad_buflist, head);
+				mutex_exit(&dev->l2ad_mtx);
 
 				cb = kmem_alloc(
 				    sizeof (l2arc_write_callback_t), KM_SLEEP);
@@ -5296,7 +5814,9 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 			buf_sz = hdr->b_size;
 			hdr->b_flags |= ARC_FLAG_HAS_L2HDR;
 
+			mutex_enter(&dev->l2ad_mtx);
 			list_insert_head(&dev->l2ad_buflist, hdr);
+			mutex_exit(&dev->l2ad_mtx);
 
 			/*
 			 * Compute and store the buffer cksum before
@@ -5310,7 +5830,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 			write_sz += buf_sz;
 		}
 
-		mutex_exit(list_lock);
+		multilist_sublist_unlock(mls);
 
 		if (full == B_TRUE)
 			break;
@@ -5319,12 +5839,13 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 	/* No buffers selected for writing? */
 	if (pio == NULL) {
 		ASSERT0(write_sz);
-		mutex_exit(&dev->l2ad_mtx);
 		ASSERT(!HDR_HAS_L1HDR(head));
 		kmem_cache_free(hdr_l2only_cache, head);
 		return (0);
 	}
 
+	mutex_enter(&dev->l2ad_mtx);
+
 	/*
 	 * Now start writing the buffers. We're starting at the write head
 	 * and work backwards, retracing the course of the buffer selector
@@ -5334,6 +5855,14 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 	    hdr = list_prev(&dev->l2ad_buflist, hdr)) {
 		uint64_t buf_sz;
 
+		/*
+		 * We rely on the L1 portion of the header below, so
+		 * it's invalid for this header to have been evicted out
+		 * of the ghost cache, prior to being written out. The
+		 * ARC_FLAG_L2_WRITING bit ensures this won't happen.
+		 */
+		ASSERT(HDR_HAS_L1HDR(hdr));
+
 		/*
 		 * We shouldn't need to lock the buffer here, since we flagged
 		 * it as ARC_FLAG_L2_WRITING in the previous step, but we must
@@ -5553,8 +6082,26 @@ l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr, enum zio_compress c)
 static void
 l2arc_release_cdata_buf(arc_buf_hdr_t *hdr)
 {
+	enum zio_compress comp = HDR_GET_COMPRESS(hdr);
+
 	ASSERT(HDR_HAS_L1HDR(hdr));
-	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_EMPTY) {
+	ASSERT(comp == ZIO_COMPRESS_OFF || L2ARC_IS_VALID_COMPRESS(comp));
+
+	if (comp == ZIO_COMPRESS_OFF) {
+		/*
+		 * In this case, b_tmp_cdata points to the same buffer
+		 * as the arc_buf_t's b_data field. We don't want to
+		 * free it, since the arc_buf_t will handle that.
+		 */
+		hdr->b_l1hdr.b_tmp_cdata = NULL;
+	} else if (comp == ZIO_COMPRESS_EMPTY) {
+		/*
+		 * In this case, b_tmp_cdata was compressed to an empty
+		 * buffer, thus there's nothing to free and b_tmp_cdata
+		 * should have been set to NULL in l2arc_write_buffers().
+		 */
+		ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
+	} else {
 		/*
 		 * If the data was compressed, then we've allocated a
 		 * temporary buffer for it, so now we need to release it.
@@ -5562,8 +6109,9 @@ l2arc_release_cdata_buf(arc_buf_hdr_t *hdr)
 		ASSERT(hdr->b_l1hdr.b_tmp_cdata != NULL);
 		zio_data_buf_free(hdr->b_l1hdr.b_tmp_cdata,
 		    hdr->b_size);
+		hdr->b_l1hdr.b_tmp_cdata = NULL;
 	}
-	hdr->b_l1hdr.b_tmp_cdata = NULL;
+
 }
 
 /*
diff --git a/uts/common/fs/zfs/dsl_pool.c b/uts/common/fs/zfs/dsl_pool.c
index 30b7b4c8f024..2e7f798485d5 100644
--- a/uts/common/fs/zfs/dsl_pool.c
+++ b/uts/common/fs/zfs/dsl_pool.c
@@ -315,7 +315,14 @@ dsl_pool_close(dsl_pool_t *dp)
 	txg_list_destroy(&dp->dp_sync_tasks);
 	txg_list_destroy(&dp->dp_dirty_dirs);
 
-	arc_flush(dp->dp_spa);
+	/*
+	 * We can't set retry to TRUE since we're explicitly specifying
+	 * a spa to flush. This is good enough; any missed buffers for
+	 * this spa won't cause trouble, and they'll eventually fall
+	 * out of the ARC just like any other unused buffer.
+	 */
+	arc_flush(dp->dp_spa, FALSE);
+
 	txg_fini(dp);
 	dsl_scan_fini(dp);
 	dmu_buf_user_evict_wait();
diff --git a/uts/common/fs/zfs/multilist.c b/uts/common/fs/zfs/multilist.c
new file mode 100644
index 000000000000..82960571bcb4
--- /dev/null
+++ b/uts/common/fs/zfs/multilist.c
@@ -0,0 +1,366 @@
+/*
+ * CDDL HEADER START
+ *
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
+ */
+
+#include <sys/zfs_context.h>
+#include <sys/multilist.h>
+
+/* needed for spa_get_random() */
+#include <sys/spa.h>
+
+/*
+ * Given the object contained on the list, return a pointer to the
+ * object's multilist_node_t structure it contains.
+ */
+static multilist_node_t *
+multilist_d2l(multilist_t *ml, void *obj)
+{
+	return ((multilist_node_t *)((char *)obj + ml->ml_offset));
+}
+
+/*
+ * Initialize a new mutlilist using the parameters specified.
+ *
+ *  - 'size' denotes the size of the structure containing the
+ *     multilist_node_t.
+ *  - 'offset' denotes the byte offset of the mutlilist_node_t within
+ *     the structure that contains it.
+ *  - 'num' specifies the number of internal sublists to create.
+ *  - 'index_func' is used to determine which sublist to insert into
+ *     when the multilist_insert() function is called; as well as which
+ *     sublist to remove from when multilist_remove() is called. The
+ *     requirements this function must meet, are the following:
+ *
+ *      - It must always return the same value when called on the same
+ *        object (to ensure the object is removed from the list it was
+ *        inserted into).
+ *
+ *      - It must return a value in the range [0, number of sublists).
+ *        The multilist_get_num_sublists() function may be used to
+ *        determine the number of sublists in the multilist.
+ *
+ *     Also, in order to reduce internal contention between the sublists
+ *     during insertion and removal, this function should choose evenly
+ *     between all available sublists when inserting. This isn't a hard
+ *     requirement, but a general rule of thumb in order to garner the
+ *     best multi-threaded performance out of the data structure.
+ */
+void
+multilist_create(multilist_t *ml, size_t size, size_t offset, unsigned int num,
+    multilist_sublist_index_func_t *index_func)
+{
+	ASSERT3P(ml, !=, NULL);
+	ASSERT3U(size, >, 0);
+	ASSERT3U(size, >=, offset + sizeof (multilist_node_t));
+	ASSERT3U(num, >, 0);
+	ASSERT3P(index_func, !=, NULL);
+
+	ml->ml_offset = offset;
+	ml->ml_num_sublists = num;
+	ml->ml_index_func = index_func;
+
+	ml->ml_sublists = kmem_zalloc(sizeof (multilist_sublist_t) *
+	    ml->ml_num_sublists, KM_SLEEP);
+
+	ASSERT3P(ml->ml_sublists, !=, NULL);
+
+	for (int i = 0; i < ml->ml_num_sublists; i++) {
+		multilist_sublist_t *mls = &ml->ml_sublists[i];
+		mutex_init(&mls->mls_lock, NULL, MUTEX_DEFAULT, NULL);
+		list_create(&mls->mls_list, size, offset);
+	}
+}
+
+/*
+ * Destroy the given multilist object, and free up any memory it holds.
+ */
+void
+multilist_destroy(multilist_t *ml)
+{
+	ASSERT(multilist_is_empty(ml));
+
+	for (int i = 0; i < ml->ml_num_sublists; i++) {
+		multilist_sublist_t *mls = &ml->ml_sublists[i];
+
+		ASSERT(list_is_empty(&mls->mls_list));
+
+		list_destroy(&mls->mls_list);
+		mutex_destroy(&mls->mls_lock);
+	}
+
+	ASSERT3P(ml->ml_sublists, !=, NULL);
+	kmem_free(ml->ml_sublists,
+	    sizeof (multilist_sublist_t) * ml->ml_num_sublists);
+
+	ml->ml_num_sublists = 0;
+	ml->ml_offset = 0;
+}
+
+/*
+ * Insert the given object into the multilist.
+ *
+ * This function will insert the object specified into the sublist
+ * determined using the function given at multilist creation time.
+ *
+ * The sublist locks are automatically acquired if not already held, to
+ * ensure consistency when inserting and removing from multiple threads.
+ */
+void
+multilist_insert(multilist_t *ml, void *obj)
+{
+	unsigned int sublist_idx = ml->ml_index_func(ml, obj);
+	multilist_sublist_t *mls;
+	boolean_t need_lock;
+
+	DTRACE_PROBE3(multilist__insert, multilist_t *, ml,
+	    unsigned int, sublist_idx, void *, obj);
+
+	ASSERT3U(sublist_idx, <, ml->ml_num_sublists);
+
+	mls = &ml->ml_sublists[sublist_idx];
+
+	/*
+	 * Note: Callers may already hold the sublist lock by calling
+	 * multilist_sublist_lock().  Here we rely on MUTEX_HELD()
+	 * returning TRUE if and only if the current thread holds the
+	 * lock.  While it's a little ugly to make the lock recursive in
+	 * this way, it works and allows the calling code to be much
+	 * simpler -- otherwise it would have to pass around a flag
+	 * indicating that it already has the lock.
+	 */
+	need_lock = !MUTEX_HELD(&mls->mls_lock);
+
+	if (need_lock)
+		mutex_enter(&mls->mls_lock);
+
+	ASSERT(!multilist_link_active(multilist_d2l(ml, obj)));
+
+	multilist_sublist_insert_head(mls, obj);
+
+	if (need_lock)
+		mutex_exit(&mls->mls_lock);
+}
+
+/*
+ * Remove the given object from the multilist.
+ *
+ * This function will remove the object specified from the sublist
+ * determined using the function given at multilist creation time.
+ *
+ * The necessary sublist locks are automatically acquired, to ensure
+ * consistency when inserting and removing from multiple threads.
+ */
+void
+multilist_remove(multilist_t *ml, void *obj)
+{
+	unsigned int sublist_idx = ml->ml_index_func(ml, obj);
+	multilist_sublist_t *mls;
+	boolean_t need_lock;
+
+	DTRACE_PROBE3(multilist__remove, multilist_t *, ml,
+	    unsigned int, sublist_idx, void *, obj);
+
+	ASSERT3U(sublist_idx, <, ml->ml_num_sublists);
+
+	mls = &ml->ml_sublists[sublist_idx];
+	/* See comment in multilist_insert(). */
+	need_lock = !MUTEX_HELD(&mls->mls_lock);
+
+	if (need_lock)
+		mutex_enter(&mls->mls_lock);
+
+	ASSERT(multilist_link_active(multilist_d2l(ml, obj)));
+
+	multilist_sublist_remove(mls, obj);
+
+	if (need_lock)
+		mutex_exit(&mls->mls_lock);
+}
+
+/*
+ * Check to see if this multilist object is empty.
+ *
+ * This will return TRUE if it finds all of the sublists of this
+ * multilist to be empty, and FALSE otherwise. Each sublist lock will be
+ * automatically acquired as necessary.
+ *
+ * If concurrent insertions and removals are occurring, the semantics
+ * of this function become a little fuzzy. Instead of locking all
+ * sublists for the entire call time of the function, each sublist is
+ * only locked as it is individually checked for emptiness. Thus, it's
+ * possible for this function to return TRUE with non-empty sublists at
+ * the time the function returns. This would be due to another thread
+ * inserting into a given sublist, after that specific sublist was check
+ * and deemed empty, but before all sublists have been checked.
+ */
+int
+multilist_is_empty(multilist_t *ml)
+{
+	for (int i = 0; i < ml->ml_num_sublists; i++) {
+		multilist_sublist_t *mls = &ml->ml_sublists[i];
+		/* See comment in multilist_insert(). */
+		boolean_t need_lock = !MUTEX_HELD(&mls->mls_lock);
+
+		if (need_lock)
+			mutex_enter(&mls->mls_lock);
+
+		if (!list_is_empty(&mls->mls_list)) {
+			if (need_lock)
+				mutex_exit(&mls->mls_lock);
+
+			return (FALSE);
+		}
+
+		if (need_lock)
+			mutex_exit(&mls->mls_lock);
+	}
+
+	return (TRUE);
+}
+
+/* Return the number of sublists composing this multilist */
+unsigned int
+multilist_get_num_sublists(multilist_t *ml)
+{
+	return (ml->ml_num_sublists);
+}
+
+/* Return a randomly selected, valid sublist index for this multilist */
+unsigned int
+multilist_get_random_index(multilist_t *ml)
+{
+	return (spa_get_random(ml->ml_num_sublists));
+}
+
+/* Lock and return the sublist specified at the given index */
+multilist_sublist_t *
+multilist_sublist_lock(multilist_t *ml, unsigned int sublist_idx)
+{
+	multilist_sublist_t *mls;
+
+	ASSERT3U(sublist_idx, <, ml->ml_num_sublists);
+	mls = &ml->ml_sublists[sublist_idx];
+	mutex_enter(&mls->mls_lock);
+
+	return (mls);
+}
+
+void
+multilist_sublist_unlock(multilist_sublist_t *mls)
+{
+	mutex_exit(&mls->mls_lock);
+}
+
+/*
+ * We're allowing any object to be inserted into this specific sublist,
+ * but this can lead to trouble if multilist_remove() is called to
+ * remove this object. Specifically, if calling ml_index_func on this
+ * object returns an index for sublist different than what is passed as
+ * a parameter here, any call to multilist_remove() with this newly
+ * inserted object is undefined! (the call to multilist_remove() will
+ * remove the object from a list that it isn't contained in)
+ */
+void
+multilist_sublist_insert_head(multilist_sublist_t *mls, void *obj)
+{
+	ASSERT(MUTEX_HELD(&mls->mls_lock));
+	list_insert_head(&mls->mls_list, obj);
+}
+
+/* please see comment above multilist_sublist_insert_head */
+void
+multilist_sublist_insert_tail(multilist_sublist_t *mls, void *obj)
+{
+	ASSERT(MUTEX_HELD(&mls->mls_lock));
+	list_insert_tail(&mls->mls_list, obj);
+}
+
+/*
+ * Move the object one element forward in the list.
+ *
+ * This function will move the given object forward in the list (towards
+ * the head) by one object. So, in essence, it will swap its position in
+ * the list with its "prev" pointer. If the given object is already at the
+ * head of the list, it cannot be moved forward any more than it already
+ * is, so no action is taken.
+ *
+ * NOTE: This function **must not** remove any object from the list other
+ *       than the object given as the parameter. This is relied upon in
+ *       arc_evict_state_impl().
+ */
+void
+multilist_sublist_move_forward(multilist_sublist_t *mls, void *obj)
+{
+	void *prev = list_prev(&mls->mls_list, obj);
+
+	ASSERT(MUTEX_HELD(&mls->mls_lock));
+	ASSERT(!list_is_empty(&mls->mls_list));
+
+	/* 'obj' must be at the head of the list, nothing to do */
+	if (prev == NULL)
+		return;
+
+	list_remove(&mls->mls_list, obj);
+	list_insert_before(&mls->mls_list, prev, obj);
+}
+
+void
+multilist_sublist_remove(multilist_sublist_t *mls, void *obj)
+{
+	ASSERT(MUTEX_HELD(&mls->mls_lock));
+	list_remove(&mls->mls_list, obj);
+}
+
+void *
+multilist_sublist_head(multilist_sublist_t *mls)
+{
+	ASSERT(MUTEX_HELD(&mls->mls_lock));
+	return (list_head(&mls->mls_list));
+}
+
+void *
+multilist_sublist_tail(multilist_sublist_t *mls)
+{
+	ASSERT(MUTEX_HELD(&mls->mls_lock));
+	return (list_tail(&mls->mls_list));
+}
+
+void *
+multilist_sublist_next(multilist_sublist_t *mls, void *obj)
+{
+	ASSERT(MUTEX_HELD(&mls->mls_lock));
+	return (list_next(&mls->mls_list, obj));
+}
+
+void *
+multilist_sublist_prev(multilist_sublist_t *mls, void *obj)
+{
+	ASSERT(MUTEX_HELD(&mls->mls_lock));
+	return (list_prev(&mls->mls_list, obj));
+}
+
+void
+multilist_link_init(multilist_node_t *link)
+{
+	list_link_init(link);
+}
+
+int
+multilist_link_active(multilist_node_t *link)
+{
+	return (list_link_active(link));
+}
diff --git a/uts/common/fs/zfs/sys/arc.h b/uts/common/fs/zfs/sys/arc.h
index 60112baaba55..843acfdab44d 100644
--- a/uts/common/fs/zfs/sys/arc.h
+++ b/uts/common/fs/zfs/sys/arc.h
@@ -37,6 +37,12 @@ extern "C" {
 #include <sys/dmu.h>
 #include <sys/spa.h>
 
+/*
+ * Used by arc_flush() to inform arc_evict_state() that it should evict
+ * all available buffers from the arc state being passed in.
+ */
+#define	ARC_EVICT_ALL	-1ULL
+
 typedef struct arc_buf_hdr arc_buf_hdr_t;
 typedef struct arc_buf arc_buf_t;
 typedef void arc_done_func_t(zio_t *zio, arc_buf_t *buf, void *private);
@@ -154,7 +160,7 @@ void arc_freed(spa_t *spa, const blkptr_t *bp);
 void arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private);
 boolean_t arc_clear_callback(arc_buf_t *buf);
 
-void arc_flush(spa_t *spa);
+void arc_flush(spa_t *spa, boolean_t retry);
 void arc_tempreserve_clear(uint64_t reserve);
 int arc_tempreserve_space(uint64_t reserve, uint64_t txg);
 
diff --git a/uts/common/fs/zfs/sys/multilist.h b/uts/common/fs/zfs/sys/multilist.h
new file mode 100644
index 000000000000..5ebb7fe1a1d5
--- /dev/null
+++ b/uts/common/fs/zfs/sys/multilist.h
@@ -0,0 +1,106 @@
+/*
+ * CDDL HEADER START
+ *
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
+ */
+
+#ifndef	_SYS_MULTILIST_H
+#define	_SYS_MULTILIST_H
+
+#include <sys/zfs_context.h>
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+typedef list_node_t multilist_node_t;
+typedef struct multilist multilist_t;
+typedef struct multilist_sublist multilist_sublist_t;
+typedef unsigned int multilist_sublist_index_func_t(multilist_t *, void *);
+
+struct multilist_sublist {
+	/*
+	 * The mutex used internally to implement thread safe insertions
+	 * and removals to this individual sublist. It can also be locked
+	 * by a consumer using multilist_sublist_{lock,unlock}, which is
+	 * useful if a consumer needs to traverse the list in a thread
+	 * safe manner.
+	 */
+	kmutex_t	mls_lock;
+	/*
+	 * The actual list object containing all objects in this sublist.
+	 */
+	list_t		mls_list;
+	/*
+	 * Pad to cache line (64 bytes), in an effort to try and prevent
+	 * cache line contention.
+	 */
+	uint8_t		mls_pad[24];
+};
+
+struct multilist {
+	/*
+	 * This is used to get to the multilist_node_t structure given
+	 * the void *object contained on the list.
+	 */
+	size_t				ml_offset;
+	/*
+	 * The number of sublists used internally by this multilist.
+	 */
+	uint64_t			ml_num_sublists;
+	/*
+	 * The array of pointers to the actual sublists.
+	 */
+	multilist_sublist_t		*ml_sublists;
+	/*
+	 * Pointer to function which determines the sublist to use
+	 * when inserting and removing objects from this multilist.
+	 * Please see the comment above multilist_create for details.
+	 */
+	multilist_sublist_index_func_t	*ml_index_func;
+};
+
+void multilist_destroy(multilist_t *);
+void multilist_create(multilist_t *, size_t, size_t, unsigned int,
+    multilist_sublist_index_func_t *);
+
+void multilist_insert(multilist_t *, void *);
+void multilist_remove(multilist_t *, void *);
+int  multilist_is_empty(multilist_t *);
+
+unsigned int multilist_get_num_sublists(multilist_t *);
+unsigned int multilist_get_random_index(multilist_t *);
+
+multilist_sublist_t *multilist_sublist_lock(multilist_t *, unsigned int);
+void multilist_sublist_unlock(multilist_sublist_t *);
+
+void multilist_sublist_insert_head(multilist_sublist_t *, void *);
+void multilist_sublist_insert_tail(multilist_sublist_t *, void *);
+void multilist_sublist_move_forward(multilist_sublist_t *mls, void *obj);
+void multilist_sublist_remove(multilist_sublist_t *, void *);
+
+void *multilist_sublist_head(multilist_sublist_t *);
+void *multilist_sublist_tail(multilist_sublist_t *);
+void *multilist_sublist_next(multilist_sublist_t *, void *);
+void *multilist_sublist_prev(multilist_sublist_t *, void *);
+
+void multilist_link_init(multilist_node_t *);
+int  multilist_link_active(multilist_node_t *);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif /* _SYS_MULTILIST_H */
diff --git a/uts/common/fs/zfs/zio_inject.c b/uts/common/fs/zfs/zio_inject.c
index 991a0a34ff6a..0a7f4e4b3b17 100644
--- a/uts/common/fs/zfs/zio_inject.c
+++ b/uts/common/fs/zfs/zio_inject.c
@@ -438,7 +438,11 @@ zio_inject_fault(char *name, int flags, int *id, zinject_record_t *record)
 	 * fault injection isn't a performance critical path.
 	 */
 	if (flags & ZINJECT_FLUSH_ARC)
-		arc_flush(NULL);
+		/*
+		 * We must use FALSE to ensure arc_flush returns, since
+		 * we're not preventing concurrent ARC insertions.
+		 */
+		arc_flush(NULL, FALSE);
 
 	return (0);
 }

From aa30cbd603b95daabeca9ab6d6dfae68911fe0b4 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sun, 19 Apr 2015 23:45:04 +0000
Subject: [PATCH 074/380] Sync Makefile.files to
 illumos-gate@244781f10dcd82684fd8163c016540667842f203.

---
 uts/common/Makefile.files | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/uts/common/Makefile.files b/uts/common/Makefile.files
index a564615ed740..8d9abca90695 100644
--- a/uts/common/Makefile.files
+++ b/uts/common/Makefile.files
@@ -21,9 +21,9 @@
 
 #
 # Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
-# Copyright (c) 2012 Nexenta Systems, Inc. All rights reserved.
 # Copyright (c) 2013 by Delphix. All rights reserved.
 # Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
+# Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
 #
 
 #
@@ -1328,6 +1328,7 @@ TRANS_OBJS += mdtrans.o trans_ioctl.o trans_log.o
 
 ZFS_COMMON_OBJS +=		\
 	arc.o			\
+	blkptr.o		\
 	bplist.o		\
 	bpobj.o			\
 	bptree.o		\
@@ -1360,6 +1361,7 @@ ZFS_COMMON_OBJS +=		\
 	lz4.o			\
 	lzjb.o			\
 	metaslab.o		\
+	multilist.o		\
 	range_tree.o		\
 	refcount.o		\
 	rrwlock.o		\
@@ -1846,7 +1848,8 @@ ZYD_OBJS += zyd.o zyd_usb.o zyd_hw.o zyd_fw.o
 
 MXFE_OBJS += mxfe.o
 
-MPTSAS_OBJS += mptsas.o mptsas_impl.o mptsas_init.o mptsas_raid.o mptsas_smhba.o
+MPTSAS_OBJS += mptsas.o mptsas_hash.o mptsas_impl.o mptsas_init.o \
+		mptsas_raid.o mptsas_smhba.o
 
 SFE_OBJS += sfe.o sfe_util.o
 
@@ -1865,6 +1868,8 @@ ATGE_OBJS += atge_main.o atge_l1e.o atge_mii.o atge_l1.o atge_l1c.o
 
 YGE_OBJS = yge.o
 
+SKD_OBJS = skd.o
+
 #
 #	Build up defines and paths.
 #

From dc01b312be04d6319c00bc3b9ce3547996f144dc Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Tue, 28 Apr 2015 07:14:08 +0000
Subject: [PATCH 075/380] 5610 zfs clone from different source and target pools
 produces coredump

Author: Alexander Eremin <a.eremin@nexenta.com>
Reviewed by: Josef 'Jeff' Sipek <josef.sipek@nexenta.com>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Approved by: Dan McDonald <danmcd@omniti.com>

illumos/illumos-gate@03b1c2971d24a9cd2c073d634f7e074fbd14e984
---
 uts/common/fs/zfs/dmu_objset.c | 13 ++-----------
 uts/common/fs/zfs/dsl_dir.c    |  3 ++-
 2 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/uts/common/fs/zfs/dmu_objset.c b/uts/common/fs/zfs/dmu_objset.c
index 8593e2c94f6c..2732d989929f 100644
--- a/uts/common/fs/zfs/dmu_objset.c
+++ b/uts/common/fs/zfs/dmu_objset.c
@@ -24,6 +24,7 @@
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
+ * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
  */
 
 /* Portions Copyright 2010 Robert Milkowski */
@@ -904,11 +905,7 @@ dmu_objset_clone_check(void *arg, dmu_tx_t *tx)
 		dsl_dir_rele(pdd, FTAG);
 		return (SET_ERROR(EEXIST));
 	}
-	/* You can't clone across pools. */
-	if (pdd->dd_pool != dp) {
-		dsl_dir_rele(pdd, FTAG);
-		return (SET_ERROR(EXDEV));
-	}
+
 	error = dsl_fs_ss_limit_check(pdd, 1, ZFS_PROP_FILESYSTEM_LIMIT, NULL,
 	    doca->doca_cred);
 	if (error != 0) {
@@ -921,12 +918,6 @@ dmu_objset_clone_check(void *arg, dmu_tx_t *tx)
 	if (error != 0)
 		return (error);
 
-	/* You can't clone across pools. */
-	if (origin->ds_dir->dd_pool != dp) {
-		dsl_dataset_rele(origin, FTAG);
-		return (SET_ERROR(EXDEV));
-	}
-
 	/* You can only clone snapshots, not the head datasets. */
 	if (!origin->ds_is_snapshot) {
 		dsl_dataset_rele(origin, FTAG);
diff --git a/uts/common/fs/zfs/dsl_dir.c b/uts/common/fs/zfs/dsl_dir.c
index ee69cc8dbffe..6a20ab3ca2ad 100644
--- a/uts/common/fs/zfs/dsl_dir.c
+++ b/uts/common/fs/zfs/dsl_dir.c
@@ -24,6 +24,7 @@
  * Copyright (c) 2013 Martin Matuska. All rights reserved.
  * Copyright (c) 2014 Joyent, Inc. All rights reserved.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
+ * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
  */
 
 #include <sys/dmu.h>
@@ -421,7 +422,7 @@ dsl_dir_hold(dsl_pool_t *dp, const char *name, void *tag,
 	/* Make sure the name is in the specified pool. */
 	spaname = spa_name(dp->dp_spa);
 	if (strcmp(buf, spaname) != 0)
-		return (SET_ERROR(EINVAL));
+		return (SET_ERROR(EXDEV));
 
 	ASSERT(dsl_pool_config_held(dp));
 

From 92b26a70ca010e260ba67398afec766b5e2ec517 Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Tue, 28 Apr 2015 07:15:34 +0000
Subject: [PATCH 076/380] 5393 spurious failures from dsl_dataset_hold_obj()

Author: Justin T. Gibbs <justing@spectralogic.com>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Will Andrews <willa@spectralogic.com>
Reviewed by: Prakash Surya <prakash.surya@delphix.com>
Reviewed by: Steven Hartland <killing@multiplay.co.uk>
Approved by: Dan McDonald <danmcd@omniti.com>

illumos/illumos-gate@e1f3c208abda53769a6531bb1141618555273460
---
 uts/common/fs/zfs/dsl_dataset.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/uts/common/fs/zfs/dsl_dataset.c b/uts/common/fs/zfs/dsl_dataset.c
index d3ca43af47d1..ec2d022aed39 100644
--- a/uts/common/fs/zfs/dsl_dataset.c
+++ b/uts/common/fs/zfs/dsl_dataset.c
@@ -405,11 +405,12 @@ dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
 		    offsetof(dmu_sendarg_t, dsa_link));
 
 		if (doi.doi_type == DMU_OTN_ZAP_METADATA) {
-			err = zap_contains(mos, dsobj, DS_FIELD_LARGE_BLOCKS);
-			if (err == 0)
+			int zaperr = zap_contains(mos, dsobj,
+			    DS_FIELD_LARGE_BLOCKS);
+			if (zaperr != ENOENT) {
+				VERIFY0(zaperr);
 				ds->ds_large_blocks = B_TRUE;
-			else
-				ASSERT3U(err, ==, ENOENT);
+			}
 		}
 
 		if (err == 0) {

From 40fcd13e64ebcf74b485bcf5c88eb050a98347bb Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Fri, 8 May 2015 13:55:24 +0000
Subject: [PATCH 077/380] 5809 Blowaway full receive in v1 pool causes kernel
 panic

Author: Paul Dagnelie <pcd@delphix.com>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Alex Reece <alex@delphix.com>
Reviewed by: Will Andrews <will@freebsd.org>
Approved by: Gordon Ross <gwr@nexenta.com>

illumos/illumos-gate@f40b29ce2a815bcc0787acf6f520a2b74258b785
---
 uts/common/fs/zfs/dmu_send.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/uts/common/fs/zfs/dmu_send.c b/uts/common/fs/zfs/dmu_send.c
index 3c156054877b..acc9f28f9d44 100644
--- a/uts/common/fs/zfs/dmu_send.c
+++ b/uts/common/fs/zfs/dmu_send.c
@@ -1108,7 +1108,8 @@ dmu_recv_begin_sync(void *arg, dmu_tx_t *tx)
 		}
 		dsobj = dsl_dataset_create_sync(ds->ds_dir, recv_clone_name,
 		    snap, crflags, drba->drba_cred, tx);
-		dsl_dataset_rele(snap, FTAG);
+		if (drba->drba_snapobj != 0)
+			dsl_dataset_rele(snap, FTAG);
 		dsl_dataset_rele(ds, FTAG);
 	} else {
 		dsl_dir_t *dd;

From 5e5cfc1d8eaae3cffecd6f5962fc12e426bde1ae Mon Sep 17 00:00:00 2001
From: markj <markj@FreeBSD.org>
Date: Mon, 25 May 2015 02:30:54 +0000
Subject: [PATCH 078/380] 3599 dtrace_dynvar tail calls can blow stack Reviewed
 by: Adam Leventhal <ahl@delphix.com> Reviewed by: Gordon Ross
 <gordon.ross@nexenta.com> Approved by: Richard Lowe <richlowe@richlowe.net>
 Author: Bryan Cantrill <bryan@joyent.com>

illumos/illumos-gate@d47448f09aae3aa1a87fc450a0c44638e7ce7b51
---
 uts/common/dtrace/dtrace.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/uts/common/dtrace/dtrace.c b/uts/common/dtrace/dtrace.c
index 0d2d6f524c2e..aa4b1d67e07d 100644
--- a/uts/common/dtrace/dtrace.c
+++ b/uts/common/dtrace/dtrace.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013, Joyent, Inc. All rights reserved.
+ * Copyright (c) 2015, Joyent, Inc. All rights reserved.
  * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
@@ -2037,7 +2037,7 @@ retry:
 	 * this hash chain, or another CPU is deleting an element from this
 	 * hash chain.  The simplest way to deal with both of these cases
 	 * (though not necessarily the most efficient) is to free our
-	 * allocated block and tail-call ourselves.  Note that the free is
+	 * allocated block and re-attempt it all.  Note that the free is
 	 * to the dirty list and _not_ to the free list.  This is to prevent
 	 * races with allocators, above.
 	 */
@@ -2050,7 +2050,7 @@ retry:
 		dvar->dtdv_next = free;
 	} while (dtrace_casptr(&dcpu->dtdsc_dirty, free, dvar) != free);
 
-	return (dtrace_dynvar(dstate, nkeys, key, dsize, op, mstate, vstate));
+	goto top;
 }
 
 /*ARGSUSED*/

From a9df4de7d460f0bbfa39e63b8bccd2ad301d4e1c Mon Sep 17 00:00:00 2001
From: markj <markj@FreeBSD.org>
Date: Mon, 25 May 2015 02:32:30 +0000
Subject: [PATCH 079/380] 5930 fasttrap_pid_enable() panics when prfind() fails
 in forking process Reviewed by: Adam Leventhal <ahl@delphix.com> Reviewed by:
 Gordon Ross <gordon.ross@nexenta.com> Approved by: Richard Lowe
 <richlowe@richlowe.net> Author: Bryan Cantrill <bryan@joyent.com>

illumos/illumos-gate@9df7e4e12eb093557252d3bec029b5c382613e36
---
 uts/common/dtrace/fasttrap.c | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/uts/common/dtrace/fasttrap.c b/uts/common/dtrace/fasttrap.c
index fc3d87574a29..215b1a411142 100644
--- a/uts/common/dtrace/fasttrap.c
+++ b/uts/common/dtrace/fasttrap.c
@@ -25,7 +25,7 @@
  */
 
 /*
- * Copyright (c) 2013, Joyent, Inc. All rights reserved.
+ * Copyright (c) 2015, Joyent, Inc. All rights reserved.
  */
 
 #include <sys/atomic.h>
@@ -928,11 +928,21 @@ fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
 		mutex_enter(&pidlock);
 		p = prfind(probe->ftp_pid);
 
+		if (p == NULL) {
+			/*
+			 * So it's not that the target process is being born,
+			 * it's that it isn't there at all (and we simply
+			 * happen to be forking).  Anyway, we know that the
+			 * target is definitely gone, so bail out.
+			 */
+			mutex_exit(&pidlock);
+			return (0);
+		}
+
 		/*
 		 * Confirm that curproc is indeed forking the process in which
 		 * we're trying to enable probes.
 		 */
-		ASSERT(p != NULL);
 		ASSERT(p->p_parent == curproc);
 		ASSERT(p->p_stat == SIDL);
 

From 91a73cecc35ff68d988ab1ed2e86ca0acdc6d78f Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Mon, 25 May 2015 15:18:32 +0000
Subject: [PATCH 080/380] 5515 dataset user hold doesn't reject empty tags

Author:	Josef 'Jeff' Sipek <josef.sipek@nexenta.com>
Reviewed by: Yuri Pankov <yuri.pankov@nexenta.com>
Reviewed by: Saso Kiselkov <saso.kiselkov@nexenta.com>
Approved by: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@752fd8dabccac68d6d09f82f3bf3561e055e400b
---
 uts/common/fs/zfs/zfs_ioctl.c | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/uts/common/fs/zfs/zfs_ioctl.c b/uts/common/fs/zfs/zfs_ioctl.c
index 20a64666a78e..3616fa2f26f8 100644
--- a/uts/common/fs/zfs/zfs_ioctl.c
+++ b/uts/common/fs/zfs/zfs_ioctl.c
@@ -22,12 +22,11 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Portions Copyright 2011 Martin Matuska
- * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2014, Joyent, Inc. All rights reserved.
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
- * Copyright (c) 2014, Nexenta Systems, Inc. All rights reserved.
  */
 
 /*
@@ -5110,6 +5109,7 @@ zfs_ioc_smb_acl(zfs_cmd_t *zc)
 static int
 zfs_ioc_hold(const char *pool, nvlist_t *args, nvlist_t *errlist)
 {
+	nvpair_t *pair;
 	nvlist_t *holds;
 	int cleanup_fd = -1;
 	int error;
@@ -5119,6 +5119,19 @@ zfs_ioc_hold(const char *pool, nvlist_t *args, nvlist_t *errlist)
 	if (error != 0)
 		return (SET_ERROR(EINVAL));
 
+	/* make sure the user didn't pass us any invalid (empty) tags */
+	for (pair = nvlist_next_nvpair(holds, NULL); pair != NULL;
+	    pair = nvlist_next_nvpair(holds, pair)) {
+		char *htag;
+
+		error = nvpair_value_string(pair, &htag);
+		if (error != 0)
+			return (SET_ERROR(error));
+
+		if (strlen(htag) == 0)
+			return (SET_ERROR(EINVAL));
+	}
+
 	if (nvlist_lookup_int32(args, "cleanup_fd", &cleanup_fd) == 0) {
 		error = zfs_onexit_fd_hold(cleanup_fd, &minor);
 		if (error != 0)

From 0160ea911d75088b4dc52ce5897dc106e3e08b83 Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Fri, 5 Jun 2015 16:49:21 +0000
Subject: [PATCH 081/380] 5818 zfs {ref}compressratio is incorrect with 4k
 sector size

Author: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: George Wilson <george@delphix.com>
Reviewed by: Richard Elling <richard.elling@richardelling.com>
Reviewed by: Steven Hartland <killing@multiplay.co.uk>
Reviewed by: Don Brady <dev.fs.zfs@gmail.com>
Approved by: Albert Lee <trisk@omniti.com>

illumos/illumos-gate@81cd5c555f505484180a62ca5a2fbb00d70c57d6
---
 uts/common/fs/zfs/spa.c           |  2 ++
 uts/common/fs/zfs/spa_misc.c      |  5 ++++-
 uts/common/fs/zfs/sys/spa_impl.h  |  4 +++-
 uts/common/fs/zfs/sys/vdev_impl.h |  4 ++--
 uts/common/fs/zfs/vdev.c          | 18 +++++++++++++++---
 uts/common/fs/zfs/zio.c           | 24 ++++++++++++++----------
 6 files changed, 40 insertions(+), 17 deletions(-)

diff --git a/uts/common/fs/zfs/spa.c b/uts/common/fs/zfs/spa.c
index 7947d25f48d1..5bc2dbcb1cda 100644
--- a/uts/common/fs/zfs/spa.c
+++ b/uts/common/fs/zfs/spa.c
@@ -2195,6 +2195,8 @@ spa_load_impl(spa_t *spa, uint64_t pool_guid, nvlist_t *config,
 		return (error);
 
 	ASSERT(spa->spa_root_vdev == rvd);
+	ASSERT3U(spa->spa_min_ashift, >=, SPA_MINBLOCKSHIFT);
+	ASSERT3U(spa->spa_max_ashift, <=, SPA_MAXBLOCKSHIFT);
 
 	if (type != SPA_IMPORT_ASSEMBLE) {
 		ASSERT(spa_guid(spa) == pool_guid);
diff --git a/uts/common/fs/zfs/spa_misc.c b/uts/common/fs/zfs/spa_misc.c
index 56ad27cf9ae7..c5af055a8608 100644
--- a/uts/common/fs/zfs/spa_misc.c
+++ b/uts/common/fs/zfs/spa_misc.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
@@ -648,6 +648,9 @@ spa_add(const char *name, nvlist_t *config, const char *altroot)
 
 	spa->spa_debug = ((zfs_flags & ZFS_DEBUG_SPA) != 0);
 
+	spa->spa_min_ashift = INT_MAX;
+	spa->spa_max_ashift = 0;
+
 	/*
 	 * As a pool is being created, treat all features as disabled by
 	 * setting SPA_FEATURE_DISABLED for all entries in the feature
diff --git a/uts/common/fs/zfs/sys/spa_impl.h b/uts/common/fs/zfs/sys/spa_impl.h
index 9bbb240e0a11..56e6adb713d2 100644
--- a/uts/common/fs/zfs/sys/spa_impl.h
+++ b/uts/common/fs/zfs/sys/spa_impl.h
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
@@ -150,6 +150,8 @@ struct spa {
 	kcondvar_t	spa_evicting_os_cv;	/* Objset Eviction Completion */
 	txg_list_t	spa_vdev_txg_list;	/* per-txg dirty vdev list */
 	vdev_t		*spa_root_vdev;		/* top-level vdev container */
+	int		spa_min_ashift;		/* of vdevs in normal class */
+	int		spa_max_ashift;		/* of vdevs in normal class */
 	uint64_t	spa_config_guid;	/* config pool guid */
 	uint64_t	spa_load_guid;		/* spa_load initialized guid */
 	uint64_t	spa_last_synced_guid;	/* last synced guid */
diff --git a/uts/common/fs/zfs/sys/vdev_impl.h b/uts/common/fs/zfs/sys/vdev_impl.h
index 3af0e2a552a7..17a18a319934 100644
--- a/uts/common/fs/zfs/sys/vdev_impl.h
+++ b/uts/common/fs/zfs/sys/vdev_impl.h
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  */
 
 #ifndef _SYS_VDEV_IMPL_H
@@ -207,7 +207,7 @@ struct vdev {
 	boolean_t	vdev_isl2cache;	/* was a l2cache device		*/
 	vdev_queue_t	vdev_queue;	/* I/O deadline schedule queue	*/
 	vdev_cache_t	vdev_cache;	/* physical block cache		*/
-	spa_aux_vdev_t	*vdev_aux;	/* for l2cache vdevs		*/
+	spa_aux_vdev_t	*vdev_aux;	/* for l2cache and spares vdevs	*/
 	zio_t		*vdev_probe_zio; /* root of current probe	*/
 	vdev_aux_t	vdev_label_aux;	/* on-disk aux state		*/
 
diff --git a/uts/common/fs/zfs/vdev.c b/uts/common/fs/zfs/vdev.c
index 6ee2a91dbd42..a2e2b36c6836 100644
--- a/uts/common/fs/zfs/vdev.c
+++ b/uts/common/fs/zfs/vdev.c
@@ -21,8 +21,8 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
- * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
+ * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -184,8 +184,9 @@ vdev_add_child(vdev_t *pvd, vdev_t *cvd)
 	size_t oldsize, newsize;
 	uint64_t id = cvd->vdev_id;
 	vdev_t **newchild;
+	spa_t *spa = cvd->vdev_spa;
 
-	ASSERT(spa_config_held(cvd->vdev_spa, SCL_ALL, RW_WRITER) == SCL_ALL);
+	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
 	ASSERT(cvd->vdev_parent == NULL);
 
 	cvd->vdev_parent = pvd;
@@ -1286,6 +1287,17 @@ vdev_open(vdev_t *vd)
 		return (error);
 	}
 
+	/*
+	 * Track the min and max ashift values for normal data devices.
+	 */
+	if (vd->vdev_top == vd && vd->vdev_ashift != 0 &&
+	    !vd->vdev_islog && vd->vdev_aux == NULL) {
+		if (vd->vdev_ashift > spa->spa_max_ashift)
+			spa->spa_max_ashift = vd->vdev_ashift;
+		if (vd->vdev_ashift < spa->spa_min_ashift)
+			spa->spa_min_ashift = vd->vdev_ashift;
+	}
+
 	/*
 	 * If a leaf vdev has a DTL, and seems healthy, then kick off a
 	 * resilver.  But don't do this if we are doing a reopen for a scrub,
diff --git a/uts/common/fs/zfs/zio.c b/uts/common/fs/zfs/zio.c
index d0a42e9af13b..7be362f59279 100644
--- a/uts/common/fs/zfs/zio.c
+++ b/uts/common/fs/zfs/zio.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2011 Nexenta Systems, Inc. All rights reserved.
  */
 
@@ -1178,19 +1178,23 @@ zio_write_bp_init(zio_t *zio)
 			return (ZIO_PIPELINE_CONTINUE);
 		} else {
 			/*
-			 * Round up compressed size to MINBLOCKSIZE and
-			 * zero the tail.
+			 * Round up compressed size up to the ashift
+			 * of the smallest-ashift device, and zero the tail.
+			 * This ensures that the compressed size of the BP
+			 * (and thus compressratio property) are correct,
+			 * in that we charge for the padding used to fill out
+			 * the last sector.
 			 */
-			size_t rounded =
-			    P2ROUNDUP(psize, (size_t)SPA_MINBLOCKSIZE);
-			if (rounded > psize) {
-				bzero((char *)cbuf + psize, rounded - psize);
-				psize = rounded;
-			}
-			if (psize == lsize) {
+			ASSERT3U(spa->spa_min_ashift, >=, SPA_MINBLOCKSHIFT);
+			size_t rounded = (size_t)P2ROUNDUP(psize,
+			    1ULL << spa->spa_min_ashift);
+			if (rounded >= lsize) {
 				compress = ZIO_COMPRESS_OFF;
 				zio_buf_free(cbuf, lsize);
+				psize = lsize;
 			} else {
+				bzero((char *)cbuf + psize, rounded - psize);
+				psize = rounded;
 				zio_push_transform(zio, cbuf,
 				    psize, lsize, NULL);
 			}

From e096467d4f902a27bcf5fa6053d7c7c26bd7e4d2 Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Fri, 5 Jun 2015 16:52:23 +0000
Subject: [PATCH 082/380] 5515 dataset user hold doesn't reject empty tags

Author: Josef 'Jeff' Sipek <josef.sipek@nexenta.com>
Reviewed by: Yuri Pankov <yuri.pankov@nexenta.com>
Reviewed by: Saso Kiselkov <saso.kiselkov@nexenta.com>
Approved by: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@752fd8dabccac68d6d09f82f3bf3561e055e400b
---
 uts/common/fs/zfs/zfs_ioctl.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/uts/common/fs/zfs/zfs_ioctl.c b/uts/common/fs/zfs/zfs_ioctl.c
index 3616fa2f26f8..1a7c55a12b83 100644
--- a/uts/common/fs/zfs/zfs_ioctl.c
+++ b/uts/common/fs/zfs/zfs_ioctl.c
@@ -5132,6 +5132,19 @@ zfs_ioc_hold(const char *pool, nvlist_t *args, nvlist_t *errlist)
 			return (SET_ERROR(EINVAL));
 	}
 
+	/* make sure the user didn't pass us any invalid (empty) tags */
+	for (pair = nvlist_next_nvpair(holds, NULL); pair != NULL;
+	    pair = nvlist_next_nvpair(holds, pair)) {
+		char *htag;
+
+		error = nvpair_value_string(pair, &htag);
+		if (error != 0)
+			return (SET_ERROR(error));
+
+		if (strlen(htag) == 0)
+			return (SET_ERROR(EINVAL));
+	}
+
 	if (nvlist_lookup_int32(args, "cleanup_fd", &cleanup_fd) == 0) {
 		error = zfs_onexit_fd_hold(cleanup_fd, &minor);
 		if (error != 0)

From 4084c996c818b9ed4805419671ebf0f263b20d9a Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Fri, 5 Jun 2015 17:04:50 +0000
Subject: [PATCH 083/380] 5438 zfs_blkptr_verify should continue after
 zfs_panic_recover

Author: Justin Gibbs <gibbs@scsiguy.com>
Reviewed by: George Wilson <george@delphix.com>
Reviewed by: Xin LI <delphij@freebsd.org>
Approved by: Dan McDonald <danmcd@omniti.com>

illumos/illumos-gate@5897eb49ccde82d19214b71984f57935e7e313d1
---
 uts/common/fs/zfs/zio.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/uts/common/fs/zfs/zio.c b/uts/common/fs/zfs/zio.c
index 7be362f59279..868192b4fcc2 100644
--- a/uts/common/fs/zfs/zio.c
+++ b/uts/common/fs/zfs/zio.c
@@ -643,18 +643,20 @@ zfs_blkptr_verify(spa_t *spa, const blkptr_t *bp)
 			zfs_panic_recover("blkptr at %p DVA %u has invalid "
 			    "VDEV %llu",
 			    bp, i, (longlong_t)vdevid);
+			continue;
 		}
 		vdev_t *vd = spa->spa_root_vdev->vdev_child[vdevid];
 		if (vd == NULL) {
 			zfs_panic_recover("blkptr at %p DVA %u has invalid "
 			    "VDEV %llu",
 			    bp, i, (longlong_t)vdevid);
+			continue;
 		}
 		if (vd->vdev_ops == &vdev_hole_ops) {
 			zfs_panic_recover("blkptr at %p DVA %u has hole "
 			    "VDEV %llu",
 			    bp, i, (longlong_t)vdevid);
-
+			continue;
 		}
 		if (vd->vdev_ops == &vdev_missing_ops) {
 			/*

From 85db2c88f721636e24f207c282ad832c26e956a4 Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Fri, 5 Jun 2015 17:06:29 +0000
Subject: [PATCH 084/380] 5961 Fix stack overflow in zfs_create_fs

Author: Gleb Smirnoff <glebius@freebsd.org>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Approved by: Dan McDonald <danmcd@omniti.com>

illumos/illumos-gate@c701fde6911c957e71b37aac4daf672bd828f4d7
---
 uts/common/fs/zfs/zfs_znode.c | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/uts/common/fs/zfs/zfs_znode.c b/uts/common/fs/zfs/zfs_znode.c
index 4664899d13f8..1321508164dd 100644
--- a/uts/common/fs/zfs/zfs_znode.c
+++ b/uts/common/fs/zfs/zfs_znode.c
@@ -1723,7 +1723,6 @@ log:
 void
 zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx)
 {
-	zfsvfs_t	zfsvfs;
 	uint64_t	moid, obj, sa_obj, version;
 	uint64_t	sense = ZFS_CASE_SENSITIVE;
 	uint64_t	norm = 0;
@@ -1731,6 +1730,7 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx)
 	int		error;
 	int		i;
 	znode_t		*rootzp = NULL;
+	zfsvfs_t	*zfsvfs;
 	vnode_t		*vp;
 	vattr_t		vattr;
 	znode_t		*zp;
@@ -1817,17 +1817,16 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx)
 	vn_reinit(vp);
 	vp->v_type = VDIR;
 
-	bzero(&zfsvfs, sizeof (zfsvfs_t));
-
-	zfsvfs.z_os = os;
-	zfsvfs.z_parent = &zfsvfs;
-	zfsvfs.z_version = version;
-	zfsvfs.z_use_fuids = USE_FUIDS(version, os);
-	zfsvfs.z_use_sa = USE_SA(version, os);
-	zfsvfs.z_norm = norm;
+	zfsvfs = kmem_zalloc(sizeof (zfsvfs_t), KM_SLEEP);
+	zfsvfs->z_os = os;
+	zfsvfs->z_parent = zfsvfs;
+	zfsvfs->z_version = version;
+	zfsvfs->z_use_fuids = USE_FUIDS(version, os);
+	zfsvfs->z_use_sa = USE_SA(version, os);
+	zfsvfs->z_norm = norm;
 
 	error = sa_setup(os, sa_obj, zfs_attr_table, ZPL_END,
-	    &zfsvfs.z_attr_table);
+	    &zfsvfs->z_attr_table);
 
 	ASSERT(error == 0);
 
@@ -1836,16 +1835,16 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx)
 	 * insensitive.
 	 */
 	if (sense == ZFS_CASE_INSENSITIVE || sense == ZFS_CASE_MIXED)
-		zfsvfs.z_norm |= U8_TEXTPREP_TOUPPER;
+		zfsvfs->z_norm |= U8_TEXTPREP_TOUPPER;
 
-	mutex_init(&zfsvfs.z_znodes_lock, NULL, MUTEX_DEFAULT, NULL);
-	list_create(&zfsvfs.z_all_znodes, sizeof (znode_t),
+	mutex_init(&zfsvfs->z_znodes_lock, NULL, MUTEX_DEFAULT, NULL);
+	list_create(&zfsvfs->z_all_znodes, sizeof (znode_t),
 	    offsetof(znode_t, z_link_node));
 
 	for (i = 0; i != ZFS_OBJ_MTX_SZ; i++)
-		mutex_init(&zfsvfs.z_hold_mtx[i], NULL, MUTEX_DEFAULT, NULL);
+		mutex_init(&zfsvfs->z_hold_mtx[i], NULL, MUTEX_DEFAULT, NULL);
 
-	rootzp->z_zfsvfs = &zfsvfs;
+	rootzp->z_zfsvfs = zfsvfs;
 	VERIFY(0 == zfs_acl_ids_create(rootzp, IS_ROOT_NODE, &vattr,
 	    cr, NULL, &acl_ids));
 	zfs_mknode(rootzp, &vattr, tx, cr, IS_ROOT_NODE, &zp, &acl_ids);
@@ -1864,12 +1863,13 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx)
 	 * Create shares directory
 	 */
 
-	error = zfs_create_share_dir(&zfsvfs, tx);
+	error = zfs_create_share_dir(zfsvfs, tx);
 
 	ASSERT(error == 0);
 
 	for (i = 0; i != ZFS_OBJ_MTX_SZ; i++)
-		mutex_destroy(&zfsvfs.z_hold_mtx[i]);
+		mutex_destroy(&zfsvfs->z_hold_mtx[i]);
+	kmem_free(zfsvfs, sizeof (zfsvfs_t));
 }
 
 #endif /* _KERNEL */

From 3af2f3e8ca3c810e726b26280937f0a025a61581 Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Fri, 5 Jun 2015 17:07:13 +0000
Subject: [PATCH 085/380] 5912 full stream can not be force-received into a
 dataset if it has a snapshot

Author: Andriy Gapon <avg@freebsd.org>
Reviewed by: Paul Dagnelie <pcd@delphix.com>
Approved by: Dan McDonald <danmcd@omniti.com>

illumos/illumos-gate@5bae108fe2364722de3aa86e04966a0d33fd027a
---
 uts/common/fs/zfs/dmu_send.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/uts/common/fs/zfs/dmu_send.c b/uts/common/fs/zfs/dmu_send.c
index acc9f28f9d44..b461d833bdcb 100644
--- a/uts/common/fs/zfs/dmu_send.c
+++ b/uts/common/fs/zfs/dmu_send.c
@@ -949,10 +949,12 @@ recv_begin_check_existing_impl(dmu_recv_begin_arg_t *drba, dsl_dataset_t *ds,
 
 		dsl_dataset_rele(snap, FTAG);
 	} else {
-		/* if full, most recent snapshot must be $ORIGIN */
-		if (dsl_dataset_phys(ds)->ds_prev_snap_txg >= TXG_INITIAL)
-			return (SET_ERROR(ENODEV));
-		drba->drba_snapobj = dsl_dataset_phys(ds)->ds_prev_snap_obj;
+		/* if full, then must be forced */
+		if (!drba->drba_cookie->drc_force)
+			return (SET_ERROR(EEXIST));
+		/* start from $ORIGIN@$ORIGIN, if supported */
+		drba->drba_snapobj = dp->dp_origin_snap != NULL ?
+		    dp->dp_origin_snap->ds_object : 0;
 	}
 
 	return (0);

From ca777f915c2949475a5c9a8d1caeff6a622726b2 Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Fri, 5 Jun 2015 17:07:51 +0000
Subject: [PATCH 086/380] 5870 dmu_recv_end_check() leaks origin_head hold if
 error happens in drc_force branch

Author: Andriy Gapon <avg@freebsd.org>
Reviewed by: Andrew Stormont <andyjstormont@gmail.com>
Approved by: Dan McDonald <danmcd@omniti.com>

illumos/illumos-gate@beddaa9ce797b9deaafc22b4f156d23f9b45c32d
---
 uts/common/fs/zfs/dmu_send.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/uts/common/fs/zfs/dmu_send.c b/uts/common/fs/zfs/dmu_send.c
index b461d833bdcb..3d837c374fd1 100644
--- a/uts/common/fs/zfs/dmu_send.c
+++ b/uts/common/fs/zfs/dmu_send.c
@@ -1953,7 +1953,7 @@ dmu_recv_end_check(void *arg, dmu_tx_t *tx)
 				error = dsl_dataset_hold_obj(dp, obj, FTAG,
 				    &snap);
 				if (error != 0)
-					return (error);
+					break;
 				if (snap->ds_dir != origin_head->ds_dir)
 					error = SET_ERROR(EINVAL);
 				if (error == 0)  {
@@ -1963,7 +1963,11 @@ dmu_recv_end_check(void *arg, dmu_tx_t *tx)
 				obj = dsl_dataset_phys(snap)->ds_prev_snap_obj;
 				dsl_dataset_rele(snap, FTAG);
 				if (error != 0)
-					return (error);
+					break;
+			}
+			if (error != 0) {
+				dsl_dataset_rele(origin_head, FTAG);
+				return (error);
 			}
 		}
 		error = dsl_dataset_clone_swap_check_impl(drc->drc_ds,

From 26701a58f3cfac81ab85e8880412f0bfd98143a2 Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Fri, 5 Jun 2015 17:08:32 +0000
Subject: [PATCH 087/380] 5909 ensure that shared snap names don't become too
 long after promotion

Author: Andriy Gapon <avg@freebsd.org>
Reviewed by: George Wilson <george@delphix.com>
Approved by: Dan McDonald <danmcd@omniti.com>

illumos/illumos-gate@cb5842f8b0caaad0ed53535bd77042e933fdbafe
---
 uts/common/fs/zfs/dsl_dataset.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/uts/common/fs/zfs/dsl_dataset.c b/uts/common/fs/zfs/dsl_dataset.c
index ec2d022aed39..5a9acd2f0bb6 100644
--- a/uts/common/fs/zfs/dsl_dataset.c
+++ b/uts/common/fs/zfs/dsl_dataset.c
@@ -2081,12 +2081,14 @@ dsl_dataset_promote_check(void *arg, dmu_tx_t *tx)
 	int err;
 	uint64_t unused;
 	uint64_t ss_mv_cnt;
+	size_t max_snap_len;
 
 	err = promote_hold(ddpa, dp, FTAG);
 	if (err != 0)
 		return (err);
 
 	hds = ddpa->ddpa_clone;
+	max_snap_len = MAXNAMELEN - strlen(ddpa->ddpa_clonename) - 1;
 
 	if (dsl_dataset_phys(hds)->ds_flags & DS_FLAG_NOPROMOTE) {
 		promote_rele(ddpa, FTAG);
@@ -2150,6 +2152,10 @@ dsl_dataset_promote_check(void *arg, dmu_tx_t *tx)
 
 		/* Check that the snapshot name does not conflict */
 		VERIFY0(dsl_dataset_get_snapname(ds));
+		if (strlen(ds->ds_snapname) >= max_snap_len) {
+			err = SET_ERROR(ENAMETOOLONG);
+			goto out;
+		}
 		err = dsl_dataset_snap_lookup(hds, ds->ds_snapname, &val);
 		if (err == 0) {
 			(void) strcpy(ddpa->err_ds, snap->ds->ds_snapname);

From f90444d14f66aced0e0d1f9b0c0a9edd509802c8 Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Fri, 5 Jun 2015 17:09:59 +0000
Subject: [PATCH 088/380] 5946 zfs_ioc_space_snaps must check that firstsnap
 and lastsnap refer to snapshots

Author: Andriy Gapon <avg@freebsd.org>
Reviewed by: Steven Hartland <killing@multiplay.co.uk>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Approved by: Gordon Ross <gordon.ross@nexenta.com>

illumos/illumos-gate@24218bebb460e4015fac2c9f2cec1902eddbcd7b
---
 uts/common/fs/zfs/dmu_send.c  | 4 ++++
 uts/common/fs/zfs/zfs_ioctl.c | 8 ++++++++
 2 files changed, 12 insertions(+)

diff --git a/uts/common/fs/zfs/dmu_send.c b/uts/common/fs/zfs/dmu_send.c
index 3d837c374fd1..4d77f626a7b5 100644
--- a/uts/common/fs/zfs/dmu_send.c
+++ b/uts/common/fs/zfs/dmu_send.c
@@ -821,6 +821,10 @@ dmu_send_estimate(dsl_dataset_t *ds, dsl_dataset_t *fromds, uint64_t *sizep)
 	if (!ds->ds_is_snapshot)
 		return (SET_ERROR(EINVAL));
 
+	/* fromsnap, if provided, must be a snapshot */
+	if (fromds != NULL && !fromds->ds_is_snapshot)
+		return (SET_ERROR(EINVAL));
+
 	/*
 	 * fromsnap must be an earlier snapshot from the same fs as tosnap,
 	 * or the origin's fs.
diff --git a/uts/common/fs/zfs/zfs_ioctl.c b/uts/common/fs/zfs/zfs_ioctl.c
index 1a7c55a12b83..a40a83af5b02 100644
--- a/uts/common/fs/zfs/zfs_ioctl.c
+++ b/uts/common/fs/zfs/zfs_ioctl.c
@@ -5258,11 +5258,19 @@ zfs_ioc_space_snaps(const char *lastsnap, nvlist_t *innvl, nvlist_t *outnvl)
 		return (error);
 
 	error = dsl_dataset_hold(dp, lastsnap, FTAG, &new);
+	if (error == 0 && !new->ds_is_snapshot) {
+		dsl_dataset_rele(new, FTAG);
+		error = SET_ERROR(EINVAL);
+	}
 	if (error != 0) {
 		dsl_pool_rele(dp, FTAG);
 		return (error);
 	}
 	error = dsl_dataset_hold(dp, firstsnap, FTAG, &old);
+	if (error == 0 && !old->ds_is_snapshot) {
+		dsl_dataset_rele(old, FTAG);
+		error = SET_ERROR(EINVAL);
+	}
 	if (error != 0) {
 		dsl_dataset_rele(new, FTAG);
 		dsl_pool_rele(dp, FTAG);

From fb649f79365fb8e02cf985f456b4a1168f0b99b3 Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Fri, 12 Jun 2015 10:50:31 +0000
Subject: [PATCH 089/380] Revert r284031: the change was already imported in
 r283534

---
 uts/common/fs/zfs/zfs_ioctl.c | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/uts/common/fs/zfs/zfs_ioctl.c b/uts/common/fs/zfs/zfs_ioctl.c
index a40a83af5b02..cea478457dd6 100644
--- a/uts/common/fs/zfs/zfs_ioctl.c
+++ b/uts/common/fs/zfs/zfs_ioctl.c
@@ -5132,19 +5132,6 @@ zfs_ioc_hold(const char *pool, nvlist_t *args, nvlist_t *errlist)
 			return (SET_ERROR(EINVAL));
 	}
 
-	/* make sure the user didn't pass us any invalid (empty) tags */
-	for (pair = nvlist_next_nvpair(holds, NULL); pair != NULL;
-	    pair = nvlist_next_nvpair(holds, pair)) {
-		char *htag;
-
-		error = nvpair_value_string(pair, &htag);
-		if (error != 0)
-			return (SET_ERROR(error));
-
-		if (strlen(htag) == 0)
-			return (SET_ERROR(EINVAL));
-	}
-
 	if (nvlist_lookup_int32(args, "cleanup_fd", &cleanup_fd) == 0) {
 		error = zfs_onexit_fd_hold(cleanup_fd, &minor);
 		if (error != 0)

From 46a8570489f28823eea17b23c8ccab9138f216b6 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sun, 14 Jun 2015 20:55:48 +0000
Subject: [PATCH 090/380] Remove support for ALLOW_DEPRECATED_ATF_TOOLS

atf-run has been deprecated upstream since 0.20, which was released over a
year ago (02/2014)
---
 share/mk/atf.test.mk | 82 --------------------------------------------
 1 file changed, 82 deletions(-)

diff --git a/share/mk/atf.test.mk b/share/mk/atf.test.mk
index bef87bd893e8..6f99d88fc78a 100644
--- a/share/mk/atf.test.mk
+++ b/share/mk/atf.test.mk
@@ -22,24 +22,6 @@ ATF_TESTS_C?=
 ATF_TESTS_CXX?=
 ATF_TESTS_SH?=
 
-# Whether to allow using the deprecated ATF tools or not.
-#
-# If 'yes', this file will generate Atffiles when requested and will also
-# support using the deprecated atf-run tool to execute the tests.
-ALLOW_DEPRECATED_ATF_TOOLS?= no
-
-# Knob to control the handling of the Atffile for this Makefile.
-#
-# If 'yes', an Atffile exists in the source tree and is installed into
-# TESTSDIR.
-#
-# If 'auto', an Atffile is automatically generated based on the list of test
-# programs built by the Makefile and is installed into TESTSDIR.  This is the
-# default and is sufficient in the majority of the cases.
-#
-# If 'no', no Atffile is installed.
-ATFFILE?= auto
-
 # Path to the prefix of the installed ATF tools, if any.
 #
 # If atf-run and atf-report are installed from ports, we automatically define a
@@ -121,67 +103,3 @@ ${_T}: ${ATF_TESTS_SH_SRC_${_T}}
 	mv ${.TARGET}.tmp ${.TARGET}
 .endfor
 .endif
-
-.if ${ALLOW_DEPRECATED_ATF_TOOLS} != "no"
-
-.if ${ATFFILE:tl} != "no"
-FILES+=	Atffile
-FILESDIR_Atffile= ${TESTSDIR}
-
-.if ${ATFFILE:tl} == "auto"
-CLEANFILES+= Atffile Atffile.tmp
-
-Atffile: Makefile
-	@{ echo 'Content-Type: application/X-atf-atffile; version="1"'; \
-	echo; \
-	echo '# Automatically generated by atf-test.mk.'; \
-	echo; \
-	echo 'prop: test-suite = "'${TESTSUITE}'"'; \
-	echo; \
-	for tp in ${ATF_TESTS_C} ${ATF_TESTS_CXX} ${ATF_TESTS_SH} \
-	    ${TESTS_SUBDIRS}; \
-	do \
-	    echo "tp: $${tp}"; \
-	done; } >Atffile.tmp
-	@mv Atffile.tmp Atffile
-.endif
-.endif
-
-ATF_REPORT?= ${ATF_PREFIX}/bin/atf-report
-ATF_RUN?= ${ATF_PREFIX}/bin/atf-run
-.if exists(${ATF_RUN}) && exists(${ATF_REPORT})
-# Definition of the "make test" target and supporting variables.
-#
-# This target, by necessity, can only work for native builds (i.e. a freeBSD
-# host building a release for the same system).  The target runs ATF, which is
-# not in the toolchain, and the tests execute code built for the target host.
-#
-# Due to the dependencies of the binaries built by the source tree and how they
-# are used by tests, it is highly possible for a execution of "make test" to
-# report bogus results unless the new binaries are put in place.
-_TESTS_FIFO= ${.OBJDIR}/atf-run.fifo
-_TESTS_LOG= ${.OBJDIR}/atf-run.log
-CLEANFILES+= ${_TESTS_FIFO} ${_TESTS_LOG}
-realregress: .PHONY
-	@set -e; \
-	if [ -z "${TESTSDIR}" ]; then \
-	    echo "*** No TESTSDIR defined; nothing to do."; \
-	    exit 0; \
-	fi; \
-	cd ${DESTDIR}${TESTSDIR}; \
-	rm -f ${_TESTS_FIFO}; \
-	mkfifo ${_TESTS_FIFO}; \
-	tee ${_TESTS_LOG} < ${_TESTS_FIFO} | ${TESTS_ENV} ${ATF_REPORT} & \
-	set +e; \
-	${TESTS_ENV} ${ATF_RUN} >> ${_TESTS_FIFO}; \
-	result=$${?}; \
-	wait; \
-	rm -f ${_TESTS_FIFO}; \
-	echo; \
-	echo "*** The verbatim output of atf-run has been saved to ${_TESTS_LOG}"; \
-	echo "***"; \
-	echo "*** WARNING: atf-run is deprecated; please install kyua instead"; \
-	exit $${result}
-.endif
-
-.endif

From f8b592f68f0ba1172767158db787a5ee561686ed Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sun, 14 Jun 2015 20:56:41 +0000
Subject: [PATCH 091/380] Note why the !.PARSEDIR case is important/exists

---
 Makefile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Makefile b/Makefile
index 44d58e8a71ef..83a3b7230817 100644
--- a/Makefile
+++ b/Makefile
@@ -99,9 +99,11 @@
 #
 # For more information, see the build(7) manual page.
 #
+
 .ifdef .PARSEDIR
 MK_META_MODE?=	yes
 .else
+# Allow bootstrapping with fmake from earlier versions of FreeBSD
 MK_META_MODE=	no
 .endif
 .if ${MK_META_MODE} == "yes"

From e745d95eb8644bd2b1366daf1040217d0f03eeb6 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sun, 14 Jun 2015 21:14:47 +0000
Subject: [PATCH 092/380] To be on the safe side, only enable MK_META_MODE if
 MAKEVERSION >= 20150606

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 83a3b7230817..408566187278 100644
--- a/Makefile
+++ b/Makefile
@@ -100,10 +100,10 @@
 # For more information, see the build(7) manual page.
 #
 
-.ifdef .PARSEDIR
+# Allow bootstrapping with fmake from earlier versions of FreeBSD
+.if defined(.PARSEDIR) && ${MAKE_VERSION} >= 20150606
 MK_META_MODE?=	yes
 .else
-# Allow bootstrapping with fmake from earlier versions of FreeBSD
 MK_META_MODE=	no
 .endif
 .if ${MK_META_MODE} == "yes"

From 8be23f7b33cdab474193f6e0035cc8b88fd9838b Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Mon, 15 Jun 2015 12:50:43 +0000
Subject: [PATCH 093/380] 5911 ZFS "hangs" while deleting file

Reviewed by: Bayard Bell <buffer.g.overflow@gmail.com>
Reviewed by: Alek Pinchuk <alek@nexenta.com>
Reviewed by: Simon Klinkert <simon.klinkert@gmail.com>
Reviewed by: Dan McDonald <danmcd@omniti.com>
Approved by: Richard Lowe <richlowe@richlowe.net>
Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@46e1baa6cf6d5432f5fd231bb588df8f9570c858
---
 uts/common/fs/zfs/dbuf.c       | 34 ++++++++-------
 uts/common/fs/zfs/dmu_tx.c     |  4 +-
 uts/common/fs/zfs/dnode.c      | 76 ++++++++++++++++++++++++++++------
 uts/common/fs/zfs/dnode_sync.c |  4 +-
 uts/common/fs/zfs/sys/dbuf.h   |  4 +-
 5 files changed, 89 insertions(+), 33 deletions(-)

diff --git a/uts/common/fs/zfs/dbuf.c b/uts/common/fs/zfs/dbuf.c
index 49c659a51b4e..57ff05c91f39 100644
--- a/uts/common/fs/zfs/dbuf.c
+++ b/uts/common/fs/zfs/dbuf.c
@@ -21,7 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
- * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
@@ -1374,6 +1374,16 @@ dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
 	dbuf_dirty_record_t *dr, **drp;
 
 	ASSERT(txg != 0);
+
+	/*
+	 * Due to our use of dn_nlevels below, this can only be called
+	 * in open context, unless we are operating on the MOS.
+	 * From syncing context, dn_nlevels may be different from the
+	 * dn_nlevels used when dbuf was dirtied.
+	 */
+	ASSERT(db->db_objset ==
+	    dmu_objset_pool(db->db_objset)->dp_meta_objset ||
+	    txg != spa_syncing_txg(dmu_objset_spa(db->db_objset)));
 	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
 	ASSERT0(db->db_level);
 	ASSERT(MUTEX_HELD(&db->db_mtx));
@@ -1396,11 +1406,8 @@ dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
 
 	ASSERT(db->db.db_size != 0);
 
-	/*
-	 * Any space we accounted for in dp_dirty_* will be cleaned up by
-	 * dsl_pool_sync().  This is relatively rare so the discrepancy
-	 * is not a big deal.
-	 */
+	dsl_pool_undirty_space(dmu_objset_pool(dn->dn_objset),
+	    dr->dr_accounted, txg);
 
 	*drp = dr->dr_next;
 
@@ -1415,7 +1422,7 @@ dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
 		list_remove(&dr->dr_parent->dt.di.dr_children, dr);
 		mutex_exit(&dr->dr_parent->dt.di.dr_mtx);
 	} else if (db->db_blkid == DMU_SPILL_BLKID ||
-	    db->db_level+1 == dn->dn_nlevels) {
+	    db->db_level + 1 == dn->dn_nlevels) {
 		ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf);
 		mutex_enter(&dn->dn_mtx);
 		list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr);
@@ -1432,11 +1439,6 @@ dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
 			VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db));
 	}
 
-	if (db->db_level != 0) {
-		mutex_destroy(&dr->dt.di.dr_mtx);
-		list_destroy(&dr->dt.di.dr_children);
-	}
-
 	kmem_free(dr, sizeof (dbuf_dirty_record_t));
 
 	ASSERT(db->db_dirtycnt > 0);
@@ -2400,7 +2402,7 @@ dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx)
 
 	zio = dr->dr_zio;
 	mutex_enter(&dr->dt.di.dr_mtx);
-	dbuf_sync_list(&dr->dt.di.dr_children, tx);
+	dbuf_sync_list(&dr->dt.di.dr_children, db->db_level - 1, tx);
 	ASSERT(list_head(&dr->dt.di.dr_children) == NULL);
 	mutex_exit(&dr->dt.di.dr_mtx);
 	zio_nowait(zio);
@@ -2542,7 +2544,7 @@ dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx)
 }
 
 void
-dbuf_sync_list(list_t *list, dmu_tx_t *tx)
+dbuf_sync_list(list_t *list, int level, dmu_tx_t *tx)
 {
 	dbuf_dirty_record_t *dr;
 
@@ -2559,6 +2561,10 @@ dbuf_sync_list(list_t *list, dmu_tx_t *tx)
 			    DMU_META_DNODE_OBJECT);
 			break;
 		}
+		if (dr->dr_dbuf->db_blkid != DMU_BONUS_BLKID &&
+		    dr->dr_dbuf->db_blkid != DMU_SPILL_BLKID) {
+			VERIFY3U(dr->dr_dbuf->db_level, ==, level);
+		}
 		list_remove(list, dr);
 		if (dr->dr_dbuf->db_level > 0)
 			dbuf_sync_indirect(dr, tx);
diff --git a/uts/common/fs/zfs/dmu_tx.c b/uts/common/fs/zfs/dmu_tx.c
index 56ce2f0c279f..60078153a549 100644
--- a/uts/common/fs/zfs/dmu_tx.c
+++ b/uts/common/fs/zfs/dmu_tx.c
@@ -21,7 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
- * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
  */
 
 #include <sys/dmu.h>
@@ -685,7 +685,7 @@ dmu_tx_hold_free(dmu_tx_t *tx, uint64_t object, uint64_t off, uint64_t len)
 			uint64_t ibyte = i << shift;
 			err = dnode_next_offset(dn, 0, &ibyte, 2, 1, 0);
 			i = ibyte >> shift;
-			if (err == ESRCH)
+			if (err == ESRCH || i > end)
 				break;
 			if (err) {
 				tx->tx_err = err;
diff --git a/uts/common/fs/zfs/dnode.c b/uts/common/fs/zfs/dnode.c
index 4e376fc4d920..bd788a5b27b5 100644
--- a/uts/common/fs/zfs/dnode.c
+++ b/uts/common/fs/zfs/dnode.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
@@ -1510,6 +1510,16 @@ out:
 		rw_downgrade(&dn->dn_struct_rwlock);
 }
 
+static void
+dnode_dirty_l1(dnode_t *dn, uint64_t l1blkid, dmu_tx_t *tx)
+{
+	dmu_buf_impl_t *db = dbuf_hold_level(dn, 1, l1blkid, FTAG);
+	if (db != NULL) {
+		dmu_buf_will_dirty(&db->db, tx);
+		dbuf_rele(db, FTAG);
+	}
+}
+
 void
 dnode_free_range(dnode_t *dn, uint64_t off, uint64_t len, dmu_tx_t *tx)
 {
@@ -1630,27 +1640,67 @@ dnode_free_range(dnode_t *dn, uint64_t off, uint64_t len, dmu_tx_t *tx)
 		nblks += 1;
 
 	/*
-	 * Dirty the first and last indirect blocks, as they (and/or their
-	 * parents) will need to be written out if they were only
-	 * partially freed.  Interior indirect blocks will be themselves freed,
-	 * by free_children(), so they need not be dirtied.  Note that these
-	 * interior blocks have already been prefetched by dmu_tx_hold_free().
+	 * Dirty all the indirect blocks in this range.  Note that only
+	 * the first and last indirect blocks can actually be written
+	 * (if they were partially freed) -- they must be dirtied, even if
+	 * they do not exist on disk yet.  The interior blocks will
+	 * be freed by free_children(), so they will not actually be written.
+	 * Even though these interior blocks will not be written, we
+	 * dirty them for two reasons:
+	 *
+	 *  - It ensures that the indirect blocks remain in memory until
+	 *    syncing context.  (They have already been prefetched by
+	 *    dmu_tx_hold_free(), so we don't have to worry about reading
+	 *    them serially here.)
+	 *
+	 *  - The dirty space accounting will put pressure on the txg sync
+	 *    mechanism to begin syncing, and to delay transactions if there
+	 *    is a large amount of freeing.  Even though these indirect
+	 *    blocks will not be written, we could need to write the same
+	 *    amount of space if we copy the freed BPs into deadlists.
 	 */
 	if (dn->dn_nlevels > 1) {
 		uint64_t first, last;
 
 		first = blkid >> epbs;
-		if (db = dbuf_hold_level(dn, 1, first, FTAG)) {
-			dmu_buf_will_dirty(&db->db, tx);
-			dbuf_rele(db, FTAG);
-		}
+		dnode_dirty_l1(dn, first, tx);
 		if (trunc)
 			last = dn->dn_maxblkid >> epbs;
 		else
 			last = (blkid + nblks - 1) >> epbs;
-		if (last > first && (db = dbuf_hold_level(dn, 1, last, FTAG))) {
-			dmu_buf_will_dirty(&db->db, tx);
-			dbuf_rele(db, FTAG);
+		if (last != first)
+			dnode_dirty_l1(dn, last, tx);
+
+		int shift = dn->dn_datablkshift + dn->dn_indblkshift -
+		    SPA_BLKPTRSHIFT;
+		for (uint64_t i = first + 1; i < last; i++) {
+			/*
+			 * Set i to the blockid of the next non-hole
+			 * level-1 indirect block at or after i.  Note
+			 * that dnode_next_offset() operates in terms of
+			 * level-0-equivalent bytes.
+			 */
+			uint64_t ibyte = i << shift;
+			int err = dnode_next_offset(dn, DNODE_FIND_HAVELOCK,
+			    &ibyte, 2, 1, 0);
+			i = ibyte >> shift;
+			if (i >= last)
+				break;
+
+			/*
+			 * Normally we should not see an error, either
+			 * from dnode_next_offset() or dbuf_hold_level()
+			 * (except for ESRCH from dnode_next_offset).
+			 * If there is an i/o error, then when we read
+			 * this block in syncing context, it will use
+			 * ZIO_FLAG_MUSTSUCCEED, and thus hang/panic according
+			 * to the "failmode" property.  dnode_next_offset()
+			 * doesn't have a flag to indicate MUSTSUCCEED.
+			 */
+			if (err != 0)
+				break;
+
+			dnode_dirty_l1(dn, i, tx);
 		}
 	}
 
diff --git a/uts/common/fs/zfs/dnode_sync.c b/uts/common/fs/zfs/dnode_sync.c
index 99ecb9e3ff73..7be35f48d122 100644
--- a/uts/common/fs/zfs/dnode_sync.c
+++ b/uts/common/fs/zfs/dnode_sync.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
@@ -700,7 +700,7 @@ dnode_sync(dnode_t *dn, dmu_tx_t *tx)
 		mutex_exit(&dn->dn_mtx);
 	}
 
-	dbuf_sync_list(list, tx);
+	dbuf_sync_list(list, dn->dn_phys->dn_nlevels - 1, tx);
 
 	if (!DMU_OBJECT_IS_SPECIAL(dn->dn_object)) {
 		ASSERT3P(list_head(list), ==, NULL);
diff --git a/uts/common/fs/zfs/sys/dbuf.h b/uts/common/fs/zfs/sys/dbuf.h
index 6da5dccc05f7..40d0344a6226 100644
--- a/uts/common/fs/zfs/sys/dbuf.h
+++ b/uts/common/fs/zfs/sys/dbuf.h
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
@@ -287,7 +287,7 @@ void dbuf_evict(dmu_buf_impl_t *db);
 
 void dbuf_setdirty(dmu_buf_impl_t *db, dmu_tx_t *tx);
 void dbuf_unoverride(dbuf_dirty_record_t *dr);
-void dbuf_sync_list(list_t *list, dmu_tx_t *tx);
+void dbuf_sync_list(list_t *list, int level, dmu_tx_t *tx);
 void dbuf_release_bp(dmu_buf_impl_t *db);
 
 void dbuf_free_range(struct dnode *dn, uint64_t start, uint64_t end,

From 1036fbb839071eac9806ebf49b655de276c4f23b Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Tue, 23 Jun 2015 15:49:12 +0000
Subject: [PATCH 094/380] Unbreak the build

---
 Makefile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Makefile b/Makefile
index f8dd6a347489..ac62b9827cd4 100644
--- a/Makefile
+++ b/Makefile
@@ -539,7 +539,6 @@ MK_STAGING= no
 # tell meta.autodep.mk to not even think about updating anything.
 UPDATE_DEPENDFILE= NO
 .export MK_META_MODE MK_STAGING UPDATE_DEPENDFILE
-.endif
 
 .if make(universe)
 # we do not want a failure of one branch abort all.

From 09ba8607158d1ddd2d819db10cba16d335eee2f7 Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Wed, 24 Jun 2015 14:48:25 +0000
Subject: [PATCH 095/380] 5269 zpool import slow

illumos/illumos-gate@12380e1e701fda28c9e9f32d01cafb54af279eb5

https://www.illumos.org/issues/5269
  When importing a pool (at boot or with zpool import) with many
  filesystem, the process can take minutes. It doesn't matter whether
  the pool has been exported cleanly or uncleanly.  The problem is that
  each dataset has its own log chain. On import, all datasets have to be
  checked if there are logs to replay.  The idea is to speed up this
  process by paralellizing it.

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: George Wilson <george@delphix.com>
Reviewed by: Dan McDonald <danmcd@omniti.com>
Approved by: Dan McDonald <danmcd@omniti.com>
Author: Arne Jansen <jansen@webgods.de>
---
 uts/common/fs/zfs/dmu_objset.c     | 221 ++++++++++++++++++++++++-----
 uts/common/fs/zfs/dsl_pool.c       |  10 +-
 uts/common/fs/zfs/spa.c            |  11 +-
 uts/common/fs/zfs/sys/dmu.h        |   1 +
 uts/common/fs/zfs/sys/dmu_objset.h |   2 +
 uts/common/fs/zfs/sys/dsl_pool.h   |   1 +
 uts/common/fs/zfs/sys/vdev.h       |   1 +
 uts/common/fs/zfs/sys/zil.h        |   9 +-
 uts/common/fs/zfs/vdev.c           |  20 +++
 uts/common/fs/zfs/zil.c            |  23 ++-
 10 files changed, 239 insertions(+), 60 deletions(-)

diff --git a/uts/common/fs/zfs/dmu_objset.c b/uts/common/fs/zfs/dmu_objset.c
index 2732d989929f..cfe9e90f1b2f 100644
--- a/uts/common/fs/zfs/dmu_objset.c
+++ b/uts/common/fs/zfs/dmu_objset.c
@@ -25,6 +25,7 @@
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
+ * Copyright (c) 2015, STRATO AG, Inc. All rights reserved.
  */
 
 /* Portions Copyright 2010 Robert Milkowski */
@@ -49,6 +50,7 @@
 #include <sys/sa.h>
 #include <sys/zfs_onexit.h>
 #include <sys/dsl_destroy.h>
+#include <sys/vdev.h>
 
 /*
  * Needed to close a window in dnode_move() that allows the objset to be freed
@@ -56,6 +58,16 @@
  */
 krwlock_t os_lock;
 
+/*
+ * Tunable to overwrite the maximum number of threads for the parallization
+ * of dmu_objset_find_dp, needed to speed up the import of pools with many
+ * datasets.
+ * Default is 4 times the number of leaf vdevs.
+ */
+int dmu_find_threads = 0;
+
+static void dmu_objset_find_dp_cb(void *arg);
+
 void
 dmu_objset_init(void)
 {
@@ -503,6 +515,25 @@ dmu_objset_hold(const char *name, void *tag, objset_t **osp)
 	return (err);
 }
 
+static int
+dmu_objset_own_impl(dsl_dataset_t *ds, dmu_objset_type_t type,
+    boolean_t readonly, void *tag, objset_t **osp)
+{
+	int err;
+
+	err = dmu_objset_from_ds(ds, osp);
+	if (err != 0) {
+		dsl_dataset_disown(ds, tag);
+	} else if (type != DMU_OST_ANY && type != (*osp)->os_phys->os_type) {
+		dsl_dataset_disown(ds, tag);
+		return (SET_ERROR(EINVAL));
+	} else if (!readonly && dsl_dataset_is_snapshot(ds)) {
+		dsl_dataset_disown(ds, tag);
+		return (SET_ERROR(EROFS));
+	}
+	return (err);
+}
+
 /*
  * dsl_pool must not be held when this is called.
  * Upon successful return, there will be a longhold on the dataset,
@@ -524,21 +555,26 @@ dmu_objset_own(const char *name, dmu_objset_type_t type,
 		dsl_pool_rele(dp, FTAG);
 		return (err);
 	}
-
-	err = dmu_objset_from_ds(ds, osp);
+	err = dmu_objset_own_impl(ds, type, readonly, tag, osp);
 	dsl_pool_rele(dp, FTAG);
-	if (err != 0) {
-		dsl_dataset_disown(ds, tag);
-	} else if (type != DMU_OST_ANY && type != (*osp)->os_phys->os_type) {
-		dsl_dataset_disown(ds, tag);
-		return (SET_ERROR(EINVAL));
-	} else if (!readonly && ds->ds_is_snapshot) {
-		dsl_dataset_disown(ds, tag);
-		return (SET_ERROR(EROFS));
-	}
+
 	return (err);
 }
 
+int
+dmu_objset_own_obj(dsl_pool_t *dp, uint64_t obj, dmu_objset_type_t type,
+    boolean_t readonly, void *tag, objset_t **osp)
+{
+	dsl_dataset_t *ds;
+	int err;
+
+	err = dsl_dataset_own_obj(dp, obj, tag, &ds);
+	if (err != 0)
+		return (err);
+
+	return (dmu_objset_own_impl(ds, type, readonly, tag, osp));
+}
+
 void
 dmu_objset_rele(objset_t *os, void *tag)
 {
@@ -1579,30 +1615,41 @@ dmu_dir_list_next(objset_t *os, int namelen, char *name,
 	return (0);
 }
 
-/*
- * Find objsets under and including ddobj, call func(ds) on each.
- */
-int
-dmu_objset_find_dp(dsl_pool_t *dp, uint64_t ddobj,
-    int func(dsl_pool_t *, dsl_dataset_t *, void *), void *arg, int flags)
+typedef struct dmu_objset_find_ctx {
+	taskq_t		*dc_tq;
+	dsl_pool_t	*dc_dp;
+	uint64_t	dc_ddobj;
+	int		(*dc_func)(dsl_pool_t *, dsl_dataset_t *, void *);
+	void		*dc_arg;
+	int		dc_flags;
+	kmutex_t	*dc_error_lock;
+	int		*dc_error;
+} dmu_objset_find_ctx_t;
+
+static void
+dmu_objset_find_dp_impl(dmu_objset_find_ctx_t *dcp)
 {
+	dsl_pool_t *dp = dcp->dc_dp;
+	dmu_objset_find_ctx_t *child_dcp;
 	dsl_dir_t *dd;
 	dsl_dataset_t *ds;
 	zap_cursor_t zc;
 	zap_attribute_t *attr;
 	uint64_t thisobj;
-	int err;
+	int err = 0;
 
-	ASSERT(dsl_pool_config_held(dp));
+	/* don't process if there already was an error */
+	if (*dcp->dc_error != 0)
+		goto out;
 
-	err = dsl_dir_hold_obj(dp, ddobj, NULL, FTAG, &dd);
+	err = dsl_dir_hold_obj(dp, dcp->dc_ddobj, NULL, FTAG, &dd);
 	if (err != 0)
-		return (err);
+		goto out;
 
 	/* Don't visit hidden ($MOS & $ORIGIN) objsets. */
 	if (dd->dd_myname[0] == '$') {
 		dsl_dir_rele(dd, FTAG);
-		return (0);
+		goto out;
 	}
 
 	thisobj = dsl_dir_phys(dd)->dd_head_dataset_obj;
@@ -1611,7 +1658,7 @@ dmu_objset_find_dp(dsl_pool_t *dp, uint64_t ddobj,
 	/*
 	 * Iterate over all children.
 	 */
-	if (flags & DS_FIND_CHILDREN) {
+	if (dcp->dc_flags & DS_FIND_CHILDREN) {
 		for (zap_cursor_init(&zc, dp->dp_meta_objset,
 		    dsl_dir_phys(dd)->dd_child_dir_zapobj);
 		    zap_cursor_retrieve(&zc, attr) == 0;
@@ -1620,24 +1667,22 @@ dmu_objset_find_dp(dsl_pool_t *dp, uint64_t ddobj,
 			    sizeof (uint64_t));
 			ASSERT3U(attr->za_num_integers, ==, 1);
 
-			err = dmu_objset_find_dp(dp, attr->za_first_integer,
-			    func, arg, flags);
-			if (err != 0)
-				break;
+			child_dcp = kmem_alloc(sizeof (*child_dcp), KM_SLEEP);
+			*child_dcp = *dcp;
+			child_dcp->dc_ddobj = attr->za_first_integer;
+			if (dcp->dc_tq != NULL)
+				(void) taskq_dispatch(dcp->dc_tq,
+				    dmu_objset_find_dp_cb, child_dcp, TQ_SLEEP);
+			else
+				dmu_objset_find_dp_impl(child_dcp);
 		}
 		zap_cursor_fini(&zc);
-
-		if (err != 0) {
-			dsl_dir_rele(dd, FTAG);
-			kmem_free(attr, sizeof (zap_attribute_t));
-			return (err);
-		}
 	}
 
 	/*
 	 * Iterate over all snapshots.
 	 */
-	if (flags & DS_FIND_SNAPSHOTS) {
+	if (dcp->dc_flags & DS_FIND_SNAPSHOTS) {
 		dsl_dataset_t *ds;
 		err = dsl_dataset_hold_obj(dp, thisobj, FTAG, &ds);
 
@@ -1658,7 +1703,7 @@ dmu_objset_find_dp(dsl_pool_t *dp, uint64_t ddobj,
 				    attr->za_first_integer, FTAG, &ds);
 				if (err != 0)
 					break;
-				err = func(dp, ds, arg);
+				err = dcp->dc_func(dp, ds, dcp->dc_arg);
 				dsl_dataset_rele(ds, FTAG);
 				if (err != 0)
 					break;
@@ -1671,17 +1716,115 @@ dmu_objset_find_dp(dsl_pool_t *dp, uint64_t ddobj,
 	kmem_free(attr, sizeof (zap_attribute_t));
 
 	if (err != 0)
-		return (err);
+		goto out;
 
 	/*
 	 * Apply to self.
 	 */
 	err = dsl_dataset_hold_obj(dp, thisobj, FTAG, &ds);
 	if (err != 0)
-		return (err);
-	err = func(dp, ds, arg);
+		goto out;
+	err = dcp->dc_func(dp, ds, dcp->dc_arg);
 	dsl_dataset_rele(ds, FTAG);
-	return (err);
+
+out:
+	if (err != 0) {
+		mutex_enter(dcp->dc_error_lock);
+		/* only keep first error */
+		if (*dcp->dc_error == 0)
+			*dcp->dc_error = err;
+		mutex_exit(dcp->dc_error_lock);
+	}
+
+	kmem_free(dcp, sizeof (*dcp));
+}
+
+static void
+dmu_objset_find_dp_cb(void *arg)
+{
+	dmu_objset_find_ctx_t *dcp = arg;
+	dsl_pool_t *dp = dcp->dc_dp;
+
+	dsl_pool_config_enter(dp, FTAG);
+
+	dmu_objset_find_dp_impl(dcp);
+
+	dsl_pool_config_exit(dp, FTAG);
+}
+
+/*
+ * Find objsets under and including ddobj, call func(ds) on each.
+ * The order for the enumeration is completely undefined.
+ * func is called with dsl_pool_config held.
+ */
+int
+dmu_objset_find_dp(dsl_pool_t *dp, uint64_t ddobj,
+    int func(dsl_pool_t *, dsl_dataset_t *, void *), void *arg, int flags)
+{
+	int error = 0;
+	taskq_t *tq = NULL;
+	int ntasks;
+	dmu_objset_find_ctx_t *dcp;
+	kmutex_t err_lock;
+
+	mutex_init(&err_lock, NULL, MUTEX_DEFAULT, NULL);
+	dcp = kmem_alloc(sizeof (*dcp), KM_SLEEP);
+	dcp->dc_tq = NULL;
+	dcp->dc_dp = dp;
+	dcp->dc_ddobj = ddobj;
+	dcp->dc_func = func;
+	dcp->dc_arg = arg;
+	dcp->dc_flags = flags;
+	dcp->dc_error_lock = &err_lock;
+	dcp->dc_error = &error;
+
+	if ((flags & DS_FIND_SERIALIZE) || dsl_pool_config_held_writer(dp)) {
+		/*
+		 * In case a write lock is held we can't make use of
+		 * parallelism, as down the stack of the worker threads
+		 * the lock is asserted via dsl_pool_config_held.
+		 * In case of a read lock this is solved by getting a read
+		 * lock in each worker thread, which isn't possible in case
+		 * of a writer lock. So we fall back to the synchronous path
+		 * here.
+		 * In the future it might be possible to get some magic into
+		 * dsl_pool_config_held in a way that it returns true for
+		 * the worker threads so that a single lock held from this
+		 * thread suffices. For now, stay single threaded.
+		 */
+		dmu_objset_find_dp_impl(dcp);
+
+		return (error);
+	}
+
+	ntasks = dmu_find_threads;
+	if (ntasks == 0)
+		ntasks = vdev_count_leaves(dp->dp_spa) * 4;
+	tq = taskq_create("dmu_objset_find", ntasks, minclsyspri, ntasks,
+	    INT_MAX, 0);
+	if (tq == NULL) {
+		kmem_free(dcp, sizeof (*dcp));
+		return (SET_ERROR(ENOMEM));
+	}
+	dcp->dc_tq = tq;
+
+	/* dcp will be freed by task */
+	(void) taskq_dispatch(tq, dmu_objset_find_dp_cb, dcp, TQ_SLEEP);
+
+	/*
+	 * PORTING: this code relies on the property of taskq_wait to wait
+	 * until no more tasks are queued and no more tasks are active. As
+	 * we always queue new tasks from within other tasks, task_wait
+	 * reliably waits for the full recursion to finish, even though we
+	 * enqueue new tasks after taskq_wait has been called.
+	 * On platforms other than illumos, taskq_wait may not have this
+	 * property.
+	 */
+	taskq_wait(tq);
+	taskq_destroy(tq);
+	mutex_destroy(&err_lock);
+
+	return (error);
 }
 
 /*
diff --git a/uts/common/fs/zfs/dsl_pool.c b/uts/common/fs/zfs/dsl_pool.c
index 2e7f798485d5..8c5d820edea2 100644
--- a/uts/common/fs/zfs/dsl_pool.c
+++ b/uts/common/fs/zfs/dsl_pool.c
@@ -758,7 +758,7 @@ dsl_pool_upgrade_clones(dsl_pool_t *dp, dmu_tx_t *tx)
 	ASSERT(dp->dp_origin_snap != NULL);
 
 	VERIFY0(dmu_objset_find_dp(dp, dp->dp_root_dir_obj, upgrade_clones_cb,
-	    tx, DS_FIND_CHILDREN));
+	    tx, DS_FIND_CHILDREN | DS_FIND_SERIALIZE));
 }
 
 /* ARGSUSED */
@@ -812,7 +812,7 @@ dsl_pool_upgrade_dir_clones(dsl_pool_t *dp, dmu_tx_t *tx)
 	VERIFY0(bpobj_open(&dp->dp_free_bpobj, dp->dp_meta_objset, obj));
 
 	VERIFY0(dmu_objset_find_dp(dp, dp->dp_root_dir_obj,
-	    upgrade_dir_clones_cb, tx, DS_FIND_CHILDREN));
+	    upgrade_dir_clones_cb, tx, DS_FIND_CHILDREN | DS_FIND_SERIALIZE));
 }
 
 void
@@ -1056,3 +1056,9 @@ dsl_pool_config_held(dsl_pool_t *dp)
 {
 	return (RRW_LOCK_HELD(&dp->dp_config_rwlock));
 }
+
+boolean_t
+dsl_pool_config_held_writer(dsl_pool_t *dp)
+{
+	return (RRW_WRITE_HELD(&dp->dp_config_rwlock));
+}
diff --git a/uts/common/fs/zfs/spa.c b/uts/common/fs/zfs/spa.c
index 5bc2dbcb1cda..19e7aa0ef642 100644
--- a/uts/common/fs/zfs/spa.c
+++ b/uts/common/fs/zfs/spa.c
@@ -1735,13 +1735,14 @@ static boolean_t
 spa_check_logs(spa_t *spa)
 {
 	boolean_t rv = B_FALSE;
+	dsl_pool_t *dp = spa_get_dsl(spa);
 
 	switch (spa->spa_log_state) {
 	case SPA_LOG_MISSING:
 		/* need to recheck in case slog has been restored */
 	case SPA_LOG_UNKNOWN:
-		rv = (dmu_objset_find(spa->spa_name, zil_check_log_chain,
-		    NULL, DS_FIND_CHILDREN) != 0);
+		rv = (dmu_objset_find_dp(dp, dp->dp_root_dir_obj,
+		    zil_check_log_chain, NULL, DS_FIND_CHILDREN) != 0);
 		if (rv)
 			spa_set_log_state(spa, SPA_LOG_MISSING);
 		break;
@@ -2714,6 +2715,7 @@ spa_load_impl(spa_t *spa, uint64_t pool_guid, nvlist_t *config,
 	    spa->spa_load_max_txg == UINT64_MAX)) {
 		dmu_tx_t *tx;
 		int need_update = B_FALSE;
+		dsl_pool_t *dp = spa_get_dsl(spa);
 
 		ASSERT(state != SPA_LOAD_TRYIMPORT);
 
@@ -2726,9 +2728,8 @@ spa_load_impl(spa_t *spa, uint64_t pool_guid, nvlist_t *config,
 		 */
 		spa->spa_claiming = B_TRUE;
 
-		tx = dmu_tx_create_assigned(spa_get_dsl(spa),
-		    spa_first_txg(spa));
-		(void) dmu_objset_find(spa_name(spa),
+		tx = dmu_tx_create_assigned(dp, spa_first_txg(spa));
+		(void) dmu_objset_find_dp(dp, dp->dp_root_dir_obj,
 		    zil_claim, tx, DS_FIND_CHILDREN);
 		dmu_tx_commit(tx);
 
diff --git a/uts/common/fs/zfs/sys/dmu.h b/uts/common/fs/zfs/sys/dmu.h
index 9b8b9ddc2016..2787be4c8090 100644
--- a/uts/common/fs/zfs/sys/dmu.h
+++ b/uts/common/fs/zfs/sys/dmu.h
@@ -243,6 +243,7 @@ void zfs_znode_byteswap(void *buf, size_t size);
 
 #define	DS_FIND_SNAPSHOTS	(1<<0)
 #define	DS_FIND_CHILDREN	(1<<1)
+#define	DS_FIND_SERIALIZE	(1<<2)
 
 /*
  * The maximum number of bytes that can be accessed as part of one
diff --git a/uts/common/fs/zfs/sys/dmu_objset.h b/uts/common/fs/zfs/sys/dmu_objset.h
index e00847e6cbba..9e98350f001f 100644
--- a/uts/common/fs/zfs/sys/dmu_objset.h
+++ b/uts/common/fs/zfs/sys/dmu_objset.h
@@ -142,6 +142,8 @@ struct objset {
 int dmu_objset_hold(const char *name, void *tag, objset_t **osp);
 int dmu_objset_own(const char *name, dmu_objset_type_t type,
     boolean_t readonly, void *tag, objset_t **osp);
+int dmu_objset_own_obj(struct dsl_pool *dp, uint64_t obj,
+    dmu_objset_type_t type, boolean_t readonly, void *tag, objset_t **osp);
 void dmu_objset_refresh_ownership(objset_t *os, void *tag);
 void dmu_objset_rele(objset_t *os, void *tag);
 void dmu_objset_disown(objset_t *os, void *tag);
diff --git a/uts/common/fs/zfs/sys/dsl_pool.h b/uts/common/fs/zfs/sys/dsl_pool.h
index a6fb201047bb..0f6d17514e45 100644
--- a/uts/common/fs/zfs/sys/dsl_pool.h
+++ b/uts/common/fs/zfs/sys/dsl_pool.h
@@ -154,6 +154,7 @@ void dsl_pool_mos_diduse_space(dsl_pool_t *dp,
 void dsl_pool_config_enter(dsl_pool_t *dp, void *tag);
 void dsl_pool_config_exit(dsl_pool_t *dp, void *tag);
 boolean_t dsl_pool_config_held(dsl_pool_t *dp);
+boolean_t dsl_pool_config_held_writer(dsl_pool_t *dp);
 boolean_t dsl_pool_need_dirty_delay(dsl_pool_t *dp);
 
 taskq_t *dsl_pool_vnrele_taskq(dsl_pool_t *dp);
diff --git a/uts/common/fs/zfs/sys/vdev.h b/uts/common/fs/zfs/sys/vdev.h
index 043ba6dd3bfb..08ce54b18c78 100644
--- a/uts/common/fs/zfs/sys/vdev.h
+++ b/uts/common/fs/zfs/sys/vdev.h
@@ -60,6 +60,7 @@ extern zio_t *vdev_probe(vdev_t *vd, zio_t *pio);
 extern boolean_t vdev_is_bootable(vdev_t *vd);
 extern vdev_t *vdev_lookup_top(spa_t *spa, uint64_t vdev);
 extern vdev_t *vdev_lookup_by_guid(vdev_t *vd, uint64_t guid);
+extern int vdev_count_leaves(spa_t *spa);
 extern void vdev_dtl_dirty(vdev_t *vd, vdev_dtl_type_t d,
     uint64_t txg, uint64_t size);
 extern boolean_t vdev_dtl_contains(vdev_t *vd, vdev_dtl_type_t d,
diff --git a/uts/common/fs/zfs/sys/zil.h b/uts/common/fs/zfs/sys/zil.h
index 895d632a2620..d3fe6e9653d2 100644
--- a/uts/common/fs/zfs/sys/zil.h
+++ b/uts/common/fs/zfs/sys/zil.h
@@ -37,6 +37,9 @@
 extern "C" {
 #endif
 
+struct dsl_pool;
+struct dsl_dataset;
+
 /*
  * Intent log format:
  *
@@ -404,8 +407,10 @@ extern void	zil_itx_assign(zilog_t *zilog, itx_t *itx, dmu_tx_t *tx);
 extern void	zil_commit(zilog_t *zilog, uint64_t oid);
 
 extern int	zil_vdev_offline(const char *osname, void *txarg);
-extern int	zil_claim(const char *osname, void *txarg);
-extern int	zil_check_log_chain(const char *osname, void *txarg);
+extern int	zil_claim(struct dsl_pool *dp,
+    struct dsl_dataset *ds, void *txarg);
+extern int 	zil_check_log_chain(struct dsl_pool *dp,
+    struct dsl_dataset *ds, void *tx);
 extern void	zil_sync(zilog_t *zilog, dmu_tx_t *tx);
 extern void	zil_clean(zilog_t *zilog, uint64_t synced_txg);
 
diff --git a/uts/common/fs/zfs/vdev.c b/uts/common/fs/zfs/vdev.c
index a2e2b36c6836..3d53bf460b0f 100644
--- a/uts/common/fs/zfs/vdev.c
+++ b/uts/common/fs/zfs/vdev.c
@@ -178,6 +178,26 @@ vdev_lookup_by_guid(vdev_t *vd, uint64_t guid)
 	return (NULL);
 }
 
+static int
+vdev_count_leaves_impl(vdev_t *vd)
+{
+	int n = 0;
+
+	if (vd->vdev_ops->vdev_op_leaf)
+		return (1);
+
+	for (int c = 0; c < vd->vdev_children; c++)
+		n += vdev_count_leaves_impl(vd->vdev_child[c]);
+
+	return (n);
+}
+
+int
+vdev_count_leaves(spa_t *spa)
+{
+	return (vdev_count_leaves_impl(spa->spa_root_vdev));
+}
+
 void
 vdev_add_child(vdev_t *pvd, vdev_t *cvd)
 {
diff --git a/uts/common/fs/zfs/zil.c b/uts/common/fs/zfs/zil.c
index f48828dc9067..38b57e012346 100644
--- a/uts/common/fs/zfs/zil.c
+++ b/uts/common/fs/zfs/zil.c
@@ -627,7 +627,7 @@ zil_destroy_sync(zilog_t *zilog, dmu_tx_t *tx)
 }
 
 int
-zil_claim(const char *osname, void *txarg)
+zil_claim(dsl_pool_t *dp, dsl_dataset_t *ds, void *txarg)
 {
 	dmu_tx_t *tx = txarg;
 	uint64_t first_txg = dmu_tx_get_txg(tx);
@@ -636,15 +636,16 @@ zil_claim(const char *osname, void *txarg)
 	objset_t *os;
 	int error;
 
-	error = dmu_objset_own(osname, DMU_OST_ANY, B_FALSE, FTAG, &os);
+	error = dmu_objset_own_obj(dp, ds->ds_object,
+	    DMU_OST_ANY, B_FALSE, FTAG, &os);
 	if (error != 0) {
 		/*
 		 * EBUSY indicates that the objset is inconsistent, in which
 		 * case it can not have a ZIL.
 		 */
 		if (error != EBUSY) {
-			cmn_err(CE_WARN, "can't open objset for %s, error %u",
-			    osname, error);
+			cmn_err(CE_WARN, "can't open objset for %llu, error %u",
+			    (unsigned long long)ds->ds_object, error);
 		}
 		return (0);
 	}
@@ -691,8 +692,9 @@ zil_claim(const char *osname, void *txarg)
  * Checksum errors are ok as they indicate the end of the chain.
  * Any other error (no device or read failure) returns an error.
  */
+/* ARGSUSED */
 int
-zil_check_log_chain(const char *osname, void *tx)
+zil_check_log_chain(dsl_pool_t *dp, dsl_dataset_t *ds, void *tx)
 {
 	zilog_t *zilog;
 	objset_t *os;
@@ -701,9 +703,10 @@ zil_check_log_chain(const char *osname, void *tx)
 
 	ASSERT(tx == NULL);
 
-	error = dmu_objset_hold(osname, FTAG, &os);
+	error = dmu_objset_from_ds(ds, &os);
 	if (error != 0) {
-		cmn_err(CE_WARN, "can't open objset for %s", osname);
+		cmn_err(CE_WARN, "can't open objset %llu, error %d",
+		    (unsigned long long)ds->ds_object, error);
 		return (0);
 	}
 
@@ -726,10 +729,8 @@ zil_check_log_chain(const char *osname, void *tx)
 			valid = vdev_log_state_valid(vd);
 		spa_config_exit(os->os_spa, SCL_STATE, FTAG);
 
-		if (!valid) {
-			dmu_objset_rele(os, FTAG);
+		if (!valid)
 			return (0);
-		}
 	}
 
 	/*
@@ -742,8 +743,6 @@ zil_check_log_chain(const char *osname, void *tx)
 	error = zil_parse(zilog, zil_claim_log_block, zil_claim_log_record, tx,
 	    zilog->zl_header->zh_claim_txg ? -1ULL : spa_first_txg(os->os_spa));
 
-	dmu_objset_rele(os, FTAG);
-
 	return ((error == ECKSUM || error == ENOENT) ? 0 : error);
 }
 

From 79225dfecccf8e56243b76c177d4e4b15f62aaa6 Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Wed, 24 Jun 2015 14:49:49 +0000
Subject: [PATCH 096/380] 5981 Deadlock in dmu_objset_find_dp

illumos/illumos-gate@1d3f896f5469c69c1339890ec3d68e9feddb0343

https://www.illumos.org/issues/5981
  When dmu_objset_find_dp gets called with a read lock held, it fans out
  the work to the task queue. Each task in turn acquires its own read
  lock before calling the callback. If during this process anyone tries
  to a acquire a write lock, it will stall all read lock requests.Thus
  the tasks will never finish, the read lock of the caller will never
  get freed and the write lock never acquired.  deadlock.

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Dan McDonald <danmcd@omniti.com>
Approved by: Robert Mustacchi <rm@joyent.com>
Author: Arne Jansen <jansen@webgods.de>
---
 uts/common/fs/zfs/dmu_objset.c   | 10 +++++++++-
 uts/common/fs/zfs/dsl_pool.c     |  7 +++++++
 uts/common/fs/zfs/rrwlock.c      | 25 ++++++++++++++++++++++---
 uts/common/fs/zfs/sys/dsl_pool.h |  1 +
 uts/common/fs/zfs/sys/rrwlock.h  |  1 +
 5 files changed, 40 insertions(+), 4 deletions(-)

diff --git a/uts/common/fs/zfs/dmu_objset.c b/uts/common/fs/zfs/dmu_objset.c
index cfe9e90f1b2f..ce3d163b1e18 100644
--- a/uts/common/fs/zfs/dmu_objset.c
+++ b/uts/common/fs/zfs/dmu_objset.c
@@ -1745,7 +1745,15 @@ dmu_objset_find_dp_cb(void *arg)
 	dmu_objset_find_ctx_t *dcp = arg;
 	dsl_pool_t *dp = dcp->dc_dp;
 
-	dsl_pool_config_enter(dp, FTAG);
+	/*
+	 * We need to get a pool_config_lock here, as there are several
+	 * asssert(pool_config_held) down the stack. Getting a lock via
+	 * dsl_pool_config_enter is risky, as it might be stalled by a
+	 * pending writer. This would deadlock, as the write lock can
+	 * only be granted when our parent thread gives up the lock.
+	 * The _prio interface gives us priority over a pending writer.
+	 */
+	dsl_pool_config_enter_prio(dp, FTAG);
 
 	dmu_objset_find_dp_impl(dcp);
 
diff --git a/uts/common/fs/zfs/dsl_pool.c b/uts/common/fs/zfs/dsl_pool.c
index 8c5d820edea2..ddd8e7344773 100644
--- a/uts/common/fs/zfs/dsl_pool.c
+++ b/uts/common/fs/zfs/dsl_pool.c
@@ -1045,6 +1045,13 @@ dsl_pool_config_enter(dsl_pool_t *dp, void *tag)
 	rrw_enter(&dp->dp_config_rwlock, RW_READER, tag);
 }
 
+void
+dsl_pool_config_enter_prio(dsl_pool_t *dp, void *tag)
+{
+	ASSERT(!rrw_held(&dp->dp_config_rwlock, RW_READER));
+	rrw_enter_read_prio(&dp->dp_config_rwlock, tag);
+}
+
 void
 dsl_pool_config_exit(dsl_pool_t *dp, void *tag)
 {
diff --git a/uts/common/fs/zfs/rrwlock.c b/uts/common/fs/zfs/rrwlock.c
index e3aa01a31c77..51394c01c431 100644
--- a/uts/common/fs/zfs/rrwlock.c
+++ b/uts/common/fs/zfs/rrwlock.c
@@ -159,8 +159,8 @@ rrw_destroy(rrwlock_t *rrl)
 	refcount_destroy(&rrl->rr_linked_rcount);
 }
 
-void
-rrw_enter_read(rrwlock_t *rrl, void *tag)
+static void
+rrw_enter_read_impl(rrwlock_t *rrl, boolean_t prio, void *tag)
 {
 	mutex_enter(&rrl->rr_lock);
 #if !defined(DEBUG) && defined(_KERNEL)
@@ -176,7 +176,7 @@ rrw_enter_read(rrwlock_t *rrl, void *tag)
 	ASSERT(refcount_count(&rrl->rr_anon_rcount) >= 0);
 
 	while (rrl->rr_writer != NULL || (rrl->rr_writer_wanted &&
-	    refcount_is_zero(&rrl->rr_anon_rcount) &&
+	    refcount_is_zero(&rrl->rr_anon_rcount) && !prio &&
 	    rrn_find(rrl) == NULL))
 		cv_wait(&rrl->rr_cv, &rrl->rr_lock);
 
@@ -191,6 +191,25 @@ rrw_enter_read(rrwlock_t *rrl, void *tag)
 	mutex_exit(&rrl->rr_lock);
 }
 
+void
+rrw_enter_read(rrwlock_t *rrl, void *tag)
+{
+	rrw_enter_read_impl(rrl, B_FALSE, tag);
+}
+
+/*
+ * take a read lock even if there are pending write lock requests. if we want
+ * to take a lock reentrantly, but from different threads (that have a
+ * relationship to each other), the normal detection mechanism to overrule
+ * the pending writer does not work, so we have to give an explicit hint here.
+ */
+void
+rrw_enter_read_prio(rrwlock_t *rrl, void *tag)
+{
+	rrw_enter_read_impl(rrl, B_TRUE, tag);
+}
+
+
 void
 rrw_enter_write(rrwlock_t *rrl)
 {
diff --git a/uts/common/fs/zfs/sys/dsl_pool.h b/uts/common/fs/zfs/sys/dsl_pool.h
index 0f6d17514e45..0e27a538208c 100644
--- a/uts/common/fs/zfs/sys/dsl_pool.h
+++ b/uts/common/fs/zfs/sys/dsl_pool.h
@@ -152,6 +152,7 @@ void dsl_pool_upgrade_dir_clones(dsl_pool_t *dp, dmu_tx_t *tx);
 void dsl_pool_mos_diduse_space(dsl_pool_t *dp,
     int64_t used, int64_t comp, int64_t uncomp);
 void dsl_pool_config_enter(dsl_pool_t *dp, void *tag);
+void dsl_pool_config_enter_prio(dsl_pool_t *dp, void *tag);
 void dsl_pool_config_exit(dsl_pool_t *dp, void *tag);
 boolean_t dsl_pool_config_held(dsl_pool_t *dp);
 boolean_t dsl_pool_config_held_writer(dsl_pool_t *dp);
diff --git a/uts/common/fs/zfs/sys/rrwlock.h b/uts/common/fs/zfs/sys/rrwlock.h
index bddf0768ae07..947b51f8728f 100644
--- a/uts/common/fs/zfs/sys/rrwlock.h
+++ b/uts/common/fs/zfs/sys/rrwlock.h
@@ -70,6 +70,7 @@ void rrw_init(rrwlock_t *rrl, boolean_t track_all);
 void rrw_destroy(rrwlock_t *rrl);
 void rrw_enter(rrwlock_t *rrl, krw_t rw, void *tag);
 void rrw_enter_read(rrwlock_t *rrl, void *tag);
+void rrw_enter_read_prio(rrwlock_t *rrl, void *tag);
 void rrw_enter_write(rrwlock_t *rrl);
 void rrw_exit(rrwlock_t *rrl, void *tag);
 boolean_t rrw_held(rrwlock_t *rrl, krw_t rw);

From a279542d9a11d1264bf2cc49cac2472ae0c1fc2d Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Thu, 2 Jul 2015 11:01:27 +0000
Subject: [PATCH 097/380] 6033 arc_adjust() should search MFU lists for oldest
 buffer when adjusting MFU size

illumos/illumos-gate@31c46cf23cd1cf4d66390a983dc5072d7d299ba2

https://www.illumos.org/issues/6033
  When we're looking for the list containing oldest buffer we never actually look
  at the MFU lists even when we try to evict from MFU.
  looks like a copy paste error, the fix is here:

Reviewed by: Saso Kiselkov <saso.kiselkov@nexenta.com>
Reviewed by: Xin Li <delphij@delphij.net>
Reviewed by: Prakash Surya <me@prakashsurya.com>
Approved by: Matthew Ahrens <mahrens@delphix.com>
Author: Alek Pinchuk <alek@nexenta.com>
Obtained from:	illumos
---
 uts/common/fs/zfs/arc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index a227c424708c..ea8bc599ce84 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -23,7 +23,7 @@
  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2014 by Saso Kiselkov. All rights reserved.
- * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
  */
 
 /*
@@ -2850,7 +2850,7 @@ arc_adjust(void)
 	 */
 	target = arc_size - arc_c;
 
-	if (arc_adjust_type(arc_mru) == ARC_BUFC_METADATA &&
+	if (arc_adjust_type(arc_mfu) == ARC_BUFC_METADATA &&
 	    arc_meta_used > arc_meta_min) {
 		bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
 		total_evicted += bytes;

From 823f826ddd1301e2ed5bd22a4d3f6b5a38061c36 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Thu, 30 Jul 2015 05:13:12 +0000
Subject: [PATCH 098/380] Revert local change accidentally committed with
 r285905

---
 usr.sbin/syslogd/syslogd.c | 31 ++++++++-----------------------
 1 file changed, 8 insertions(+), 23 deletions(-)

diff --git a/usr.sbin/syslogd/syslogd.c b/usr.sbin/syslogd/syslogd.c
index 14cb75f3892a..ff3e5249fd4c 100644
--- a/usr.sbin/syslogd/syslogd.c
+++ b/usr.sbin/syslogd/syslogd.c
@@ -349,22 +349,6 @@ static int	waitdaemon(int, int, int);
 static void	timedout(int);
 static void	increase_rcvbuf(int);
 
-static void
-close_filed(struct filed *f)
-{
-	int saved_errno;
-
-	if (f == NULL || f->f_file == -1)
-		return;
-
-	saved_errno = errno;
-
-	close(f->f_file);
-	f->f_file = -1;
-	f->f_type = F_UNUSED;
-	errno = saved_errno;
-}
-
 int
 main(int argc, char *argv[])
 {
@@ -1041,7 +1025,6 @@ logmsg(int pri, const char *msg, const char *from, int flags)
 				sizeof(f->f_lasttime));
 			fprintlog(f, flags, msg);
 			(void)close(f->f_file);
-			f->f_file = -1;
 		}
 		(void)sigsetmask(omask);
 		return;
@@ -1330,7 +1313,8 @@ fprintlog(struct filed *f, int flags, const char *msg)
 			 */
 			if (errno != ENOSPC) {
 				int e = errno;
-				close_filed(f);
+				(void)close(f->f_file);
+				f->f_type = F_UNUSED;
 				errno = e;
 				logerror(f->f_un.f_fname);
 			}
@@ -1353,11 +1337,13 @@ fprintlog(struct filed *f, int flags, const char *msg)
 			}
 		}
 		if (writev(f->f_file, iov, IOV_SIZE) < 0) {
-			close_filed(f);
+			int e = errno;
+			(void)close(f->f_file);
 			if (f->f_un.f_pipe.f_pid > 0)
 				deadq_enter(f->f_un.f_pipe.f_pid,
 					    f->f_un.f_pipe.f_pname);
 			f->f_un.f_pipe.f_pid = 0;
+			errno = e;
 			logerror(f->f_un.f_pipe.f_pname);
 		}
 		break;
@@ -1460,7 +1446,7 @@ reapchild(int signo __unused)
 		for (f = Files; f; f = f->f_next)
 			if (f->f_type == F_PIPE &&
 			    f->f_un.f_pipe.f_pid == pid) {
-				close_filed(f);
+				(void)close(f->f_file);
 				f->f_un.f_pipe.f_pid = 0;
 				log_deadchild(pid, status,
 					      f->f_un.f_pipe.f_pname);
@@ -1564,7 +1550,7 @@ die(int signo)
 		if (f->f_prevcount)
 			fprintlog(f, 0, (char *)NULL);
 		if (f->f_type == F_PIPE && f->f_un.f_pipe.f_pid > 0) {
-			close_filed(f);
+			(void)close(f->f_file);
 			f->f_un.f_pipe.f_pid = 0;
 		}
 	}
@@ -1649,11 +1635,10 @@ init(int signo)
 		case F_CONSOLE:
 		case F_TTY:
 			(void)close(f->f_file);
-			f->f_file = -1;
 			break;
 		case F_PIPE:
 			if (f->f_un.f_pipe.f_pid > 0) {
-				close_filed(f);
+				(void)close(f->f_file);
 				deadq_enter(f->f_un.f_pipe.f_pid,
 					    f->f_un.f_pipe.f_pname);
 			}

From 0e953a7368ba8667ac64805fe886cfd1d0ca380e Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Mon, 3 Aug 2015 10:10:49 +0000
Subject: [PATCH 099/380] 5695 dmu_sync'ed holes do not retain birth time

illumos/illumos-gate@70163ac57e58ace1c5c94dfbe85dca5a974eff36

https://www.illumos.org/issues/5695
  In dmu_sync_ready(), a hole block pointer will have it's logical size
  explicitly set as it's necessary for replay purposes. To "undo" this,
  dmu_sync_done() will zero out any hole that it finds. This becomes a
  problem when using the "hole_birth" feature, as this will also wipe out
  any birth time that might have happened to be set on the hole.
  ...
  As a fix, the logic to zero out a hole is only applied to old style
  holes with a birth time of zero. Holes created with the "hole_birth"
  feature enabled will have a non-zero birth time, and will be skipped
  (thus preserving the ltime, type, and level information as well).
  In addition, zdb was updated to also print the ltime, type, and level
  information for these new style holes. Previously, only the logical
  birth time would be printed.

Author: Prakash Surya <prakash.surya@delphix.com>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: George Wilson <george@delphix.com>
Reviewed by: Christopher Siden <christopher.siden@delphix.com>
Reviewed by: Bayard Bell <buffer.g.overflow@gmail.com>
Approved by: Dan McDonald <danmcd@omniti.com>
---
 uts/common/fs/zfs/dmu.c     | 14 +++++++++++++-
 uts/common/fs/zfs/sys/spa.h | 13 +++++++------
 2 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/uts/common/fs/zfs/dmu.c b/uts/common/fs/zfs/dmu.c
index c541e04d5429..218f2d0b0681 100644
--- a/uts/common/fs/zfs/dmu.c
+++ b/uts/common/fs/zfs/dmu.c
@@ -1416,7 +1416,19 @@ dmu_sync_done(zio_t *zio, arc_buf_t *buf, void *varg)
 		dr->dt.dl.dr_overridden_by = *zio->io_bp;
 		dr->dt.dl.dr_override_state = DR_OVERRIDDEN;
 		dr->dt.dl.dr_copies = zio->io_prop.zp_copies;
-		if (BP_IS_HOLE(&dr->dt.dl.dr_overridden_by))
+
+		/*
+		 * Old style holes are filled with all zeros, whereas
+		 * new-style holes maintain their lsize, type, level,
+		 * and birth time (see zio_write_compress). While we
+		 * need to reset the BP_SET_LSIZE() call that happened
+		 * in dmu_sync_ready for old style holes, we do *not*
+		 * want to wipe out the information contained in new
+		 * style holes. Thus, only zero out the block pointer if
+		 * it's an old style hole.
+		 */
+		if (BP_IS_HOLE(&dr->dt.dl.dr_overridden_by) &&
+		    dr->dt.dl.dr_overridden_by.blk_birth == 0)
 			BP_ZERO(&dr->dt.dl.dr_overridden_by);
 	} else {
 		dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
diff --git a/uts/common/fs/zfs/sys/spa.h b/uts/common/fs/zfs/sys/spa.h
index 4dd6ce69af91..d5a8e7f2fd78 100644
--- a/uts/common/fs/zfs/sys/spa.h
+++ b/uts/common/fs/zfs/sys/spa.h
@@ -507,12 +507,13 @@ _NOTE(CONSTCOND) } while (0)
 	if (bp == NULL) {						\
 		len += func(buf + len, size - len, "<NULL>");		\
 	} else if (BP_IS_HOLE(bp)) {					\
-		len += func(buf + len, size - len, "<hole>");		\
-		if (bp->blk_birth > 0) {				\
-			len += func(buf + len, size - len,		\
-			    " birth=%lluL",				\
-			    (u_longlong_t)bp->blk_birth);		\
-		}							\
+		len += func(buf + len, size - len,			\
+		    "HOLE [L%llu %s] "					\
+		    "size=%llxL birth=%lluL",				\
+		    (u_longlong_t)BP_GET_LEVEL(bp),			\
+		    type,						\
+		    (u_longlong_t)BP_GET_LSIZE(bp),			\
+		    (u_longlong_t)bp->blk_birth);			\
 	} else if (BP_IS_EMBEDDED(bp)) {				\
 		len = func(buf + len, size - len,			\
 		    "EMBEDDED [L%llu %s] et=%u %s "			\

From bd7c1b51ac6947d69892aa1da169d674f88cf47b Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Sun, 9 Aug 2015 19:12:43 +0000
Subject: [PATCH 100/380] 5562 ZFS sa_handle's violate kmem invariants, debug
 kernels panic on boot

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Robert Mustacchi <rm@fingolfin.org>
Reviewed by: George Wilson <george@delphix.com>
Reviewed by: Rich Lowe <richlowe@richlowe.net>
Approved by: Dan McDonald <danmcd@omniti.com>
Author: Justin T. Gibbs <justing@spectralogic.com>

illumos/illumos-gate@0fda3cc5c1c5a1d9bdea6d52637bef6e781549c9
---
 uts/common/fs/zfs/sa.c | 20 ++++++--------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/uts/common/fs/zfs/sa.c b/uts/common/fs/zfs/sa.c
index 2e3156c00a4d..0b5b37f5fba4 100644
--- a/uts/common/fs/zfs/sa.c
+++ b/uts/common/fs/zfs/sa.c
@@ -211,13 +211,6 @@ sa_cache_constructor(void *buf, void *unused, int kmflag)
 {
 	sa_handle_t *hdl = buf;
 
-	hdl->sa_dbu.dbu_evict_func = NULL;
-	hdl->sa_bonus_tab = NULL;
-	hdl->sa_spill_tab = NULL;
-	hdl->sa_os = NULL;
-	hdl->sa_userp = NULL;
-	hdl->sa_bonus = NULL;
-	hdl->sa_spill = NULL;
 	mutex_init(&hdl->sa_lock, NULL, MUTEX_DEFAULT, NULL);
 	return (0);
 }
@@ -227,7 +220,6 @@ static void
 sa_cache_destructor(void *buf, void *unused)
 {
 	sa_handle_t *hdl = buf;
-	hdl->sa_dbu.dbu_evict_func = NULL;
 	mutex_destroy(&hdl->sa_lock);
 }
 
@@ -1351,14 +1343,11 @@ sa_handle_destroy(sa_handle_t *hdl)
 	mutex_enter(&hdl->sa_lock);
 	(void) dmu_buf_remove_user(db, &hdl->sa_dbu);
 
-	if (hdl->sa_bonus_tab) {
+	if (hdl->sa_bonus_tab)
 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_bonus_tab);
-		hdl->sa_bonus_tab = NULL;
-	}
-	if (hdl->sa_spill_tab) {
+
+	if (hdl->sa_spill_tab)
 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_spill_tab);
-		hdl->sa_spill_tab = NULL;
-	}
 
 	dmu_buf_rele(hdl->sa_bonus, NULL);
 
@@ -1392,10 +1381,13 @@ sa_handle_get_from_db(objset_t *os, dmu_buf_t *db, void *userp,
 		sa_handle_t *winner = NULL;
 
 		handle = kmem_cache_alloc(sa_cache, KM_SLEEP);
+		handle->sa_dbu.dbu_evict_func = NULL;
 		handle->sa_userp = userp;
 		handle->sa_bonus = db;
 		handle->sa_os = os;
 		handle->sa_spill = NULL;
+		handle->sa_bonus_tab = NULL;
+		handle->sa_spill_tab = NULL;
 
 		error = sa_build_index(handle, SA_BONUS);
 

From 476aedc3b7ea314cd38f434875b6684439bee099 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Sun, 9 Aug 2015 19:25:40 +0000
Subject: [PATCH 101/380] 5531 NULL pointer dereference in dsl_prop_get_ds()

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Dan McDonald <danmcd@omniti.com>
Reviewed by: George Wilson <george@delphix.com>
Reviewed by: Bayard Bell <buffer.g.overflow@gmail.com>
Approved by: Robert Mustacchi <rm@joyent.com>
Author: Justin T. Gibbs <justing@spectralogic.com>

illumos/illumos-gate@e57a022b8f718889ffa92adbde47a8f08abcdb25
---
 uts/common/fs/zfs/dbuf.c            | 50 ++++++++++++++++++++++++++---
 uts/common/fs/zfs/dnode_sync.c      |  3 +-
 uts/common/fs/zfs/dsl_dataset.c     |  7 ++++
 uts/common/fs/zfs/dsl_prop.c        | 47 ++++++++++++++++++++++-----
 uts/common/fs/zfs/sys/dbuf.h        |  5 ++-
 uts/common/fs/zfs/sys/dmu.h         | 16 +++++++++
 uts/common/fs/zfs/sys/dsl_dataset.h |  2 ++
 7 files changed, 115 insertions(+), 15 deletions(-)

diff --git a/uts/common/fs/zfs/dbuf.c b/uts/common/fs/zfs/dbuf.c
index 57ff05c91f39..621bb0fae5c8 100644
--- a/uts/common/fs/zfs/dbuf.c
+++ b/uts/common/fs/zfs/dbuf.c
@@ -125,11 +125,9 @@ dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid)
 	(dbuf)->db_blkid == (blkid))
 
 dmu_buf_impl_t *
-dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid)
+dbuf_find(objset_t *os, uint64_t obj, uint8_t level, uint64_t blkid)
 {
 	dbuf_hash_table_t *h = &dbuf_hash_table;
-	objset_t *os = dn->dn_objset;
-	uint64_t obj = dn->dn_object;
 	uint64_t hv = DBUF_HASH(os, obj, level, blkid);
 	uint64_t idx = hv & h->hash_table_mask;
 	dmu_buf_impl_t *db;
@@ -149,6 +147,24 @@ dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid)
 	return (NULL);
 }
 
+static dmu_buf_impl_t *
+dbuf_find_bonus(objset_t *os, uint64_t object)
+{
+	dnode_t *dn;
+	dmu_buf_impl_t *db = NULL;
+
+	if (dnode_hold(os, object, FTAG, &dn) == 0) {
+		rw_enter(&dn->dn_struct_rwlock, RW_READER);
+		if (dn->dn_bonus != NULL) {
+			db = dn->dn_bonus;
+			mutex_enter(&db->db_mtx);
+		}
+		rw_exit(&dn->dn_struct_rwlock);
+		dnode_rele(dn, FTAG);
+	}
+	return (db);
+}
+
 /*
  * Insert an entry into the hash table.  If there is already an element
  * equal to elem in the hash table, then the already existing element
@@ -1927,7 +1943,7 @@ dbuf_prefetch(dnode_t *dn, uint64_t blkid, zio_priority_t prio)
 		return;
 
 	/* dbuf_find() returns with db_mtx held */
-	if (db = dbuf_find(dn, 0, blkid)) {
+	if (db = dbuf_find(dn->dn_objset, dn->dn_object, 0, blkid)) {
 		/*
 		 * This dbuf is already in the cache.  We assume that
 		 * it is already CACHED, or else about to be either
@@ -1974,7 +1990,7 @@ dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse,
 	*dbp = NULL;
 top:
 	/* dbuf_find() returns with db_mtx held */
-	db = dbuf_find(dn, level, blkid);
+	db = dbuf_find(dn->dn_objset, dn->dn_object, level, blkid);
 
 	if (db == NULL) {
 		blkptr_t *bp = NULL;
@@ -2110,6 +2126,30 @@ dbuf_add_ref(dmu_buf_impl_t *db, void *tag)
 	ASSERT(holds > 1);
 }
 
+#pragma weak dmu_buf_try_add_ref = dbuf_try_add_ref
+boolean_t
+dbuf_try_add_ref(dmu_buf_t *db_fake, objset_t *os, uint64_t obj, uint64_t blkid,
+    void *tag)
+{
+	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
+	dmu_buf_impl_t *found_db;
+	boolean_t result = B_FALSE;
+
+	if (db->db_blkid == DMU_BONUS_BLKID)
+		found_db = dbuf_find_bonus(os, obj);
+	else
+		found_db = dbuf_find(os, obj, 0, blkid);
+
+	if (found_db != NULL) {
+		if (db == found_db && dbuf_refcount(db) > db->db_dirtycnt) {
+			(void) refcount_add(&db->db_holds, tag);
+			result = B_TRUE;
+		}
+		mutex_exit(&db->db_mtx);
+	}
+	return (result);
+}
+
 /*
  * If you call dbuf_rele() you had better not be referencing the dnode handle
  * unless you have some other direct or indirect hold on the dnode. (An indirect
diff --git a/uts/common/fs/zfs/dnode_sync.c b/uts/common/fs/zfs/dnode_sync.c
index 7be35f48d122..d95bac99390b 100644
--- a/uts/common/fs/zfs/dnode_sync.c
+++ b/uts/common/fs/zfs/dnode_sync.c
@@ -77,7 +77,8 @@ dnode_increase_indirection(dnode_t *dn, dmu_tx_t *tx)
 
 	/* set dbuf's parent pointers to new indirect buf */
 	for (i = 0; i < nblkptr; i++) {
-		dmu_buf_impl_t *child = dbuf_find(dn, old_toplvl, i);
+		dmu_buf_impl_t *child =
+		    dbuf_find(dn->dn_objset, dn->dn_object, old_toplvl, i);
 
 		if (child == NULL)
 			continue;
diff --git a/uts/common/fs/zfs/dsl_dataset.c b/uts/common/fs/zfs/dsl_dataset.c
index 5a9acd2f0bb6..bc9a346d5820 100644
--- a/uts/common/fs/zfs/dsl_dataset.c
+++ b/uts/common/fs/zfs/dsl_dataset.c
@@ -360,6 +360,13 @@ dsl_dataset_snap_remove(dsl_dataset_t *ds, const char *name, dmu_tx_t *tx,
 	return (err);
 }
 
+boolean_t
+dsl_dataset_try_add_ref(dsl_pool_t *dp, dsl_dataset_t *ds, void *tag)
+{
+	return (dmu_buf_try_add_ref(ds->ds_dbuf, dp->dp_meta_objset,
+	    ds->ds_object, DMU_BONUS_BLKID, tag));
+}
+
 int
 dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
     dsl_dataset_t **dsp)
diff --git a/uts/common/fs/zfs/dsl_prop.c b/uts/common/fs/zfs/dsl_prop.c
index e795d735c6f0..09eec21604f4 100644
--- a/uts/common/fs/zfs/dsl_prop.c
+++ b/uts/common/fs/zfs/dsl_prop.c
@@ -441,9 +441,31 @@ dsl_prop_notify_all_cb(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg)
 	    cbr = list_next(&dd->dd_prop_cbs, cbr)) {
 		uint64_t value;
 
+		/*
+		 * Callback entries do not have holds on their datasets
+		 * so that datasets with registered callbacks are still
+		 * eligible for eviction.  Unlike operations on callbacks
+		 * for a single dataset, we are performing a recursive
+		 * descent of related datasets and the calling context
+		 * for this iteration only has a dataset hold on the root.
+		 * Without a hold, the callback's pointer to the dataset
+		 * could be invalidated by eviction at any time.
+		 *
+		 * Use dsl_dataset_try_add_ref() to verify that the
+		 * dataset has not begun eviction processing and to
+		 * prevent eviction from occurring for the duration
+		 * of the callback.  If the hold attempt fails, this
+		 * object is already being evicted and the callback can
+		 * be safely ignored.
+		 */
+		if (!dsl_dataset_try_add_ref(dp, cbr->cbr_ds, FTAG))
+			continue;
+
 		if (dsl_prop_get_ds(cbr->cbr_ds, cbr->cbr_propname,
 		    sizeof (value), 1, &value, NULL) == 0)
 			cbr->cbr_func(cbr->cbr_arg, value);
+
+		dsl_dataset_rele(cbr->cbr_ds, FTAG);
 	}
 	mutex_exit(&dd->dd_lock);
 
@@ -496,19 +518,28 @@ dsl_prop_changed_notify(dsl_pool_t *dp, uint64_t ddobj,
 	mutex_enter(&dd->dd_lock);
 	for (cbr = list_head(&dd->dd_prop_cbs); cbr;
 	    cbr = list_next(&dd->dd_prop_cbs, cbr)) {
-		uint64_t propobj = dsl_dataset_phys(cbr->cbr_ds)->ds_props_obj;
-
-		if (strcmp(cbr->cbr_propname, propname) != 0)
-			continue;
+		uint64_t propobj;
 
 		/*
-		 * If the property is set on this ds, then it is not
-		 * inherited here; don't call the callback.
+		 * cbr->cbf_ds may be invalidated due to eviction,
+		 * requiring the use of dsl_dataset_try_add_ref().
+		 * See comment block in dsl_prop_notify_all_cb()
+		 * for details.
 		 */
-		if (propobj && 0 == zap_contains(mos, propobj, propname))
+		if (strcmp(cbr->cbr_propname, propname) != 0 ||
+		    !dsl_dataset_try_add_ref(dp, cbr->cbr_ds, FTAG))
 			continue;
 
-		cbr->cbr_func(cbr->cbr_arg, value);
+		propobj = dsl_dataset_phys(cbr->cbr_ds)->ds_props_obj;
+
+		/*
+		 * If the property is not set on this ds, then it is
+		 * inherited here; call the callback.
+		 */
+		if (propobj == 0 || zap_contains(mos, propobj, propname) != 0)
+			cbr->cbr_func(cbr->cbr_arg, value);
+
+		dsl_dataset_rele(cbr->cbr_ds, FTAG);
 	}
 	mutex_exit(&dd->dd_lock);
 
diff --git a/uts/common/fs/zfs/sys/dbuf.h b/uts/common/fs/zfs/sys/dbuf.h
index 40d0344a6226..2e07185f4cab 100644
--- a/uts/common/fs/zfs/sys/dbuf.h
+++ b/uts/common/fs/zfs/sys/dbuf.h
@@ -264,12 +264,15 @@ int dbuf_hold_impl(struct dnode *dn, uint8_t level, uint64_t blkid, int create,
 void dbuf_prefetch(struct dnode *dn, uint64_t blkid, zio_priority_t prio);
 
 void dbuf_add_ref(dmu_buf_impl_t *db, void *tag);
+boolean_t dbuf_try_add_ref(dmu_buf_t *db, objset_t *os, uint64_t obj,
+    uint64_t blkid, void *tag);
 uint64_t dbuf_refcount(dmu_buf_impl_t *db);
 
 void dbuf_rele(dmu_buf_impl_t *db, void *tag);
 void dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag);
 
-dmu_buf_impl_t *dbuf_find(struct dnode *dn, uint8_t level, uint64_t blkid);
+dmu_buf_impl_t *dbuf_find(struct objset *os, uint64_t object, uint8_t level,
+    uint64_t blkid);
 
 int dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags);
 void dmu_buf_will_not_fill(dmu_buf_t *db, dmu_tx_t *tx);
diff --git a/uts/common/fs/zfs/sys/dmu.h b/uts/common/fs/zfs/sys/dmu.h
index 2787be4c8090..0b3dd9b2f108 100644
--- a/uts/common/fs/zfs/sys/dmu.h
+++ b/uts/common/fs/zfs/sys/dmu.h
@@ -458,7 +458,23 @@ int dmu_spill_hold_existing(dmu_buf_t *bonus, void *tag, dmu_buf_t **dbp);
  */
 int dmu_buf_hold(objset_t *os, uint64_t object, uint64_t offset,
     void *tag, dmu_buf_t **, int flags);
+
+/*
+ * Add a reference to a dmu buffer that has already been held via
+ * dmu_buf_hold() in the current context.
+ */
 void dmu_buf_add_ref(dmu_buf_t *db, void* tag);
+
+/*
+ * Attempt to add a reference to a dmu buffer that is in an unknown state,
+ * using a pointer that may have been invalidated by eviction processing.
+ * The request will succeed if the passed in dbuf still represents the
+ * same os/object/blkid, is ineligible for eviction, and has at least
+ * one hold by a user other than the syncer.
+ */
+boolean_t dmu_buf_try_add_ref(dmu_buf_t *, objset_t *os, uint64_t object,
+    uint64_t blkid, void *tag);
+
 void dmu_buf_rele(dmu_buf_t *db, void *tag);
 uint64_t dmu_buf_refcount(dmu_buf_t *db);
 
diff --git a/uts/common/fs/zfs/sys/dsl_dataset.h b/uts/common/fs/zfs/sys/dsl_dataset.h
index f0c78c971bef..5bfc34ea6380 100644
--- a/uts/common/fs/zfs/sys/dsl_dataset.h
+++ b/uts/common/fs/zfs/sys/dsl_dataset.h
@@ -206,6 +206,8 @@ dsl_dataset_phys(dsl_dataset_t *ds)
 
 int dsl_dataset_hold(struct dsl_pool *dp, const char *name, void *tag,
     dsl_dataset_t **dsp);
+boolean_t dsl_dataset_try_add_ref(struct dsl_pool *dp, dsl_dataset_t *ds,
+    void *tag);
 int dsl_dataset_hold_obj(struct dsl_pool *dp, uint64_t dsobj, void *tag,
     dsl_dataset_t **);
 void dsl_dataset_rele(dsl_dataset_t *ds, void *tag);

From 26a57d0eaddf0f8afeb71f93bb00851509f9497e Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Sun, 9 Aug 2015 19:28:31 +0000
Subject: [PATCH 102/380] 5592 NULL pointer dereference in
 dsl_prop_notify_all_cb()

Reviewed by: Dan McDonald <danmcd@omniti.com>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: George Wilson <george@delphix.com>
Reviewed by: Will Andrews <will@freebsd.org>
Approved by: Robert Mustacchi <rm@joyent.com>

illumos/illumos-gate@9d47dec0481d8cd53b2c1053c96bfa3f78357d6a
---
 uts/common/fs/zfs/dsl_dataset.c | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/uts/common/fs/zfs/dsl_dataset.c b/uts/common/fs/zfs/dsl_dataset.c
index bc9a346d5820..a11926b58f1a 100644
--- a/uts/common/fs/zfs/dsl_dataset.c
+++ b/uts/common/fs/zfs/dsl_dataset.c
@@ -363,8 +363,19 @@ dsl_dataset_snap_remove(dsl_dataset_t *ds, const char *name, dmu_tx_t *tx,
 boolean_t
 dsl_dataset_try_add_ref(dsl_pool_t *dp, dsl_dataset_t *ds, void *tag)
 {
-	return (dmu_buf_try_add_ref(ds->ds_dbuf, dp->dp_meta_objset,
-	    ds->ds_object, DMU_BONUS_BLKID, tag));
+	dmu_buf_t *dbuf = ds->ds_dbuf;
+	boolean_t result = B_FALSE;
+
+	if (dbuf != NULL && dmu_buf_try_add_ref(dbuf, dp->dp_meta_objset,
+	    ds->ds_object, DMU_BONUS_BLKID, tag)) {
+
+		if (ds == dmu_buf_get_user(dbuf))
+			result = B_TRUE;
+		else
+			dmu_buf_rele(dbuf, tag);
+	}
+
+	return (result);
 }
 
 int

From 0dff95d155ce4670f6db75f3eb3356f286c41f79 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Sun, 9 Aug 2015 19:35:08 +0000
Subject: [PATCH 103/380] 5630 stale bonus buffer in recycled dnode_t leads to
 data corruption

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: George Wilson <george@delphix.com>
Reviewed by: Will Andrews <will@freebsd.org>
Approved by: Robert Mustacchi <rm@joyent.com>
Author: Justin T. Gibbs <justing@spectralogic.com>
---
 uts/common/fs/zfs/dbuf.c       | 61 ++++++++++++++++++++++++++++------
 uts/common/fs/zfs/dnode.c      |  8 ++++-
 uts/common/fs/zfs/dnode_sync.c |  6 ++++
 uts/common/fs/zfs/sys/dnode.h  |  2 ++
 4 files changed, 65 insertions(+), 12 deletions(-)

diff --git a/uts/common/fs/zfs/dbuf.c b/uts/common/fs/zfs/dbuf.c
index 621bb0fae5c8..7914800d4c61 100644
--- a/uts/common/fs/zfs/dbuf.c
+++ b/uts/common/fs/zfs/dbuf.c
@@ -2203,21 +2203,60 @@ dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag)
 
 	if (holds == 0) {
 		if (db->db_blkid == DMU_BONUS_BLKID) {
+			dnode_t *dn;
+
+			/*
+			 * If the dnode moves here, we cannot cross this
+			 * barrier until the move completes.
+			 */
+			DB_DNODE_ENTER(db);
+
+			dn = DB_DNODE(db);
+			atomic_dec_32(&dn->dn_dbufs_count);
+
+			/*
+			 * Decrementing the dbuf count means that the bonus
+			 * buffer's dnode hold is no longer discounted in
+			 * dnode_move(). The dnode cannot move until after
+			 * the dnode_rele_and_unlock() below.
+			 */
+			DB_DNODE_EXIT(db);
+
+			/*
+			 * Do not reference db after its lock is dropped.
+			 * Another thread may evict it.
+			 */
 			mutex_exit(&db->db_mtx);
 
 			/*
-			 * If the dnode moves here, we cannot cross this barrier
-			 * until the move completes.
+			 * If the dnode has been freed, evict the bonus
+			 * buffer immediately.	The data in the bonus
+			 * buffer is no longer relevant and this prevents
+			 * a stale bonus buffer from being associated
+			 * with this dnode_t should the dnode_t be reused
+			 * prior to being destroyed.
 			 */
-			DB_DNODE_ENTER(db);
-			atomic_dec_32(&DB_DNODE(db)->dn_dbufs_count);
-			DB_DNODE_EXIT(db);
-			/*
-			 * The bonus buffer's dnode hold is no longer discounted
-			 * in dnode_move(). The dnode cannot move until after
-			 * the dnode_rele().
-			 */
-			dnode_rele(DB_DNODE(db), db);
+			mutex_enter(&dn->dn_mtx);
+			if (dn->dn_type == DMU_OT_NONE ||
+			    dn->dn_free_txg != 0) {
+				/*
+				 * Drop dn_mtx.  It is a leaf lock and
+				 * cannot be held when dnode_evict_bonus()
+				 * acquires other locks in order to
+				 * perform the eviction.
+				 *
+				 * Freed dnodes cannot be reused until the
+				 * last hold is released.  Since this bonus
+				 * buffer has a hold, the dnode will remain
+				 * in the free state, even without dn_mtx
+				 * held, until the dnode_rele_and_unlock()
+				 * below.
+				 */
+				mutex_exit(&dn->dn_mtx);
+				dnode_evict_bonus(dn);
+				mutex_enter(&dn->dn_mtx);
+			}
+			dnode_rele_and_unlock(dn, db);
 		} else if (db->db_buf == NULL) {
 			/*
 			 * This is a special case: we never associated this
diff --git a/uts/common/fs/zfs/dnode.c b/uts/common/fs/zfs/dnode.c
index bd788a5b27b5..397cc5c0ed04 100644
--- a/uts/common/fs/zfs/dnode.c
+++ b/uts/common/fs/zfs/dnode.c
@@ -1222,13 +1222,19 @@ dnode_add_ref(dnode_t *dn, void *tag)
 
 void
 dnode_rele(dnode_t *dn, void *tag)
+{
+	mutex_enter(&dn->dn_mtx);
+	dnode_rele_and_unlock(dn, tag);
+}
+
+void
+dnode_rele_and_unlock(dnode_t *dn, void *tag)
 {
 	uint64_t refs;
 	/* Get while the hold prevents the dnode from moving. */
 	dmu_buf_impl_t *db = dn->dn_dbuf;
 	dnode_handle_t *dnh = dn->dn_handle;
 
-	mutex_enter(&dn->dn_mtx);
 	refs = refcount_remove(&dn->dn_holds, tag);
 	mutex_exit(&dn->dn_mtx);
 
diff --git a/uts/common/fs/zfs/dnode_sync.c b/uts/common/fs/zfs/dnode_sync.c
index d95bac99390b..06336045fc82 100644
--- a/uts/common/fs/zfs/dnode_sync.c
+++ b/uts/common/fs/zfs/dnode_sync.c
@@ -430,6 +430,12 @@ dnode_evict_dbufs(dnode_t *dn)
 	}
 	mutex_exit(&dn->dn_dbufs_mtx);
 
+	dnode_evict_bonus(dn);
+}
+
+void
+dnode_evict_bonus(dnode_t *dn)
+{
 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
 	if (dn->dn_bonus && refcount_is_zero(&dn->dn_bonus->db_holds)) {
 		mutex_enter(&dn->dn_bonus->db_mtx);
diff --git a/uts/common/fs/zfs/sys/dnode.h b/uts/common/fs/zfs/sys/dnode.h
index 406954a65323..69cc54dc272b 100644
--- a/uts/common/fs/zfs/sys/dnode.h
+++ b/uts/common/fs/zfs/sys/dnode.h
@@ -281,6 +281,7 @@ int dnode_hold_impl(struct objset *dd, uint64_t object, int flag,
     void *ref, dnode_t **dnp);
 boolean_t dnode_add_ref(dnode_t *dn, void *ref);
 void dnode_rele(dnode_t *dn, void *ref);
+void dnode_rele_and_unlock(dnode_t *dn, void *tag);
 void dnode_setdirty(dnode_t *dn, dmu_tx_t *tx);
 void dnode_sync(dnode_t *dn, dmu_tx_t *tx);
 void dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs,
@@ -302,6 +303,7 @@ void dnode_fini(void);
 int dnode_next_offset(dnode_t *dn, int flags, uint64_t *off,
     int minlvl, uint64_t blkfill, uint64_t txg);
 void dnode_evict_dbufs(dnode_t *dn);
+void dnode_evict_bonus(dnode_t *dn);
 
 #ifdef ZFS_DEBUG
 

From 3a5d8c0c1a86857924b26a34d7e9e327ba3eaa57 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Sun, 9 Aug 2015 20:01:22 +0000
Subject: [PATCH 104/380] 5661 ZFS: "compression = on" should use lz4 if
 feature is enabled

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
Reviewed by: Xin LI <delphij@freebsd.org>
Approved by: Robert Mustacchi <rm@joyent.com>
Author: Justin T. Gibbs <justing@spectralogic.com>

illumos/illumos-gate@db1741f555ec79def5e9846e6bfd132248514ffe
---
 uts/common/fs/zfs/dmu.c          | 19 ++++++++-----------
 uts/common/fs/zfs/dmu_objset.c   |  5 +++--
 uts/common/fs/zfs/sys/zio.h      | 18 +++++++++++-------
 uts/common/fs/zfs/zio_compress.c | 23 ++++++++++++++++-------
 4 files changed, 38 insertions(+), 27 deletions(-)

diff --git a/uts/common/fs/zfs/dmu.c b/uts/common/fs/zfs/dmu.c
index 218f2d0b0681..6704df044604 100644
--- a/uts/common/fs/zfs/dmu.c
+++ b/uts/common/fs/zfs/dmu.c
@@ -1725,19 +1725,15 @@ dmu_write_policy(objset_t *os, dnode_t *dn, int level, int wp, zio_prop_t *zp)
 	 *	 3. all other level 0 blocks
 	 */
 	if (ismd) {
-		/*
-		 * XXX -- we should design a compression algorithm
-		 * that specializes in arrays of bps.
-		 */
-		boolean_t lz4_ac = spa_feature_is_active(os->os_spa,
-		    SPA_FEATURE_LZ4_COMPRESS);
-
 		if (zfs_mdcomp_disable) {
 			compress = ZIO_COMPRESS_EMPTY;
-		} else if (lz4_ac) {
-			compress = ZIO_COMPRESS_LZ4;
 		} else {
-			compress = ZIO_COMPRESS_LZJB;
+			/*
+			 * XXX -- we should design a compression algorithm
+			 * that specializes in arrays of bps.
+			 */
+			compress = zio_compress_select(os->os_spa,
+			    ZIO_COMPRESS_ON, ZIO_COMPRESS_ON);
 		}
 
 		/*
@@ -1770,7 +1766,8 @@ dmu_write_policy(objset_t *os, dnode_t *dn, int level, int wp, zio_prop_t *zp)
 		compress = ZIO_COMPRESS_OFF;
 		checksum = ZIO_CHECKSUM_NOPARITY;
 	} else {
-		compress = zio_compress_select(dn->dn_compress, compress);
+		compress = zio_compress_select(os->os_spa, dn->dn_compress,
+		    compress);
 
 		checksum = (dedup_checksum == ZIO_CHECKSUM_OFF) ?
 		    zio_checksum_select(dn->dn_checksum, checksum) :
diff --git a/uts/common/fs/zfs/dmu_objset.c b/uts/common/fs/zfs/dmu_objset.c
index ce3d163b1e18..9ad18f0d11fb 100644
--- a/uts/common/fs/zfs/dmu_objset.c
+++ b/uts/common/fs/zfs/dmu_objset.c
@@ -164,7 +164,8 @@ compression_changed_cb(void *arg, uint64_t newval)
 	 */
 	ASSERT(newval != ZIO_COMPRESS_INHERIT);
 
-	os->os_compress = zio_compress_select(newval, ZIO_COMPRESS_ON_VALUE);
+	os->os_compress = zio_compress_select(os->os_spa, newval,
+	    ZIO_COMPRESS_ON);
 }
 
 static void
@@ -421,7 +422,7 @@ dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
 	} else {
 		/* It's the meta-objset. */
 		os->os_checksum = ZIO_CHECKSUM_FLETCHER_4;
-		os->os_compress = ZIO_COMPRESS_LZJB;
+		os->os_compress = ZIO_COMPRESS_ON;
 		os->os_copies = spa_max_replication(spa);
 		os->os_dedup_checksum = ZIO_CHECKSUM_OFF;
 		os->os_dedup_verify = B_FALSE;
diff --git a/uts/common/fs/zfs/sys/zio.h b/uts/common/fs/zfs/sys/zio.h
index 71bb6b96a01c..198dc9238709 100644
--- a/uts/common/fs/zfs/sys/zio.h
+++ b/uts/common/fs/zfs/sys/zio.h
@@ -125,15 +125,19 @@ enum zio_compress {
  */
 #define	ZIO_COMPRESS_LEGACY_FUNCTIONS ZIO_COMPRESS_LZ4
 
-/* N.B. when altering this value, also change BOOTFS_COMPRESS_VALID below */
-#define	ZIO_COMPRESS_ON_VALUE	ZIO_COMPRESS_LZJB
-#define	ZIO_COMPRESS_DEFAULT	ZIO_COMPRESS_OFF
+/*
+ * The meaning of "compress = on" selected by the compression features enabled
+ * on a given pool.
+ */
+#define	ZIO_COMPRESS_LEGACY_ON_VALUE	ZIO_COMPRESS_LZJB
+#define	ZIO_COMPRESS_LZ4_ON_VALUE	ZIO_COMPRESS_LZ4
+
+#define	ZIO_COMPRESS_DEFAULT		ZIO_COMPRESS_OFF
 
 #define	BOOTFS_COMPRESS_VALID(compress)			\
 	((compress) == ZIO_COMPRESS_LZJB ||		\
 	(compress) == ZIO_COMPRESS_LZ4 ||		\
-	((compress) == ZIO_COMPRESS_ON &&		\
-	ZIO_COMPRESS_ON_VALUE == ZIO_COMPRESS_LZJB) ||	\
+	(compress) == ZIO_COMPRESS_ON ||		\
 	(compress) == ZIO_COMPRESS_OFF)
 
 #define	ZIO_FAILURE_MODE_WAIT		0
@@ -538,8 +542,8 @@ extern enum zio_checksum zio_checksum_select(enum zio_checksum child,
     enum zio_checksum parent);
 extern enum zio_checksum zio_checksum_dedup_select(spa_t *spa,
     enum zio_checksum child, enum zio_checksum parent);
-extern enum zio_compress zio_compress_select(enum zio_compress child,
-    enum zio_compress parent);
+extern enum zio_compress zio_compress_select(spa_t *spa,
+    enum zio_compress child, enum zio_compress parent);
 
 extern void zio_suspend(spa_t *spa, zio_t *zio);
 extern int zio_resume(spa_t *spa);
diff --git a/uts/common/fs/zfs/zio_compress.c b/uts/common/fs/zfs/zio_compress.c
index 0d77f246c4a7..4e2d645572eb 100644
--- a/uts/common/fs/zfs/zio_compress.c
+++ b/uts/common/fs/zfs/zio_compress.c
@@ -34,6 +34,7 @@
 #include <sys/zfs_context.h>
 #include <sys/compress.h>
 #include <sys/spa.h>
+#include <sys/zfeature.h>
 #include <sys/zio.h>
 #include <sys/zio_compress.h>
 
@@ -61,19 +62,27 @@ zio_compress_info_t zio_compress_table[ZIO_COMPRESS_FUNCTIONS] = {
 };
 
 enum zio_compress
-zio_compress_select(enum zio_compress child, enum zio_compress parent)
+zio_compress_select(spa_t *spa, enum zio_compress child,
+    enum zio_compress parent)
 {
+	enum zio_compress result;
+
 	ASSERT(child < ZIO_COMPRESS_FUNCTIONS);
 	ASSERT(parent < ZIO_COMPRESS_FUNCTIONS);
-	ASSERT(parent != ZIO_COMPRESS_INHERIT && parent != ZIO_COMPRESS_ON);
+	ASSERT(parent != ZIO_COMPRESS_INHERIT);
 
-	if (child == ZIO_COMPRESS_INHERIT)
-		return (parent);
+	result = child;
+	if (result == ZIO_COMPRESS_INHERIT)
+		result = parent;
 
-	if (child == ZIO_COMPRESS_ON)
-		return (ZIO_COMPRESS_ON_VALUE);
+	if (result == ZIO_COMPRESS_ON) {
+		if (spa_feature_is_active(spa, SPA_FEATURE_LZ4_COMPRESS))
+			result = ZIO_COMPRESS_LZ4_ON_VALUE;
+		else
+			result = ZIO_COMPRESS_LEGACY_ON_VALUE;
+	}
 
-	return (child);
+	return (result);
 }
 
 size_t

From 740101574693208732480b6e0c58274741a35df0 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Sun, 9 Aug 2015 20:08:14 +0000
Subject: [PATCH 105/380] 5693 ztest fails in dbuf_verify: buf[i] == 0, due to
 dedup and bp_override

Reviewed by: George Wilson <george@delphix.com>
Reviewed by: Christopher Siden <christopher.siden@delphix.com>
Reviewed by: Bayard Bell <buffer.g.overflow@gmail.com>
Approved by: Dan McDonald <danmcd@omniti.com>
Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@7f7ace370074e350853da254c65688fd43ddc695
---
 uts/common/fs/zfs/zio.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/uts/common/fs/zfs/zio.c b/uts/common/fs/zfs/zio.c
index 868192b4fcc2..aa0f2945dd94 100644
--- a/uts/common/fs/zfs/zio.c
+++ b/uts/common/fs/zfs/zio.c
@@ -1130,8 +1130,6 @@ zio_write_bp_init(zio_t *zio)
 			zio->io_pipeline |= ZIO_STAGE_DDT_WRITE;
 			return (ZIO_PIPELINE_CONTINUE);
 		}
-		zio->io_bp_override = NULL;
-		BP_ZERO(bp);
 	}
 
 	if (!BP_IS_HOLE(bp) && bp->blk_birth == zio->io_txg) {

From a6174766d6c122c6355ac752f01422219cdfe5b8 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Sun, 9 Aug 2015 20:22:59 +0000
Subject: [PATCH 106/380] 5694 traverse_prefetcher does not prefetch enough

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Alex Reece <alex@delphix.com>
Reviewed by: Christopher Siden <christopher.siden@delphix.com>
Reviewed by: Josef 'Jeff' Sipek <josef.sipek@nexenta.com>
Reviewed by: Bayard Bell <buffer.g.overflow@gmail.com>
Approved by: Garrett D'Amore <garrett@damore.org>
Author: George Wilson <george.wilson@delphix.com>

illumos/illumos-gate@34d7ce052c4565b078f73b95ccbd49274e98edaa
---
 uts/common/fs/zfs/dmu_traverse.c | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/uts/common/fs/zfs/dmu_traverse.c b/uts/common/fs/zfs/dmu_traverse.c
index 14eef2e7516c..8e49d2862b06 100644
--- a/uts/common/fs/zfs/dmu_traverse.c
+++ b/uts/common/fs/zfs/dmu_traverse.c
@@ -38,13 +38,12 @@
 #include <sys/callb.h>
 #include <sys/zfeature.h>
 
-int zfs_pd_blks_max = 100;
+int32_t zfs_pd_bytes_max = 50 * 1024 * 1024;	/* 50MB */
 
 typedef struct prefetch_data {
 	kmutex_t pd_mtx;
 	kcondvar_t pd_cv;
-	int pd_blks_max;
-	int pd_blks_fetched;
+	int32_t pd_bytes_fetched;
 	int pd_flags;
 	boolean_t pd_cancel;
 	boolean_t pd_exited;
@@ -249,11 +248,12 @@ traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
 	}
 
 	if (pd != NULL && !pd->pd_exited && prefetch_needed(pd, bp)) {
+		uint64_t size = BP_GET_LSIZE(bp);
 		mutex_enter(&pd->pd_mtx);
-		ASSERT(pd->pd_blks_fetched >= 0);
-		while (pd->pd_blks_fetched == 0 && !pd->pd_exited)
+		ASSERT(pd->pd_bytes_fetched >= 0);
+		while (pd->pd_bytes_fetched < size && !pd->pd_exited)
 			cv_wait(&pd->pd_cv, &pd->pd_mtx);
-		pd->pd_blks_fetched--;
+		pd->pd_bytes_fetched -= size;
 		cv_broadcast(&pd->pd_cv);
 		mutex_exit(&pd->pd_mtx);
 	}
@@ -444,7 +444,7 @@ traverse_prefetcher(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 	prefetch_data_t *pfd = arg;
 	arc_flags_t aflags = ARC_FLAG_NOWAIT | ARC_FLAG_PREFETCH;
 
-	ASSERT(pfd->pd_blks_fetched >= 0);
+	ASSERT(pfd->pd_bytes_fetched >= 0);
 	if (pfd->pd_cancel)
 		return (SET_ERROR(EINTR));
 
@@ -452,9 +452,9 @@ traverse_prefetcher(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 		return (0);
 
 	mutex_enter(&pfd->pd_mtx);
-	while (!pfd->pd_cancel && pfd->pd_blks_fetched >= pfd->pd_blks_max)
+	while (!pfd->pd_cancel && pfd->pd_bytes_fetched >= zfs_pd_bytes_max)
 		cv_wait(&pfd->pd_cv, &pfd->pd_mtx);
-	pfd->pd_blks_fetched++;
+	pfd->pd_bytes_fetched += BP_GET_LSIZE(bp);
 	cv_broadcast(&pfd->pd_cv);
 	mutex_exit(&pfd->pd_mtx);
 
@@ -526,7 +526,6 @@ traverse_impl(spa_t *spa, dsl_dataset_t *ds, uint64_t objset, blkptr_t *rootbp,
 		td.td_hole_birth_enabled_txg = 0;
 	}
 
-	pd.pd_blks_max = zfs_pd_blks_max;
 	pd.pd_flags = flags;
 	mutex_init(&pd.pd_mtx, NULL, MUTEX_DEFAULT, NULL);
 	cv_init(&pd.pd_cv, NULL, CV_DEFAULT, NULL);

From c536e59f04fe4de4c4f5d2d502b7627abe3902c3 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Sun, 9 Aug 2015 20:29:25 +0000
Subject: [PATCH 107/380] 5770 Add load_nvlist() error handling

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: George Wilson <george@delphix.com>
Reviewed by: Prakash Surya <prakash.surya@delphix.com>
Reviewed by: Richard Elling <richard.elling@richardelling.com>
Reviewed by: Richard PALO <richard@NetBSD.org>
Approved by: Richard Lowe <richlowe@richlowe.net>
Author: Brian Behlendorf <behlendorf1@llnl.gov>

illumos/illumos-gate@a45f1c3c5ee26f908655e7762d7e1e2aecdfefbc
---
 uts/common/fs/zfs/spa.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/uts/common/fs/zfs/spa.c b/uts/common/fs/zfs/spa.c
index 19e7aa0ef642..2820ba535997 100644
--- a/uts/common/fs/zfs/spa.c
+++ b/uts/common/fs/zfs/spa.c
@@ -1575,7 +1575,10 @@ load_nvlist(spa_t *spa, uint64_t obj, nvlist_t **value)
 	int error;
 	*value = NULL;
 
-	VERIFY(0 == dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db));
+	error = dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db);
+	if (error != 0)
+		return (error);
+
 	nvsize = *(uint64_t *)db->db_data;
 	dmu_buf_rele(db, FTAG);
 

From 8c21602e72048520561d42bee48e937dd14983da Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Sun, 9 Aug 2015 20:32:10 +0000
Subject: [PATCH 108/380] 5769 Cast 'zfs bad bloc' to ULL for x86

Reviewed by: Prakash Surya <prakash.surya@delphix.com>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Paul Dagnelie <paul.dagnelie@delphix.com>
Reviewed by: Richard PALO <richard@NetBSD.org>
Approved by: Dan McDonald <danmcd@omniti.com>

illumos/illumos-gate@8c76e0763bcf0029556e106377da859f6492a7ee
---
 uts/common/fs/zfs/dmu_send.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/uts/common/fs/zfs/dmu_send.c b/uts/common/fs/zfs/dmu_send.c
index 4d77f626a7b5..6fa3c00f4f32 100644
--- a/uts/common/fs/zfs/dmu_send.c
+++ b/uts/common/fs/zfs/dmu_send.c
@@ -511,7 +511,7 @@ backup_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 				for (ptr = abuf->b_data;
 				    (char *)ptr < (char *)abuf->b_data + blksz;
 				    ptr++)
-					*ptr = 0x2f5baddb10c;
+					*ptr = 0x2f5baddb10cULL;
 			} else {
 				return (SET_ERROR(EIO));
 			}

From 65e51be3ab7a3b266925033d02676d95d823c653 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Sun, 9 Aug 2015 20:41:23 +0000
Subject: [PATCH 109/380] Avoid 128K kmem allocations in mzap_upgrade()

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Prakash Surya <prakash.surya@delphix.com>
Reviewed by: George Wilson <george@delphix.com>
Reviewed by: Steven Hartland <killing@multiplay.co.uk>
Approved by: Rich Lowe <richlowe@richlowe.net>

illumos/illumos-gate@be3e2ab906b80af79c7b22885f279e45ad8fb995
---
 uts/common/fs/zfs/zap_micro.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/uts/common/fs/zfs/zap_micro.c b/uts/common/fs/zfs/zap_micro.c
index 3742471f78e6..689c7958de6d 100644
--- a/uts/common/fs/zfs/zap_micro.c
+++ b/uts/common/fs/zfs/zap_micro.c
@@ -538,7 +538,7 @@ mzap_upgrade(zap_t **zapp, dmu_tx_t *tx, zap_flags_t flags)
 	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));
 
 	sz = zap->zap_dbuf->db_size;
-	mzp = kmem_alloc(sz, KM_SLEEP);
+	mzp = zio_buf_alloc(sz);
 	bcopy(zap->zap_dbuf->db_data, mzp, sz);
 	nchunks = zap->zap_m.zap_num_chunks;
 
@@ -546,7 +546,7 @@ mzap_upgrade(zap_t **zapp, dmu_tx_t *tx, zap_flags_t flags)
 		err = dmu_object_set_blocksize(zap->zap_objset, zap->zap_object,
 		    1ULL << fzap_default_block_shift, 0, tx);
 		if (err) {
-			kmem_free(mzp, sz);
+			zio_buf_free(mzp, sz);
 			return (err);
 		}
 	}
@@ -572,7 +572,7 @@ mzap_upgrade(zap_t **zapp, dmu_tx_t *tx, zap_flags_t flags)
 		if (err)
 			break;
 	}
-	kmem_free(mzp, sz);
+	zio_buf_free(mzp, sz);
 	*zapp = zap;
 	return (err);
 }

From 3ccc4ae7a356056e62f66dae7f14ca30b3bc8459 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Mon, 10 Aug 2015 19:32:30 +0000
Subject: [PATCH 110/380] 5746 more checksumming in zfs send

Reviewed by: Christopher Siden <christopher.siden@delphix.com>
Reviewed by: George Wilson <george.wilson@delphix.com>
Reviewed by: Bayard Bell <buffer.g.overflow@gmail.com>
Approved by: Albert Lee <trisk@omniti.com>
Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@98110f08fa182032082d98be2ddb9391fcd62bf1
---
 uts/common/fs/zfs/dmu_send.c         | 508 +++++++++++++++------------
 uts/common/fs/zfs/sys/spa.h          |  13 +
 uts/common/fs/zfs/sys/zfs_ioctl.h    |  18 +-
 uts/common/fs/zfs/sys/zio_checksum.h |   7 +-
 4 files changed, 323 insertions(+), 223 deletions(-)

diff --git a/uts/common/fs/zfs/dmu_send.c b/uts/common/fs/zfs/dmu_send.c
index 6fa3c00f4f32..d2ad529d7930 100644
--- a/uts/common/fs/zfs/dmu_send.c
+++ b/uts/common/fs/zfs/dmu_send.c
@@ -66,7 +66,6 @@ dump_bytes(dmu_sendarg_t *dsp, void *buf, int len)
 	ssize_t resid; /* have to get resid to get detailed errno */
 	ASSERT0(len % 8);
 
-	fletcher_4_incremental_native(buf, len, &dsp->dsa_zc);
 	dsp->dsa_err = vn_rdwr(UIO_WRITE, dsp->dsa_vp,
 	    (caddr_t)buf, len,
 	    0, UIO_SYSSPACE, FAPPEND, RLIM64_INFINITY, CRED(), &resid);
@@ -78,6 +77,38 @@ dump_bytes(dmu_sendarg_t *dsp, void *buf, int len)
 	return (dsp->dsa_err);
 }
 
+/*
+ * For all record types except BEGIN, fill in the checksum (overlaid in
+ * drr_u.drr_checksum.drr_checksum).  The checksum verifies everything
+ * up to the start of the checksum itself.
+ */
+static int
+dump_record(dmu_sendarg_t *dsp, void *payload, int payload_len)
+{
+	ASSERT3U(offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
+	    ==, sizeof (dmu_replay_record_t) - sizeof (zio_cksum_t));
+	fletcher_4_incremental_native(dsp->dsa_drr,
+	    offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
+	    &dsp->dsa_zc);
+	if (dsp->dsa_drr->drr_type != DRR_BEGIN) {
+		ASSERT(ZIO_CHECKSUM_IS_ZERO(&dsp->dsa_drr->drr_u.
+		    drr_checksum.drr_checksum));
+		dsp->dsa_drr->drr_u.drr_checksum.drr_checksum = dsp->dsa_zc;
+	}
+	fletcher_4_incremental_native(&dsp->dsa_drr->
+	    drr_u.drr_checksum.drr_checksum,
+	    sizeof (zio_cksum_t), &dsp->dsa_zc);
+	if (dump_bytes(dsp, dsp->dsa_drr, sizeof (dmu_replay_record_t)) != 0)
+		return (SET_ERROR(EINTR));
+	if (payload_len != 0) {
+		fletcher_4_incremental_native(payload, payload_len,
+		    &dsp->dsa_zc);
+		if (dump_bytes(dsp, payload, payload_len) != 0)
+			return (SET_ERROR(EINTR));
+	}
+	return (0);
+}
+
 static int
 dump_free(dmu_sendarg_t *dsp, uint64_t object, uint64_t offset,
     uint64_t length)
@@ -122,8 +153,7 @@ dump_free(dmu_sendarg_t *dsp, uint64_t object, uint64_t offset,
 	 */
 	if (dsp->dsa_pending_op != PENDING_NONE &&
 	    dsp->dsa_pending_op != PENDING_FREE) {
-		if (dump_bytes(dsp, dsp->dsa_drr,
-		    sizeof (dmu_replay_record_t)) != 0)
+		if (dump_record(dsp, NULL, 0) != 0)
 			return (SET_ERROR(EINTR));
 		dsp->dsa_pending_op = PENDING_NONE;
 	}
@@ -146,8 +176,7 @@ dump_free(dmu_sendarg_t *dsp, uint64_t object, uint64_t offset,
 			return (0);
 		} else {
 			/* not a continuation.  Push out pending record */
-			if (dump_bytes(dsp, dsp->dsa_drr,
-			    sizeof (dmu_replay_record_t)) != 0)
+			if (dump_record(dsp, NULL, 0) != 0)
 				return (SET_ERROR(EINTR));
 			dsp->dsa_pending_op = PENDING_NONE;
 		}
@@ -160,8 +189,7 @@ dump_free(dmu_sendarg_t *dsp, uint64_t object, uint64_t offset,
 	drrf->drr_length = length;
 	drrf->drr_toguid = dsp->dsa_toguid;
 	if (length == -1ULL) {
-		if (dump_bytes(dsp, dsp->dsa_drr,
-		    sizeof (dmu_replay_record_t)) != 0)
+		if (dump_record(dsp, NULL, 0) != 0)
 			return (SET_ERROR(EINTR));
 	} else {
 		dsp->dsa_pending_op = PENDING_FREE;
@@ -193,12 +221,11 @@ dump_write(dmu_sendarg_t *dsp, dmu_object_type_t type,
 	 * of different types.
 	 */
 	if (dsp->dsa_pending_op != PENDING_NONE) {
-		if (dump_bytes(dsp, dsp->dsa_drr,
-		    sizeof (dmu_replay_record_t)) != 0)
+		if (dump_record(dsp, NULL, 0) != 0)
 			return (SET_ERROR(EINTR));
 		dsp->dsa_pending_op = PENDING_NONE;
 	}
-	/* write a DATA record */
+	/* write a WRITE record */
 	bzero(dsp->dsa_drr, sizeof (dmu_replay_record_t));
 	dsp->dsa_drr->drr_type = DRR_WRITE;
 	drrw->drr_object = object;
@@ -224,9 +251,7 @@ dump_write(dmu_sendarg_t *dsp, dmu_object_type_t type,
 		drrw->drr_key.ddk_cksum = bp->blk_cksum;
 	}
 
-	if (dump_bytes(dsp, dsp->dsa_drr, sizeof (dmu_replay_record_t)) != 0)
-		return (SET_ERROR(EINTR));
-	if (dump_bytes(dsp, data, blksz) != 0)
+	if (dump_record(dsp, data, blksz) != 0)
 		return (SET_ERROR(EINTR));
 	return (0);
 }
@@ -240,8 +265,7 @@ dump_write_embedded(dmu_sendarg_t *dsp, uint64_t object, uint64_t offset,
 	    &(dsp->dsa_drr->drr_u.drr_write_embedded);
 
 	if (dsp->dsa_pending_op != PENDING_NONE) {
-		if (dump_bytes(dsp, dsp->dsa_drr,
-		    sizeof (dmu_replay_record_t)) != 0)
+		if (dump_record(dsp, NULL, 0) != 0)
 			return (EINTR);
 		dsp->dsa_pending_op = PENDING_NONE;
 	}
@@ -261,9 +285,7 @@ dump_write_embedded(dmu_sendarg_t *dsp, uint64_t object, uint64_t offset,
 
 	decode_embedded_bp_compressed(bp, buf);
 
-	if (dump_bytes(dsp, dsp->dsa_drr, sizeof (dmu_replay_record_t)) != 0)
-		return (EINTR);
-	if (dump_bytes(dsp, buf, P2ROUNDUP(drrw->drr_psize, 8)) != 0)
+	if (dump_record(dsp, buf, P2ROUNDUP(drrw->drr_psize, 8)) != 0)
 		return (EINTR);
 	return (0);
 }
@@ -274,8 +296,7 @@ dump_spill(dmu_sendarg_t *dsp, uint64_t object, int blksz, void *data)
 	struct drr_spill *drrs = &(dsp->dsa_drr->drr_u.drr_spill);
 
 	if (dsp->dsa_pending_op != PENDING_NONE) {
-		if (dump_bytes(dsp, dsp->dsa_drr,
-		    sizeof (dmu_replay_record_t)) != 0)
+		if (dump_record(dsp, NULL, 0) != 0)
 			return (SET_ERROR(EINTR));
 		dsp->dsa_pending_op = PENDING_NONE;
 	}
@@ -287,9 +308,7 @@ dump_spill(dmu_sendarg_t *dsp, uint64_t object, int blksz, void *data)
 	drrs->drr_length = blksz;
 	drrs->drr_toguid = dsp->dsa_toguid;
 
-	if (dump_bytes(dsp, dsp->dsa_drr, sizeof (dmu_replay_record_t)))
-		return (SET_ERROR(EINTR));
-	if (dump_bytes(dsp, data, blksz))
+	if (dump_record(dsp, data, blksz) != 0)
 		return (SET_ERROR(EINTR));
 	return (0);
 }
@@ -312,8 +331,7 @@ dump_freeobjects(dmu_sendarg_t *dsp, uint64_t firstobj, uint64_t numobjs)
 	 */
 	if (dsp->dsa_pending_op != PENDING_NONE &&
 	    dsp->dsa_pending_op != PENDING_FREEOBJECTS) {
-		if (dump_bytes(dsp, dsp->dsa_drr,
-		    sizeof (dmu_replay_record_t)) != 0)
+		if (dump_record(dsp, NULL, 0) != 0)
 			return (SET_ERROR(EINTR));
 		dsp->dsa_pending_op = PENDING_NONE;
 	}
@@ -327,8 +345,7 @@ dump_freeobjects(dmu_sendarg_t *dsp, uint64_t firstobj, uint64_t numobjs)
 			return (0);
 		} else {
 			/* can't be aggregated.  Push out pending record */
-			if (dump_bytes(dsp, dsp->dsa_drr,
-			    sizeof (dmu_replay_record_t)) != 0)
+			if (dump_record(dsp, NULL, 0) != 0)
 				return (SET_ERROR(EINTR));
 			dsp->dsa_pending_op = PENDING_NONE;
 		}
@@ -355,8 +372,7 @@ dump_dnode(dmu_sendarg_t *dsp, uint64_t object, dnode_phys_t *dnp)
 		return (dump_freeobjects(dsp, object, 1));
 
 	if (dsp->dsa_pending_op != PENDING_NONE) {
-		if (dump_bytes(dsp, dsp->dsa_drr,
-		    sizeof (dmu_replay_record_t)) != 0)
+		if (dump_record(dsp, NULL, 0) != 0)
 			return (SET_ERROR(EINTR));
 		dsp->dsa_pending_op = PENDING_NONE;
 	}
@@ -377,11 +393,10 @@ dump_dnode(dmu_sendarg_t *dsp, uint64_t object, dnode_phys_t *dnp)
 	    drro->drr_blksz > SPA_OLD_MAXBLOCKSIZE)
 		drro->drr_blksz = SPA_OLD_MAXBLOCKSIZE;
 
-	if (dump_bytes(dsp, dsp->dsa_drr, sizeof (dmu_replay_record_t)) != 0)
-		return (SET_ERROR(EINTR));
-
-	if (dump_bytes(dsp, DN_BONUS(dnp), P2ROUNDUP(dnp->dn_bonuslen, 8)) != 0)
+	if (dump_record(dsp, DN_BONUS(dnp),
+	    P2ROUNDUP(dnp->dn_bonuslen, 8)) != 0) {
 		return (SET_ERROR(EINTR));
+	}
 
 	/* Free anything past the end of the file. */
 	if (dump_free(dsp, object, (dnp->dn_maxblkid + 1) *
@@ -625,7 +640,6 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *ds,
 	dsp->dsa_os = os;
 	dsp->dsa_off = off;
 	dsp->dsa_toguid = dsl_dataset_phys(ds)->ds_guid;
-	ZIO_SET_CHECKSUM(&dsp->dsa_zc, 0, 0, 0, 0);
 	dsp->dsa_pending_op = PENDING_NONE;
 	dsp->dsa_incremental = (fromzb != NULL);
 	dsp->dsa_featureflags = featureflags;
@@ -637,7 +651,7 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *ds,
 	dsl_dataset_long_hold(ds, FTAG);
 	dsl_pool_rele(dp, tag);
 
-	if (dump_bytes(dsp, drr, sizeof (dmu_replay_record_t)) != 0) {
+	if (dump_record(dsp, NULL, 0) != 0) {
 		err = dsp->dsa_err;
 		goto out;
 	}
@@ -646,7 +660,7 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *ds,
 	    backup_cb, dsp);
 
 	if (dsp->dsa_pending_op != PENDING_NONE)
-		if (dump_bytes(dsp, drr, sizeof (dmu_replay_record_t)) != 0)
+		if (dump_record(dsp, NULL, 0) != 0)
 			err = SET_ERROR(EINTR);
 
 	if (err != 0) {
@@ -660,7 +674,7 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *ds,
 	drr->drr_u.drr_end.drr_checksum = dsp->dsa_zc;
 	drr->drr_u.drr_end.drr_toguid = dsp->dsa_toguid;
 
-	if (dump_bytes(dsp, drr, sizeof (dmu_replay_record_t)) != 0) {
+	if (dump_record(dsp, NULL, 0) != 0) {
 		err = dsp->dsa_err;
 		goto out;
 	}
@@ -1217,13 +1231,19 @@ dmu_recv_begin(char *tofs, char *tosnap, struct drr_begin *drrb,
 }
 
 struct restorearg {
+	objset_t *os;
 	int err;
 	boolean_t byteswap;
 	vnode_t *vp;
-	char *buf;
 	uint64_t voff;
 	int bufsize; /* amount of memory allocated for buf */
+
+	dmu_replay_record_t *drr;
+	dmu_replay_record_t *next_drr;
+	char *buf;
 	zio_cksum_t cksum;
+	zio_cksum_t prev_cksum;
+
 	avl_tree_t *guid_to_ds_map;
 };
 
@@ -1262,14 +1282,11 @@ free_guid_map_onexit(void *arg)
 	kmem_free(ca, sizeof (avl_tree_t));
 }
 
-static void *
-restore_read(struct restorearg *ra, int len, char *buf)
+static int
+restore_read(struct restorearg *ra, int len, void *buf)
 {
 	int done = 0;
 
-	if (buf == NULL)
-		buf = ra->buf;
-
 	/* some things will require 8-byte alignment, so everything must */
 	ASSERT0(len % 8);
 	ASSERT3U(len, <=, ra->bufsize);
@@ -1278,7 +1295,7 @@ restore_read(struct restorearg *ra, int len, char *buf)
 		ssize_t resid;
 
 		ra->err = vn_rdwr(UIO_READ, ra->vp,
-		    buf + done, len - done,
+		    (char *)buf + done, len - done,
 		    ra->voff, UIO_SYSSPACE, FAPPEND,
 		    RLIM64_INFINITY, CRED(), &resid);
 
@@ -1287,24 +1304,21 @@ restore_read(struct restorearg *ra, int len, char *buf)
 		ra->voff += len - done - resid;
 		done = len - resid;
 		if (ra->err != 0)
-			return (NULL);
+			return (ra->err);
 	}
 
 	ASSERT3U(done, ==, len);
-	if (ra->byteswap)
-		fletcher_4_incremental_byteswap(buf, len, &ra->cksum);
-	else
-		fletcher_4_incremental_native(buf, len, &ra->cksum);
-	return (buf);
+	return (0);
 }
 
 static void
-backup_byteswap(dmu_replay_record_t *drr)
+byteswap_record(dmu_replay_record_t *drr)
 {
 #define	DO64(X) (drr->drr_u.X = BSWAP_64(drr->drr_u.X))
 #define	DO32(X) (drr->drr_u.X = BSWAP_32(drr->drr_u.X))
 	drr->drr_type = BSWAP_32(drr->drr_type);
 	drr->drr_payloadlen = BSWAP_32(drr->drr_payloadlen);
+
 	switch (drr->drr_type) {
 	case DRR_BEGIN:
 		DO64(drr_begin.drr_magic);
@@ -1334,10 +1348,7 @@ backup_byteswap(dmu_replay_record_t *drr)
 		DO64(drr_write.drr_offset);
 		DO64(drr_write.drr_length);
 		DO64(drr_write.drr_toguid);
-		DO64(drr_write.drr_key.ddk_cksum.zc_word[0]);
-		DO64(drr_write.drr_key.ddk_cksum.zc_word[1]);
-		DO64(drr_write.drr_key.ddk_cksum.zc_word[2]);
-		DO64(drr_write.drr_key.ddk_cksum.zc_word[3]);
+		ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_write.drr_key.ddk_cksum);
 		DO64(drr_write.drr_key.ddk_prop);
 		break;
 	case DRR_WRITE_BYREF:
@@ -1348,10 +1359,8 @@ backup_byteswap(dmu_replay_record_t *drr)
 		DO64(drr_write_byref.drr_refguid);
 		DO64(drr_write_byref.drr_refobject);
 		DO64(drr_write_byref.drr_refoffset);
-		DO64(drr_write_byref.drr_key.ddk_cksum.zc_word[0]);
-		DO64(drr_write_byref.drr_key.ddk_cksum.zc_word[1]);
-		DO64(drr_write_byref.drr_key.ddk_cksum.zc_word[2]);
-		DO64(drr_write_byref.drr_key.ddk_cksum.zc_word[3]);
+		ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_write_byref.
+		    drr_key.ddk_cksum);
 		DO64(drr_write_byref.drr_key.ddk_prop);
 		break;
 	case DRR_WRITE_EMBEDDED:
@@ -1374,13 +1383,15 @@ backup_byteswap(dmu_replay_record_t *drr)
 		DO64(drr_spill.drr_toguid);
 		break;
 	case DRR_END:
-		DO64(drr_end.drr_checksum.zc_word[0]);
-		DO64(drr_end.drr_checksum.zc_word[1]);
-		DO64(drr_end.drr_checksum.zc_word[2]);
-		DO64(drr_end.drr_checksum.zc_word[3]);
 		DO64(drr_end.drr_toguid);
+		ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_end.drr_checksum);
 		break;
 	}
+
+	if (drr->drr_type != DRR_BEGIN) {
+		ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_checksum.drr_checksum);
+	}
+
 #undef DO64
 #undef DO32
 }
@@ -1397,11 +1408,10 @@ deduce_nblkptr(dmu_object_type_t bonus_type, uint64_t bonus_size)
 }
 
 static int
-restore_object(struct restorearg *ra, objset_t *os, struct drr_object *drro)
+restore_object(struct restorearg *ra, struct drr_object *drro, void *data)
 {
 	dmu_object_info_t doi;
 	dmu_tx_t *tx;
-	void *data = NULL;
 	uint64_t object;
 	int err;
 
@@ -1412,23 +1422,17 @@ restore_object(struct restorearg *ra, objset_t *os, struct drr_object *drro)
 	    drro->drr_compress >= ZIO_COMPRESS_FUNCTIONS ||
 	    P2PHASE(drro->drr_blksz, SPA_MINBLOCKSIZE) ||
 	    drro->drr_blksz < SPA_MINBLOCKSIZE ||
-	    drro->drr_blksz > spa_maxblocksize(dmu_objset_spa(os)) ||
+	    drro->drr_blksz > spa_maxblocksize(dmu_objset_spa(ra->os)) ||
 	    drro->drr_bonuslen > DN_MAX_BONUSLEN) {
 		return (SET_ERROR(EINVAL));
 	}
 
-	err = dmu_object_info(os, drro->drr_object, &doi);
+	err = dmu_object_info(ra->os, drro->drr_object, &doi);
 
 	if (err != 0 && err != ENOENT)
 		return (SET_ERROR(EINVAL));
 	object = err == 0 ? drro->drr_object : DMU_NEW_OBJECT;
 
-	if (drro->drr_bonuslen) {
-		data = restore_read(ra, P2ROUNDUP(drro->drr_bonuslen, 8), NULL);
-		if (ra->err != 0)
-			return (ra->err);
-	}
-
 	/*
 	 * If we are losing blkptrs or changing the block size this must
 	 * be a new file instance.  We must clear out the previous file
@@ -1442,14 +1446,14 @@ restore_object(struct restorearg *ra, objset_t *os, struct drr_object *drro)
 
 		if (drro->drr_blksz != doi.doi_data_block_size ||
 		    nblkptr < doi.doi_nblkptr) {
-			err = dmu_free_long_range(os, drro->drr_object,
+			err = dmu_free_long_range(ra->os, drro->drr_object,
 			    0, DMU_OBJECT_END);
 			if (err != 0)
 				return (SET_ERROR(EINVAL));
 		}
 	}
 
-	tx = dmu_tx_create(os);
+	tx = dmu_tx_create(ra->os);
 	dmu_tx_hold_bonus(tx, object);
 	err = dmu_tx_assign(tx, TXG_WAIT);
 	if (err != 0) {
@@ -1459,7 +1463,7 @@ restore_object(struct restorearg *ra, objset_t *os, struct drr_object *drro)
 
 	if (object == DMU_NEW_OBJECT) {
 		/* currently free, want to be allocated */
-		err = dmu_object_claim(os, drro->drr_object,
+		err = dmu_object_claim(ra->os, drro->drr_object,
 		    drro->drr_type, drro->drr_blksz,
 		    drro->drr_bonustype, drro->drr_bonuslen, tx);
 	} else if (drro->drr_type != doi.doi_type ||
@@ -1467,7 +1471,7 @@ restore_object(struct restorearg *ra, objset_t *os, struct drr_object *drro)
 	    drro->drr_bonustype != doi.doi_bonus_type ||
 	    drro->drr_bonuslen != doi.doi_bonus_size) {
 		/* currently allocated, but with different properties */
-		err = dmu_object_reclaim(os, drro->drr_object,
+		err = dmu_object_reclaim(ra->os, drro->drr_object,
 		    drro->drr_type, drro->drr_blksz,
 		    drro->drr_bonustype, drro->drr_bonuslen, tx);
 	}
@@ -1476,14 +1480,15 @@ restore_object(struct restorearg *ra, objset_t *os, struct drr_object *drro)
 		return (SET_ERROR(EINVAL));
 	}
 
-	dmu_object_set_checksum(os, drro->drr_object, drro->drr_checksumtype,
-	    tx);
-	dmu_object_set_compress(os, drro->drr_object, drro->drr_compress, tx);
+	dmu_object_set_checksum(ra->os, drro->drr_object,
+	    drro->drr_checksumtype, tx);
+	dmu_object_set_compress(ra->os, drro->drr_object,
+	    drro->drr_compress, tx);
 
 	if (data != NULL) {
 		dmu_buf_t *db;
 
-		VERIFY(0 == dmu_bonus_hold(os, drro->drr_object, FTAG, &db));
+		VERIFY0(dmu_bonus_hold(ra->os, drro->drr_object, FTAG, &db));
 		dmu_buf_will_dirty(db, tx);
 
 		ASSERT3U(db->db_size, >=, drro->drr_bonuslen);
@@ -1502,7 +1507,7 @@ restore_object(struct restorearg *ra, objset_t *os, struct drr_object *drro)
 
 /* ARGSUSED */
 static int
-restore_freeobjects(struct restorearg *ra, objset_t *os,
+restore_freeobjects(struct restorearg *ra,
     struct drr_freeobjects *drrfo)
 {
 	uint64_t obj;
@@ -1512,13 +1517,13 @@ restore_freeobjects(struct restorearg *ra, objset_t *os,
 
 	for (obj = drrfo->drr_firstobj;
 	    obj < drrfo->drr_firstobj + drrfo->drr_numobjs;
-	    (void) dmu_object_next(os, &obj, FALSE, 0)) {
+	    (void) dmu_object_next(ra->os, &obj, FALSE, 0)) {
 		int err;
 
-		if (dmu_object_info(os, obj, NULL) != 0)
+		if (dmu_object_info(ra->os, obj, NULL) != 0)
 			continue;
 
-		err = dmu_free_long_object(os, obj);
+		err = dmu_free_long_object(ra->os, obj);
 		if (err != 0)
 			return (err);
 	}
@@ -1526,49 +1531,37 @@ restore_freeobjects(struct restorearg *ra, objset_t *os,
 }
 
 static int
-restore_write(struct restorearg *ra, objset_t *os,
-    struct drr_write *drrw)
+restore_write(struct restorearg *ra, struct drr_write *drrw, arc_buf_t *abuf)
 {
 	dmu_tx_t *tx;
-	void *data;
 	int err;
 
 	if (drrw->drr_offset + drrw->drr_length < drrw->drr_offset ||
 	    !DMU_OT_IS_VALID(drrw->drr_type))
 		return (SET_ERROR(EINVAL));
 
-	if (dmu_object_info(os, drrw->drr_object, NULL) != 0)
+	if (dmu_object_info(ra->os, drrw->drr_object, NULL) != 0)
 		return (SET_ERROR(EINVAL));
 
-	dmu_buf_t *bonus;
-	if (dmu_bonus_hold(os, drrw->drr_object, FTAG, &bonus) != 0)
-		return (SET_ERROR(EINVAL));
-
-	arc_buf_t *abuf = dmu_request_arcbuf(bonus, drrw->drr_length);
-
-	data = restore_read(ra, drrw->drr_length, abuf->b_data);
-	if (data == NULL) {
-		dmu_return_arcbuf(abuf);
-		dmu_buf_rele(bonus, FTAG);
-		return (ra->err);
-	}
-
-	tx = dmu_tx_create(os);
+	tx = dmu_tx_create(ra->os);
 
 	dmu_tx_hold_write(tx, drrw->drr_object,
 	    drrw->drr_offset, drrw->drr_length);
 	err = dmu_tx_assign(tx, TXG_WAIT);
 	if (err != 0) {
-		dmu_return_arcbuf(abuf);
-		dmu_buf_rele(bonus, FTAG);
 		dmu_tx_abort(tx);
 		return (err);
 	}
 	if (ra->byteswap) {
 		dmu_object_byteswap_t byteswap =
 		    DMU_OT_BYTESWAP(drrw->drr_type);
-		dmu_ot_byteswap[byteswap].ob_func(data, drrw->drr_length);
+		dmu_ot_byteswap[byteswap].ob_func(abuf->b_data,
+		    drrw->drr_length);
 	}
+
+	dmu_buf_t *bonus;
+	if (dmu_bonus_hold(ra->os, drrw->drr_object, FTAG, &bonus) != 0)
+		return (SET_ERROR(EINVAL));
 	dmu_assign_arcbuf(bonus, drrw->drr_offset, abuf, tx);
 	dmu_tx_commit(tx);
 	dmu_buf_rele(bonus, FTAG);
@@ -1583,8 +1576,7 @@ restore_write(struct restorearg *ra, objset_t *os,
  * data from the stream to fulfill this write.
  */
 static int
-restore_write_byref(struct restorearg *ra, objset_t *os,
-    struct drr_write_byref *drrwbr)
+restore_write_byref(struct restorearg *ra, struct drr_write_byref *drrwbr)
 {
 	dmu_tx_t *tx;
 	int err;
@@ -1610,7 +1602,7 @@ restore_write_byref(struct restorearg *ra, objset_t *os,
 		if (dmu_objset_from_ds(gmep->gme_ds, &ref_os))
 			return (SET_ERROR(EINVAL));
 	} else {
-		ref_os = os;
+		ref_os = ra->os;
 	}
 
 	err = dmu_buf_hold(ref_os, drrwbr->drr_refobject,
@@ -1618,7 +1610,7 @@ restore_write_byref(struct restorearg *ra, objset_t *os,
 	if (err != 0)
 		return (err);
 
-	tx = dmu_tx_create(os);
+	tx = dmu_tx_create(ra->os);
 
 	dmu_tx_hold_write(tx, drrwbr->drr_object,
 	    drrwbr->drr_offset, drrwbr->drr_length);
@@ -1627,7 +1619,7 @@ restore_write_byref(struct restorearg *ra, objset_t *os,
 		dmu_tx_abort(tx);
 		return (err);
 	}
-	dmu_write(os, drrwbr->drr_object,
+	dmu_write(ra->os, drrwbr->drr_object,
 	    drrwbr->drr_offset, drrwbr->drr_length, dbp->db_data, tx);
 	dmu_buf_rele(dbp, FTAG);
 	dmu_tx_commit(tx);
@@ -1635,12 +1627,11 @@ restore_write_byref(struct restorearg *ra, objset_t *os,
 }
 
 static int
-restore_write_embedded(struct restorearg *ra, objset_t *os,
-    struct drr_write_embedded *drrwnp)
+restore_write_embedded(struct restorearg *ra,
+    struct drr_write_embedded *drrwnp, void *data)
 {
 	dmu_tx_t *tx;
 	int err;
-	void *data;
 
 	if (drrwnp->drr_offset + drrwnp->drr_length < drrwnp->drr_offset)
 		return (EINVAL);
@@ -1653,11 +1644,7 @@ restore_write_embedded(struct restorearg *ra, objset_t *os,
 	if (drrwnp->drr_compression >= ZIO_COMPRESS_FUNCTIONS)
 		return (EINVAL);
 
-	data = restore_read(ra, P2ROUNDUP(drrwnp->drr_psize, 8), NULL);
-	if (data == NULL)
-		return (ra->err);
-
-	tx = dmu_tx_create(os);
+	tx = dmu_tx_create(ra->os);
 
 	dmu_tx_hold_write(tx, drrwnp->drr_object,
 	    drrwnp->drr_offset, drrwnp->drr_length);
@@ -1667,7 +1654,7 @@ restore_write_embedded(struct restorearg *ra, objset_t *os,
 		return (err);
 	}
 
-	dmu_write_embedded(os, drrwnp->drr_object,
+	dmu_write_embedded(ra->os, drrwnp->drr_object,
 	    drrwnp->drr_offset, data, drrwnp->drr_etype,
 	    drrwnp->drr_compression, drrwnp->drr_lsize, drrwnp->drr_psize,
 	    ra->byteswap ^ ZFS_HOST_BYTEORDER, tx);
@@ -1677,31 +1664,26 @@ restore_write_embedded(struct restorearg *ra, objset_t *os,
 }
 
 static int
-restore_spill(struct restorearg *ra, objset_t *os, struct drr_spill *drrs)
+restore_spill(struct restorearg *ra, struct drr_spill *drrs, void *data)
 {
 	dmu_tx_t *tx;
-	void *data;
 	dmu_buf_t *db, *db_spill;
 	int err;
 
 	if (drrs->drr_length < SPA_MINBLOCKSIZE ||
-	    drrs->drr_length > spa_maxblocksize(dmu_objset_spa(os)))
+	    drrs->drr_length > spa_maxblocksize(dmu_objset_spa(ra->os)))
 		return (SET_ERROR(EINVAL));
 
-	data = restore_read(ra, drrs->drr_length, NULL);
-	if (data == NULL)
-		return (ra->err);
-
-	if (dmu_object_info(os, drrs->drr_object, NULL) != 0)
+	if (dmu_object_info(ra->os, drrs->drr_object, NULL) != 0)
 		return (SET_ERROR(EINVAL));
 
-	VERIFY(0 == dmu_bonus_hold(os, drrs->drr_object, FTAG, &db));
+	VERIFY0(dmu_bonus_hold(ra->os, drrs->drr_object, FTAG, &db));
 	if ((err = dmu_spill_hold_by_bonus(db, FTAG, &db_spill)) != 0) {
 		dmu_buf_rele(db, FTAG);
 		return (err);
 	}
 
-	tx = dmu_tx_create(os);
+	tx = dmu_tx_create(ra->os);
 
 	dmu_tx_hold_spill(tx, db->db_object);
 
@@ -1728,8 +1710,7 @@ restore_spill(struct restorearg *ra, objset_t *os, struct drr_spill *drrs)
 
 /* ARGSUSED */
 static int
-restore_free(struct restorearg *ra, objset_t *os,
-    struct drr_free *drrf)
+restore_free(struct restorearg *ra, struct drr_free *drrf)
 {
 	int err;
 
@@ -1737,10 +1718,10 @@ restore_free(struct restorearg *ra, objset_t *os,
 	    drrf->drr_offset + drrf->drr_length < drrf->drr_offset)
 		return (SET_ERROR(EINVAL));
 
-	if (dmu_object_info(os, drrf->drr_object, NULL) != 0)
+	if (dmu_object_info(ra->os, drrf->drr_object, NULL) != 0)
 		return (SET_ERROR(EINVAL));
 
-	err = dmu_free_long_range(os, drrf->drr_object,
+	err = dmu_free_long_range(ra->os, drrf->drr_object,
 	    drrf->drr_offset, drrf->drr_length);
 	return (err);
 }
@@ -1755,6 +1736,155 @@ dmu_recv_cleanup_ds(dmu_recv_cookie_t *drc)
 	(void) dsl_destroy_head(name);
 }
 
+static void
+restore_cksum(struct restorearg *ra, int len, void *buf)
+{
+	if (ra->byteswap) {
+		fletcher_4_incremental_byteswap(buf, len, &ra->cksum);
+	} else {
+		fletcher_4_incremental_native(buf, len, &ra->cksum);
+	}
+}
+
+/*
+ * If len != 0, read payload into buf.
+ * Read next record's header into ra->next_drr.
+ * Verify checksum of payload and next record.
+ */
+static int
+restore_read_payload_and_next_header(struct restorearg *ra, int len, void *buf)
+{
+	int err;
+
+	if (len != 0) {
+		ASSERT3U(len, <=, ra->bufsize);
+		err = restore_read(ra, len, buf);
+		if (err != 0)
+			return (err);
+		restore_cksum(ra, len, buf);
+	}
+
+	ra->prev_cksum = ra->cksum;
+
+	err = restore_read(ra, sizeof (*ra->next_drr), ra->next_drr);
+	if (err != 0)
+		return (err);
+	if (ra->next_drr->drr_type == DRR_BEGIN)
+		return (SET_ERROR(EINVAL));
+
+	/*
+	 * Note: checksum is of everything up to but not including the
+	 * checksum itself.
+	 */
+	ASSERT3U(offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
+	    ==, sizeof (dmu_replay_record_t) - sizeof (zio_cksum_t));
+	restore_cksum(ra,
+	    offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
+	    ra->next_drr);
+
+	zio_cksum_t cksum_orig = ra->next_drr->drr_u.drr_checksum.drr_checksum;
+	zio_cksum_t *cksump = &ra->next_drr->drr_u.drr_checksum.drr_checksum;
+
+	if (ra->byteswap)
+		byteswap_record(ra->next_drr);
+
+	if ((!ZIO_CHECKSUM_IS_ZERO(cksump)) &&
+	    !ZIO_CHECKSUM_EQUAL(ra->cksum, *cksump))
+		return (SET_ERROR(ECKSUM));
+
+	restore_cksum(ra, sizeof (cksum_orig), &cksum_orig);
+
+	return (0);
+}
+
+static int
+restore_process_record(struct restorearg *ra)
+{
+	int err;
+
+	switch (ra->drr->drr_type) {
+	case DRR_OBJECT:
+	{
+		struct drr_object *drro = &ra->drr->drr_u.drr_object;
+		err = restore_read_payload_and_next_header(ra,
+		    P2ROUNDUP(drro->drr_bonuslen, 8), ra->buf);
+		if (err != 0)
+			return (err);
+		return (restore_object(ra, drro, ra->buf));
+	}
+	case DRR_FREEOBJECTS:
+	{
+		struct drr_freeobjects *drrfo =
+		    &ra->drr->drr_u.drr_freeobjects;
+		err = restore_read_payload_and_next_header(ra, 0, NULL);
+		if (err != 0)
+			return (err);
+		return (restore_freeobjects(ra, drrfo));
+	}
+	case DRR_WRITE:
+	{
+		struct drr_write *drrw = &ra->drr->drr_u.drr_write;
+		arc_buf_t *abuf = arc_loan_buf(dmu_objset_spa(ra->os),
+		    drrw->drr_length);
+
+		err = restore_read_payload_and_next_header(ra,
+		    drrw->drr_length, abuf->b_data);
+		if (err != 0)
+			return (err);
+		err = restore_write(ra, drrw, abuf);
+		/* if restore_write() is successful, it consumes the arc_buf */
+		if (err != 0)
+			dmu_return_arcbuf(abuf);
+		return (err);
+	}
+	case DRR_WRITE_BYREF:
+	{
+		struct drr_write_byref *drrwbr =
+		    &ra->drr->drr_u.drr_write_byref;
+		err = restore_read_payload_and_next_header(ra, 0, NULL);
+		if (err != 0)
+			return (err);
+		return (restore_write_byref(ra, drrwbr));
+	}
+	case DRR_WRITE_EMBEDDED:
+	{
+		struct drr_write_embedded *drrwe =
+		    &ra->drr->drr_u.drr_write_embedded;
+		err = restore_read_payload_and_next_header(ra,
+		    P2ROUNDUP(drrwe->drr_psize, 8), ra->buf);
+		if (err != 0)
+			return (err);
+		return (restore_write_embedded(ra, drrwe, ra->buf));
+	}
+	case DRR_FREE:
+	{
+		struct drr_free *drrf = &ra->drr->drr_u.drr_free;
+		err = restore_read_payload_and_next_header(ra, 0, NULL);
+		if (err != 0)
+			return (err);
+		return (restore_free(ra, drrf));
+	}
+	case DRR_END:
+	{
+		struct drr_end *drre = &ra->drr->drr_u.drr_end;
+		if (!ZIO_CHECKSUM_EQUAL(ra->prev_cksum, drre->drr_checksum))
+			return (SET_ERROR(EINVAL));
+		return (0);
+	}
+	case DRR_SPILL:
+	{
+		struct drr_spill *drrs = &ra->drr->drr_u.drr_spill;
+		err = restore_read_payload_and_next_header(ra,
+		    drrs->drr_length, ra->buf);
+		if (err != 0)
+			return (err);
+		return (restore_spill(ra, drrs, ra->buf));
+	}
+	default:
+		return (SET_ERROR(EINVAL));
+	}
+}
+
 /*
  * NB: callers *must* call dmu_recv_end() if this succeeds.
  */
@@ -1762,10 +1892,8 @@ int
 dmu_recv_stream(dmu_recv_cookie_t *drc, vnode_t *vp, offset_t *voffp,
     int cleanup_fd, uint64_t *action_handlep)
 {
+	int err = 0;
 	struct restorearg ra = { 0 };
-	dmu_replay_record_t *drr;
-	objset_t *os;
-	zio_cksum_t pcksum;
 	int featureflags;
 
 	ra.byteswap = drc->drc_byteswap;
@@ -1773,7 +1901,9 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, vnode_t *vp, offset_t *voffp,
 	ra.vp = vp;
 	ra.voff = *voffp;
 	ra.bufsize = SPA_MAXBLOCKSIZE;
+	ra.drr = kmem_alloc(sizeof (*ra.drr), KM_SLEEP);
 	ra.buf = kmem_alloc(ra.bufsize, KM_SLEEP);
+	ra.next_drr = kmem_alloc(sizeof (*ra.next_drr), KM_SLEEP);
 
 	/* these were verified in dmu_recv_begin */
 	ASSERT3U(DMU_GET_STREAM_HDRTYPE(drc->drc_drrb->drr_versioninfo), ==,
@@ -1783,7 +1913,7 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, vnode_t *vp, offset_t *voffp,
 	/*
 	 * Open the objset we are modifying.
 	 */
-	VERIFY0(dmu_objset_from_ds(drc->drc_ds, &os));
+	VERIFY0(dmu_objset_from_ds(drc->drc_ds, &ra.os));
 
 	ASSERT(dsl_dataset_phys(drc->drc_ds)->ds_flags & DS_FLAG_INCONSISTENT);
 
@@ -1809,13 +1939,13 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, vnode_t *vp, offset_t *voffp,
 			avl_create(ra.guid_to_ds_map, guid_compare,
 			    sizeof (guid_map_entry_t),
 			    offsetof(guid_map_entry_t, avlnode));
-			ra.err = zfs_onexit_add_cb(minor,
+			err = zfs_onexit_add_cb(minor,
 			    free_guid_map_onexit, ra.guid_to_ds_map,
 			    action_handlep);
 			if (ra.err != 0)
 				goto out;
 		} else {
-			ra.err = zfs_onexit_cb_data(minor, *action_handlep,
+			err = zfs_onexit_cb_data(minor, *action_handlep,
 			    (void **)&ra.guid_to_ds_map);
 			if (ra.err != 0)
 				goto out;
@@ -1824,96 +1954,34 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, vnode_t *vp, offset_t *voffp,
 		drc->drc_guid_to_ds_map = ra.guid_to_ds_map;
 	}
 
-	/*
-	 * Read records and process them.
-	 */
-	pcksum = ra.cksum;
-	while (ra.err == 0 &&
-	    NULL != (drr = restore_read(&ra, sizeof (*drr), NULL))) {
+	err = restore_read_payload_and_next_header(&ra, 0, NULL);
+	if (err != 0)
+		goto out;
+	for (;;) {
+		void *tmp;
+
 		if (issig(JUSTLOOKING) && issig(FORREAL)) {
-			ra.err = SET_ERROR(EINTR);
-			goto out;
+			err = SET_ERROR(EINTR);
+			break;
 		}
 
-		if (ra.byteswap)
-			backup_byteswap(drr);
+		tmp = ra.next_drr;
+		ra.next_drr = ra.drr;
+		ra.drr = tmp;
 
-		switch (drr->drr_type) {
-		case DRR_OBJECT:
-		{
-			/*
-			 * We need to make a copy of the record header,
-			 * because restore_{object,write} may need to
-			 * restore_read(), which will invalidate drr.
-			 */
-			struct drr_object drro = drr->drr_u.drr_object;
-			ra.err = restore_object(&ra, os, &drro);
+		/* process ra.drr, read in ra.next_drr */
+		err = restore_process_record(&ra);
+		if (err != 0)
 			break;
-		}
-		case DRR_FREEOBJECTS:
-		{
-			struct drr_freeobjects drrfo =
-			    drr->drr_u.drr_freeobjects;
-			ra.err = restore_freeobjects(&ra, os, &drrfo);
+		if (ra.drr->drr_type == DRR_END)
 			break;
-		}
-		case DRR_WRITE:
-		{
-			struct drr_write drrw = drr->drr_u.drr_write;
-			ra.err = restore_write(&ra, os, &drrw);
-			break;
-		}
-		case DRR_WRITE_BYREF:
-		{
-			struct drr_write_byref drrwbr =
-			    drr->drr_u.drr_write_byref;
-			ra.err = restore_write_byref(&ra, os, &drrwbr);
-			break;
-		}
-		case DRR_WRITE_EMBEDDED:
-		{
-			struct drr_write_embedded drrwe =
-			    drr->drr_u.drr_write_embedded;
-			ra.err = restore_write_embedded(&ra, os, &drrwe);
-			break;
-		}
-		case DRR_FREE:
-		{
-			struct drr_free drrf = drr->drr_u.drr_free;
-			ra.err = restore_free(&ra, os, &drrf);
-			break;
-		}
-		case DRR_END:
-		{
-			struct drr_end drre = drr->drr_u.drr_end;
-			/*
-			 * We compare against the *previous* checksum
-			 * value, because the stored checksum is of
-			 * everything before the DRR_END record.
-			 */
-			if (!ZIO_CHECKSUM_EQUAL(drre.drr_checksum, pcksum))
-				ra.err = SET_ERROR(ECKSUM);
-			goto out;
-		}
-		case DRR_SPILL:
-		{
-			struct drr_spill drrs = drr->drr_u.drr_spill;
-			ra.err = restore_spill(&ra, os, &drrs);
-			break;
-		}
-		default:
-			ra.err = SET_ERROR(EINVAL);
-			goto out;
-		}
-		pcksum = ra.cksum;
 	}
-	ASSERT(ra.err != 0);
 
 out:
 	if ((featureflags & DMU_BACKUP_FEATURE_DEDUP) && (cleanup_fd != -1))
 		zfs_onexit_fd_rele(cleanup_fd);
 
-	if (ra.err != 0) {
+	if (err != 0) {
 		/*
 		 * destroy what we created, so we don't leave it in the
 		 * inconsistent restoring state.
@@ -1921,9 +1989,11 @@ out:
 		dmu_recv_cleanup_ds(drc);
 	}
 
+	kmem_free(ra.drr, sizeof (*ra.drr));
 	kmem_free(ra.buf, ra.bufsize);
+	kmem_free(ra.next_drr, sizeof (*ra.next_drr));
 	*voffp = ra.voff;
-	return (ra.err);
+	return (err);
 }
 
 static int
diff --git a/uts/common/fs/zfs/sys/spa.h b/uts/common/fs/zfs/sys/spa.h
index d5a8e7f2fd78..2c4887b6ca50 100644
--- a/uts/common/fs/zfs/sys/spa.h
+++ b/uts/common/fs/zfs/sys/spa.h
@@ -443,6 +443,19 @@ _NOTE(CONSTCOND) } while (0)
 	((zc1).zc_word[2] - (zc2).zc_word[2]) | \
 	((zc1).zc_word[3] - (zc2).zc_word[3])))
 
+#define	ZIO_CHECKSUM_IS_ZERO(zc) \
+	(0 == ((zc)->zc_word[0] | (zc)->zc_word[1] | \
+	(zc)->zc_word[2] | (zc)->zc_word[3]))
+
+#define	ZIO_CHECKSUM_BSWAP(zcp)					\
+{								\
+	(zcp)->zc_word[0] = BSWAP_64((zcp)->zc_word[0]);	\
+	(zcp)->zc_word[1] = BSWAP_64((zcp)->zc_word[1]);	\
+	(zcp)->zc_word[2] = BSWAP_64((zcp)->zc_word[2]);	\
+	(zcp)->zc_word[3] = BSWAP_64((zcp)->zc_word[3]);	\
+}
+
+
 #define	DVA_IS_VALID(dva)	(DVA_GET_ASIZE(dva) != 0)
 
 #define	ZIO_SET_CHECKSUM(zcp, w0, w1, w2, w3)	\
diff --git a/uts/common/fs/zfs/sys/zfs_ioctl.h b/uts/common/fs/zfs/sys/zfs_ioctl.h
index 62f6ff997d16..1e88212425d9 100644
--- a/uts/common/fs/zfs/sys/zfs_ioctl.h
+++ b/uts/common/fs/zfs/sys/zfs_ioctl.h
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #ifndef	_SYS_ZFS_IOCTL_H
@@ -226,6 +226,22 @@ typedef struct dmu_replay_record {
 			uint32_t drr_psize; /* compr. (real) size of payload */
 			/* (possibly compressed) content follows */
 		} drr_write_embedded;
+
+		/*
+		 * Nore: drr_checksum is overlaid with all record types
+		 * except DRR_BEGIN.  Therefore its (non-pad) members
+		 * must not overlap with members from the other structs.
+		 * We accomplish this by putting its members at the very
+		 * end of the struct.
+		 */
+		struct drr_checksum {
+			uint64_t drr_pad[34];
+			/*
+			 * fletcher-4 checksum of everything preceding the
+			 * checksum.
+			 */
+			zio_cksum_t drr_checksum;
+		} drr_checksum;
 	} drr_u;
 } dmu_replay_record_t;
 
diff --git a/uts/common/fs/zfs/sys/zio_checksum.h b/uts/common/fs/zfs/sys/zio_checksum.h
index 0956c04ab1b4..a921a2f1396e 100644
--- a/uts/common/fs/zfs/sys/zio_checksum.h
+++ b/uts/common/fs/zfs/sys/zio_checksum.h
@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014 by Delphix. All rights reserved.
  */
 
 #ifndef _SYS_ZIO_CHECKSUM_H
@@ -34,13 +35,13 @@ extern "C" {
 /*
  * Signature for checksum functions.
  */
-typedef void zio_checksum_t(const void *data, uint64_t size, zio_cksum_t *zcp);
+typedef void zio_checksum_func_t(const void *, uint64_t, zio_cksum_t *);
 
 /*
  * Information about each checksum function.
  */
 typedef struct zio_checksum_info {
-	zio_checksum_t	*ci_func[2]; /* checksum function for each byteorder */
+	zio_checksum_func_t *ci_func[2]; /* checksum function per byteorder */
 	int		ci_correctable;	/* number of correctable bits	*/
 	int		ci_eck;		/* uses zio embedded checksum? */
 	int		ci_dedup;	/* strong enough for dedup? */
@@ -61,7 +62,7 @@ extern zio_checksum_info_t zio_checksum_table[ZIO_CHECKSUM_FUNCTIONS];
 /*
  * Checksum routines.
  */
-extern zio_checksum_t zio_checksum_SHA256;
+extern zio_checksum_func_t zio_checksum_SHA256;
 
 extern void zio_checksum_compute(zio_t *zio, enum zio_checksum checksum,
     void *data, uint64_t size);

From 3b21799b3ed99227465e2f0cec4b8d001b29c78e Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Mon, 10 Aug 2015 19:37:43 +0000
Subject: [PATCH 111/380] 5820 verify failed in zio_done(): BP_EQUAL(bp,
 io_bp_orig)

Reviewed by: Alex Reece <alex@delphix.com>
Reviewed by: George Wilson <george@delphix.com>
Reviewed by: Steven Hartland <killing@multiplay.co.uk>
Approved by: Garrett D'Amore <garrett@damore.org>
Author: Matthew Ahrens <mahrens@delphix.com>

illumod/illumos-gate@34e8acef009195effafdcf6417aec385e241796e
---
 uts/common/fs/zfs/dmu.c | 35 ++++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/uts/common/fs/zfs/dmu.c b/uts/common/fs/zfs/dmu.c
index 6704df044604..a8cb22ff0269 100644
--- a/uts/common/fs/zfs/dmu.c
+++ b/uts/common/fs/zfs/dmu.c
@@ -1596,19 +1596,32 @@ dmu_sync(zio_t *pio, uint64_t txg, dmu_sync_cb_t *done, zgd_t *zgd)
 	ASSERT(dr->dr_next == NULL || dr->dr_next->dr_txg < txg);
 
 	/*
-	 * Assume the on-disk data is X, the current syncing data is Y,
-	 * and the current in-memory data is Z (currently in dmu_sync).
-	 * X and Z are identical but Y is has been modified. Normally,
-	 * when X and Z are the same we will perform a nopwrite but if Y
-	 * is different we must disable nopwrite since the resulting write
-	 * of Y to disk can free the block containing X. If we allowed a
-	 * nopwrite to occur the block pointing to Z would reference a freed
-	 * block. Since this is a rare case we simplify this by disabling
-	 * nopwrite if the current dmu_sync-ing dbuf has been modified in
-	 * a previous transaction.
+	 * Assume the on-disk data is X, the current syncing data (in
+	 * txg - 1) is Y, and the current in-memory data is Z (currently
+	 * in dmu_sync).
+	 *
+	 * We usually want to perform a nopwrite if X and Z are the
+	 * same.  However, if Y is different (i.e. the BP is going to
+	 * change before this write takes effect), then a nopwrite will
+	 * be incorrect - we would override with X, which could have
+	 * been freed when Y was written.
+	 *
+	 * (Note that this is not a concern when we are nop-writing from
+	 * syncing context, because X and Y must be identical, because
+	 * all previous txgs have been synced.)
+	 *
+	 * Therefore, we disable nopwrite if the current BP could change
+	 * before this TXG.  There are two ways it could change: by
+	 * being dirty (dr_next is non-NULL), or by being freed
+	 * (dnode_block_freed()).  This behavior is verified by
+	 * zio_done(), which VERIFYs that the override BP is identical
+	 * to the on-disk BP.
 	 */
-	if (dr->dr_next)
+	DB_DNODE_ENTER(db);
+	dn = DB_DNODE(db);
+	if (dr->dr_next != NULL || dnode_block_freed(dn, db->db_blkid))
 		zp.zp_nopwrite = B_FALSE;
+	DB_DNODE_EXIT(db);
 
 	ASSERT(dr->dr_txg == txg);
 	if (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC ||

From 648d112108325f3c887d85ed1fca7a1fd4d0498b Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Mon, 10 Aug 2015 21:13:31 +0000
Subject: [PATCH 112/380] 5701 zpool list reports incorrect "alloc" value for
 cache devices

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: George Wilson <george@delphix.com>
Reviewed by: Alek Pinchuk <alek.pinchuk@nexenta.com>
Approved by: Dan McDonald <danmcd@omniti.com>
Author: Prakash Surya <prakash.surya@delphix.com>

illumos/illumos-gate@a52fc310ba80fa3b2006110936198de7f828cd94
---
 uts/common/fs/zfs/arc.c | 183 +++++++++++++++++++++++++++++++---------
 1 file changed, 141 insertions(+), 42 deletions(-)

diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index ea8bc599ce84..3efe13b1f34b 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -864,6 +864,16 @@ uint64_t zfs_crc64_table[256];
 #define	L2ARC_FEED_SECS		1		/* caching interval secs */
 #define	L2ARC_FEED_MIN_MS	200		/* min caching interval ms */
 
+/*
+ * Used to distinguish headers that are being process by
+ * l2arc_write_buffers(), but have yet to be assigned to a l2arc disk
+ * address. This can happen when the header is added to the l2arc's list
+ * of buffers to write in the first stage of l2arc_write_buffers(), but
+ * has not yet been written out which happens in the second stage of
+ * l2arc_write_buffers().
+ */
+#define	L2ARC_ADDR_UNSET	((uint64_t)(-1))
+
 #define	l2arc_writes_sent	ARCSTAT(arcstat_l2_writes_sent)
 #define	l2arc_writes_done	ARCSTAT(arcstat_l2_writes_done)
 
@@ -887,12 +897,12 @@ struct l2arc_dev {
 	uint64_t		l2ad_hand;	/* next write location */
 	uint64_t		l2ad_start;	/* first addr on device */
 	uint64_t		l2ad_end;	/* last addr on device */
-	uint64_t		l2ad_evict;	/* last addr eviction reached */
 	boolean_t		l2ad_first;	/* first sweep through */
 	boolean_t		l2ad_writing;	/* currently writing */
 	kmutex_t		l2ad_mtx;	/* lock for buffer list */
 	list_t			l2ad_buflist;	/* buffer list */
 	list_node_t		l2ad_node;	/* device list node */
+	refcount_t		l2ad_alloc;	/* allocated bytes */
 };
 
 static list_t L2ARC_dev_list;			/* device list */
@@ -1265,6 +1275,7 @@ arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
 	buf_hash_remove(hdr);
 
 	bcopy(hdr, nhdr, HDR_L2ONLY_SIZE);
+
 	if (new == hdr_full_cache) {
 		nhdr->b_flags |= ARC_FLAG_HAS_L1HDR;
 		/*
@@ -1321,6 +1332,20 @@ arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
 
 	mutex_exit(&dev->l2ad_mtx);
 
+	/*
+	 * Since we're using the pointer address as the tag when
+	 * incrementing and decrementing the l2ad_alloc refcount, we
+	 * must remove the old pointer (that we're about to destroy) and
+	 * add the new pointer to the refcount. Otherwise we'd remove
+	 * the wrong pointer address when calling arc_hdr_destroy() later.
+	 */
+
+	(void) refcount_remove_many(&dev->l2ad_alloc,
+	    hdr->b_l2hdr.b_asize, hdr);
+
+	(void) refcount_add_many(&dev->l2ad_alloc,
+	    nhdr->b_l2hdr.b_asize, nhdr);
+
 	buf_discard_identity(hdr);
 	hdr->b_freeze_cksum = NULL;
 	kmem_cache_free(old, hdr);
@@ -2046,6 +2071,57 @@ arc_buf_destroy(arc_buf_t *buf, boolean_t remove)
 	kmem_cache_free(buf_cache, buf);
 }
 
+static void
+arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr)
+{
+	l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
+	l2arc_dev_t *dev = l2hdr->b_dev;
+
+	ASSERT(MUTEX_HELD(&dev->l2ad_mtx));
+	ASSERT(HDR_HAS_L2HDR(hdr));
+
+	list_remove(&dev->l2ad_buflist, hdr);
+
+	/*
+	 * We don't want to leak the b_tmp_cdata buffer that was
+	 * allocated in l2arc_write_buffers()
+	 */
+	arc_buf_l2_cdata_free(hdr);
+
+	/*
+	 * If the l2hdr's b_daddr is equal to L2ARC_ADDR_UNSET, then
+	 * this header is being processed by l2arc_write_buffers() (i.e.
+	 * it's in the first stage of l2arc_write_buffers()).
+	 * Re-affirming that truth here, just to serve as a reminder. If
+	 * b_daddr does not equal L2ARC_ADDR_UNSET, then the header may or
+	 * may not have its HDR_L2_WRITING flag set. (the write may have
+	 * completed, in which case HDR_L2_WRITING will be false and the
+	 * b_daddr field will point to the address of the buffer on disk).
+	 */
+	IMPLY(l2hdr->b_daddr == L2ARC_ADDR_UNSET, HDR_L2_WRITING(hdr));
+
+	/*
+	 * If b_daddr is equal to L2ARC_ADDR_UNSET, we're racing with
+	 * l2arc_write_buffers(). Since we've just removed this header
+	 * from the l2arc buffer list, this header will never reach the
+	 * second stage of l2arc_write_buffers(), which increments the
+	 * accounting stats for this header. Thus, we must be careful
+	 * not to decrement them for this header either.
+	 */
+	if (l2hdr->b_daddr != L2ARC_ADDR_UNSET) {
+		ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
+		ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
+
+		vdev_space_update(dev->l2ad_vdev,
+		    -l2hdr->b_asize, 0, 0);
+
+		(void) refcount_remove_many(&dev->l2ad_alloc,
+		    l2hdr->b_asize, hdr);
+	}
+
+	hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
+}
+
 static void
 arc_hdr_destroy(arc_buf_hdr_t *hdr)
 {
@@ -2059,29 +2135,26 @@ arc_hdr_destroy(arc_buf_hdr_t *hdr)
 	ASSERT(!HDR_IN_HASH_TABLE(hdr));
 
 	if (HDR_HAS_L2HDR(hdr)) {
-		l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
-		boolean_t buflist_held = MUTEX_HELD(&l2hdr->b_dev->l2ad_mtx);
-
-		if (!buflist_held) {
-			mutex_enter(&l2hdr->b_dev->l2ad_mtx);
-			l2hdr = &hdr->b_l2hdr;
-		}
-
-		list_remove(&l2hdr->b_dev->l2ad_buflist, hdr);
-
-		/*
-		 * We don't want to leak the b_tmp_cdata buffer that was
-		 * allocated in l2arc_write_buffers()
-		 */
-		arc_buf_l2_cdata_free(hdr);
-
-		ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
-		ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
+		l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
+		boolean_t buflist_held = MUTEX_HELD(&dev->l2ad_mtx);
 
 		if (!buflist_held)
-			mutex_exit(&l2hdr->b_dev->l2ad_mtx);
+			mutex_enter(&dev->l2ad_mtx);
 
-		hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
+		/*
+		 * Even though we checked this conditional above, we
+		 * need to check this again now that we have the
+		 * l2ad_mtx. This is because we could be racing with
+		 * another thread calling l2arc_evict() which might have
+		 * destroyed this header's L2 portion as we were waiting
+		 * to acquire the l2ad_mtx. If that happens, we don't
+		 * want to re-destroy the header's L2 portion.
+		 */
+		if (HDR_HAS_L2HDR(hdr))
+			arc_hdr_l2hdr_destroy(hdr);
+
+		if (!buflist_held)
+			mutex_exit(&dev->l2ad_mtx);
 	}
 
 	if (!BUF_EMPTY(hdr))
@@ -4319,21 +4392,20 @@ arc_release(arc_buf_t *buf, void *tag)
 	ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) > 0);
 
 	if (HDR_HAS_L2HDR(hdr)) {
-		ARCSTAT_INCR(arcstat_l2_asize, -hdr->b_l2hdr.b_asize);
-		ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
-
 		mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);
-		list_remove(&hdr->b_l2hdr.b_dev->l2ad_buflist, hdr);
 
 		/*
-		 * We don't want to leak the b_tmp_cdata buffer that was
-		 * allocated in l2arc_write_buffers()
+		 * We have to recheck this conditional again now that
+		 * we're holding the l2ad_mtx to prevent a race with
+		 * another thread which might be concurrently calling
+		 * l2arc_evict(). In that case, l2arc_evict() might have
+		 * destroyed the header's L2 portion as we were waiting
+		 * to acquire the l2ad_mtx.
 		 */
-		arc_buf_l2_cdata_free(hdr);
+		if (HDR_HAS_L2HDR(hdr))
+			arc_hdr_l2hdr_destroy(hdr);
 
 		mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);
-
-		hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
 	}
 
 	/*
@@ -5407,6 +5479,10 @@ top:
 
 			ARCSTAT_INCR(arcstat_l2_asize, -hdr->b_l2hdr.b_asize);
 			ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
+
+			bytes_dropped += hdr->b_l2hdr.b_asize;
+			(void) refcount_remove_many(&dev->l2ad_alloc,
+			    hdr->b_l2hdr.b_asize, hdr);
 		}
 
 		/*
@@ -5565,7 +5641,6 @@ l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
 	arc_buf_hdr_t *hdr, *hdr_prev;
 	kmutex_t *hash_lock;
 	uint64_t taddr;
-	int64_t bytes_evicted = 0;
 
 	buflist = &dev->l2ad_buflist;
 
@@ -5656,22 +5731,15 @@ top:
 				hdr->b_flags |= ARC_FLAG_L2_EVICTED;
 			}
 
-			/* Tell ARC this no longer exists in L2ARC. */
-			ARCSTAT_INCR(arcstat_l2_asize, -hdr->b_l2hdr.b_asize);
-			ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
-			hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
-			list_remove(buflist, hdr);
-
 			/* Ensure this header has finished being written */
 			ASSERT(!HDR_L2_WRITING(hdr));
 			ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
+
+			arc_hdr_l2hdr_destroy(hdr);
 		}
 		mutex_exit(hash_lock);
 	}
 	mutex_exit(&dev->l2ad_mtx);
-
-	vdev_space_update(dev->l2ad_vdev, -bytes_evicted, 0, 0);
-	dev->l2ad_evict = taddr;
 }
 
 /*
@@ -5811,6 +5879,29 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 			hdr->b_l2hdr.b_asize = hdr->b_size;
 			hdr->b_l1hdr.b_tmp_cdata = hdr->b_l1hdr.b_buf->b_data;
 
+			/*
+			 * Explicitly set the b_daddr field to a known
+			 * value which means "invalid address". This
+			 * enables us to differentiate which stage of
+			 * l2arc_write_buffers() the particular header
+			 * is in (e.g. this loop, or the one below).
+			 * ARC_FLAG_L2_WRITING is not enough to make
+			 * this distinction, and we need to know in
+			 * order to do proper l2arc vdev accounting in
+			 * arc_release() and arc_hdr_destroy().
+			 *
+			 * Note, we can't use a new flag to distinguish
+			 * the two stages because we don't hold the
+			 * header's hash_lock below, in the second stage
+			 * of this function. Thus, we can't simply
+			 * change the b_flags field to denote that the
+			 * IO has been sent. We can change the b_daddr
+			 * field of the L2 portion, though, since we'll
+			 * be holding the l2ad_mtx; which is why we're
+			 * using it to denote the header's state change.
+			 */
+			hdr->b_l2hdr.b_daddr = L2ARC_ADDR_UNSET;
+
 			buf_sz = hdr->b_size;
 			hdr->b_flags |= ARC_FLAG_HAS_L2HDR;
 
@@ -5890,6 +5981,13 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 		buf_data = hdr->b_l1hdr.b_tmp_cdata;
 		buf_sz = hdr->b_l2hdr.b_asize;
 
+		/*
+		 * We need to do this regardless if buf_sz is zero or
+		 * not, otherwise, when this l2hdr is evicted we'll
+		 * remove a reference that was never added.
+		 */
+		(void) refcount_add_many(&dev->l2ad_alloc, buf_sz, hdr);
+
 		/* Compression may have squashed the buffer to zero length. */
 		if (buf_sz != 0) {
 			uint64_t buf_p_sz;
@@ -5904,6 +6002,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 			(void) zio_nowait(wzio);
 
 			write_asize += buf_sz;
+
 			/*
 			 * Keep the clock hand suitably device-aligned.
 			 */
@@ -5928,7 +6027,6 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 	 */
 	if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) {
 		dev->l2ad_hand = dev->l2ad_start;
-		dev->l2ad_evict = dev->l2ad_start;
 		dev->l2ad_first = B_FALSE;
 	}
 
@@ -6248,7 +6346,6 @@ l2arc_add_vdev(spa_t *spa, vdev_t *vd)
 	adddev->l2ad_start = VDEV_LABEL_START_SIZE;
 	adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd);
 	adddev->l2ad_hand = adddev->l2ad_start;
-	adddev->l2ad_evict = adddev->l2ad_start;
 	adddev->l2ad_first = B_TRUE;
 	adddev->l2ad_writing = B_FALSE;
 
@@ -6261,6 +6358,7 @@ l2arc_add_vdev(spa_t *spa, vdev_t *vd)
 	    offsetof(arc_buf_hdr_t, b_l2hdr.b_l2node));
 
 	vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand);
+	refcount_create(&adddev->l2ad_alloc);
 
 	/*
 	 * Add device to global list
@@ -6306,6 +6404,7 @@ l2arc_remove_vdev(vdev_t *vd)
 	l2arc_evict(remdev, 0, B_TRUE);
 	list_destroy(&remdev->l2ad_buflist);
 	mutex_destroy(&remdev->l2ad_mtx);
+	refcount_destroy(&remdev->l2ad_alloc);
 	kmem_free(remdev, sizeof (l2arc_dev_t));
 }
 

From 1e53768c772f7c4c3334de26906d5e6beab05341 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Mon, 10 Aug 2015 21:19:16 +0000
Subject: [PATCH 113/380] 5808 spa_check_logs is not necessary on readonly
 pools

Reviewed by: George Wilson <george@delphix.com>
Reviewed by: Paul Dagnelie <paul.dagnelie@delphix.com>
Reviewed by: Simon Klinkert <simon.klinkert@gmail.com>
Reviewed by: Will Andrews <will@freebsd.org>
Approved by: Gordon Ross <gwr@nexenta.com>
Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@23367a2f2caec1ccb4d918bdd0f2fc2c9cadcd06
---
 uts/common/fs/zfs/spa.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/uts/common/fs/zfs/spa.c b/uts/common/fs/zfs/spa.c
index 2820ba535997..e9d2432a8179 100644
--- a/uts/common/fs/zfs/spa.c
+++ b/uts/common/fs/zfs/spa.c
@@ -2687,7 +2687,7 @@ spa_load_impl(spa_t *spa, uint64_t pool_guid, nvlist_t *config,
 		if (rvd->vdev_state <= VDEV_STATE_CANT_OPEN)
 			return (SET_ERROR(ENXIO));
 
-		if (spa_check_logs(spa)) {
+		if (spa_writeable(spa) && spa_check_logs(spa)) {
 			*ereport = FM_EREPORT_ZFS_LOG_REPLAY;
 			return (spa_vdev_err(rvd, VDEV_AUX_BAD_LOG, ENXIO));
 		}

From 4741407eb0dcf33424d008f9e4d57a5143870d73 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Mon, 10 Aug 2015 21:32:07 +0000
Subject: [PATCH 114/380] 5810 zdb should print details of bpobj

Reviewed by: Prakash Surya <prakash.surya@delphix.com>
Reviewed by: Alex Reece <alex@delphix.com>
Reviewed by: George Wilson <george@delphix.com>
Reviewed by: Will Andrews <will@freebsd.org>
Reviewed by: Simon Klinkert <simon.klinkert@gmail.com>
Approved by: Gordon Ross <gwr@nexenta.com>
Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@732885fca09e11183dd0ea69aaaab5588fb7dbff
---
 uts/common/fs/zfs/bpobj.c     | 5 ++---
 uts/common/fs/zfs/sys/bpobj.h | 3 +--
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/uts/common/fs/zfs/bpobj.c b/uts/common/fs/zfs/bpobj.c
index da4d38a3a9cf..7d200965b51e 100644
--- a/uts/common/fs/zfs/bpobj.c
+++ b/uts/common/fs/zfs/bpobj.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  */
 
 #include <sys/bpobj.h>
@@ -256,9 +256,8 @@ bpobj_iterate_impl(bpobj_t *bpo, bpobj_itor_t func, void *arg, dmu_tx_t *tx,
 		dbuf = NULL;
 	}
 	if (free) {
-		i++;
 		VERIFY3U(0, ==, dmu_free_range(bpo->bpo_os, bpo->bpo_object,
-		    i * sizeof (blkptr_t), -1ULL, tx));
+		    (i + 1) * sizeof (blkptr_t), -1ULL, tx));
 	}
 	if (err || !bpo->bpo_havesubobj || bpo->bpo_phys->bpo_subobjs == 0)
 		goto out;
diff --git a/uts/common/fs/zfs/sys/bpobj.h b/uts/common/fs/zfs/sys/bpobj.h
index af975c734560..2a365199ce44 100644
--- a/uts/common/fs/zfs/sys/bpobj.h
+++ b/uts/common/fs/zfs/sys/bpobj.h
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #ifndef	_SYS_BPOBJ_H
@@ -77,7 +77,6 @@ void bpobj_close(bpobj_t *bpo);
 
 int bpobj_iterate(bpobj_t *bpo, bpobj_itor_t func, void *arg, dmu_tx_t *tx);
 int bpobj_iterate_nofree(bpobj_t *bpo, bpobj_itor_t func, void *, dmu_tx_t *);
-int bpobj_iterate_dbg(bpobj_t *bpo, uint64_t *itorp, blkptr_t *bp);
 
 void bpobj_enqueue_subobj(bpobj_t *bpo, uint64_t subobj, dmu_tx_t *tx);
 void bpobj_enqueue(bpobj_t *bpo, const blkptr_t *bp, dmu_tx_t *tx);

From 26de6cb8296295d889ab7a064dce103d6f15ec54 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Mon, 10 Aug 2015 21:36:10 +0000
Subject: [PATCH 115/380] 5812 assertion failed in zrl_tryenter():
 zr_owner==NULL

Reviewed by: George Wilson <george@delphix.com>
Reviewed by: Alex Reece <alex@delphix.com>
Reviewed by: Will Andrews <will@freebsd.org>
Approved by: Gordon Ross <gwr@nexenta.com>
Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@8df173054ca442cd8845a7364c3edad9d6822351
---
 uts/common/fs/zfs/zrlock.c | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/uts/common/fs/zfs/zrlock.c b/uts/common/fs/zfs/zrlock.c
index ec94b08555be..22151843e030 100644
--- a/uts/common/fs/zfs/zrlock.c
+++ b/uts/common/fs/zfs/zrlock.c
@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014 by Delphix. All rights reserved.
  */
 
 /*
@@ -42,7 +43,7 @@
  * A ZRL can be locked only while there are zero references, so ZRL_LOCKED is
  * treated as zero references.
  */
-#define	ZRL_LOCKED	((uint32_t)-1)
+#define	ZRL_LOCKED	-1
 #define	ZRL_DESTROYED	-2
 
 void
@@ -60,7 +61,7 @@ zrl_init(zrlock_t *zrl)
 void
 zrl_destroy(zrlock_t *zrl)
 {
-	ASSERT(zrl->zr_refcount == 0);
+	ASSERT0(zrl->zr_refcount);
 
 	mutex_destroy(&zrl->zr_mtx);
 	zrl->zr_refcount = ZRL_DESTROYED;
@@ -80,7 +81,7 @@ zrl_add(zrlock_t *zrl)
 		uint32_t cas = atomic_cas_32(
 		    (uint32_t *)&zrl->zr_refcount, n, n + 1);
 		if (cas == n) {
-			ASSERT((int32_t)n >= 0);
+			ASSERT3S((int32_t)n, >=, 0);
 #ifdef	ZFS_DEBUG
 			if (zrl->zr_owner == curthread) {
 				DTRACE_PROBE2(zrlock__reentry,
@@ -98,7 +99,7 @@ zrl_add(zrlock_t *zrl)
 	while (zrl->zr_refcount == ZRL_LOCKED) {
 		cv_wait(&zrl->zr_cv, &zrl->zr_mtx);
 	}
-	ASSERT(zrl->zr_refcount >= 0);
+	ASSERT3S(zrl->zr_refcount, >=, 0);
 	zrl->zr_refcount++;
 #ifdef	ZFS_DEBUG
 	zrl->zr_owner = curthread;
@@ -112,14 +113,14 @@ zrl_remove(zrlock_t *zrl)
 {
 	uint32_t n;
 
-	n = atomic_dec_32_nv((uint32_t *)&zrl->zr_refcount);
-	ASSERT((int32_t)n >= 0);
 #ifdef	ZFS_DEBUG
 	if (zrl->zr_owner == curthread) {
 		zrl->zr_owner = NULL;
 		zrl->zr_caller = NULL;
 	}
 #endif
+	n = atomic_dec_32_nv((uint32_t *)&zrl->zr_refcount);
+	ASSERT3S((int32_t)n, >=, 0);
 }
 
 int
@@ -132,14 +133,14 @@ zrl_tryenter(zrlock_t *zrl)
 		    (uint32_t *)&zrl->zr_refcount, 0, ZRL_LOCKED);
 		if (cas == 0) {
 #ifdef	ZFS_DEBUG
-			ASSERT(zrl->zr_owner == NULL);
+			ASSERT3P(zrl->zr_owner, ==, NULL);
 			zrl->zr_owner = curthread;
 #endif
 			return (1);
 		}
 	}
 
-	ASSERT((int32_t)n > ZRL_DESTROYED);
+	ASSERT3S((int32_t)n, >, ZRL_DESTROYED);
 
 	return (0);
 }
@@ -147,11 +148,11 @@ zrl_tryenter(zrlock_t *zrl)
 void
 zrl_exit(zrlock_t *zrl)
 {
-	ASSERT(zrl->zr_refcount == ZRL_LOCKED);
+	ASSERT3S(zrl->zr_refcount, ==, ZRL_LOCKED);
 
 	mutex_enter(&zrl->zr_mtx);
 #ifdef	ZFS_DEBUG
-	ASSERT(zrl->zr_owner == curthread);
+	ASSERT3P(zrl->zr_owner, ==, curthread);
 	zrl->zr_owner = NULL;
 	membar_producer();	/* make sure the owner store happens first */
 #endif
@@ -163,7 +164,7 @@ zrl_exit(zrlock_t *zrl)
 int
 zrl_refcount(zrlock_t *zrl)
 {
-	ASSERT(zrl->zr_refcount > ZRL_DESTROYED);
+	ASSERT3S(zrl->zr_refcount, >, ZRL_DESTROYED);
 
 	int n = (int)zrl->zr_refcount;
 	return (n <= 0 ? 0 : n);
@@ -172,7 +173,7 @@ zrl_refcount(zrlock_t *zrl)
 int
 zrl_is_zero(zrlock_t *zrl)
 {
-	ASSERT(zrl->zr_refcount > ZRL_DESTROYED);
+	ASSERT3S(zrl->zr_refcount, >, ZRL_DESTROYED);
 
 	return (zrl->zr_refcount <= 0);
 }
@@ -180,7 +181,7 @@ zrl_is_zero(zrlock_t *zrl)
 int
 zrl_is_locked(zrlock_t *zrl)
 {
-	ASSERT(zrl->zr_refcount > ZRL_DESTROYED);
+	ASSERT3S(zrl->zr_refcount, >, ZRL_DESTROYED);
 
 	return (zrl->zr_refcount == ZRL_LOCKED);
 }

From 6e5b75b290aea8619a8adc23c042b0afdac5449e Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Wed, 12 Aug 2015 18:08:40 +0000
Subject: [PATCH 116/380] 5765 add support for estimating send stream size with
 lzc_send_space when so urce is a bookmark

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Christopher Siden <christopher.siden@delphix.com>
Reviewed by: Steven Hartland <killing@multiplay.co.uk>
Reviewed by: Bayard Bell <buffer.g.overflow@gmail.com>
Approved by: Albert Lee <trisk@nexenta.com>

illumos/illumos-gate@643da460c8ca583e39ce053081754e24087f84c8
---
 uts/common/fs/zfs/dmu_send.c        | 112 +++++++++++++++++++++-------
 uts/common/fs/zfs/sys/dmu_send.h    |   4 +-
 uts/common/fs/zfs/sys/dsl_dataset.h |   3 +
 uts/common/fs/zfs/zfs_ioctl.c       |  52 ++++++++++---
 4 files changed, 133 insertions(+), 38 deletions(-)

diff --git a/uts/common/fs/zfs/dmu_send.c b/uts/common/fs/zfs/dmu_send.c
index d2ad529d7930..7d728c6877e5 100644
--- a/uts/common/fs/zfs/dmu_send.c
+++ b/uts/common/fs/zfs/dmu_send.c
@@ -822,6 +822,40 @@ dmu_send(const char *tosnap, const char *fromsnap,
 	return (err);
 }
 
+static int
+dmu_adjust_send_estimate_for_indirects(dsl_dataset_t *ds, uint64_t size,
+    uint64_t *sizep)
+{
+	int err;
+	/*
+	 * Assume that space (both on-disk and in-stream) is dominated by
+	 * data.  We will adjust for indirect blocks and the copies property,
+	 * but ignore per-object space used (eg, dnodes and DRR_OBJECT records).
+	 */
+
+	/*
+	 * Subtract out approximate space used by indirect blocks.
+	 * Assume most space is used by data blocks (non-indirect, non-dnode).
+	 * Assume all blocks are recordsize.  Assume ditto blocks and
+	 * internal fragmentation counter out compression.
+	 *
+	 * Therefore, space used by indirect blocks is sizeof(blkptr_t) per
+	 * block, which we observe in practice.
+	 */
+	uint64_t recordsize;
+	err = dsl_prop_get_int_ds(ds, "recordsize", &recordsize);
+	if (err != 0)
+		return (err);
+	size -= size / recordsize * sizeof (blkptr_t);
+
+	/* Add in the space for the record associated with each block. */
+	size += size / recordsize * sizeof (dmu_replay_record_t);
+
+	*sizep = size;
+
+	return (0);
+}
+
 int
 dmu_send_estimate(dsl_dataset_t *ds, dsl_dataset_t *fromds, uint64_t *sizep)
 {
@@ -857,35 +891,63 @@ dmu_send_estimate(dsl_dataset_t *ds, dsl_dataset_t *fromds, uint64_t *sizep)
 			return (err);
 	}
 
-	/*
-	 * Assume that space (both on-disk and in-stream) is dominated by
-	 * data.  We will adjust for indirect blocks and the copies property,
-	 * but ignore per-object space used (eg, dnodes and DRR_OBJECT records).
-	 */
-
-	/*
-	 * Subtract out approximate space used by indirect blocks.
-	 * Assume most space is used by data blocks (non-indirect, non-dnode).
-	 * Assume all blocks are recordsize.  Assume ditto blocks and
-	 * internal fragmentation counter out compression.
-	 *
-	 * Therefore, space used by indirect blocks is sizeof(blkptr_t) per
-	 * block, which we observe in practice.
-	 */
-	uint64_t recordsize;
-	err = dsl_prop_get_int_ds(ds, "recordsize", &recordsize);
-	if (err != 0)
-		return (err);
-	size -= size / recordsize * sizeof (blkptr_t);
-
-	/* Add in the space for the record associated with each block. */
-	size += size / recordsize * sizeof (dmu_replay_record_t);
-
-	*sizep = size;
+	err = dmu_adjust_send_estimate_for_indirects(ds, size, sizep);
+	return (err);
+}
 
+/*
+ * Simple callback used to traverse the blocks of a snapshot and sum their
+ * uncompressed size
+ */
+/* ARGSUSED */
+static int
+dmu_calculate_send_traversal(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
+    const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg)
+{
+	uint64_t *spaceptr = arg;
+	if (bp != NULL && !BP_IS_HOLE(bp)) {
+		*spaceptr += BP_GET_UCSIZE(bp);
+	}
 	return (0);
 }
 
+/*
+ * Given a desination snapshot and a TXG, calculate the approximate size of a
+ * send stream sent from that TXG. from_txg may be zero, indicating that the
+ * whole snapshot will be sent.
+ */
+int
+dmu_send_estimate_from_txg(dsl_dataset_t *ds, uint64_t from_txg,
+    uint64_t *sizep)
+{
+	dsl_pool_t *dp = ds->ds_dir->dd_pool;
+	int err;
+	uint64_t size = 0;
+
+	ASSERT(dsl_pool_config_held(dp));
+
+	/* tosnap must be a snapshot */
+	if (!dsl_dataset_is_snapshot(ds))
+		return (SET_ERROR(EINVAL));
+
+	/* verify that from_txg is before the provided snapshot was taken */
+	if (from_txg >= dsl_dataset_phys(ds)->ds_creation_txg) {
+		return (SET_ERROR(EXDEV));
+	}
+
+	/*
+	 * traverse the blocks of the snapshot with birth times after
+	 * from_txg, summing their uncompressed size
+	 */
+	err = traverse_dataset(ds, from_txg, TRAVERSE_POST,
+	    dmu_calculate_send_traversal, &size);
+	if (err)
+		return (err);
+
+	err = dmu_adjust_send_estimate_for_indirects(ds, size, sizep);
+	return (err);
+}
+
 typedef struct dmu_recv_begin_arg {
 	const char *drba_origin;
 	dmu_recv_cookie_t *drba_cookie;
diff --git a/uts/common/fs/zfs/sys/dmu_send.h b/uts/common/fs/zfs/sys/dmu_send.h
index 3a8dc89abd4a..2442a1f8aab1 100644
--- a/uts/common/fs/zfs/sys/dmu_send.h
+++ b/uts/common/fs/zfs/sys/dmu_send.h
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  */
@@ -42,6 +42,8 @@ int dmu_send(const char *tosnap, const char *fromsnap,
     int outfd, struct vnode *vp, offset_t *off);
 int dmu_send_estimate(struct dsl_dataset *ds, struct dsl_dataset *fromds,
     uint64_t *sizep);
+int dmu_send_estimate_from_txg(struct dsl_dataset *ds, uint64_t fromtxg,
+    uint64_t *sizep);
 int dmu_send_obj(const char *pool, uint64_t tosnap, uint64_t fromsnap,
     boolean_t embedok, boolean_t large_block_ok,
     int outfd, struct vnode *vp, offset_t *off);
diff --git a/uts/common/fs/zfs/sys/dsl_dataset.h b/uts/common/fs/zfs/sys/dsl_dataset.h
index 5bfc34ea6380..7d490ec60318 100644
--- a/uts/common/fs/zfs/sys/dsl_dataset.h
+++ b/uts/common/fs/zfs/sys/dsl_dataset.h
@@ -201,6 +201,9 @@ dsl_dataset_phys(dsl_dataset_t *ds)
  */
 #define	MAX_TAG_PREFIX_LEN	17
 
+#define	dsl_dataset_is_snapshot(ds) \
+	(dsl_dataset_phys(ds)->ds_num_children != 0)
+
 #define	DS_UNIQUE_IS_ACCURATE(ds)	\
 	((dsl_dataset_phys(ds)->ds_flags & DS_FLAG_UNIQUE_ACCURATE) != 0)
 
diff --git a/uts/common/fs/zfs/zfs_ioctl.c b/uts/common/fs/zfs/zfs_ioctl.c
index cea478457dd6..1826c8aaacdf 100644
--- a/uts/common/fs/zfs/zfs_ioctl.c
+++ b/uts/common/fs/zfs/zfs_ioctl.c
@@ -5325,7 +5325,8 @@ zfs_ioc_send_new(const char *snapname, nvlist_t *innvl, nvlist_t *outnvl)
  * of bytes that will be written to the fd supplied to zfs_ioc_send_new().
  *
  * innvl: {
- *     (optional) "fromsnap" -> full snap name to send an incremental from
+ *     (optional) "from" -> full snap or bookmark name to send an incremental
+ *                          from
  * }
  *
  * outnvl: {
@@ -5336,7 +5337,6 @@ static int
 zfs_ioc_send_space(const char *snapname, nvlist_t *innvl, nvlist_t *outnvl)
 {
 	dsl_pool_t *dp;
-	dsl_dataset_t *fromsnap = NULL;
 	dsl_dataset_t *tosnap;
 	int error;
 	char *fromname;
@@ -5352,27 +5352,55 @@ zfs_ioc_send_space(const char *snapname, nvlist_t *innvl, nvlist_t *outnvl)
 		return (error);
 	}
 
-	error = nvlist_lookup_string(innvl, "fromsnap", &fromname);
+	error = nvlist_lookup_string(innvl, "from", &fromname);
 	if (error == 0) {
-		error = dsl_dataset_hold(dp, fromname, FTAG, &fromsnap);
-		if (error != 0) {
-			dsl_dataset_rele(tosnap, FTAG);
-			dsl_pool_rele(dp, FTAG);
-			return (error);
+		if (strchr(fromname, '@') != NULL) {
+			/*
+			 * If from is a snapshot, hold it and use the more
+			 * efficient dmu_send_estimate to estimate send space
+			 * size using deadlists.
+			 */
+			dsl_dataset_t *fromsnap;
+			error = dsl_dataset_hold(dp, fromname, FTAG, &fromsnap);
+			if (error != 0)
+				goto out;
+			error = dmu_send_estimate(tosnap, fromsnap, &space);
+			dsl_dataset_rele(fromsnap, FTAG);
+		} else if (strchr(fromname, '#') != NULL) {
+			/*
+			 * If from is a bookmark, fetch the creation TXG of the
+			 * snapshot it was created from and use that to find
+			 * blocks that were born after it.
+			 */
+			zfs_bookmark_phys_t frombm;
+
+			error = dsl_bookmark_lookup(dp, fromname, tosnap,
+			    &frombm);
+			if (error != 0)
+				goto out;
+			error = dmu_send_estimate_from_txg(tosnap,
+			    frombm.zbm_creation_txg, &space);
+		} else {
+			/*
+			 * from is not properly formatted as a snapshot or
+			 * bookmark
+			 */
+			error = SET_ERROR(EINVAL);
+			goto out;
 		}
+	} else {
+		// If estimating the size of a full send, use dmu_send_estimate
+		error = dmu_send_estimate(tosnap, NULL, &space);
 	}
 
-	error = dmu_send_estimate(tosnap, fromsnap, &space);
 	fnvlist_add_uint64(outnvl, "space", space);
 
-	if (fromsnap != NULL)
-		dsl_dataset_rele(fromsnap, FTAG);
+out:
 	dsl_dataset_rele(tosnap, FTAG);
 	dsl_pool_rele(dp, FTAG);
 	return (error);
 }
 
-
 static zfs_ioc_vec_t zfs_ioc_vec[ZFS_IOC_LAST - ZFS_IOC_FIRST];
 
 static void

From 1ec2f7edceaaba7c9f99abe62e6d7a7fd88c4c1f Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Wed, 12 Aug 2015 22:36:02 +0000
Subject: [PATCH 117/380] 5960 zfs recv should prefetch indirect blocks 5925
 zfs receive -o origin=

Reviewed by: Prakash Surya <prakash.surya@delphix.com>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Author: Paul Dagnelie <pcd@delphix.com>

While running 'zfs recv' we noticed that every 128th 8K block required a
read. We were seeing that restore_write() was calling dmu_tx_hold_write()
and the indirect block was not cached. We should prefetch upcoming indirect
blocks to avoid having to go to disk and blocking the restore_write().
---
 uts/common/Makefile.files            |   5 +-
 uts/common/fs/zfs/bptree.c           |   2 +-
 uts/common/fs/zfs/bqueue.c           | 111 ++++
 uts/common/fs/zfs/dbuf.c             | 265 +++++++--
 uts/common/fs/zfs/dmu.c              |  44 +-
 uts/common/fs/zfs/dmu_diff.c         |   2 +-
 uts/common/fs/zfs/dmu_object.c       |   5 +
 uts/common/fs/zfs/dmu_send.c         | 815 ++++++++++++++++++++-------
 uts/common/fs/zfs/dmu_traverse.c     |  28 +-
 uts/common/fs/zfs/dmu_tx.c           |   6 +-
 uts/common/fs/zfs/dmu_zfetch.c       |   3 +-
 uts/common/fs/zfs/dnode.c            |  18 +-
 uts/common/fs/zfs/dnode_sync.c       |   6 +-
 uts/common/fs/zfs/dsl_dataset.c      |  28 +-
 uts/common/fs/zfs/dsl_destroy.c      |   2 +-
 uts/common/fs/zfs/dsl_scan.c         |   3 +-
 uts/common/fs/zfs/spa.c              |   2 +-
 uts/common/fs/zfs/space_map.c        |   4 +-
 uts/common/fs/zfs/sys/bqueue.h       |  54 ++
 uts/common/fs/zfs/sys/dbuf.h         |   9 +-
 uts/common/fs/zfs/sys/dmu.h          |   5 +-
 uts/common/fs/zfs/sys/dsl_dataset.h  |   2 +-
 uts/common/fs/zfs/sys/zio.h          |  22 +-
 uts/common/fs/zfs/sys/zio_checksum.h |   2 +-
 uts/common/fs/zfs/sys/zio_priority.h |  40 ++
 uts/common/fs/zfs/zap.c              |  13 +-
 uts/common/fs/zfs/zfs_vfsops.c       |   4 +-
 uts/common/fs/zfs/zfs_vnops.c        |   3 +-
 uts/common/fs/zfs/zio.c              | 147 ++++-
 uts/common/fs/zfs/zvol.c             |   2 +-
 30 files changed, 1299 insertions(+), 353 deletions(-)
 create mode 100644 uts/common/fs/zfs/bqueue.c
 create mode 100644 uts/common/fs/zfs/sys/bqueue.h
 create mode 100644 uts/common/fs/zfs/sys/zio_priority.h

diff --git a/uts/common/Makefile.files b/uts/common/Makefile.files
index 8d9abca90695..cc8b16a70787 100644
--- a/uts/common/Makefile.files
+++ b/uts/common/Makefile.files
@@ -21,7 +21,9 @@
 
 #
 # Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
-# Copyright (c) 2013 by Delphix. All rights reserved.
+# Copyright (c) 2012 Nexenta Systems, Inc. All rights reserved.
+# Copyright (c) 2012 Joyent, Inc.  All rights reserved.
+# Copyright (c) 2011, 2014 by Delphix. All rights reserved.
 # Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
 # Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
 #
@@ -1332,6 +1334,7 @@ ZFS_COMMON_OBJS +=		\
 	bplist.o		\
 	bpobj.o			\
 	bptree.o		\
+	bqueue.o		\
 	dbuf.o			\
 	ddt.o			\
 	ddt_zap.o		\
diff --git a/uts/common/fs/zfs/bptree.c b/uts/common/fs/zfs/bptree.c
index 5f7d76f0e2ad..b2b98877420a 100644
--- a/uts/common/fs/zfs/bptree.c
+++ b/uts/common/fs/zfs/bptree.c
@@ -154,7 +154,7 @@ bptree_visit_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 	int err;
 	struct bptree_args *ba = arg;
 
-	if (BP_IS_HOLE(bp))
+	if (bp == NULL || BP_IS_HOLE(bp))
 		return (0);
 
 	err = ba->ba_func(ba->ba_arg, bp, ba->ba_tx);
diff --git a/uts/common/fs/zfs/bqueue.c b/uts/common/fs/zfs/bqueue.c
new file mode 100644
index 000000000000..1ddc697b5424
--- /dev/null
+++ b/uts/common/fs/zfs/bqueue.c
@@ -0,0 +1,111 @@
+/*
+ * CDDL HEADER START
+ *
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2014 by Delphix. All rights reserved.
+ */
+
+#include	<sys/bqueue.h>
+#include	<sys/zfs_context.h>
+
+static inline bqueue_node_t *
+obj2node(bqueue_t *q, void *data)
+{
+	return ((bqueue_node_t *)((char *)data + q->bq_node_offset));
+}
+
+/*
+ * Initialize a blocking queue  The maximum capacity of the queue is set to
+ * size.  Types that want to be stored in a bqueue must contain a bqueue_node_t,
+ * and offset should give its offset from the start of the struct.  Return 0 on
+ * success, or -1 on failure.
+ */
+int
+bqueue_init(bqueue_t *q, uint64_t size, size_t node_offset)
+{
+	list_create(&q->bq_list, node_offset + sizeof (bqueue_node_t),
+	    node_offset + offsetof(bqueue_node_t, bqn_node));
+	cv_init(&q->bq_add_cv, NULL, CV_DEFAULT, NULL);
+	cv_init(&q->bq_pop_cv, NULL, CV_DEFAULT, NULL);
+	mutex_init(&q->bq_lock, NULL, MUTEX_DEFAULT, NULL);
+	q->bq_node_offset = node_offset;
+	q->bq_size = 0;
+	q->bq_maxsize = size;
+	return (0);
+}
+
+/*
+ * Destroy a blocking queue.  This function asserts that there are no
+ * elements in the queue, and no one is blocked on the condition
+ * variables.
+ */
+void
+bqueue_destroy(bqueue_t *q)
+{
+	ASSERT0(q->bq_size);
+	cv_destroy(&q->bq_add_cv);
+	cv_destroy(&q->bq_pop_cv);
+	mutex_destroy(&q->bq_lock);
+	list_destroy(&q->bq_list);
+}
+
+/*
+ * Add data to q, consuming size units of capacity.  If there is insufficient
+ * capacity to consume size units, block until capacity exists.  Asserts size is
+ * > 0.
+ */
+void
+bqueue_enqueue(bqueue_t *q, void *data, uint64_t item_size)
+{
+	ASSERT3U(item_size, >, 0);
+	ASSERT3U(item_size, <, q->bq_maxsize);
+	mutex_enter(&q->bq_lock);
+	obj2node(q, data)->bqn_size = item_size;
+	while (q->bq_size + item_size > q->bq_maxsize) {
+		cv_wait(&q->bq_add_cv, &q->bq_lock);
+	}
+	q->bq_size += item_size;
+	list_insert_tail(&q->bq_list, data);
+	cv_signal(&q->bq_pop_cv);
+	mutex_exit(&q->bq_lock);
+}
+/*
+ * Take the first element off of q.  If there are no elements on the queue, wait
+ * until one is put there.  Return the removed element.
+ */
+void *
+bqueue_dequeue(bqueue_t *q)
+{
+	void *ret;
+	uint64_t item_size;
+	mutex_enter(&q->bq_lock);
+	while (q->bq_size == 0) {
+		cv_wait(&q->bq_pop_cv, &q->bq_lock);
+	}
+	ret = list_remove_head(&q->bq_list);
+	item_size = obj2node(q, ret)->bqn_size;
+	q->bq_size -= item_size;
+	mutex_exit(&q->bq_lock);
+	cv_signal(&q->bq_add_cv);
+	return (ret);
+}
+
+/*
+ * Returns true if the space used is 0.
+ */
+boolean_t
+bqueue_empty(bqueue_t *q)
+{
+	return (q->bq_size == 0);
+}
diff --git a/uts/common/fs/zfs/dbuf.c b/uts/common/fs/zfs/dbuf.c
index 7914800d4c61..623d57ab4404 100644
--- a/uts/common/fs/zfs/dbuf.c
+++ b/uts/common/fs/zfs/dbuf.c
@@ -548,11 +548,35 @@ dbuf_loan_arcbuf(dmu_buf_impl_t *db)
 	return (abuf);
 }
 
+/*
+ * Calculate which level n block references the data at the level 0 offset
+ * provided.
+ */
 uint64_t
-dbuf_whichblock(dnode_t *dn, uint64_t offset)
+dbuf_whichblock(dnode_t *dn, int64_t level, uint64_t offset)
 {
-	if (dn->dn_datablkshift) {
-		return (offset >> dn->dn_datablkshift);
+	if (dn->dn_datablkshift != 0 && dn->dn_indblkshift != 0) {
+		/*
+		 * The level n blkid is equal to the level 0 blkid divided by
+		 * the number of level 0s in a level n block.
+		 *
+		 * The level 0 blkid is offset >> datablkshift =
+		 * offset / 2^datablkshift.
+		 *
+		 * The number of level 0s in a level n is the number of block
+		 * pointers in an indirect block, raised to the power of level.
+		 * This is 2^(indblkshift - SPA_BLKPTRSHIFT)^level =
+		 * 2^(level*(indblkshift - SPA_BLKPTRSHIFT)).
+		 *
+		 * Thus, the level n blkid is: offset /
+		 * ((2^datablkshift)*(2^(level*(indblkshift - SPA_BLKPTRSHIFT)))
+		 * = offset / 2^(datablkshift + level *
+		 *   (indblkshift - SPA_BLKPTRSHIFT))
+		 * = offset >> (datablkshift + level *
+		 *   (indblkshift - SPA_BLKPTRSHIFT))
+		 */
+		return (offset >> (dn->dn_datablkshift + level *
+		    (dn->dn_indblkshift - SPA_BLKPTRSHIFT)));
 	} else {
 		ASSERT3U(offset, <, dn->dn_datablksz);
 		return (0);
@@ -1715,6 +1739,12 @@ dbuf_clear(dmu_buf_impl_t *db)
 		dbuf_rele(parent, db);
 }
 
+/*
+ * Note: While bpp will always be updated if the function returns success,
+ * parentp will not be updated if the dnode does not have dn_dbuf filled in;
+ * this happens when the dnode is the meta-dnode, or a userused or groupused
+ * object.
+ */
 static int
 dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse,
     dmu_buf_impl_t **parentp, blkptr_t **bpp)
@@ -1755,7 +1785,7 @@ dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse,
 	} else if (level < nlevels-1) {
 		/* this block is referenced from an indirect block */
 		int err = dbuf_hold_impl(dn, level+1,
-		    blkid >> epbs, fail_sparse, NULL, parentp);
+		    blkid >> epbs, fail_sparse, FALSE, NULL, parentp);
 		if (err)
 			return (err);
 		err = dbuf_read(*parentp, NULL,
@@ -1930,11 +1960,96 @@ dbuf_destroy(dmu_buf_impl_t *db)
 	arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER);
 }
 
-void
-dbuf_prefetch(dnode_t *dn, uint64_t blkid, zio_priority_t prio)
+typedef struct dbuf_prefetch_arg {
+	spa_t *dpa_spa;	/* The spa to issue the prefetch in. */
+	zbookmark_phys_t dpa_zb; /* The target block to prefetch. */
+	int dpa_epbs; /* Entries (blkptr_t's) Per Block Shift. */
+	int dpa_curlevel; /* The current level that we're reading */
+	zio_priority_t dpa_prio; /* The priority I/Os should be issued at. */
+	zio_t *dpa_zio; /* The parent zio_t for all prefetches. */
+	arc_flags_t dpa_aflags; /* Flags to pass to the final prefetch. */
+} dbuf_prefetch_arg_t;
+
+/*
+ * Actually issue the prefetch read for the block given.
+ */
+static void
+dbuf_issue_final_prefetch(dbuf_prefetch_arg_t *dpa, blkptr_t *bp)
 {
-	dmu_buf_impl_t *db = NULL;
-	blkptr_t *bp = NULL;
+	if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp))
+		return;
+
+	arc_flags_t aflags =
+	    dpa->dpa_aflags | ARC_FLAG_NOWAIT | ARC_FLAG_PREFETCH;
+
+	ASSERT3U(dpa->dpa_curlevel, ==, BP_GET_LEVEL(bp));
+	ASSERT3U(dpa->dpa_curlevel, ==, dpa->dpa_zb.zb_level);
+	ASSERT(dpa->dpa_zio != NULL);
+	(void) arc_read(dpa->dpa_zio, dpa->dpa_spa, bp, NULL, NULL,
+	    dpa->dpa_prio, ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE,
+	    &aflags, &dpa->dpa_zb);
+}
+
+/*
+ * Called when an indirect block above our prefetch target is read in.  This
+ * will either read in the next indirect block down the tree or issue the actual
+ * prefetch if the next block down is our target.
+ */
+static void
+dbuf_prefetch_indirect_done(zio_t *zio, arc_buf_t *abuf, void *private)
+{
+	dbuf_prefetch_arg_t *dpa = private;
+
+	ASSERT3S(dpa->dpa_zb.zb_level, <, dpa->dpa_curlevel);
+	ASSERT3S(dpa->dpa_curlevel, >, 0);
+	if (zio != NULL) {
+		ASSERT3S(BP_GET_LEVEL(zio->io_bp), ==, dpa->dpa_curlevel);
+		ASSERT3U(BP_GET_LSIZE(zio->io_bp), ==, zio->io_size);
+		ASSERT3P(zio->io_spa, ==, dpa->dpa_spa);
+	}
+
+	dpa->dpa_curlevel--;
+
+	uint64_t nextblkid = dpa->dpa_zb.zb_blkid >>
+	    (dpa->dpa_epbs * (dpa->dpa_curlevel - dpa->dpa_zb.zb_level));
+	blkptr_t *bp = ((blkptr_t *)abuf->b_data) +
+	    P2PHASE(nextblkid, 1ULL << dpa->dpa_epbs);
+	if (BP_IS_HOLE(bp) || (zio != NULL && zio->io_error != 0)) {
+		kmem_free(dpa, sizeof (*dpa));
+	} else if (dpa->dpa_curlevel == dpa->dpa_zb.zb_level) {
+		ASSERT3U(nextblkid, ==, dpa->dpa_zb.zb_blkid);
+		dbuf_issue_final_prefetch(dpa, bp);
+		kmem_free(dpa, sizeof (*dpa));
+	} else {
+		arc_flags_t iter_aflags = ARC_FLAG_NOWAIT;
+		zbookmark_phys_t zb;
+
+		ASSERT3U(dpa->dpa_curlevel, ==, BP_GET_LEVEL(bp));
+
+		SET_BOOKMARK(&zb, dpa->dpa_zb.zb_objset,
+		    dpa->dpa_zb.zb_object, dpa->dpa_curlevel, nextblkid);
+
+		(void) arc_read(dpa->dpa_zio, dpa->dpa_spa,
+		    bp, dbuf_prefetch_indirect_done, dpa, dpa->dpa_prio,
+		    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE,
+		    &iter_aflags, &zb);
+	}
+	(void) arc_buf_remove_ref(abuf, private);
+}
+
+/*
+ * Issue prefetch reads for the given block on the given level.  If the indirect
+ * blocks above that block are not in memory, we will read them in
+ * asynchronously.  As a result, this call never blocks waiting for a read to
+ * complete.
+ */
+void
+dbuf_prefetch(dnode_t *dn, int64_t level, uint64_t blkid, zio_priority_t prio,
+    arc_flags_t aflags)
+{
+	blkptr_t bp;
+	int epbs, nlevels, curlevel;
+	uint64_t curblkid;
 
 	ASSERT(blkid != DMU_BONUS_BLKID);
 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
@@ -1942,35 +2057,104 @@ dbuf_prefetch(dnode_t *dn, uint64_t blkid, zio_priority_t prio)
 	if (dnode_block_freed(dn, blkid))
 		return;
 
-	/* dbuf_find() returns with db_mtx held */
-	if (db = dbuf_find(dn->dn_objset, dn->dn_object, 0, blkid)) {
-		/*
-		 * This dbuf is already in the cache.  We assume that
-		 * it is already CACHED, or else about to be either
-		 * read or filled.
-		 */
+	/*
+	 * This dnode hasn't been written to disk yet, so there's nothing to
+	 * prefetch.
+	 */
+	nlevels = dn->dn_phys->dn_nlevels;
+	if (level >= nlevels || dn->dn_phys->dn_nblkptr == 0)
+		return;
+
+	epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
+	if (dn->dn_phys->dn_maxblkid < blkid << (epbs * level))
+		return;
+
+	dmu_buf_impl_t *db = dbuf_find(dn->dn_objset, dn->dn_object,
+	    level, blkid);
+	if (db != NULL) {
 		mutex_exit(&db->db_mtx);
+		/*
+		 * This dbuf already exists.  It is either CACHED, or
+		 * (we assume) about to be read or filled.
+		 */
 		return;
 	}
 
-	if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) {
-		if (bp && !BP_IS_HOLE(bp) && !BP_IS_EMBEDDED(bp)) {
-			dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;
-			arc_flags_t aflags =
-			    ARC_FLAG_NOWAIT | ARC_FLAG_PREFETCH;
-			zbookmark_phys_t zb;
+	/*
+	 * Find the closest ancestor (indirect block) of the target block
+	 * that is present in the cache.  In this indirect block, we will
+	 * find the bp that is at curlevel, curblkid.
+	 */
+	curlevel = level;
+	curblkid = blkid;
+	while (curlevel < nlevels - 1) {
+		int parent_level = curlevel + 1;
+		uint64_t parent_blkid = curblkid >> epbs;
+		dmu_buf_impl_t *db;
 
-			SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET,
-			    dn->dn_object, 0, blkid);
-
-			(void) arc_read(NULL, dn->dn_objset->os_spa,
-			    bp, NULL, NULL, prio,
-			    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE,
-			    &aflags, &zb);
+		if (dbuf_hold_impl(dn, parent_level, parent_blkid,
+		    FALSE, TRUE, FTAG, &db) == 0) {
+			blkptr_t *bpp = db->db_buf->b_data;
+			bp = bpp[P2PHASE(curblkid, 1 << epbs)];
+			dbuf_rele(db, FTAG);
+			break;
 		}
-		if (db)
-			dbuf_rele(db, NULL);
+
+		curlevel = parent_level;
+		curblkid = parent_blkid;
 	}
+
+	if (curlevel == nlevels - 1) {
+		/* No cached indirect blocks found. */
+		ASSERT3U(curblkid, <, dn->dn_phys->dn_nblkptr);
+		bp = dn->dn_phys->dn_blkptr[curblkid];
+	}
+	if (BP_IS_HOLE(&bp))
+		return;
+
+	ASSERT3U(curlevel, ==, BP_GET_LEVEL(&bp));
+
+	zio_t *pio = zio_root(dmu_objset_spa(dn->dn_objset), NULL, NULL,
+	    ZIO_FLAG_CANFAIL);
+
+	dbuf_prefetch_arg_t *dpa = kmem_zalloc(sizeof (*dpa), KM_SLEEP);
+	dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;
+	SET_BOOKMARK(&dpa->dpa_zb, ds != NULL ? ds->ds_object : DMU_META_OBJSET,
+	    dn->dn_object, level, blkid);
+	dpa->dpa_curlevel = curlevel;
+	dpa->dpa_prio = prio;
+	dpa->dpa_aflags = aflags;
+	dpa->dpa_spa = dn->dn_objset->os_spa;
+	dpa->dpa_epbs = epbs;
+	dpa->dpa_zio = pio;
+
+	/*
+	 * If we have the indirect just above us, no need to do the asynchronous
+	 * prefetch chain; we'll just run the last step ourselves.  If we're at
+	 * a higher level, though, we want to issue the prefetches for all the
+	 * indirect blocks asynchronously, so we can go on with whatever we were
+	 * doing.
+	 */
+	if (curlevel == level) {
+		ASSERT3U(curblkid, ==, blkid);
+		dbuf_issue_final_prefetch(dpa, &bp);
+		kmem_free(dpa, sizeof (*dpa));
+	} else {
+		arc_flags_t iter_aflags = ARC_FLAG_NOWAIT;
+		zbookmark_phys_t zb;
+
+		SET_BOOKMARK(&zb, ds != NULL ? ds->ds_object : DMU_META_OBJSET,
+		    dn->dn_object, curlevel, curblkid);
+		(void) arc_read(dpa->dpa_zio, dpa->dpa_spa,
+		    &bp, dbuf_prefetch_indirect_done, dpa, prio,
+		    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE,
+		    &iter_aflags, &zb);
+	}
+	/*
+	 * We use pio here instead of dpa_zio since it's possible that
+	 * dpa may have already been freed.
+	 */
+	zio_nowait(pio);
 }
 
 /*
@@ -1978,7 +2162,8 @@ dbuf_prefetch(dnode_t *dn, uint64_t blkid, zio_priority_t prio)
  * Note: dn_struct_rwlock must be held.
  */
 int
-dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse,
+dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid,
+    boolean_t fail_sparse, boolean_t fail_uncached,
     void *tag, dmu_buf_impl_t **dbp)
 {
 	dmu_buf_impl_t *db, *parent = NULL;
@@ -1996,6 +2181,9 @@ top:
 		blkptr_t *bp = NULL;
 		int err;
 
+		if (fail_uncached)
+			return (SET_ERROR(ENOENT));
+
 		ASSERT3P(parent, ==, NULL);
 		err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp);
 		if (fail_sparse) {
@@ -2012,6 +2200,11 @@ top:
 		db = dbuf_create(dn, level, blkid, parent, bp);
 	}
 
+	if (fail_uncached && db->db_state != DB_CACHED) {
+		mutex_exit(&db->db_mtx);
+		return (SET_ERROR(ENOENT));
+	}
+
 	if (db->db_buf && refcount_is_zero(&db->db_holds)) {
 		arc_buf_add_ref(db->db_buf, db);
 		if (db->db_buf->b_data == NULL) {
@@ -2067,16 +2260,14 @@ top:
 dmu_buf_impl_t *
 dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag)
 {
-	dmu_buf_impl_t *db;
-	int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db);
-	return (err ? NULL : db);
+	return (dbuf_hold_level(dn, 0, blkid, tag));
 }
 
 dmu_buf_impl_t *
 dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag)
 {
 	dmu_buf_impl_t *db;
-	int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db);
+	int err = dbuf_hold_impl(dn, level, blkid, FALSE, FALSE, tag, &db);
 	return (err ? NULL : db);
 }
 
@@ -2429,8 +2620,8 @@ dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db)
 		if (parent == NULL) {
 			mutex_exit(&db->db_mtx);
 			rw_enter(&dn->dn_struct_rwlock, RW_READER);
-			(void) dbuf_hold_impl(dn, db->db_level+1,
-			    db->db_blkid >> epbs, FALSE, db, &parent);
+			parent = dbuf_hold_level(dn, db->db_level + 1,
+			    db->db_blkid >> epbs, db);
 			rw_exit(&dn->dn_struct_rwlock);
 			mutex_enter(&db->db_mtx);
 			db->db_parent = parent;
diff --git a/uts/common/fs/zfs/dmu.c b/uts/common/fs/zfs/dmu.c
index a8cb22ff0269..841651fa3bff 100644
--- a/uts/common/fs/zfs/dmu.c
+++ b/uts/common/fs/zfs/dmu.c
@@ -138,7 +138,7 @@ dmu_buf_hold_noread(objset_t *os, uint64_t object, uint64_t offset,
 	err = dnode_hold(os, object, FTAG, &dn);
 	if (err)
 		return (err);
-	blkid = dbuf_whichblock(dn, offset);
+	blkid = dbuf_whichblock(dn, 0, offset);
 	rw_enter(&dn->dn_struct_rwlock, RW_READER);
 	db = dbuf_hold(dn, blkid, tag);
 	rw_exit(&dn->dn_struct_rwlock);
@@ -421,7 +421,7 @@ dmu_buf_hold_array_by_dnode(dnode_t *dn, uint64_t offset, uint64_t length,
 	dbp = kmem_zalloc(sizeof (dmu_buf_t *) * nblks, KM_SLEEP);
 
 	zio = zio_root(dn->dn_objset->os_spa, NULL, NULL, ZIO_FLAG_CANFAIL);
-	blkid = dbuf_whichblock(dn, offset);
+	blkid = dbuf_whichblock(dn, 0, offset);
 	for (i = 0; i < nblks; i++) {
 		dmu_buf_impl_t *db = dbuf_hold(dn, blkid+i, tag);
 		if (db == NULL) {
@@ -522,17 +522,16 @@ dmu_buf_rele_array(dmu_buf_t **dbp_fake, int numbufs, void *tag)
 }
 
 /*
- * Issue prefetch i/os for the given blocks.
+ * Issue prefetch i/os for the given blocks.  If level is greater than 0, the
+ * indirect blocks prefeteched will be those that point to the blocks containing
+ * the data starting at offset, and continuing to offset + len.
  *
- * Note: The assumption is that we *know* these blocks will be needed
- * almost immediately.  Therefore, the prefetch i/os will be issued at
- * ZIO_PRIORITY_SYNC_READ
- *
- * Note: indirect blocks and other metadata will be read synchronously,
- * causing this function to block if they are not already cached.
+ * Note that if the indirect blocks above the blocks being prefetched are not in
+ * cache, they will be asychronously read in.
  */
 void
-dmu_prefetch(objset_t *os, uint64_t object, uint64_t offset, uint64_t len)
+dmu_prefetch(objset_t *os, uint64_t object, int64_t level, uint64_t offset,
+    uint64_t len, zio_priority_t pri)
 {
 	dnode_t *dn;
 	uint64_t blkid;
@@ -548,8 +547,9 @@ dmu_prefetch(objset_t *os, uint64_t object, uint64_t offset, uint64_t len)
 			return;
 
 		rw_enter(&dn->dn_struct_rwlock, RW_READER);
-		blkid = dbuf_whichblock(dn, object * sizeof (dnode_phys_t));
-		dbuf_prefetch(dn, blkid, ZIO_PRIORITY_SYNC_READ);
+		blkid = dbuf_whichblock(dn, level,
+		    object * sizeof (dnode_phys_t));
+		dbuf_prefetch(dn, level, blkid, pri, 0);
 		rw_exit(&dn->dn_struct_rwlock);
 		return;
 	}
@@ -564,18 +564,24 @@ dmu_prefetch(objset_t *os, uint64_t object, uint64_t offset, uint64_t len)
 		return;
 
 	rw_enter(&dn->dn_struct_rwlock, RW_READER);
-	if (dn->dn_datablkshift) {
-		int blkshift = dn->dn_datablkshift;
-		nblks = (P2ROUNDUP(offset + len, 1 << blkshift) -
-		    P2ALIGN(offset, 1 << blkshift)) >> blkshift;
+	/*
+	 * offset + len - 1 is the last byte we want to prefetch for, and offset
+	 * is the first.  Then dbuf_whichblk(dn, level, off + len - 1) is the
+	 * last block we want to prefetch, and dbuf_whichblock(dn, level,
+	 * offset)  is the first.  Then the number we need to prefetch is the
+	 * last - first + 1.
+	 */
+	if (level > 0 || dn->dn_datablkshift != 0) {
+		nblks = dbuf_whichblock(dn, level, offset + len - 1) -
+		    dbuf_whichblock(dn, level, offset) + 1;
 	} else {
 		nblks = (offset < dn->dn_datablksz);
 	}
 
 	if (nblks != 0) {
-		blkid = dbuf_whichblock(dn, offset);
+		blkid = dbuf_whichblock(dn, level, offset);
 		for (int i = 0; i < nblks; i++)
-			dbuf_prefetch(dn, blkid + i, ZIO_PRIORITY_SYNC_READ);
+			dbuf_prefetch(dn, level, blkid + i, pri, 0);
 	}
 
 	rw_exit(&dn->dn_struct_rwlock);
@@ -1325,7 +1331,7 @@ dmu_assign_arcbuf(dmu_buf_t *handle, uint64_t offset, arc_buf_t *buf,
 	DB_DNODE_ENTER(dbuf);
 	dn = DB_DNODE(dbuf);
 	rw_enter(&dn->dn_struct_rwlock, RW_READER);
-	blkid = dbuf_whichblock(dn, offset);
+	blkid = dbuf_whichblock(dn, 0, offset);
 	VERIFY((db = dbuf_hold(dn, blkid, FTAG)) != NULL);
 	rw_exit(&dn->dn_struct_rwlock);
 	DB_DNODE_EXIT(dbuf);
diff --git a/uts/common/fs/zfs/dmu_diff.c b/uts/common/fs/zfs/dmu_diff.c
index 91415d0d2dcb..7665d1ca591d 100644
--- a/uts/common/fs/zfs/dmu_diff.c
+++ b/uts/common/fs/zfs/dmu_diff.c
@@ -115,7 +115,7 @@ diff_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 	if (issig(JUSTLOOKING) && issig(FORREAL))
 		return (SET_ERROR(EINTR));
 
-	if (zb->zb_object != DMU_META_DNODE_OBJECT)
+	if (bp == NULL || zb->zb_object != DMU_META_DNODE_OBJECT)
 		return (0);
 
 	if (BP_IS_HOLE(bp)) {
diff --git a/uts/common/fs/zfs/dmu_object.c b/uts/common/fs/zfs/dmu_object.c
index 808864a57607..6ca021eecb1d 100644
--- a/uts/common/fs/zfs/dmu_object.c
+++ b/uts/common/fs/zfs/dmu_object.c
@@ -148,6 +148,11 @@ dmu_object_free(objset_t *os, uint64_t object, dmu_tx_t *tx)
 	return (0);
 }
 
+/*
+ * Return (in *objectp) the next object which is allocated (or a hole)
+ * after *object, taking into account only objects that may have been modified
+ * after the specified txg.
+ */
 int
 dmu_object_next(objset_t *os, uint64_t *objectp, boolean_t hole, uint64_t txg)
 {
diff --git a/uts/common/fs/zfs/dmu_send.c b/uts/common/fs/zfs/dmu_send.c
index 7d728c6877e5..ca09d10bd806 100644
--- a/uts/common/fs/zfs/dmu_send.c
+++ b/uts/common/fs/zfs/dmu_send.c
@@ -52,13 +52,38 @@
 #include <sys/blkptr.h>
 #include <sys/dsl_bookmark.h>
 #include <sys/zfeature.h>
+#include <sys/bqueue.h>
 
 /* Set this tunable to TRUE to replace corrupt data with 0x2f5baddb10c */
 int zfs_send_corrupt_data = B_FALSE;
+int zfs_send_queue_length = 16 * 1024 * 1024;
+int zfs_recv_queue_length = 16 * 1024 * 1024;
 
 static char *dmu_recv_tag = "dmu_recv_tag";
 static const char *recv_clone_name = "%recv";
 
+#define	BP_SPAN(datablkszsec, indblkshift, level) \
+	(((uint64_t)datablkszsec) << (SPA_MINBLOCKSHIFT + \
+	(level) * (indblkshift - SPA_BLKPTRSHIFT)))
+
+struct send_thread_arg {
+	bqueue_t	q;
+	dsl_dataset_t	*ds;		/* Dataset to traverse */
+	uint64_t	fromtxg;	/* Traverse from this txg */
+	int		flags;		/* flags to pass to traverse_dataset */
+	int		error_code;
+	boolean_t	cancel;
+};
+
+struct send_block_record {
+	boolean_t		eos_marker; /* Marks the end of the stream */
+	blkptr_t		bp;
+	zbookmark_phys_t	zb;
+	uint8_t			indblkshift;
+	uint16_t		datablkszsec;
+	bqueue_node_t		ln;
+};
+
 static int
 dump_bytes(dmu_sendarg_t *dsp, void *buf, int len)
 {
@@ -434,58 +459,115 @@ backup_do_embed(dmu_sendarg_t *dsp, const blkptr_t *bp)
 	return (B_FALSE);
 }
 
-#define	BP_SPAN(dnp, level) \
-	(((uint64_t)dnp->dn_datablkszsec) << (SPA_MINBLOCKSHIFT + \
-	(level) * (dnp->dn_indblkshift - SPA_BLKPTRSHIFT)))
-
-/* ARGSUSED */
+/*
+ * This is the callback function to traverse_dataset that acts as the worker
+ * thread for dmu_send_impl.
+ */
+/*ARGSUSED*/
 static int
-backup_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
-    const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg)
+send_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
+    const zbookmark_phys_t *zb, const struct dnode_phys *dnp, void *arg)
 {
-	dmu_sendarg_t *dsp = arg;
+	struct send_thread_arg *sta = arg;
+	struct send_block_record *record;
+	uint64_t record_size;
+	int err = 0;
+
+	if (sta->cancel)
+		return (SET_ERROR(EINTR));
+
+	if (bp == NULL) {
+		ASSERT3U(zb->zb_level, ==, ZB_DNODE_LEVEL);
+		return (0);
+	} else if (zb->zb_level < 0) {
+		return (0);
+	}
+
+	record = kmem_zalloc(sizeof (struct send_block_record), KM_SLEEP);
+	record->eos_marker = B_FALSE;
+	record->bp = *bp;
+	record->zb = *zb;
+	record->indblkshift = dnp->dn_indblkshift;
+	record->datablkszsec = dnp->dn_datablkszsec;
+	record_size = dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT;
+	bqueue_enqueue(&sta->q, record, record_size);
+
+	return (err);
+}
+
+/*
+ * This function kicks off the traverse_dataset.  It also handles setting the
+ * error code of the thread in case something goes wrong, and pushes the End of
+ * Stream record when the traverse_dataset call has finished.  If there is no
+ * dataset to traverse, the thread immediately pushes End of Stream marker.
+ */
+static void
+send_traverse_thread(void *arg)
+{
+	struct send_thread_arg *st_arg = arg;
+	int err;
+	struct send_block_record *data;
+
+	if (st_arg->ds != NULL) {
+		err = traverse_dataset(st_arg->ds, st_arg->fromtxg,
+		    st_arg->flags, send_cb, arg);
+		if (err != EINTR)
+			st_arg->error_code = err;
+	}
+	data = kmem_zalloc(sizeof (*data), KM_SLEEP);
+	data->eos_marker = B_TRUE;
+	bqueue_enqueue(&st_arg->q, data, 1);
+}
+
+/*
+ * This function actually handles figuring out what kind of record needs to be
+ * dumped, reading the data (which has hopefully been prefetched), and calling
+ * the appropriate helper function.
+ */
+static int
+do_dump(dmu_sendarg_t *dsa, struct send_block_record *data)
+{
+	dsl_dataset_t *ds = dmu_objset_ds(dsa->dsa_os);
+	const blkptr_t *bp = &data->bp;
+	const zbookmark_phys_t *zb = &data->zb;
+	uint8_t indblkshift = data->indblkshift;
+	uint16_t dblkszsec = data->datablkszsec;
+	spa_t *spa = ds->ds_dir->dd_pool->dp_spa;
 	dmu_object_type_t type = bp ? BP_GET_TYPE(bp) : DMU_OT_NONE;
 	int err = 0;
 
-	if (issig(JUSTLOOKING) && issig(FORREAL))
-		return (SET_ERROR(EINTR));
+	ASSERT3U(zb->zb_level, >=, 0);
 
 	if (zb->zb_object != DMU_META_DNODE_OBJECT &&
 	    DMU_OBJECT_IS_SPECIAL(zb->zb_object)) {
 		return (0);
-	} else if (zb->zb_level == ZB_ZIL_LEVEL) {
-		/*
-		 * If we are sending a non-snapshot (which is allowed on
-		 * read-only pools), it may have a ZIL, which must be ignored.
-		 */
-		return (0);
 	} else if (BP_IS_HOLE(bp) &&
 	    zb->zb_object == DMU_META_DNODE_OBJECT) {
-		uint64_t span = BP_SPAN(dnp, zb->zb_level);
+		uint64_t span = BP_SPAN(dblkszsec, indblkshift, zb->zb_level);
 		uint64_t dnobj = (zb->zb_blkid * span) >> DNODE_SHIFT;
-		err = dump_freeobjects(dsp, dnobj, span >> DNODE_SHIFT);
+		err = dump_freeobjects(dsa, dnobj, span >> DNODE_SHIFT);
 	} else if (BP_IS_HOLE(bp)) {
-		uint64_t span = BP_SPAN(dnp, zb->zb_level);
-		err = dump_free(dsp, zb->zb_object, zb->zb_blkid * span, span);
+		uint64_t span = BP_SPAN(dblkszsec, indblkshift, zb->zb_level);
+		uint64_t offset = zb->zb_blkid * span;
+		err = dump_free(dsa, zb->zb_object, offset, span);
 	} else if (zb->zb_level > 0 || type == DMU_OT_OBJSET) {
 		return (0);
 	} else if (type == DMU_OT_DNODE) {
-		dnode_phys_t *blk;
-		int i;
 		int blksz = BP_GET_LSIZE(bp);
 		arc_flags_t aflags = ARC_FLAG_WAIT;
 		arc_buf_t *abuf;
 
+		ASSERT0(zb->zb_level);
+
 		if (arc_read(NULL, spa, bp, arc_getbuf_func, &abuf,
 		    ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL,
 		    &aflags, zb) != 0)
 			return (SET_ERROR(EIO));
 
-		blk = abuf->b_data;
-		for (i = 0; i < blksz >> DNODE_SHIFT; i++) {
-			uint64_t dnobj = (zb->zb_blkid <<
-			    (DNODE_BLOCK_SHIFT - DNODE_SHIFT)) + i;
-			err = dump_dnode(dsp, dnobj, blk+i);
+		dnode_phys_t *blk = abuf->b_data;
+		uint64_t dnobj = zb->zb_blkid * (blksz >> DNODE_SHIFT);
+		for (int i = 0; i < blksz >> DNODE_SHIFT; i++) {
+			err = dump_dnode(dsa, dnobj + i, blk + i);
 			if (err != 0)
 				break;
 		}
@@ -500,20 +582,21 @@ backup_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 		    &aflags, zb) != 0)
 			return (SET_ERROR(EIO));
 
-		err = dump_spill(dsp, zb->zb_object, blksz, abuf->b_data);
+		err = dump_spill(dsa, zb->zb_object, blksz, abuf->b_data);
 		(void) arc_buf_remove_ref(abuf, &abuf);
-	} else if (backup_do_embed(dsp, bp)) {
+	} else if (backup_do_embed(dsa, bp)) {
 		/* it's an embedded level-0 block of a regular object */
-		int blksz = dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT;
-		err = dump_write_embedded(dsp, zb->zb_object,
+		int blksz = dblkszsec << SPA_MINBLOCKSHIFT;
+		ASSERT0(zb->zb_level);
+		err = dump_write_embedded(dsa, zb->zb_object,
 		    zb->zb_blkid * blksz, blksz, bp);
-	} else { /* it's a level-0 block of a regular object */
+	} else {
+		/* it's a level-0 block of a regular object */
 		arc_flags_t aflags = ARC_FLAG_WAIT;
 		arc_buf_t *abuf;
-		int blksz = BP_GET_LSIZE(bp);
+		int blksz = dblkszsec << SPA_MINBLOCKSHIFT;
 		uint64_t offset;
 
-		ASSERT3U(blksz, ==, dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT);
 		ASSERT0(zb->zb_level);
 		if (arc_read(NULL, spa, bp, arc_getbuf_func, &abuf,
 		    ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL,
@@ -534,20 +617,20 @@ backup_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 
 		offset = zb->zb_blkid * blksz;
 
-		if (!(dsp->dsa_featureflags &
+		if (!(dsa->dsa_featureflags &
 		    DMU_BACKUP_FEATURE_LARGE_BLOCKS) &&
 		    blksz > SPA_OLD_MAXBLOCKSIZE) {
 			char *buf = abuf->b_data;
 			while (blksz > 0 && err == 0) {
 				int n = MIN(blksz, SPA_OLD_MAXBLOCKSIZE);
-				err = dump_write(dsp, type, zb->zb_object,
+				err = dump_write(dsa, type, zb->zb_object,
 				    offset, n, NULL, buf);
 				offset += n;
 				buf += n;
 				blksz -= n;
 			}
 		} else {
-			err = dump_write(dsp, type, zb->zb_object,
+			err = dump_write(dsa, type, zb->zb_object,
 			    offset, blksz, bp, abuf->b_data);
 		}
 		(void) arc_buf_remove_ref(abuf, &abuf);
@@ -558,11 +641,24 @@ backup_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 }
 
 /*
- * Releases dp using the specified tag.
+ * Pop the new data off the queue, and free the old data.
+ */
+static struct send_block_record *
+get_next_record(bqueue_t *bq, struct send_block_record *data)
+{
+	struct send_block_record *tmp = bqueue_dequeue(bq);
+	kmem_free(data, sizeof (*data));
+	return (tmp);
+}
+
+/*
+ * Actually do the bulk of the work in a zfs send.
+ *
+ * Note: Releases dp using the specified tag.
  */
 static int
-dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *ds,
-    zfs_bookmark_phys_t *fromzb, boolean_t is_clone, boolean_t embedok,
+dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *to_ds,
+    zfs_bookmark_phys_t *ancestor_zb, boolean_t is_clone, boolean_t embedok,
     boolean_t large_block_ok, int outfd, vnode_t *vp, offset_t *off)
 {
 	objset_t *os;
@@ -571,8 +667,9 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *ds,
 	int err;
 	uint64_t fromtxg = 0;
 	uint64_t featureflags = 0;
+	struct send_thread_arg to_arg;
 
-	err = dmu_objset_from_ds(ds, &os);
+	err = dmu_objset_from_ds(to_ds, &os);
 	if (err != 0) {
 		dsl_pool_rele(dp, tag);
 		return (err);
@@ -598,35 +695,34 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *ds,
 	}
 #endif
 
-	if (large_block_ok && ds->ds_large_blocks)
+	if (large_block_ok && to_ds->ds_large_blocks)
 		featureflags |= DMU_BACKUP_FEATURE_LARGE_BLOCKS;
 	if (embedok &&
 	    spa_feature_is_active(dp->dp_spa, SPA_FEATURE_EMBEDDED_DATA)) {
 		featureflags |= DMU_BACKUP_FEATURE_EMBED_DATA;
 		if (spa_feature_is_active(dp->dp_spa, SPA_FEATURE_LZ4_COMPRESS))
 			featureflags |= DMU_BACKUP_FEATURE_EMBED_DATA_LZ4;
-	} else {
-		embedok = B_FALSE;
 	}
 
 	DMU_SET_FEATUREFLAGS(drr->drr_u.drr_begin.drr_versioninfo,
 	    featureflags);
 
 	drr->drr_u.drr_begin.drr_creation_time =
-	    dsl_dataset_phys(ds)->ds_creation_time;
+	    dsl_dataset_phys(to_ds)->ds_creation_time;
 	drr->drr_u.drr_begin.drr_type = dmu_objset_type(os);
 	if (is_clone)
 		drr->drr_u.drr_begin.drr_flags |= DRR_FLAG_CLONE;
-	drr->drr_u.drr_begin.drr_toguid = dsl_dataset_phys(ds)->ds_guid;
-	if (dsl_dataset_phys(ds)->ds_flags & DS_FLAG_CI_DATASET)
+	drr->drr_u.drr_begin.drr_toguid = dsl_dataset_phys(to_ds)->ds_guid;
+	if (dsl_dataset_phys(to_ds)->ds_flags & DS_FLAG_CI_DATASET)
 		drr->drr_u.drr_begin.drr_flags |= DRR_FLAG_CI_DATA;
 
-	if (fromzb != NULL) {
-		drr->drr_u.drr_begin.drr_fromguid = fromzb->zbm_guid;
-		fromtxg = fromzb->zbm_creation_txg;
+	if (ancestor_zb != NULL) {
+		drr->drr_u.drr_begin.drr_fromguid =
+		    ancestor_zb->zbm_guid;
+		fromtxg = ancestor_zb->zbm_creation_txg;
 	}
-	dsl_dataset_name(ds, drr->drr_u.drr_begin.drr_toname);
-	if (!ds->ds_is_snapshot) {
+	dsl_dataset_name(to_ds, drr->drr_u.drr_begin.drr_toname);
+	if (!to_ds->ds_is_snapshot) {
 		(void) strlcat(drr->drr_u.drr_begin.drr_toname, "@--head--",
 		    sizeof (drr->drr_u.drr_begin.drr_toname));
 	}
@@ -639,16 +735,16 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *ds,
 	dsp->dsa_proc = curproc;
 	dsp->dsa_os = os;
 	dsp->dsa_off = off;
-	dsp->dsa_toguid = dsl_dataset_phys(ds)->ds_guid;
+	dsp->dsa_toguid = dsl_dataset_phys(to_ds)->ds_guid;
 	dsp->dsa_pending_op = PENDING_NONE;
-	dsp->dsa_incremental = (fromzb != NULL);
+	dsp->dsa_incremental = (ancestor_zb != NULL);
 	dsp->dsa_featureflags = featureflags;
 
-	mutex_enter(&ds->ds_sendstream_lock);
-	list_insert_head(&ds->ds_sendstreams, dsp);
-	mutex_exit(&ds->ds_sendstream_lock);
+	mutex_enter(&to_ds->ds_sendstream_lock);
+	list_insert_head(&to_ds->ds_sendstreams, dsp);
+	mutex_exit(&to_ds->ds_sendstream_lock);
 
-	dsl_dataset_long_hold(ds, FTAG);
+	dsl_dataset_long_hold(to_ds, FTAG);
 	dsl_pool_rele(dp, tag);
 
 	if (dump_record(dsp, NULL, 0) != 0) {
@@ -656,8 +752,41 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *ds,
 		goto out;
 	}
 
-	err = traverse_dataset(ds, fromtxg, TRAVERSE_PRE | TRAVERSE_PREFETCH,
-	    backup_cb, dsp);
+	err = bqueue_init(&to_arg.q, zfs_send_queue_length,
+	    offsetof(struct send_block_record, ln));
+	to_arg.error_code = 0;
+	to_arg.cancel = B_FALSE;
+	to_arg.ds = to_ds;
+	to_arg.fromtxg = fromtxg;
+	to_arg.flags = TRAVERSE_PRE | TRAVERSE_PREFETCH;
+	(void) thread_create(NULL, 0, send_traverse_thread, &to_arg, 0, curproc,
+	    TS_RUN, minclsyspri);
+
+	struct send_block_record *to_data;
+	to_data = bqueue_dequeue(&to_arg.q);
+
+	while (!to_data->eos_marker && err == 0) {
+		err = do_dump(dsp, to_data);
+		to_data = get_next_record(&to_arg.q, to_data);
+		if (issig(JUSTLOOKING) && issig(FORREAL))
+			err = EINTR;
+	}
+
+	if (err != 0) {
+		to_arg.cancel = B_TRUE;
+		while (!to_data->eos_marker) {
+			to_data = get_next_record(&to_arg.q, to_data);
+		}
+	}
+	kmem_free(to_data, sizeof (*to_data));
+
+	bqueue_destroy(&to_arg.q);
+
+	if (err == 0 && to_arg.error_code != 0)
+		err = to_arg.error_code;
+
+	if (err != 0)
+		goto out;
 
 	if (dsp->dsa_pending_op != PENDING_NONE)
 		if (dump_record(dsp, NULL, 0) != 0)
@@ -674,20 +803,18 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *ds,
 	drr->drr_u.drr_end.drr_checksum = dsp->dsa_zc;
 	drr->drr_u.drr_end.drr_toguid = dsp->dsa_toguid;
 
-	if (dump_record(dsp, NULL, 0) != 0) {
+	if (dump_record(dsp, NULL, 0) != 0)
 		err = dsp->dsa_err;
-		goto out;
-	}
 
 out:
-	mutex_enter(&ds->ds_sendstream_lock);
-	list_remove(&ds->ds_sendstreams, dsp);
-	mutex_exit(&ds->ds_sendstream_lock);
+	mutex_enter(&to_ds->ds_sendstream_lock);
+	list_remove(&to_ds->ds_sendstreams, dsp);
+	mutex_exit(&to_ds->ds_sendstream_lock);
 
 	kmem_free(drr, sizeof (dmu_replay_record_t));
 	kmem_free(dsp, sizeof (dmu_sendarg_t));
 
-	dsl_dataset_long_rele(ds, FTAG);
+	dsl_dataset_long_rele(to_ds, FTAG);
 
 	return (err);
 }
@@ -1110,7 +1237,8 @@ dmu_recv_begin_check(void *arg, dmu_tx_t *tx)
 		 * If it's a non-clone incremental, we are missing the
 		 * target fs, so fail the recv.
 		 */
-		if (fromguid != 0 && !(flags & DRR_FLAG_CLONE))
+		if (fromguid != 0 && !(flags & DRR_FLAG_CLONE ||
+		    drba->drba_origin))
 			return (SET_ERROR(ENOENT));
 
 		/* Open the parent of tofs */
@@ -1292,21 +1420,57 @@ dmu_recv_begin(char *tofs, char *tosnap, struct drr_begin *drrb,
 	    &drba, 5, ZFS_SPACE_CHECK_NORMAL));
 }
 
-struct restorearg {
-	objset_t *os;
-	int err;
-	boolean_t byteswap;
-	vnode_t *vp;
-	uint64_t voff;
-	int bufsize; /* amount of memory allocated for buf */
+struct receive_record_arg {
+	dmu_replay_record_t header;
+	void *payload; /* Pointer to a buffer containing the payload */
+	/*
+	 * If the record is a write, pointer to the arc_buf_t containing the
+	 * payload.
+	 */
+	arc_buf_t *write_buf;
+	int payload_size;
+	boolean_t eos_marker; /* Marks the end of the stream */
+	bqueue_node_t node;
+};
 
-	dmu_replay_record_t *drr;
-	dmu_replay_record_t *next_drr;
-	char *buf;
+struct receive_writer_arg {
+	objset_t *os;
+	boolean_t byteswap;
+	bqueue_t q;
+	/*
+	 * These three args are used to signal to the main thread that we're
+	 * done.
+	 */
+	kmutex_t mutex;
+	kcondvar_t cv;
+	boolean_t done;
+	int err;
+	/* A map from guid to dataset to help handle dedup'd streams. */
+	avl_tree_t *guid_to_ds_map;
+};
+
+struct receive_arg  {
+	objset_t *os;
+	vnode_t *vp; /* The vnode to read the stream from */
+	uint64_t voff; /* The current offset in the stream */
+	/*
+	 * A record that has had its payload read in, but hasn't yet been handed
+	 * off to the worker thread.
+	 */
+	struct receive_record_arg *rrd;
+	/* A record that has had its header read in, but not its payload. */
+	struct receive_record_arg *next_rrd;
 	zio_cksum_t cksum;
 	zio_cksum_t prev_cksum;
+	int err;
+	boolean_t byteswap;
+	/* Sorted list of objects not to issue prefetches for. */
+	list_t ignore_obj_list;
+};
 
-	avl_tree_t *guid_to_ds_map;
+struct receive_ign_obj_node {
+	list_node_t node;
+	uint64_t object;
 };
 
 typedef struct guid_map_entry {
@@ -1345,13 +1509,12 @@ free_guid_map_onexit(void *arg)
 }
 
 static int
-restore_read(struct restorearg *ra, int len, void *buf)
+receive_read(struct receive_arg *ra, int len, void *buf)
 {
 	int done = 0;
 
 	/* some things will require 8-byte alignment, so everything must */
 	ASSERT0(len % 8);
-	ASSERT3U(len, <=, ra->bufsize);
 
 	while (done < len) {
 		ssize_t resid;
@@ -1470,7 +1633,8 @@ deduce_nblkptr(dmu_object_type_t bonus_type, uint64_t bonus_size)
 }
 
 static int
-restore_object(struct restorearg *ra, struct drr_object *drro, void *data)
+receive_object(struct receive_writer_arg *rwa, struct drr_object *drro,
+    void *data)
 {
 	dmu_object_info_t doi;
 	dmu_tx_t *tx;
@@ -1484,12 +1648,12 @@ restore_object(struct restorearg *ra, struct drr_object *drro, void *data)
 	    drro->drr_compress >= ZIO_COMPRESS_FUNCTIONS ||
 	    P2PHASE(drro->drr_blksz, SPA_MINBLOCKSIZE) ||
 	    drro->drr_blksz < SPA_MINBLOCKSIZE ||
-	    drro->drr_blksz > spa_maxblocksize(dmu_objset_spa(ra->os)) ||
+	    drro->drr_blksz > spa_maxblocksize(dmu_objset_spa(rwa->os)) ||
 	    drro->drr_bonuslen > DN_MAX_BONUSLEN) {
 		return (SET_ERROR(EINVAL));
 	}
 
-	err = dmu_object_info(ra->os, drro->drr_object, &doi);
+	err = dmu_object_info(rwa->os, drro->drr_object, &doi);
 
 	if (err != 0 && err != ENOENT)
 		return (SET_ERROR(EINVAL));
@@ -1508,14 +1672,14 @@ restore_object(struct restorearg *ra, struct drr_object *drro, void *data)
 
 		if (drro->drr_blksz != doi.doi_data_block_size ||
 		    nblkptr < doi.doi_nblkptr) {
-			err = dmu_free_long_range(ra->os, drro->drr_object,
+			err = dmu_free_long_range(rwa->os, drro->drr_object,
 			    0, DMU_OBJECT_END);
 			if (err != 0)
 				return (SET_ERROR(EINVAL));
 		}
 	}
 
-	tx = dmu_tx_create(ra->os);
+	tx = dmu_tx_create(rwa->os);
 	dmu_tx_hold_bonus(tx, object);
 	err = dmu_tx_assign(tx, TXG_WAIT);
 	if (err != 0) {
@@ -1525,7 +1689,7 @@ restore_object(struct restorearg *ra, struct drr_object *drro, void *data)
 
 	if (object == DMU_NEW_OBJECT) {
 		/* currently free, want to be allocated */
-		err = dmu_object_claim(ra->os, drro->drr_object,
+		err = dmu_object_claim(rwa->os, drro->drr_object,
 		    drro->drr_type, drro->drr_blksz,
 		    drro->drr_bonustype, drro->drr_bonuslen, tx);
 	} else if (drro->drr_type != doi.doi_type ||
@@ -1533,7 +1697,7 @@ restore_object(struct restorearg *ra, struct drr_object *drro, void *data)
 	    drro->drr_bonustype != doi.doi_bonus_type ||
 	    drro->drr_bonuslen != doi.doi_bonus_size) {
 		/* currently allocated, but with different properties */
-		err = dmu_object_reclaim(ra->os, drro->drr_object,
+		err = dmu_object_reclaim(rwa->os, drro->drr_object,
 		    drro->drr_type, drro->drr_blksz,
 		    drro->drr_bonustype, drro->drr_bonuslen, tx);
 	}
@@ -1542,20 +1706,20 @@ restore_object(struct restorearg *ra, struct drr_object *drro, void *data)
 		return (SET_ERROR(EINVAL));
 	}
 
-	dmu_object_set_checksum(ra->os, drro->drr_object,
+	dmu_object_set_checksum(rwa->os, drro->drr_object,
 	    drro->drr_checksumtype, tx);
-	dmu_object_set_compress(ra->os, drro->drr_object,
+	dmu_object_set_compress(rwa->os, drro->drr_object,
 	    drro->drr_compress, tx);
 
 	if (data != NULL) {
 		dmu_buf_t *db;
 
-		VERIFY0(dmu_bonus_hold(ra->os, drro->drr_object, FTAG, &db));
+		VERIFY0(dmu_bonus_hold(rwa->os, drro->drr_object, FTAG, &db));
 		dmu_buf_will_dirty(db, tx);
 
 		ASSERT3U(db->db_size, >=, drro->drr_bonuslen);
 		bcopy(data, db->db_data, drro->drr_bonuslen);
-		if (ra->byteswap) {
+		if (rwa->byteswap) {
 			dmu_object_byteswap_t byteswap =
 			    DMU_OT_BYTESWAP(drro->drr_bonustype);
 			dmu_ot_byteswap[byteswap].ob_func(db->db_data,
@@ -1569,7 +1733,7 @@ restore_object(struct restorearg *ra, struct drr_object *drro, void *data)
 
 /* ARGSUSED */
 static int
-restore_freeobjects(struct restorearg *ra,
+receive_freeobjects(struct receive_writer_arg *rwa,
     struct drr_freeobjects *drrfo)
 {
 	uint64_t obj;
@@ -1579,13 +1743,13 @@ restore_freeobjects(struct restorearg *ra,
 
 	for (obj = drrfo->drr_firstobj;
 	    obj < drrfo->drr_firstobj + drrfo->drr_numobjs;
-	    (void) dmu_object_next(ra->os, &obj, FALSE, 0)) {
+	    (void) dmu_object_next(rwa->os, &obj, FALSE, 0)) {
 		int err;
 
-		if (dmu_object_info(ra->os, obj, NULL) != 0)
+		if (dmu_object_info(rwa->os, obj, NULL) != 0)
 			continue;
 
-		err = dmu_free_long_object(ra->os, obj);
+		err = dmu_free_long_object(rwa->os, obj);
 		if (err != 0)
 			return (err);
 	}
@@ -1593,7 +1757,8 @@ restore_freeobjects(struct restorearg *ra,
 }
 
 static int
-restore_write(struct restorearg *ra, struct drr_write *drrw, arc_buf_t *abuf)
+receive_write(struct receive_writer_arg *rwa, struct drr_write *drrw,
+    arc_buf_t *abuf)
 {
 	dmu_tx_t *tx;
 	int err;
@@ -1602,10 +1767,10 @@ restore_write(struct restorearg *ra, struct drr_write *drrw, arc_buf_t *abuf)
 	    !DMU_OT_IS_VALID(drrw->drr_type))
 		return (SET_ERROR(EINVAL));
 
-	if (dmu_object_info(ra->os, drrw->drr_object, NULL) != 0)
+	if (dmu_object_info(rwa->os, drrw->drr_object, NULL) != 0)
 		return (SET_ERROR(EINVAL));
 
-	tx = dmu_tx_create(ra->os);
+	tx = dmu_tx_create(rwa->os);
 
 	dmu_tx_hold_write(tx, drrw->drr_object,
 	    drrw->drr_offset, drrw->drr_length);
@@ -1614,7 +1779,7 @@ restore_write(struct restorearg *ra, struct drr_write *drrw, arc_buf_t *abuf)
 		dmu_tx_abort(tx);
 		return (err);
 	}
-	if (ra->byteswap) {
+	if (rwa->byteswap) {
 		dmu_object_byteswap_t byteswap =
 		    DMU_OT_BYTESWAP(drrw->drr_type);
 		dmu_ot_byteswap[byteswap].ob_func(abuf->b_data,
@@ -1622,7 +1787,7 @@ restore_write(struct restorearg *ra, struct drr_write *drrw, arc_buf_t *abuf)
 	}
 
 	dmu_buf_t *bonus;
-	if (dmu_bonus_hold(ra->os, drrw->drr_object, FTAG, &bonus) != 0)
+	if (dmu_bonus_hold(rwa->os, drrw->drr_object, FTAG, &bonus) != 0)
 		return (SET_ERROR(EINVAL));
 	dmu_assign_arcbuf(bonus, drrw->drr_offset, abuf, tx);
 	dmu_tx_commit(tx);
@@ -1638,7 +1803,8 @@ restore_write(struct restorearg *ra, struct drr_write *drrw, arc_buf_t *abuf)
  * data from the stream to fulfill this write.
  */
 static int
-restore_write_byref(struct restorearg *ra, struct drr_write_byref *drrwbr)
+receive_write_byref(struct receive_writer_arg *rwa,
+    struct drr_write_byref *drrwbr)
 {
 	dmu_tx_t *tx;
 	int err;
@@ -1657,14 +1823,14 @@ restore_write_byref(struct restorearg *ra, struct drr_write_byref *drrwbr)
 	 */
 	if (drrwbr->drr_toguid != drrwbr->drr_refguid) {
 		gmesrch.guid = drrwbr->drr_refguid;
-		if ((gmep = avl_find(ra->guid_to_ds_map, &gmesrch,
+		if ((gmep = avl_find(rwa->guid_to_ds_map, &gmesrch,
 		    &where)) == NULL) {
 			return (SET_ERROR(EINVAL));
 		}
 		if (dmu_objset_from_ds(gmep->gme_ds, &ref_os))
 			return (SET_ERROR(EINVAL));
 	} else {
-		ref_os = ra->os;
+		ref_os = rwa->os;
 	}
 
 	err = dmu_buf_hold(ref_os, drrwbr->drr_refobject,
@@ -1672,7 +1838,7 @@ restore_write_byref(struct restorearg *ra, struct drr_write_byref *drrwbr)
 	if (err != 0)
 		return (err);
 
-	tx = dmu_tx_create(ra->os);
+	tx = dmu_tx_create(rwa->os);
 
 	dmu_tx_hold_write(tx, drrwbr->drr_object,
 	    drrwbr->drr_offset, drrwbr->drr_length);
@@ -1681,7 +1847,7 @@ restore_write_byref(struct restorearg *ra, struct drr_write_byref *drrwbr)
 		dmu_tx_abort(tx);
 		return (err);
 	}
-	dmu_write(ra->os, drrwbr->drr_object,
+	dmu_write(rwa->os, drrwbr->drr_object,
 	    drrwbr->drr_offset, drrwbr->drr_length, dbp->db_data, tx);
 	dmu_buf_rele(dbp, FTAG);
 	dmu_tx_commit(tx);
@@ -1689,7 +1855,7 @@ restore_write_byref(struct restorearg *ra, struct drr_write_byref *drrwbr)
 }
 
 static int
-restore_write_embedded(struct restorearg *ra,
+receive_write_embedded(struct receive_writer_arg *rwa,
     struct drr_write_embedded *drrwnp, void *data)
 {
 	dmu_tx_t *tx;
@@ -1706,7 +1872,7 @@ restore_write_embedded(struct restorearg *ra,
 	if (drrwnp->drr_compression >= ZIO_COMPRESS_FUNCTIONS)
 		return (EINVAL);
 
-	tx = dmu_tx_create(ra->os);
+	tx = dmu_tx_create(rwa->os);
 
 	dmu_tx_hold_write(tx, drrwnp->drr_object,
 	    drrwnp->drr_offset, drrwnp->drr_length);
@@ -1716,36 +1882,37 @@ restore_write_embedded(struct restorearg *ra,
 		return (err);
 	}
 
-	dmu_write_embedded(ra->os, drrwnp->drr_object,
+	dmu_write_embedded(rwa->os, drrwnp->drr_object,
 	    drrwnp->drr_offset, data, drrwnp->drr_etype,
 	    drrwnp->drr_compression, drrwnp->drr_lsize, drrwnp->drr_psize,
-	    ra->byteswap ^ ZFS_HOST_BYTEORDER, tx);
+	    rwa->byteswap ^ ZFS_HOST_BYTEORDER, tx);
 
 	dmu_tx_commit(tx);
 	return (0);
 }
 
 static int
-restore_spill(struct restorearg *ra, struct drr_spill *drrs, void *data)
+receive_spill(struct receive_writer_arg *rwa, struct drr_spill *drrs,
+    void *data)
 {
 	dmu_tx_t *tx;
 	dmu_buf_t *db, *db_spill;
 	int err;
 
 	if (drrs->drr_length < SPA_MINBLOCKSIZE ||
-	    drrs->drr_length > spa_maxblocksize(dmu_objset_spa(ra->os)))
+	    drrs->drr_length > spa_maxblocksize(dmu_objset_spa(rwa->os)))
 		return (SET_ERROR(EINVAL));
 
-	if (dmu_object_info(ra->os, drrs->drr_object, NULL) != 0)
+	if (dmu_object_info(rwa->os, drrs->drr_object, NULL) != 0)
 		return (SET_ERROR(EINVAL));
 
-	VERIFY0(dmu_bonus_hold(ra->os, drrs->drr_object, FTAG, &db));
+	VERIFY0(dmu_bonus_hold(rwa->os, drrs->drr_object, FTAG, &db));
 	if ((err = dmu_spill_hold_by_bonus(db, FTAG, &db_spill)) != 0) {
 		dmu_buf_rele(db, FTAG);
 		return (err);
 	}
 
-	tx = dmu_tx_create(ra->os);
+	tx = dmu_tx_create(rwa->os);
 
 	dmu_tx_hold_spill(tx, db->db_object);
 
@@ -1772,7 +1939,7 @@ restore_spill(struct restorearg *ra, struct drr_spill *drrs, void *data)
 
 /* ARGSUSED */
 static int
-restore_free(struct restorearg *ra, struct drr_free *drrf)
+receive_free(struct receive_writer_arg *rwa, struct drr_free *drrf)
 {
 	int err;
 
@@ -1780,11 +1947,12 @@ restore_free(struct restorearg *ra, struct drr_free *drrf)
 	    drrf->drr_offset + drrf->drr_length < drrf->drr_offset)
 		return (SET_ERROR(EINVAL));
 
-	if (dmu_object_info(ra->os, drrf->drr_object, NULL) != 0)
+	if (dmu_object_info(rwa->os, drrf->drr_object, NULL) != 0)
 		return (SET_ERROR(EINVAL));
 
-	err = dmu_free_long_range(ra->os, drrf->drr_object,
+	err = dmu_free_long_range(rwa->os, drrf->drr_object,
 	    drrf->drr_offset, drrf->drr_length);
+
 	return (err);
 }
 
@@ -1799,7 +1967,7 @@ dmu_recv_cleanup_ds(dmu_recv_cookie_t *drc)
 }
 
 static void
-restore_cksum(struct restorearg *ra, int len, void *buf)
+receive_cksum(struct receive_arg *ra, int len, void *buf)
 {
 	if (ra->byteswap) {
 		fletcher_4_incremental_byteswap(buf, len, &ra->cksum);
@@ -1809,30 +1977,42 @@ restore_cksum(struct restorearg *ra, int len, void *buf)
 }
 
 /*
- * If len != 0, read payload into buf.
- * Read next record's header into ra->next_drr.
+ * Read the payload into a buffer of size len, and update the current record's
+ * payload field.
+ * Allocate ra->next_rrd and read the next record's header into
+ * ra->next_rrd->header.
  * Verify checksum of payload and next record.
  */
 static int
-restore_read_payload_and_next_header(struct restorearg *ra, int len, void *buf)
+receive_read_payload_and_next_header(struct receive_arg *ra, int len, void *buf)
 {
 	int err;
 
 	if (len != 0) {
-		ASSERT3U(len, <=, ra->bufsize);
-		err = restore_read(ra, len, buf);
+		ASSERT3U(len, <=, SPA_MAXBLOCKSIZE);
+		ra->rrd->payload = buf;
+		ra->rrd->payload_size = len;
+		err = receive_read(ra, len, ra->rrd->payload);
 		if (err != 0)
 			return (err);
-		restore_cksum(ra, len, buf);
+		receive_cksum(ra, len, ra->rrd->payload);
 	}
 
 	ra->prev_cksum = ra->cksum;
 
-	err = restore_read(ra, sizeof (*ra->next_drr), ra->next_drr);
-	if (err != 0)
+	ra->next_rrd = kmem_zalloc(sizeof (*ra->next_rrd), KM_SLEEP);
+	err = receive_read(ra, sizeof (ra->next_rrd->header),
+	    &ra->next_rrd->header);
+	if (err != 0) {
+		kmem_free(ra->next_rrd, sizeof (*ra->next_rrd));
+		ra->next_rrd = NULL;
 		return (err);
-	if (ra->next_drr->drr_type == DRR_BEGIN)
+	}
+	if (ra->next_rrd->header.drr_type == DRR_BEGIN) {
+		kmem_free(ra->next_rrd, sizeof (*ra->next_rrd));
+		ra->next_rrd = NULL;
 		return (SET_ERROR(EINVAL));
+	}
 
 	/*
 	 * Note: checksum is of everything up to but not including the
@@ -1840,107 +2020,180 @@ restore_read_payload_and_next_header(struct restorearg *ra, int len, void *buf)
 	 */
 	ASSERT3U(offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
 	    ==, sizeof (dmu_replay_record_t) - sizeof (zio_cksum_t));
-	restore_cksum(ra,
+	receive_cksum(ra,
 	    offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
-	    ra->next_drr);
+	    &ra->next_rrd->header);
 
-	zio_cksum_t cksum_orig = ra->next_drr->drr_u.drr_checksum.drr_checksum;
-	zio_cksum_t *cksump = &ra->next_drr->drr_u.drr_checksum.drr_checksum;
+	zio_cksum_t cksum_orig =
+	    ra->next_rrd->header.drr_u.drr_checksum.drr_checksum;
+	zio_cksum_t *cksump =
+	    &ra->next_rrd->header.drr_u.drr_checksum.drr_checksum;
 
 	if (ra->byteswap)
-		byteswap_record(ra->next_drr);
+		byteswap_record(&ra->next_rrd->header);
 
 	if ((!ZIO_CHECKSUM_IS_ZERO(cksump)) &&
-	    !ZIO_CHECKSUM_EQUAL(ra->cksum, *cksump))
+	    !ZIO_CHECKSUM_EQUAL(ra->cksum, *cksump)) {
+		kmem_free(ra->next_rrd, sizeof (*ra->next_rrd));
+		ra->next_rrd = NULL;
 		return (SET_ERROR(ECKSUM));
+	}
 
-	restore_cksum(ra, sizeof (cksum_orig), &cksum_orig);
+	receive_cksum(ra, sizeof (cksum_orig), &cksum_orig);
 
 	return (0);
 }
 
+/*
+ * Issue the prefetch reads for any necessary indirect blocks.
+ *
+ * We use the object ignore list to tell us whether or not to issue prefetches
+ * for a given object.  We do this for both correctness (in case the blocksize
+ * of an object has changed) and performance (if the object doesn't exist, don't
+ * needlessly try to issue prefetches).  We also trim the list as we go through
+ * the stream to prevent it from growing to an unbounded size.
+ *
+ * The object numbers within will always be in sorted order, and any write
+ * records we see will also be in sorted order, but they're not sorted with
+ * respect to each other (i.e. we can get several object records before
+ * receiving each object's write records).  As a result, once we've reached a
+ * given object number, we can safely remove any reference to lower object
+ * numbers in the ignore list. In practice, we receive up to 32 object records
+ * before receiving write records, so the list can have up to 32 nodes in it.
+ */
+/* ARGSUSED */
+static void
+receive_read_prefetch(struct receive_arg *ra,
+    uint64_t object, uint64_t offset, uint64_t length)
+{
+	struct receive_ign_obj_node *node = list_head(&ra->ignore_obj_list);
+	while (node != NULL && node->object < object) {
+		VERIFY3P(node, ==, list_remove_head(&ra->ignore_obj_list));
+		kmem_free(node, sizeof (*node));
+		node = list_head(&ra->ignore_obj_list);
+	}
+	if (node == NULL || node->object > object) {
+		dmu_prefetch(ra->os, object, 1, offset, length,
+		    ZIO_PRIORITY_SYNC_READ);
+	}
+}
+
+/*
+ * Read records off the stream, issuing any necessary prefetches.
+ */
 static int
-restore_process_record(struct restorearg *ra)
+receive_read_record(struct receive_arg *ra)
 {
 	int err;
 
-	switch (ra->drr->drr_type) {
+	switch (ra->rrd->header.drr_type) {
 	case DRR_OBJECT:
 	{
-		struct drr_object *drro = &ra->drr->drr_u.drr_object;
-		err = restore_read_payload_and_next_header(ra,
-		    P2ROUNDUP(drro->drr_bonuslen, 8), ra->buf);
-		if (err != 0)
+		struct drr_object *drro = &ra->rrd->header.drr_u.drr_object;
+		uint32_t size = P2ROUNDUP(drro->drr_bonuslen, 8);
+		void *buf = kmem_zalloc(size, KM_SLEEP);
+		dmu_object_info_t doi;
+		err = receive_read_payload_and_next_header(ra, size, buf);
+		if (err != 0) {
+			kmem_free(buf, size);
 			return (err);
-		return (restore_object(ra, drro, ra->buf));
+		}
+		err = dmu_object_info(ra->os, drro->drr_object, &doi);
+		/*
+		 * See receive_read_prefetch for an explanation why we're
+		 * storing this object in the ignore_obj_list.
+		 */
+		if (err == ENOENT ||
+		    (err == 0 && doi.doi_data_block_size != drro->drr_blksz)) {
+			struct receive_ign_obj_node *node =
+			    kmem_zalloc(sizeof (*node),
+			    KM_SLEEP);
+			node->object = drro->drr_object;
+#ifdef ZFS_DEBUG
+			struct receive_ign_obj_node *last_object =
+			    list_tail(&ra->ignore_obj_list);
+			uint64_t last_objnum = (last_object != NULL ?
+			    last_object->object : 0);
+			ASSERT3U(node->object, >, last_objnum);
+#endif
+			list_insert_tail(&ra->ignore_obj_list, node);
+			err = 0;
+		}
+		return (err);
 	}
 	case DRR_FREEOBJECTS:
 	{
-		struct drr_freeobjects *drrfo =
-		    &ra->drr->drr_u.drr_freeobjects;
-		err = restore_read_payload_and_next_header(ra, 0, NULL);
-		if (err != 0)
-			return (err);
-		return (restore_freeobjects(ra, drrfo));
+		err = receive_read_payload_and_next_header(ra, 0, NULL);
+		return (err);
 	}
 	case DRR_WRITE:
 	{
-		struct drr_write *drrw = &ra->drr->drr_u.drr_write;
+		struct drr_write *drrw = &ra->rrd->header.drr_u.drr_write;
 		arc_buf_t *abuf = arc_loan_buf(dmu_objset_spa(ra->os),
 		    drrw->drr_length);
 
-		err = restore_read_payload_and_next_header(ra,
+		err = receive_read_payload_and_next_header(ra,
 		    drrw->drr_length, abuf->b_data);
-		if (err != 0)
-			return (err);
-		err = restore_write(ra, drrw, abuf);
-		/* if restore_write() is successful, it consumes the arc_buf */
-		if (err != 0)
+		if (err != 0) {
 			dmu_return_arcbuf(abuf);
+			return (err);
+		}
+		ra->rrd->write_buf = abuf;
+		receive_read_prefetch(ra, drrw->drr_object, drrw->drr_offset,
+		    drrw->drr_length);
 		return (err);
 	}
 	case DRR_WRITE_BYREF:
 	{
-		struct drr_write_byref *drrwbr =
-		    &ra->drr->drr_u.drr_write_byref;
-		err = restore_read_payload_and_next_header(ra, 0, NULL);
-		if (err != 0)
-			return (err);
-		return (restore_write_byref(ra, drrwbr));
+		struct drr_write_byref *drrwb =
+		    &ra->rrd->header.drr_u.drr_write_byref;
+		err = receive_read_payload_and_next_header(ra, 0, NULL);
+		receive_read_prefetch(ra, drrwb->drr_object, drrwb->drr_offset,
+		    drrwb->drr_length);
+		return (err);
 	}
 	case DRR_WRITE_EMBEDDED:
 	{
 		struct drr_write_embedded *drrwe =
-		    &ra->drr->drr_u.drr_write_embedded;
-		err = restore_read_payload_and_next_header(ra,
-		    P2ROUNDUP(drrwe->drr_psize, 8), ra->buf);
-		if (err != 0)
+		    &ra->rrd->header.drr_u.drr_write_embedded;
+		uint32_t size = P2ROUNDUP(drrwe->drr_psize, 8);
+		void *buf = kmem_zalloc(size, KM_SLEEP);
+
+		err = receive_read_payload_and_next_header(ra, size, buf);
+		if (err != 0) {
+			kmem_free(buf, size);
 			return (err);
-		return (restore_write_embedded(ra, drrwe, ra->buf));
+		}
+
+		receive_read_prefetch(ra, drrwe->drr_object, drrwe->drr_offset,
+		    drrwe->drr_length);
+		return (err);
 	}
 	case DRR_FREE:
 	{
-		struct drr_free *drrf = &ra->drr->drr_u.drr_free;
-		err = restore_read_payload_and_next_header(ra, 0, NULL);
-		if (err != 0)
-			return (err);
-		return (restore_free(ra, drrf));
+		/*
+		 * It might be beneficial to prefetch indirect blocks here, but
+		 * we don't really have the data to decide for sure.
+		 */
+		err = receive_read_payload_and_next_header(ra, 0, NULL);
+		return (err);
 	}
 	case DRR_END:
 	{
-		struct drr_end *drre = &ra->drr->drr_u.drr_end;
+		struct drr_end *drre = &ra->rrd->header.drr_u.drr_end;
 		if (!ZIO_CHECKSUM_EQUAL(ra->prev_cksum, drre->drr_checksum))
 			return (SET_ERROR(EINVAL));
 		return (0);
 	}
 	case DRR_SPILL:
 	{
-		struct drr_spill *drrs = &ra->drr->drr_u.drr_spill;
-		err = restore_read_payload_and_next_header(ra,
-		    drrs->drr_length, ra->buf);
+		struct drr_spill *drrs = &ra->rrd->header.drr_u.drr_spill;
+		void *buf = kmem_zalloc(drrs->drr_length, KM_SLEEP);
+		err = receive_read_payload_and_next_header(ra, drrs->drr_length,
+		    buf);
 		if (err != 0)
-			return (err);
-		return (restore_spill(ra, drrs, ra->buf));
+			kmem_free(buf, drrs->drr_length);
+		return (err);
 	}
 	default:
 		return (SET_ERROR(EINVAL));
@@ -1948,6 +2201,118 @@ restore_process_record(struct restorearg *ra)
 }
 
 /*
+ * Commit the records to the pool.
+ */
+static int
+receive_process_record(struct receive_writer_arg *rwa,
+    struct receive_record_arg *rrd)
+{
+	int err;
+
+	switch (rrd->header.drr_type) {
+	case DRR_OBJECT:
+	{
+		struct drr_object *drro = &rrd->header.drr_u.drr_object;
+		err = receive_object(rwa, drro, rrd->payload);
+		kmem_free(rrd->payload, rrd->payload_size);
+		rrd->payload = NULL;
+		return (err);
+	}
+	case DRR_FREEOBJECTS:
+	{
+		struct drr_freeobjects *drrfo =
+		    &rrd->header.drr_u.drr_freeobjects;
+		return (receive_freeobjects(rwa, drrfo));
+	}
+	case DRR_WRITE:
+	{
+		struct drr_write *drrw = &rrd->header.drr_u.drr_write;
+		err = receive_write(rwa, drrw, rrd->write_buf);
+		/* if receive_write() is successful, it consumes the arc_buf */
+		if (err != 0)
+			dmu_return_arcbuf(rrd->write_buf);
+		rrd->write_buf = NULL;
+		rrd->payload = NULL;
+		return (err);
+	}
+	case DRR_WRITE_BYREF:
+	{
+		struct drr_write_byref *drrwbr =
+		    &rrd->header.drr_u.drr_write_byref;
+		return (receive_write_byref(rwa, drrwbr));
+	}
+	case DRR_WRITE_EMBEDDED:
+	{
+		struct drr_write_embedded *drrwe =
+		    &rrd->header.drr_u.drr_write_embedded;
+		err = receive_write_embedded(rwa, drrwe, rrd->payload);
+		kmem_free(rrd->payload, rrd->payload_size);
+		rrd->payload = NULL;
+		return (err);
+	}
+	case DRR_FREE:
+	{
+		struct drr_free *drrf = &rrd->header.drr_u.drr_free;
+		return (receive_free(rwa, drrf));
+	}
+	case DRR_SPILL:
+	{
+		struct drr_spill *drrs = &rrd->header.drr_u.drr_spill;
+		err = receive_spill(rwa, drrs, rrd->payload);
+		kmem_free(rrd->payload, rrd->payload_size);
+		rrd->payload = NULL;
+		return (err);
+	}
+	default:
+		return (SET_ERROR(EINVAL));
+	}
+}
+
+/*
+ * dmu_recv_stream's worker thread; pull records off the queue, and then call
+ * receive_process_record  When we're done, signal the main thread and exit.
+ */
+static void
+receive_writer_thread(void *arg)
+{
+	struct receive_writer_arg *rwa = arg;
+	struct receive_record_arg *rrd;
+	for (rrd = bqueue_dequeue(&rwa->q); !rrd->eos_marker;
+	    rrd = bqueue_dequeue(&rwa->q)) {
+		/*
+		 * If there's an error, the main thread will stop putting things
+		 * on the queue, but we need to clear everything in it before we
+		 * can exit.
+		 */
+		if (rwa->err == 0) {
+			rwa->err = receive_process_record(rwa, rrd);
+		} else if (rrd->write_buf != NULL) {
+			dmu_return_arcbuf(rrd->write_buf);
+			rrd->write_buf = NULL;
+			rrd->payload = NULL;
+		} else if (rrd->payload != NULL) {
+			kmem_free(rrd->payload, rrd->payload_size);
+			rrd->payload = NULL;
+		}
+		kmem_free(rrd, sizeof (*rrd));
+	}
+	kmem_free(rrd, sizeof (*rrd));
+	mutex_enter(&rwa->mutex);
+	rwa->done = B_TRUE;
+	cv_signal(&rwa->cv);
+	mutex_exit(&rwa->mutex);
+}
+
+/*
+ * Read in the stream's records, one by one, and apply them to the pool.  There
+ * are two threads involved; the thread that calls this function will spin up a
+ * worker thread, read the records off the stream one by one, and issue
+ * prefetches for any necessary indirect blocks.  It will then push the records
+ * onto an internal blocking queue.  The worker thread will pull the records off
+ * the queue, and actually write the data into the DMU.  This way, the worker
+ * thread doesn't have to wait for reads to complete, since everything it needs
+ * (the indirect blocks) will be prefetched.
+ *
  * NB: callers *must* call dmu_recv_end() if this succeeds.
  */
 int
@@ -1955,17 +2320,16 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, vnode_t *vp, offset_t *voffp,
     int cleanup_fd, uint64_t *action_handlep)
 {
 	int err = 0;
-	struct restorearg ra = { 0 };
+	struct receive_arg ra = { 0 };
+	struct receive_writer_arg rwa = { 0 };
 	int featureflags;
 
 	ra.byteswap = drc->drc_byteswap;
 	ra.cksum = drc->drc_cksum;
 	ra.vp = vp;
 	ra.voff = *voffp;
-	ra.bufsize = SPA_MAXBLOCKSIZE;
-	ra.drr = kmem_alloc(sizeof (*ra.drr), KM_SLEEP);
-	ra.buf = kmem_alloc(ra.bufsize, KM_SLEEP);
-	ra.next_drr = kmem_alloc(sizeof (*ra.next_drr), KM_SLEEP);
+	list_create(&ra.ignore_obj_list, sizeof (struct receive_ign_obj_node),
+	    offsetof(struct receive_ign_obj_node, node));
 
 	/* these were verified in dmu_recv_begin */
 	ASSERT3U(DMU_GET_STREAM_HDRTYPE(drc->drc_drrb->drr_versioninfo), ==,
@@ -1996,48 +2360,92 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, vnode_t *vp, offset_t *voffp,
 		}
 
 		if (*action_handlep == 0) {
-			ra.guid_to_ds_map =
+			rwa.guid_to_ds_map =
 			    kmem_alloc(sizeof (avl_tree_t), KM_SLEEP);
-			avl_create(ra.guid_to_ds_map, guid_compare,
+			avl_create(rwa.guid_to_ds_map, guid_compare,
 			    sizeof (guid_map_entry_t),
 			    offsetof(guid_map_entry_t, avlnode));
 			err = zfs_onexit_add_cb(minor,
-			    free_guid_map_onexit, ra.guid_to_ds_map,
+			    free_guid_map_onexit, rwa.guid_to_ds_map,
 			    action_handlep);
 			if (ra.err != 0)
 				goto out;
 		} else {
 			err = zfs_onexit_cb_data(minor, *action_handlep,
-			    (void **)&ra.guid_to_ds_map);
+			    (void **)&rwa.guid_to_ds_map);
 			if (ra.err != 0)
 				goto out;
 		}
 
-		drc->drc_guid_to_ds_map = ra.guid_to_ds_map;
+		drc->drc_guid_to_ds_map = rwa.guid_to_ds_map;
 	}
 
-	err = restore_read_payload_and_next_header(&ra, 0, NULL);
-	if (err != 0)
+	err = receive_read_payload_and_next_header(&ra, 0, NULL);
+	if (err)
 		goto out;
-	for (;;) {
-		void *tmp;
 
+	(void) bqueue_init(&rwa.q, zfs_recv_queue_length,
+	    offsetof(struct receive_record_arg, node));
+	cv_init(&rwa.cv, NULL, CV_DEFAULT, NULL);
+	mutex_init(&rwa.mutex, NULL, MUTEX_DEFAULT, NULL);
+	rwa.os = ra.os;
+	rwa.byteswap = drc->drc_byteswap;
+
+	(void) thread_create(NULL, 0, receive_writer_thread, &rwa, 0, curproc,
+	    TS_RUN, minclsyspri);
+	/*
+	 * We're reading rwa.err without locks, which is safe since we are the
+	 * only reader, and the worker thread is the only writer.  It's ok if we
+	 * miss a write for an iteration or two of the loop, since the writer
+	 * thread will keep freeing records we send it until we send it an eos
+	 * marker.
+	 *
+	 * We can leave this loop in 3 ways:  First, if rwa.err is
+	 * non-zero.  In that case, the writer thread will free the rrd we just
+	 * pushed.  Second, if  we're interrupted; in that case, either it's the
+	 * first loop and ra.rrd was never allocated, or it's later, and ra.rrd
+	 * has been handed off to the writer thread who will free it.  Finally,
+	 * if receive_read_record fails or we're at the end of the stream, then
+	 * we free ra.rrd and exit.
+	 */
+	while (rwa.err == 0) {
 		if (issig(JUSTLOOKING) && issig(FORREAL)) {
 			err = SET_ERROR(EINTR);
 			break;
 		}
 
-		tmp = ra.next_drr;
-		ra.next_drr = ra.drr;
-		ra.drr = tmp;
+		ASSERT3P(ra.rrd, ==, NULL);
+		ra.rrd = ra.next_rrd;
+		ra.next_rrd = NULL;
+		/* Allocates and loads header into ra.next_rrd */
+		err = receive_read_record(&ra);
 
-		/* process ra.drr, read in ra.next_drr */
-		err = restore_process_record(&ra);
-		if (err != 0)
-			break;
-		if (ra.drr->drr_type == DRR_END)
+		if (ra.rrd->header.drr_type == DRR_END || err != 0) {
+			kmem_free(ra.rrd, sizeof (*ra.rrd));
+			ra.rrd = NULL;
 			break;
+		}
+
+		bqueue_enqueue(&rwa.q, ra.rrd,
+		    sizeof (struct receive_record_arg) + ra.rrd->payload_size);
+		ra.rrd = NULL;
 	}
+	if (ra.next_rrd == NULL)
+		ra.next_rrd = kmem_zalloc(sizeof (*ra.next_rrd), KM_SLEEP);
+	ra.next_rrd->eos_marker = B_TRUE;
+	bqueue_enqueue(&rwa.q, ra.next_rrd, 1);
+
+	mutex_enter(&rwa.mutex);
+	while (!rwa.done) {
+		cv_wait(&rwa.cv, &rwa.mutex);
+	}
+	mutex_exit(&rwa.mutex);
+
+	cv_destroy(&rwa.cv);
+	mutex_destroy(&rwa.mutex);
+	bqueue_destroy(&rwa.q);
+	if (err == 0)
+		err = rwa.err;
 
 out:
 	if ((featureflags & DMU_BACKUP_FEATURE_DEDUP) && (cleanup_fd != -1))
@@ -2051,10 +2459,13 @@ out:
 		dmu_recv_cleanup_ds(drc);
 	}
 
-	kmem_free(ra.drr, sizeof (*ra.drr));
-	kmem_free(ra.buf, ra.bufsize);
-	kmem_free(ra.next_drr, sizeof (*ra.next_drr));
 	*voffp = ra.voff;
+	for (struct receive_ign_obj_node *n =
+	    list_remove_head(&ra.ignore_obj_list); n != NULL;
+	    n = list_remove_head(&ra.ignore_obj_list)) {
+		kmem_free(n, sizeof (*n));
+	}
+	list_destroy(&ra.ignore_obj_list);
 	return (err);
 }
 
diff --git a/uts/common/fs/zfs/dmu_traverse.c b/uts/common/fs/zfs/dmu_traverse.c
index 8e49d2862b06..9222b54d624c 100644
--- a/uts/common/fs/zfs/dmu_traverse.c
+++ b/uts/common/fs/zfs/dmu_traverse.c
@@ -157,7 +157,7 @@ resume_skip_check(traverse_data_t *td, const dnode_phys_t *dnp,
 		 * If we already visited this bp & everything below,
 		 * don't bother doing it again.
 		 */
-		if (zbookmark_is_before(dnp, zb, td->td_resume))
+		if (zbookmark_subtree_completed(dnp, zb, td->td_resume))
 			return (RESUME_SKIP_ALL);
 
 		/*
@@ -422,6 +422,17 @@ traverse_dnode(traverse_data_t *td, const dnode_phys_t *dnp,
 	int j, err = 0;
 	zbookmark_phys_t czb;
 
+	if (td->td_flags & TRAVERSE_PRE) {
+		SET_BOOKMARK(&czb, objset, object, ZB_DNODE_LEVEL,
+		    ZB_DNODE_BLKID);
+		err = td->td_func(td->td_spa, NULL, NULL, &czb, dnp,
+		    td->td_arg);
+		if (err == TRAVERSE_VISIT_NO_CHILDREN)
+			return (0);
+		if (err != 0)
+			return (err);
+	}
+
 	for (j = 0; j < dnp->dn_nblkptr; j++) {
 		SET_BOOKMARK(&czb, objset, object, dnp->dn_nlevels - 1, j);
 		err = traverse_visitbp(td, dnp, &dnp->dn_blkptr[j], &czb);
@@ -429,10 +440,21 @@ traverse_dnode(traverse_data_t *td, const dnode_phys_t *dnp,
 			break;
 	}
 
-	if (err == 0 && dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
+	if (err == 0 && (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) {
 		SET_BOOKMARK(&czb, objset, object, 0, DMU_SPILL_BLKID);
 		err = traverse_visitbp(td, dnp, &dnp->dn_spill, &czb);
 	}
+
+	if (err == 0 && (td->td_flags & TRAVERSE_POST)) {
+		SET_BOOKMARK(&czb, objset, object, ZB_DNODE_LEVEL,
+		    ZB_DNODE_BLKID);
+		err = td->td_func(td->td_spa, NULL, NULL, &czb, dnp,
+		    td->td_arg);
+		if (err == TRAVERSE_VISIT_NO_CHILDREN)
+			return (0);
+		if (err != 0)
+			return (err);
+	}
 	return (err);
 }
 
@@ -445,6 +467,8 @@ traverse_prefetcher(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 	arc_flags_t aflags = ARC_FLAG_NOWAIT | ARC_FLAG_PREFETCH;
 
 	ASSERT(pfd->pd_bytes_fetched >= 0);
+	if (bp == NULL)
+		return (0);
 	if (pfd->pd_cancel)
 		return (SET_ERROR(EINTR));
 
diff --git a/uts/common/fs/zfs/dmu_tx.c b/uts/common/fs/zfs/dmu_tx.c
index 60078153a549..50885acafc9c 100644
--- a/uts/common/fs/zfs/dmu_tx.c
+++ b/uts/common/fs/zfs/dmu_tx.c
@@ -315,7 +315,8 @@ dmu_tx_count_write(dmu_tx_hold_t *txh, uint64_t off, uint64_t len)
 			dmu_buf_impl_t *db;
 
 			rw_enter(&dn->dn_struct_rwlock, RW_READER);
-			err = dbuf_hold_impl(dn, 0, start, FALSE, FTAG, &db);
+			err = dbuf_hold_impl(dn, 0, start,
+			    FALSE, FALSE, FTAG, &db);
 			rw_exit(&dn->dn_struct_rwlock);
 
 			if (err) {
@@ -516,7 +517,8 @@ dmu_tx_count_free(dmu_tx_hold_t *txh, uint64_t off, uint64_t len)
 		blkoff = P2PHASE(blkid, epb);
 		tochk = MIN(epb - blkoff, nblks);
 
-		err = dbuf_hold_impl(dn, 1, blkid >> epbs, FALSE, FTAG, &dbuf);
+		err = dbuf_hold_impl(dn, 1, blkid >> epbs,
+		    FALSE, FALSE, FTAG, &dbuf);
 		if (err) {
 			txh->txh_tx->tx_err = err;
 			break;
diff --git a/uts/common/fs/zfs/dmu_zfetch.c b/uts/common/fs/zfs/dmu_zfetch.c
index 3cc3d67677cb..70930ae84587 100644
--- a/uts/common/fs/zfs/dmu_zfetch.c
+++ b/uts/common/fs/zfs/dmu_zfetch.c
@@ -291,7 +291,8 @@ dmu_zfetch_fetch(dnode_t *dn, uint64_t blkid, uint64_t nblks)
 	fetchsz = dmu_zfetch_fetchsz(dn, blkid, nblks);
 
 	for (i = 0; i < fetchsz; i++) {
-		dbuf_prefetch(dn, blkid + i, ZIO_PRIORITY_ASYNC_READ);
+		dbuf_prefetch(dn, 0, blkid + i, ZIO_PRIORITY_ASYNC_READ,
+		    ARC_FLAG_PREFETCH);
 	}
 
 	return (fetchsz);
diff --git a/uts/common/fs/zfs/dnode.c b/uts/common/fs/zfs/dnode.c
index 397cc5c0ed04..17edd95b287c 100644
--- a/uts/common/fs/zfs/dnode.c
+++ b/uts/common/fs/zfs/dnode.c
@@ -1111,7 +1111,7 @@ dnode_hold_impl(objset_t *os, uint64_t object, int flag,
 		drop_struct_lock = TRUE;
 	}
 
-	blk = dbuf_whichblock(mdn, object * sizeof (dnode_phys_t));
+	blk = dbuf_whichblock(mdn, 0, object * sizeof (dnode_phys_t));
 
 	db = dbuf_hold(mdn, blk, FTAG);
 	if (drop_struct_lock)
@@ -1408,7 +1408,7 @@ dnode_set_blksz(dnode_t *dn, uint64_t size, int ibs, dmu_tx_t *tx)
 		goto fail;
 
 	/* resize the old block */
-	err = dbuf_hold_impl(dn, 0, 0, TRUE, FTAG, &db);
+	err = dbuf_hold_impl(dn, 0, 0, TRUE, FALSE, FTAG, &db);
 	if (err == 0)
 		dbuf_new_size(db, size, tx);
 	else if (err != ENOENT)
@@ -1581,8 +1581,8 @@ dnode_free_range(dnode_t *dn, uint64_t off, uint64_t len, dmu_tx_t *tx)
 		ASSERT3U(blkoff + head, ==, blksz);
 		if (len < head)
 			head = len;
-		if (dbuf_hold_impl(dn, 0, dbuf_whichblock(dn, off), TRUE,
-		    FTAG, &db) == 0) {
+		if (dbuf_hold_impl(dn, 0, dbuf_whichblock(dn, 0, off),
+		    TRUE, FALSE, FTAG, &db) == 0) {
 			caddr_t data;
 
 			/* don't dirty if it isn't on disk and isn't dirty */
@@ -1619,8 +1619,8 @@ dnode_free_range(dnode_t *dn, uint64_t off, uint64_t len, dmu_tx_t *tx)
 	if (tail) {
 		if (len < tail)
 			tail = len;
-		if (dbuf_hold_impl(dn, 0, dbuf_whichblock(dn, off+len),
-		    TRUE, FTAG, &db) == 0) {
+		if (dbuf_hold_impl(dn, 0, dbuf_whichblock(dn, 0, off+len),
+		    TRUE, FALSE, FTAG, &db) == 0) {
 			/* don't dirty if not on disk and not dirty */
 			if (db->db_last_dirty ||
 			    (db->db_blkptr && !BP_IS_HOLE(db->db_blkptr))) {
@@ -1849,7 +1849,7 @@ dnode_willuse_space(dnode_t *dn, int64_t space, dmu_tx_t *tx)
  */
 static int
 dnode_next_offset_level(dnode_t *dn, int flags, uint64_t *offset,
-	int lvl, uint64_t blkfill, uint64_t txg)
+    int lvl, uint64_t blkfill, uint64_t txg)
 {
 	dmu_buf_impl_t *db = NULL;
 	void *data = NULL;
@@ -1871,8 +1871,8 @@ dnode_next_offset_level(dnode_t *dn, int flags, uint64_t *offset,
 		epb = dn->dn_phys->dn_nblkptr;
 		data = dn->dn_phys->dn_blkptr;
 	} else {
-		uint64_t blkid = dbuf_whichblock(dn, *offset) >> (epbs * lvl);
-		error = dbuf_hold_impl(dn, lvl, blkid, TRUE, FTAG, &db);
+		uint64_t blkid = dbuf_whichblock(dn, lvl, *offset);
+		error = dbuf_hold_impl(dn, lvl, blkid, TRUE, FALSE, FTAG, &db);
 		if (error) {
 			if (error != ENOENT)
 				return (error);
diff --git a/uts/common/fs/zfs/dnode_sync.c b/uts/common/fs/zfs/dnode_sync.c
index 06336045fc82..0787885229d4 100644
--- a/uts/common/fs/zfs/dnode_sync.c
+++ b/uts/common/fs/zfs/dnode_sync.c
@@ -188,7 +188,7 @@ free_verify(dmu_buf_impl_t *db, uint64_t start, uint64_t end, dmu_tx_t *tx)
 
 		rw_enter(&dn->dn_struct_rwlock, RW_READER);
 		err = dbuf_hold_impl(dn, db->db_level-1,
-		    (db->db_blkid << epbs) + i, TRUE, FTAG, &child);
+		    (db->db_blkid << epbs) + i, TRUE, FALSE, FTAG, &child);
 		rw_exit(&dn->dn_struct_rwlock);
 		if (err == ENOENT)
 			continue;
@@ -284,7 +284,7 @@ free_children(dmu_buf_impl_t *db, uint64_t blkid, uint64_t nblks,
 				continue;
 			rw_enter(&dn->dn_struct_rwlock, RW_READER);
 			VERIFY0(dbuf_hold_impl(dn, db->db_level - 1,
-			    i, B_TRUE, FTAG, &subdb));
+			    i, TRUE, FALSE, FTAG, &subdb));
 			rw_exit(&dn->dn_struct_rwlock);
 			ASSERT3P(bp, ==, subdb->db_blkptr);
 
@@ -357,7 +357,7 @@ dnode_sync_free_range_impl(dnode_t *dn, uint64_t blkid, uint64_t nblks,
 				continue;
 			rw_enter(&dn->dn_struct_rwlock, RW_READER);
 			VERIFY0(dbuf_hold_impl(dn, dnlevel - 1, i,
-			    TRUE, FTAG, &db));
+			    TRUE, FALSE, FTAG, &db));
 			rw_exit(&dn->dn_struct_rwlock);
 
 			free_children(db, blkid, nblks, tx);
diff --git a/uts/common/fs/zfs/dsl_dataset.c b/uts/common/fs/zfs/dsl_dataset.c
index a11926b58f1a..8f419d682875 100644
--- a/uts/common/fs/zfs/dsl_dataset.c
+++ b/uts/common/fs/zfs/dsl_dataset.c
@@ -530,6 +530,7 @@ dsl_dataset_hold(dsl_pool_t *dp, const char *name,
 	const char *snapname;
 	uint64_t obj;
 	int err = 0;
+	dsl_dataset_t *ds;
 
 	err = dsl_dir_hold(dp, name, FTAG, &dd, &snapname);
 	if (err != 0)
@@ -538,36 +539,37 @@ dsl_dataset_hold(dsl_pool_t *dp, const char *name,
 	ASSERT(dsl_pool_config_held(dp));
 	obj = dsl_dir_phys(dd)->dd_head_dataset_obj;
 	if (obj != 0)
-		err = dsl_dataset_hold_obj(dp, obj, tag, dsp);
+		err = dsl_dataset_hold_obj(dp, obj, tag, &ds);
 	else
 		err = SET_ERROR(ENOENT);
 
 	/* we may be looking for a snapshot */
 	if (err == 0 && snapname != NULL) {
-		dsl_dataset_t *ds;
+		dsl_dataset_t *snap_ds;
 
 		if (*snapname++ != '@') {
-			dsl_dataset_rele(*dsp, tag);
+			dsl_dataset_rele(ds, tag);
 			dsl_dir_rele(dd, FTAG);
 			return (SET_ERROR(ENOENT));
 		}
 
 		dprintf("looking for snapshot '%s'\n", snapname);
-		err = dsl_dataset_snap_lookup(*dsp, snapname, &obj);
+		err = dsl_dataset_snap_lookup(ds, snapname, &obj);
 		if (err == 0)
-			err = dsl_dataset_hold_obj(dp, obj, tag, &ds);
-		dsl_dataset_rele(*dsp, tag);
+			err = dsl_dataset_hold_obj(dp, obj, tag, &snap_ds);
+		dsl_dataset_rele(ds, tag);
 
 		if (err == 0) {
-			mutex_enter(&ds->ds_lock);
-			if (ds->ds_snapname[0] == 0)
-				(void) strlcpy(ds->ds_snapname, snapname,
-				    sizeof (ds->ds_snapname));
-			mutex_exit(&ds->ds_lock);
-			*dsp = ds;
+			mutex_enter(&snap_ds->ds_lock);
+			if (snap_ds->ds_snapname[0] == 0)
+				(void) strlcpy(snap_ds->ds_snapname, snapname,
+				    sizeof (snap_ds->ds_snapname));
+			mutex_exit(&snap_ds->ds_lock);
+			ds = snap_ds;
 		}
 	}
-
+	if (err == 0)
+		*dsp = ds;
 	dsl_dir_rele(dd, FTAG);
 	return (err);
 }
diff --git a/uts/common/fs/zfs/dsl_destroy.c b/uts/common/fs/zfs/dsl_destroy.c
index 7f9046957f9c..40e34cbf9deb 100644
--- a/uts/common/fs/zfs/dsl_destroy.c
+++ b/uts/common/fs/zfs/dsl_destroy.c
@@ -552,7 +552,7 @@ kill_blkptr(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 	struct killarg *ka = arg;
 	dmu_tx_t *tx = ka->tx;
 
-	if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp))
+	if (bp == NULL || BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp))
 		return (0);
 
 	if (zb->zb_level == ZB_ZIL_LEVEL) {
diff --git a/uts/common/fs/zfs/dsl_scan.c b/uts/common/fs/zfs/dsl_scan.c
index 0894ebd45451..ba397b1bb0cb 100644
--- a/uts/common/fs/zfs/dsl_scan.c
+++ b/uts/common/fs/zfs/dsl_scan.c
@@ -575,7 +575,8 @@ dsl_scan_check_resume(dsl_scan_t *scn, const dnode_phys_t *dnp,
 		 * If we already visited this bp & everything below (in
 		 * a prior txg sync), don't bother doing it again.
 		 */
-		if (zbookmark_is_before(dnp, zb, &scn->scn_phys.scn_bookmark))
+		if (zbookmark_subtree_completed(dnp, zb,
+		    &scn->scn_phys.scn_bookmark))
 			return (B_TRUE);
 
 		/*
diff --git a/uts/common/fs/zfs/spa.c b/uts/common/fs/zfs/spa.c
index e9d2432a8179..5e664d7d61ac 100644
--- a/uts/common/fs/zfs/spa.c
+++ b/uts/common/fs/zfs/spa.c
@@ -1874,7 +1874,7 @@ static int
 spa_load_verify_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
     const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg)
 {
-	if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp))
+	if (bp == NULL || BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp))
 		return (0);
 	/*
 	 * Note: normally this routine will not be called if
diff --git a/uts/common/fs/zfs/space_map.c b/uts/common/fs/zfs/space_map.c
index 94790b9eaee5..e0fe6ac2604e 100644
--- a/uts/common/fs/zfs/space_map.c
+++ b/uts/common/fs/zfs/space_map.c
@@ -76,8 +76,8 @@ space_map_load(space_map_t *sm, range_tree_t *rt, maptype_t maptype)
 
 	mutex_exit(sm->sm_lock);
 	if (end > bufsize) {
-		dmu_prefetch(sm->sm_os, space_map_object(sm), bufsize,
-		    end - bufsize);
+		dmu_prefetch(sm->sm_os, space_map_object(sm), 0, bufsize,
+		    end - bufsize, ZIO_PRIORITY_SYNC_READ);
 	}
 	mutex_enter(sm->sm_lock);
 
diff --git a/uts/common/fs/zfs/sys/bqueue.h b/uts/common/fs/zfs/sys/bqueue.h
new file mode 100644
index 000000000000..63722df1bbf3
--- /dev/null
+++ b/uts/common/fs/zfs/sys/bqueue.h
@@ -0,0 +1,54 @@
+/*
+ * CDDL HEADER START
+ *
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2014 by Delphix. All rights reserved.
+ */
+
+#ifndef	_BQUEUE_H
+#define	_BQUEUE_H
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#include	<sys/zfs_context.h>
+
+typedef struct bqueue {
+	list_t bq_list;
+	kmutex_t bq_lock;
+	kcondvar_t bq_add_cv;
+	kcondvar_t bq_pop_cv;
+	uint64_t bq_size;
+	uint64_t bq_maxsize;
+	size_t bq_node_offset;
+} bqueue_t;
+
+typedef struct bqueue_node {
+	list_node_t bqn_node;
+	uint64_t bqn_size;
+} bqueue_node_t;
+
+
+int bqueue_init(bqueue_t *, uint64_t, size_t);
+void bqueue_destroy(bqueue_t *);
+void bqueue_enqueue(bqueue_t *, void *, uint64_t);
+void *bqueue_dequeue(bqueue_t *);
+boolean_t bqueue_empty(bqueue_t *);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _BQUEUE_H */
diff --git a/uts/common/fs/zfs/sys/dbuf.h b/uts/common/fs/zfs/sys/dbuf.h
index 2e07185f4cab..482ccb01dacb 100644
--- a/uts/common/fs/zfs/sys/dbuf.h
+++ b/uts/common/fs/zfs/sys/dbuf.h
@@ -245,8 +245,7 @@ typedef struct dbuf_hash_table {
 	kmutex_t hash_mutexes[DBUF_MUTEXES];
 } dbuf_hash_table_t;
 
-
-uint64_t dbuf_whichblock(struct dnode *di, uint64_t offset);
+uint64_t dbuf_whichblock(struct dnode *di, int64_t level, uint64_t offset);
 
 dmu_buf_impl_t *dbuf_create_tlib(struct dnode *dn, char *data);
 void dbuf_create_bonus(struct dnode *dn);
@@ -258,10 +257,12 @@ void dbuf_rm_spill(struct dnode *dn, dmu_tx_t *tx);
 dmu_buf_impl_t *dbuf_hold(struct dnode *dn, uint64_t blkid, void *tag);
 dmu_buf_impl_t *dbuf_hold_level(struct dnode *dn, int level, uint64_t blkid,
     void *tag);
-int dbuf_hold_impl(struct dnode *dn, uint8_t level, uint64_t blkid, int create,
+int dbuf_hold_impl(struct dnode *dn, uint8_t level, uint64_t blkid,
+    boolean_t fail_sparse, boolean_t fail_uncached,
     void *tag, dmu_buf_impl_t **dbp);
 
-void dbuf_prefetch(struct dnode *dn, uint64_t blkid, zio_priority_t prio);
+void dbuf_prefetch(struct dnode *dn, int64_t level, uint64_t blkid,
+    zio_priority_t prio, arc_flags_t aflags);
 
 void dbuf_add_ref(dmu_buf_impl_t *db, void *tag);
 boolean_t dbuf_try_add_ref(dmu_buf_t *db, objset_t *os, uint64_t obj,
diff --git a/uts/common/fs/zfs/sys/dmu.h b/uts/common/fs/zfs/sys/dmu.h
index 0b3dd9b2f108..d7d4d6f519af 100644
--- a/uts/common/fs/zfs/sys/dmu.h
+++ b/uts/common/fs/zfs/sys/dmu.h
@@ -46,6 +46,7 @@
 #include <sys/inttypes.h>
 #include <sys/cred.h>
 #include <sys/fs/zfs.h>
+#include <sys/zio_priority.h>
 
 #ifdef	__cplusplus
 extern "C" {
@@ -739,8 +740,8 @@ extern int zfs_max_recordsize;
 /*
  * Asynchronously try to read in the data.
  */
-void dmu_prefetch(objset_t *os, uint64_t object, uint64_t offset,
-    uint64_t len);
+void dmu_prefetch(objset_t *os, uint64_t object, int64_t level, uint64_t offset,
+    uint64_t len, enum zio_priority pri);
 
 typedef struct dmu_object_info {
 	/* All sizes are in bytes unless otherwise indicated. */
diff --git a/uts/common/fs/zfs/sys/dsl_dataset.h b/uts/common/fs/zfs/sys/dsl_dataset.h
index 7d490ec60318..81757b5217d4 100644
--- a/uts/common/fs/zfs/sys/dsl_dataset.h
+++ b/uts/common/fs/zfs/sys/dsl_dataset.h
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
diff --git a/uts/common/fs/zfs/sys/zio.h b/uts/common/fs/zfs/sys/zio.h
index 198dc9238709..d160c9468659 100644
--- a/uts/common/fs/zfs/sys/zio.h
+++ b/uts/common/fs/zfs/sys/zio.h
@@ -30,6 +30,7 @@
 #ifndef _ZIO_H
 #define	_ZIO_H
 
+#include <sys/zio_priority.h>
 #include <sys/zfs_context.h>
 #include <sys/spa.h>
 #include <sys/txg.h>
@@ -144,17 +145,6 @@ enum zio_compress {
 #define	ZIO_FAILURE_MODE_CONTINUE	1
 #define	ZIO_FAILURE_MODE_PANIC		2
 
-typedef enum zio_priority {
-	ZIO_PRIORITY_SYNC_READ,
-	ZIO_PRIORITY_SYNC_WRITE,	/* ZIL */
-	ZIO_PRIORITY_ASYNC_READ,	/* prefetch */
-	ZIO_PRIORITY_ASYNC_WRITE,	/* spa_sync() */
-	ZIO_PRIORITY_SCRUB,		/* asynchronous scrub/resilver reads */
-	ZIO_PRIORITY_NUM_QUEUEABLE,
-
-	ZIO_PRIORITY_NOW		/* non-queued i/os (e.g. free) */
-} zio_priority_t;
-
 enum zio_flag {
 	/*
 	 * Flags inherited by gang, ddt, and vdev children,
@@ -258,6 +248,7 @@ extern const char *zio_type_name[ZIO_TYPES];
  * Root blocks (objset_phys_t) are object 0, level -1:  <objset, 0, -1, 0>.
  * ZIL blocks are bookmarked <objset, 0, -2, blkid == ZIL sequence number>.
  * dmu_sync()ed ZIL data blocks are bookmarked <objset, object, -2, blkid>.
+ * dnode visit bookmarks are <objset, object id of dnode, -3, 0>.
  *
  * Note: this structure is called a bookmark because its original purpose
  * was to remember where to resume a pool-wide traverse.
@@ -290,6 +281,9 @@ typedef struct zbookmark_phys {
 #define	ZB_ZIL_OBJECT		(0ULL)
 #define	ZB_ZIL_LEVEL		(-2LL)
 
+#define	ZB_DNODE_LEVEL		(-3LL)
+#define	ZB_DNODE_BLKID		(0ULL)
+
 #define	ZB_IS_ZERO(zb)						\
 	((zb)->zb_objset == 0 && (zb)->zb_object == 0 &&	\
 	(zb)->zb_level == 0 && (zb)->zb_blkid == 0)
@@ -592,8 +586,10 @@ extern void zfs_ereport_post_checksum(spa_t *spa, vdev_t *vd,
 extern void spa_handle_ignored_writes(spa_t *spa);
 
 /* zbookmark_phys functions */
-boolean_t zbookmark_is_before(const struct dnode_phys *dnp,
-    const zbookmark_phys_t *zb1, const zbookmark_phys_t *zb2);
+boolean_t zbookmark_subtree_completed(const struct dnode_phys *dnp,
+    const zbookmark_phys_t *subtree_root, const zbookmark_phys_t *last_block);
+int zbookmark_compare(uint16_t dbss1, uint8_t ibs1, uint16_t dbss2,
+    uint8_t ibs2, const zbookmark_phys_t *zb1, const zbookmark_phys_t *zb2);
 
 #ifdef	__cplusplus
 }
diff --git a/uts/common/fs/zfs/sys/zio_checksum.h b/uts/common/fs/zfs/sys/zio_checksum.h
index a921a2f1396e..0c293ab20e2c 100644
--- a/uts/common/fs/zfs/sys/zio_checksum.h
+++ b/uts/common/fs/zfs/sys/zio_checksum.h
@@ -44,7 +44,7 @@ typedef struct zio_checksum_info {
 	zio_checksum_func_t *ci_func[2]; /* checksum function per byteorder */
 	int		ci_correctable;	/* number of correctable bits	*/
 	int		ci_eck;		/* uses zio embedded checksum? */
-	int		ci_dedup;	/* strong enough for dedup? */
+	boolean_t	ci_dedup;	/* strong enough for dedup? */
 	char		*ci_name;	/* descriptive name */
 } zio_checksum_info_t;
 
diff --git a/uts/common/fs/zfs/sys/zio_priority.h b/uts/common/fs/zfs/sys/zio_priority.h
new file mode 100644
index 000000000000..e33b9585b1c0
--- /dev/null
+++ b/uts/common/fs/zfs/sys/zio_priority.h
@@ -0,0 +1,40 @@
+/*
+ * CDDL HEADER START
+ *
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2014 by Delphix. All rights reserved.
+ */
+#ifndef	_ZIO_PRIORITY_H
+#define	_ZIO_PRIORITY_H
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+typedef enum zio_priority {
+	ZIO_PRIORITY_SYNC_READ,
+	ZIO_PRIORITY_SYNC_WRITE,	/* ZIL */
+	ZIO_PRIORITY_ASYNC_READ,	/* prefetch */
+	ZIO_PRIORITY_ASYNC_WRITE,	/* spa_sync() */
+	ZIO_PRIORITY_SCRUB,		/* asynchronous scrub/resilver reads */
+	ZIO_PRIORITY_NUM_QUEUEABLE,
+
+	ZIO_PRIORITY_NOW		/* non-queued i/os (e.g. free) */
+} zio_priority_t;
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _ZIO_PRIORITY_H */
diff --git a/uts/common/fs/zfs/zap.c b/uts/common/fs/zfs/zap.c
index 20f349651000..2ea6b12219f0 100644
--- a/uts/common/fs/zfs/zap.c
+++ b/uts/common/fs/zfs/zap.c
@@ -162,8 +162,9 @@ zap_table_grow(zap_t *zap, zap_table_phys_t *tbl,
 		newblk = zap_allocate_blocks(zap, tbl->zt_numblks * 2);
 		tbl->zt_nextblk = newblk;
 		ASSERT0(tbl->zt_blks_copied);
-		dmu_prefetch(zap->zap_objset, zap->zap_object,
-		    tbl->zt_blk << bs, tbl->zt_numblks << bs);
+		dmu_prefetch(zap->zap_objset, zap->zap_object, 0,
+		    tbl->zt_blk << bs, tbl->zt_numblks << bs,
+		    ZIO_PRIORITY_SYNC_READ);
 	}
 
 	/*
@@ -939,7 +940,8 @@ fzap_prefetch(zap_name_t *zn)
 	if (zap_idx_to_blk(zap, idx, &blk) != 0)
 		return;
 	bs = FZAP_BLOCK_SHIFT(zap);
-	dmu_prefetch(zap->zap_objset, zap->zap_object, blk << bs, 1 << bs);
+	dmu_prefetch(zap->zap_objset, zap->zap_object, 0, blk << bs, 1 << bs,
+	    ZIO_PRIORITY_SYNC_READ);
 }
 
 /*
@@ -1285,9 +1287,10 @@ fzap_get_stats(zap_t *zap, zap_stats_t *zs)
 	} else {
 		int b;
 
-		dmu_prefetch(zap->zap_objset, zap->zap_object,
+		dmu_prefetch(zap->zap_objset, zap->zap_object, 0,
 		    zap_f_phys(zap)->zap_ptrtbl.zt_blk << bs,
-		    zap_f_phys(zap)->zap_ptrtbl.zt_numblks << bs);
+		    zap_f_phys(zap)->zap_ptrtbl.zt_numblks << bs,
+		    ZIO_PRIORITY_SYNC_READ);
 
 		for (b = 0; b < zap_f_phys(zap)->zap_ptrtbl.zt_numblks;
 		    b++) {
diff --git a/uts/common/fs/zfs/zfs_vfsops.c b/uts/common/fs/zfs/zfs_vfsops.c
index 8cf83b399a15..3559b9b7b32d 100644
--- a/uts/common/fs/zfs/zfs_vfsops.c
+++ b/uts/common/fs/zfs/zfs_vfsops.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 /* Portions Copyright 2010 Robert Milkowski */
@@ -949,7 +949,7 @@ zfsvfs_create(const char *osname, zfsvfs_t **zfvp)
 		error = zap_lookup(os, MASTER_NODE_OBJ, ZFS_SA_ATTRS, 8, 1,
 		    &sa_obj);
 		if (error)
-			return (error);
+			goto out;
 	} else {
 		/*
 		 * Pre SA versions file systems should never touch
diff --git a/uts/common/fs/zfs/zfs_vnops.c b/uts/common/fs/zfs/zfs_vnops.c
index 5060412afd9b..518e276dff07 100644
--- a/uts/common/fs/zfs/zfs_vnops.c
+++ b/uts/common/fs/zfs/zfs_vnops.c
@@ -2357,7 +2357,8 @@ zfs_readdir(vnode_t *vp, uio_t *uio, cred_t *cr, int *eofp,
 
 		/* Prefetch znode */
 		if (prefetch)
-			dmu_prefetch(os, objnum, 0, 0);
+			dmu_prefetch(os, objnum, 0, 0, 0,
+			    ZIO_PRIORITY_SYNC_READ);
 
 	skip_entry:
 		/*
diff --git a/uts/common/fs/zfs/zio.c b/uts/common/fs/zfs/zio.c
index aa0f2945dd94..f92ecb4739b2 100644
--- a/uts/common/fs/zfs/zio.c
+++ b/uts/common/fs/zfs/zio.c
@@ -67,6 +67,9 @@ extern vmem_t *zio_alloc_arena;
 #define	ZIO_PIPELINE_CONTINUE		0x100
 #define	ZIO_PIPELINE_STOP		0x101
 
+#define	BP_SPANB(indblkshift, level) \
+	(((uint64_t)1) << ((level) * ((indblkshift) - SPA_BLKPTRSHIFT)))
+#define	COMPARE_META_LEVEL	0x80000000ul
 /*
  * The following actions directly effect the spa's sync-to-convergence logic.
  * The values below define the sync pass when we start performing the action.
@@ -3316,37 +3319,127 @@ static zio_pipe_stage_t *zio_pipeline[] = {
 	zio_done
 };
 
-/* dnp is the dnode for zb1->zb_object */
-boolean_t
-zbookmark_is_before(const dnode_phys_t *dnp, const zbookmark_phys_t *zb1,
-    const zbookmark_phys_t *zb2)
-{
-	uint64_t zb1nextL0, zb2thisobj;
 
-	ASSERT(zb1->zb_objset == zb2->zb_objset);
-	ASSERT(zb2->zb_level == 0);
+
+
+/*
+ * Compare two zbookmark_phys_t's to see which we would reach first in a
+ * pre-order traversal of the object tree.
+ *
+ * This is simple in every case aside from the meta-dnode object. For all other
+ * objects, we traverse them in order (object 1 before object 2, and so on).
+ * However, all of these objects are traversed while traversing object 0, since
+ * the data it points to is the list of objects.  Thus, we need to convert to a
+ * canonical representation so we can compare meta-dnode bookmarks to
+ * non-meta-dnode bookmarks.
+ *
+ * We do this by calculating "equivalents" for each field of the zbookmark.
+ * zbookmarks outside of the meta-dnode use their own object and level, and
+ * calculate the level 0 equivalent (the first L0 blkid that is contained in the
+ * blocks this bookmark refers to) by multiplying their blkid by their span
+ * (the number of L0 blocks contained within one block at their level).
+ * zbookmarks inside the meta-dnode calculate their object equivalent
+ * (which is L0equiv * dnodes per data block), use 0 for their L0equiv, and use
+ * level + 1<<31 (any value larger than a level could ever be) for their level.
+ * This causes them to always compare before a bookmark in their object
+ * equivalent, compare appropriately to bookmarks in other objects, and to
+ * compare appropriately to other bookmarks in the meta-dnode.
+ */
+int
+zbookmark_compare(uint16_t dbss1, uint8_t ibs1, uint16_t dbss2, uint8_t ibs2,
+    const zbookmark_phys_t *zb1, const zbookmark_phys_t *zb2)
+{
+	/*
+	 * These variables represent the "equivalent" values for the zbookmark,
+	 * after converting zbookmarks inside the meta dnode to their
+	 * normal-object equivalents.
+	 */
+	uint64_t zb1obj, zb2obj;
+	uint64_t zb1L0, zb2L0;
+	uint64_t zb1level, zb2level;
+
+	if (zb1->zb_object == zb2->zb_object &&
+	    zb1->zb_level == zb2->zb_level &&
+	    zb1->zb_blkid == zb2->zb_blkid)
+		return (0);
+
+	/*
+	 * BP_SPANB calculates the span in blocks.
+	 */
+	zb1L0 = (zb1->zb_blkid) * BP_SPANB(ibs1, zb1->zb_level);
+	zb2L0 = (zb2->zb_blkid) * BP_SPANB(ibs2, zb2->zb_level);
+
+	if (zb1->zb_object == DMU_META_DNODE_OBJECT) {
+		zb1obj = zb1L0 * (dbss1 << (SPA_MINBLOCKSHIFT - DNODE_SHIFT));
+		zb1L0 = 0;
+		zb1level = zb1->zb_level + COMPARE_META_LEVEL;
+	} else {
+		zb1obj = zb1->zb_object;
+		zb1level = zb1->zb_level;
+	}
+
+	if (zb2->zb_object == DMU_META_DNODE_OBJECT) {
+		zb2obj = zb2L0 * (dbss2 << (SPA_MINBLOCKSHIFT - DNODE_SHIFT));
+		zb2L0 = 0;
+		zb2level = zb2->zb_level + COMPARE_META_LEVEL;
+	} else {
+		zb2obj = zb2->zb_object;
+		zb2level = zb2->zb_level;
+	}
+
+	/* Now that we have a canonical representation, do the comparison. */
+	if (zb1obj != zb2obj)
+		return (zb1obj < zb2obj ? -1 : 1);
+	else if (zb1L0 != zb2L0)
+		return (zb1L0 < zb2L0 ? -1 : 1);
+	else if (zb1level != zb2level)
+		return (zb1level > zb2level ? -1 : 1);
+	/*
+	 * This can (theoretically) happen if the bookmarks have the same object
+	 * and level, but different blkids, if the block sizes are not the same.
+	 * There is presently no way to change the indirect block sizes
+	 */
+	return (0);
+}
+
+/*
+ *  This function checks the following: given that last_block is the place that
+ *  our traversal stopped last time, does that guarantee that we've visited
+ *  every node under subtree_root?  Therefore, we can't just use the raw output
+ *  of zbookmark_compare.  We have to pass in a modified version of
+ *  subtree_root; by incrementing the block id, and then checking whether
+ *  last_block is before or equal to that, we can tell whether or not having
+ *  visited last_block implies that all of subtree_root's children have been
+ *  visited.
+ */
+boolean_t
+zbookmark_subtree_completed(const dnode_phys_t *dnp,
+    const zbookmark_phys_t *subtree_root, const zbookmark_phys_t *last_block)
+{
+	zbookmark_phys_t mod_zb = *subtree_root;
+	mod_zb.zb_blkid++;
+	ASSERT(last_block->zb_level == 0);
 
 	/* The objset_phys_t isn't before anything. */
 	if (dnp == NULL)
 		return (B_FALSE);
 
-	zb1nextL0 = (zb1->zb_blkid + 1) <<
-	    ((zb1->zb_level) * (dnp->dn_indblkshift - SPA_BLKPTRSHIFT));
-
-	zb2thisobj = zb2->zb_object ? zb2->zb_object :
-	    zb2->zb_blkid << (DNODE_BLOCK_SHIFT - DNODE_SHIFT);
-
-	if (zb1->zb_object == DMU_META_DNODE_OBJECT) {
-		uint64_t nextobj = zb1nextL0 *
-		    (dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT) >> DNODE_SHIFT;
-		return (nextobj <= zb2thisobj);
-	}
-
-	if (zb1->zb_object < zb2thisobj)
-		return (B_TRUE);
-	if (zb1->zb_object > zb2thisobj)
-		return (B_FALSE);
-	if (zb2->zb_object == DMU_META_DNODE_OBJECT)
-		return (B_FALSE);
-	return (zb1nextL0 <= zb2->zb_blkid);
+	/*
+	 * We pass in 1ULL << (DNODE_BLOCK_SHIFT - SPA_MINBLOCKSHIFT) for the
+	 * data block size in sectors, because that variable is only used if
+	 * the bookmark refers to a block in the meta-dnode.  Since we don't
+	 * know without examining it what object it refers to, and there's no
+	 * harm in passing in this value in other cases, we always pass it in.
+	 *
+	 * We pass in 0 for the indirect block size shift because zb2 must be
+	 * level 0.  The indirect block size is only used to calculate the span
+	 * of the bookmark, but since the bookmark must be level 0, the span is
+	 * always 1, so the math works out.
+	 *
+	 * If you make changes to how the zbookmark_compare code works, be sure
+	 * to make sure that this code still works afterwards.
+	 */
+	return (zbookmark_compare(dnp->dn_datablkszsec, dnp->dn_indblkshift,
+	    1ULL << (DNODE_BLOCK_SHIFT - SPA_MINBLOCKSHIFT), 0, &mod_zb,
+	    last_block) <= 0);
 }
diff --git a/uts/common/fs/zfs/zvol.c b/uts/common/fs/zfs/zvol.c
index 80888103fe48..101566a1ea55 100644
--- a/uts/common/fs/zfs/zvol.c
+++ b/uts/common/fs/zfs/zvol.c
@@ -255,7 +255,7 @@ zvol_map_block(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 	zvol_extent_t *ze;
 	int bs = ma->ma_zv->zv_volblocksize;
 
-	if (BP_IS_HOLE(bp) ||
+	if (bp == NULL || BP_IS_HOLE(bp) ||
 	    zb->zb_object != ZVOL_OBJ || zb->zb_level != 0)
 		return (0);
 

From 963975c9c7f39822220fc1fd0a58a960508c603b Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Wed, 12 Aug 2015 23:38:58 +0000
Subject: [PATCH 118/380] 5959 clean up per-dataset feature count code

Reviewed by: Toomas Soome <tsoome@me.com>
Reviewed by: George Wilson <george@delphix.com>
Reviewed by: Alex Reece <alex@delphix.com>
Approved by: Richard Lowe <richlowe@richlowe.net>
Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@ca0cc3918a1789fa839194af2a9245f801a06b1a

A ZFS feature flags (large blocks) tracks its refcounts as the number of
datasets that have ever used the feature. Several features of this type
are planned to be added (new checksum functions). This code should be made
common infrastructure rather than duplicating the code for each feature.
---
 common/zfs/zfeature_common.c        |  54 ++++----
 common/zfs/zfeature_common.h        |  18 ++-
 uts/common/fs/zfs/dbuf.c            |   5 +
 uts/common/fs/zfs/dmu_send.c        |  11 +-
 uts/common/fs/zfs/dsl_dataset.c     | 188 ++++++++++++++--------------
 uts/common/fs/zfs/dsl_destroy.c     |  22 ++--
 uts/common/fs/zfs/sys/dsl_dataset.h |  22 +++-
 uts/common/fs/zfs/zfeature.c        |  15 +--
 8 files changed, 182 insertions(+), 153 deletions(-)

diff --git a/common/zfs/zfeature_common.c b/common/zfs/zfeature_common.c
index 3358e5eb87ba..025c227e677f 100644
--- a/common/zfs/zfeature_common.c
+++ b/common/zfs/zfeature_common.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  * Copyright (c) 2014, Nexenta Systems, Inc. All rights reserved.
@@ -130,15 +130,15 @@ zfeature_depends_on(spa_feature_t fid, spa_feature_t check) {
 
 static void
 zfeature_register(spa_feature_t fid, const char *guid, const char *name,
-    const char *desc, boolean_t readonly, boolean_t mos,
-    boolean_t activate_on_enable, const spa_feature_t *deps)
+    const char *desc, zfeature_flags_t flags, const spa_feature_t *deps)
 {
 	zfeature_info_t *feature = &spa_feature_table[fid];
 	static spa_feature_t nodeps[] = { SPA_FEATURE_NONE };
 
 	ASSERT(name != NULL);
 	ASSERT(desc != NULL);
-	ASSERT(!readonly || !mos);
+	ASSERT((flags & ZFEATURE_FLAG_READONLY_COMPAT) == 0 ||
+	    (flags & ZFEATURE_FLAG_MOS) == 0);
 	ASSERT3U(fid, <, SPA_FEATURES);
 	ASSERT(zfeature_is_valid_guid(guid));
 
@@ -149,9 +149,7 @@ zfeature_register(spa_feature_t fid, const char *guid, const char *name,
 	feature->fi_guid = guid;
 	feature->fi_uname = name;
 	feature->fi_desc = desc;
-	feature->fi_can_readonly = readonly;
-	feature->fi_mos = mos;
-	feature->fi_activate_on_enable = activate_on_enable;
+	feature->fi_flags = flags;
 	feature->fi_depends = deps;
 }
 
@@ -160,45 +158,46 @@ zpool_feature_init(void)
 {
 	zfeature_register(SPA_FEATURE_ASYNC_DESTROY,
 	    "com.delphix:async_destroy", "async_destroy",
-	    "Destroy filesystems asynchronously.", B_TRUE, B_FALSE,
-	    B_FALSE, NULL);
+	    "Destroy filesystems asynchronously.",
+	    ZFEATURE_FLAG_READONLY_COMPAT, NULL);
 
 	zfeature_register(SPA_FEATURE_EMPTY_BPOBJ,
 	    "com.delphix:empty_bpobj", "empty_bpobj",
-	    "Snapshots use less space.", B_TRUE, B_FALSE,
-	    B_FALSE, NULL);
+	    "Snapshots use less space.",
+	    ZFEATURE_FLAG_READONLY_COMPAT, NULL);
 
 	zfeature_register(SPA_FEATURE_LZ4_COMPRESS,
 	    "org.illumos:lz4_compress", "lz4_compress",
-	    "LZ4 compression algorithm support.", B_FALSE, B_FALSE,
-	    B_TRUE, NULL);
+	    "LZ4 compression algorithm support.",
+	    ZFEATURE_FLAG_ACTIVATE_ON_ENABLE, NULL);
 
 	zfeature_register(SPA_FEATURE_MULTI_VDEV_CRASH_DUMP,
 	    "com.joyent:multi_vdev_crash_dump", "multi_vdev_crash_dump",
-	    "Crash dumps to multiple vdev pools.", B_FALSE, B_FALSE,
-	    B_FALSE, NULL);
+	    "Crash dumps to multiple vdev pools.",
+	    0, NULL);
 
 	zfeature_register(SPA_FEATURE_SPACEMAP_HISTOGRAM,
 	    "com.delphix:spacemap_histogram", "spacemap_histogram",
-	    "Spacemaps maintain space histograms.", B_TRUE, B_FALSE,
-	    B_FALSE, NULL);
+	    "Spacemaps maintain space histograms.",
+	    ZFEATURE_FLAG_READONLY_COMPAT, NULL);
 
 	zfeature_register(SPA_FEATURE_ENABLED_TXG,
 	    "com.delphix:enabled_txg", "enabled_txg",
-	    "Record txg at which a feature is enabled", B_TRUE, B_FALSE,
-	    B_FALSE, NULL);
+	    "Record txg at which a feature is enabled",
+	    ZFEATURE_FLAG_READONLY_COMPAT, NULL);
 
 	static spa_feature_t hole_birth_deps[] = { SPA_FEATURE_ENABLED_TXG,
 	    SPA_FEATURE_NONE };
 	zfeature_register(SPA_FEATURE_HOLE_BIRTH,
 	    "com.delphix:hole_birth", "hole_birth",
 	    "Retain hole birth txg for more precise zfs send",
-	    B_FALSE, B_TRUE, B_TRUE, hole_birth_deps);
+	    ZFEATURE_FLAG_MOS | ZFEATURE_FLAG_ACTIVATE_ON_ENABLE,
+	    hole_birth_deps);
 
 	zfeature_register(SPA_FEATURE_EXTENSIBLE_DATASET,
 	    "com.delphix:extensible_dataset", "extensible_dataset",
 	    "Enhanced dataset functionality, used by other features.",
-	    B_FALSE, B_FALSE, B_FALSE, NULL);
+	    0, NULL);
 
 	static const spa_feature_t bookmarks_deps[] = {
 		SPA_FEATURE_EXTENSIBLE_DATASET,
@@ -207,7 +206,7 @@ zpool_feature_init(void)
 	zfeature_register(SPA_FEATURE_BOOKMARKS,
 	    "com.delphix:bookmarks", "bookmarks",
 	    "\"zfs bookmark\" command",
-	    B_TRUE, B_FALSE, B_FALSE, bookmarks_deps);
+	    ZFEATURE_FLAG_READONLY_COMPAT, bookmarks_deps);
 
 	static const spa_feature_t filesystem_limits_deps[] = {
 	    SPA_FEATURE_EXTENSIBLE_DATASET,
@@ -215,13 +214,14 @@ zpool_feature_init(void)
 	};
 	zfeature_register(SPA_FEATURE_FS_SS_LIMIT,
 	    "com.joyent:filesystem_limits", "filesystem_limits",
-	    "Filesystem and snapshot limits.", B_TRUE, B_FALSE, B_FALSE,
-	    filesystem_limits_deps);
+	    "Filesystem and snapshot limits.",
+	    ZFEATURE_FLAG_READONLY_COMPAT, filesystem_limits_deps);
 
 	zfeature_register(SPA_FEATURE_EMBEDDED_DATA,
 	    "com.delphix:embedded_data", "embedded_data",
 	    "Blocks which compress very well use even less space.",
-	    B_FALSE, B_TRUE, B_TRUE, NULL);
+	    ZFEATURE_FLAG_MOS | ZFEATURE_FLAG_ACTIVATE_ON_ENABLE,
+	    NULL);
 
 	static const spa_feature_t large_blocks_deps[] = {
 		SPA_FEATURE_EXTENSIBLE_DATASET,
@@ -229,6 +229,6 @@ zpool_feature_init(void)
 	};
 	zfeature_register(SPA_FEATURE_LARGE_BLOCKS,
 	    "org.open-zfs:large_blocks", "large_blocks",
-	    "Support for blocks larger than 128KB.", B_FALSE, B_FALSE, B_FALSE,
-	    large_blocks_deps);
+	    "Support for blocks larger than 128KB.",
+	    ZFEATURE_FLAG_PER_DATASET, large_blocks_deps);
 }
diff --git a/common/zfs/zfeature_common.h b/common/zfs/zfeature_common.h
index 3f54b392be42..794290bfe0c5 100644
--- a/common/zfs/zfeature_common.h
+++ b/common/zfs/zfeature_common.h
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  */
@@ -57,15 +57,23 @@ typedef enum spa_feature {
 
 #define	SPA_FEATURE_DISABLED	(-1ULL)
 
+typedef enum zfeature_flags {
+	/* Can open pool readonly even if this feature is not supported. */
+	ZFEATURE_FLAG_READONLY_COMPAT =		(1 << 0),
+	/* Is this feature necessary to read the MOS? */
+	ZFEATURE_FLAG_MOS =			(1 << 1),
+	/* Activate this feature at the same time it is enabled. */
+	ZFEATURE_FLAG_ACTIVATE_ON_ENABLE =	(1 << 2),
+	/* Each dataset has a field set if it has ever used this feature. */
+	ZFEATURE_FLAG_PER_DATASET =		(1 << 3)
+} zfeature_flags_t;
+
 typedef struct zfeature_info {
 	spa_feature_t fi_feature;
 	const char *fi_uname;	/* User-facing feature name */
 	const char *fi_guid;	/* On-disk feature identifier */
 	const char *fi_desc;	/* Feature description */
-	boolean_t fi_can_readonly; /* Can open pool readonly w/o support? */
-	boolean_t fi_mos;	/* Is the feature necessary to read the MOS? */
-	/* Activate this feature at the same time it is enabled */
-	boolean_t fi_activate_on_enable;
+	zfeature_flags_t fi_flags;
 	/* array of dependencies, terminated by SPA_FEATURE_NONE */
 	const spa_feature_t *fi_depends;
 } zfeature_info_t;
diff --git a/uts/common/fs/zfs/dbuf.c b/uts/common/fs/zfs/dbuf.c
index 623d57ab4404..e2ed92aac70c 100644
--- a/uts/common/fs/zfs/dbuf.c
+++ b/uts/common/fs/zfs/dbuf.c
@@ -1573,6 +1573,11 @@ dmu_buf_write_embedded(dmu_buf_t *dbuf, void *data,
 	struct dirty_leaf *dl;
 	dmu_object_type_t type;
 
+	if (etype == BP_EMBEDDED_TYPE_DATA) {
+		ASSERT(spa_feature_is_active(dmu_objset_spa(db->db_objset),
+		    SPA_FEATURE_EMBEDDED_DATA));
+	}
+
 	DB_DNODE_ENTER(db);
 	type = DB_DNODE(db)->dn_type;
 	DB_DNODE_EXIT(db);
diff --git a/uts/common/fs/zfs/dmu_send.c b/uts/common/fs/zfs/dmu_send.c
index ca09d10bd806..0b57da6b8aaa 100644
--- a/uts/common/fs/zfs/dmu_send.c
+++ b/uts/common/fs/zfs/dmu_send.c
@@ -21,7 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
- * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2014, Joyent, Inc. All rights reserved.
  * Copyright 2014 HybridCluster. All rights reserved.
  */
@@ -695,7 +695,7 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *to_ds,
 	}
 #endif
 
-	if (large_block_ok && to_ds->ds_large_blocks)
+	if (large_block_ok && to_ds->ds_feature_inuse[SPA_FEATURE_LARGE_BLOCKS])
 		featureflags |= DMU_BACKUP_FEATURE_LARGE_BLOCKS;
 	if (embedok &&
 	    spa_feature_is_active(dp->dp_spa, SPA_FEATURE_EMBEDDED_DATA)) {
@@ -1344,13 +1344,6 @@ dmu_recv_begin_sync(void *arg, dmu_tx_t *tx)
 	}
 	VERIFY0(dsl_dataset_own_obj(dp, dsobj, dmu_recv_tag, &newds));
 
-	if ((DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) &
-	    DMU_BACKUP_FEATURE_LARGE_BLOCKS) &&
-	    !newds->ds_large_blocks) {
-		dsl_dataset_activate_large_blocks_sync_impl(dsobj, tx);
-		newds->ds_large_blocks = B_TRUE;
-	}
-
 	dmu_buf_will_dirty(newds->ds_dbuf, tx);
 	dsl_dataset_phys(newds)->ds_flags |= DS_FLAG_INCONSISTENT;
 
diff --git a/uts/common/fs/zfs/dsl_dataset.c b/uts/common/fs/zfs/dsl_dataset.c
index 8f419d682875..7f63cd97bcde 100644
--- a/uts/common/fs/zfs/dsl_dataset.c
+++ b/uts/common/fs/zfs/dsl_dataset.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2014, Joyent, Inc. All rights reserved.
  * Copyright (c) 2014 RackTop Systems.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
@@ -124,8 +124,10 @@ dsl_dataset_block_born(dsl_dataset_t *ds, const blkptr_t *bp, dmu_tx_t *tx)
 	dsl_dataset_phys(ds)->ds_compressed_bytes += compressed;
 	dsl_dataset_phys(ds)->ds_uncompressed_bytes += uncompressed;
 	dsl_dataset_phys(ds)->ds_unique_bytes += used;
-	if (BP_GET_LSIZE(bp) > SPA_OLD_MAXBLOCKSIZE)
-		ds->ds_need_large_blocks = B_TRUE;
+	if (BP_GET_LSIZE(bp) > SPA_OLD_MAXBLOCKSIZE) {
+		ds->ds_feature_activation_needed[SPA_FEATURE_LARGE_BLOCKS] =
+		    B_TRUE;
+	}
 	mutex_exit(&ds->ds_lock);
 	dsl_dir_diduse_space(ds->ds_dir, DD_USED_HEAD, delta,
 	    compressed, uncompressed, tx);
@@ -423,19 +425,23 @@ dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
 		    offsetof(dmu_sendarg_t, dsa_link));
 
 		if (doi.doi_type == DMU_OTN_ZAP_METADATA) {
-			int zaperr = zap_contains(mos, dsobj,
-			    DS_FIELD_LARGE_BLOCKS);
-			if (zaperr != ENOENT) {
-				VERIFY0(zaperr);
-				ds->ds_large_blocks = B_TRUE;
+			for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
+				if (!(spa_feature_table[f].fi_flags &
+				    ZFEATURE_FLAG_PER_DATASET))
+					continue;
+				err = zap_contains(mos, dsobj,
+				    spa_feature_table[f].fi_guid);
+				if (err == 0) {
+					ds->ds_feature_inuse[f] = B_TRUE;
+				} else {
+					ASSERT3U(err, ==, ENOENT);
+					err = 0;
+				}
 			}
 		}
 
-		if (err == 0) {
-			err = dsl_dir_hold_obj(dp,
-			    dsl_dataset_phys(ds)->ds_dir_obj, NULL, ds,
-			    &ds->ds_dir);
-		}
+		err = dsl_dir_hold_obj(dp,
+		    dsl_dataset_phys(ds)->ds_dir_obj, NULL, ds, &ds->ds_dir);
 		if (err != 0) {
 			mutex_destroy(&ds->ds_lock);
 			mutex_destroy(&ds->ds_opening_lock);
@@ -691,6 +697,34 @@ dsl_dataset_tryown(dsl_dataset_t *ds, void *tag)
 	return (gotit);
 }
 
+static void
+dsl_dataset_activate_feature(uint64_t dsobj, spa_feature_t f, dmu_tx_t *tx)
+{
+	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
+	objset_t *mos = dmu_tx_pool(tx)->dp_meta_objset;
+	uint64_t zero = 0;
+
+	VERIFY(spa_feature_table[f].fi_flags & ZFEATURE_FLAG_PER_DATASET);
+
+	spa_feature_incr(spa, f, tx);
+	dmu_object_zapify(mos, dsobj, DMU_OT_DSL_DATASET, tx);
+
+	VERIFY0(zap_add(mos, dsobj, spa_feature_table[f].fi_guid,
+	    sizeof (zero), 1, &zero, tx));
+}
+
+void
+dsl_dataset_deactivate_feature(uint64_t dsobj, spa_feature_t f, dmu_tx_t *tx)
+{
+	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
+	objset_t *mos = dmu_tx_pool(tx)->dp_meta_objset;
+
+	VERIFY(spa_feature_table[f].fi_flags & ZFEATURE_FLAG_PER_DATASET);
+
+	VERIFY0(zap_remove(mos, dsobj, spa_feature_table[f].fi_guid, tx));
+	spa_feature_decr(spa, f, tx);
+}
+
 uint64_t
 dsl_dataset_create_sync_dd(dsl_dir_t *dd, dsl_dataset_t *origin,
     uint64_t flags, dmu_tx_t *tx)
@@ -749,8 +783,10 @@ dsl_dataset_create_sync_dd(dsl_dir_t *dd, dsl_dataset_t *origin,
 		dsphys->ds_flags |= dsl_dataset_phys(origin)->ds_flags &
 		    (DS_FLAG_INCONSISTENT | DS_FLAG_CI_DATASET);
 
-		if (origin->ds_large_blocks)
-			dsl_dataset_activate_large_blocks_sync_impl(dsobj, tx);
+		for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
+			if (origin->ds_feature_inuse[f])
+				dsl_dataset_activate_feature(dsobj, f, tx);
+		}
 
 		dmu_buf_will_dirty(origin->ds_dbuf, tx);
 		dsl_dataset_phys(origin)->ds_num_children++;
@@ -1270,8 +1306,10 @@ dsl_dataset_snapshot_sync_impl(dsl_dataset_t *ds, const char *snapname,
 	dsphys->ds_bp = dsl_dataset_phys(ds)->ds_bp;
 	dmu_buf_rele(dbuf, FTAG);
 
-	if (ds->ds_large_blocks)
-		dsl_dataset_activate_large_blocks_sync_impl(dsobj, tx);
+	for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
+		if (ds->ds_feature_inuse[f])
+			dsl_dataset_activate_feature(dsobj, f, tx);
+	}
 
 	ASSERT3U(ds->ds_prev != 0, ==,
 	    dsl_dataset_phys(ds)->ds_prev_snap_obj != 0);
@@ -1552,9 +1590,13 @@ dsl_dataset_sync(dsl_dataset_t *ds, zio_t *zio, dmu_tx_t *tx)
 
 	dmu_objset_sync(ds->ds_objset, zio, tx);
 
-	if (ds->ds_need_large_blocks && !ds->ds_large_blocks) {
-		dsl_dataset_activate_large_blocks_sync_impl(ds->ds_object, tx);
-		ds->ds_large_blocks = B_TRUE;
+	for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
+		if (ds->ds_feature_activation_needed[f]) {
+			if (ds->ds_feature_inuse[f])
+				continue;
+			dsl_dataset_activate_feature(ds->ds_object, f, tx);
+			ds->ds_feature_inuse[f] = B_TRUE;
+		}
 	}
 }
 
@@ -2675,6 +2717,40 @@ dsl_dataset_clone_swap_sync_impl(dsl_dataset_t *clone,
 	    dsl_dataset_phys(clone)->ds_unique_bytes <= origin_head->ds_quota);
 	ASSERT3P(clone->ds_prev, ==, origin_head->ds_prev);
 
+	/*
+	 * Swap per-dataset feature flags.
+	 */
+	for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
+		if (!(spa_feature_table[f].fi_flags &
+		    ZFEATURE_FLAG_PER_DATASET)) {
+			ASSERT(!clone->ds_feature_inuse[f]);
+			ASSERT(!origin_head->ds_feature_inuse[f]);
+			continue;
+		}
+
+		boolean_t clone_inuse = clone->ds_feature_inuse[f];
+		boolean_t origin_head_inuse = origin_head->ds_feature_inuse[f];
+
+		if (clone_inuse) {
+			dsl_dataset_deactivate_feature(clone->ds_object, f, tx);
+			clone->ds_feature_inuse[f] = B_FALSE;
+		}
+		if (origin_head_inuse) {
+			dsl_dataset_deactivate_feature(origin_head->ds_object,
+			    f, tx);
+			origin_head->ds_feature_inuse[f] = B_FALSE;
+		}
+		if (clone_inuse) {
+			dsl_dataset_activate_feature(origin_head->ds_object,
+			    f, tx);
+			origin_head->ds_feature_inuse[f] = B_TRUE;
+		}
+		if (origin_head_inuse) {
+			dsl_dataset_activate_feature(clone->ds_object, f, tx);
+			clone->ds_feature_inuse[f] = B_TRUE;
+		}
+	}
+
 	dmu_buf_will_dirty(clone->ds_dbuf, tx);
 	dmu_buf_will_dirty(origin_head->ds_dbuf, tx);
 
@@ -3229,77 +3305,6 @@ dsl_dataset_space_wouldfree(dsl_dataset_t *firstsnap,
 	return (err);
 }
 
-static int
-dsl_dataset_activate_large_blocks_check(void *arg, dmu_tx_t *tx)
-{
-	const char *dsname = arg;
-	dsl_dataset_t *ds;
-	dsl_pool_t *dp = dmu_tx_pool(tx);
-	int error = 0;
-
-	if (!spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_LARGE_BLOCKS))
-		return (SET_ERROR(ENOTSUP));
-
-	ASSERT(spa_feature_is_enabled(dp->dp_spa,
-	    SPA_FEATURE_EXTENSIBLE_DATASET));
-
-	error = dsl_dataset_hold(dp, dsname, FTAG, &ds);
-	if (error != 0)
-		return (error);
-
-	if (ds->ds_large_blocks)
-		error = EALREADY;
-	dsl_dataset_rele(ds, FTAG);
-
-	return (error);
-}
-
-void
-dsl_dataset_activate_large_blocks_sync_impl(uint64_t dsobj, dmu_tx_t *tx)
-{
-	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
-	objset_t *mos = dmu_tx_pool(tx)->dp_meta_objset;
-	uint64_t zero = 0;
-
-	spa_feature_incr(spa, SPA_FEATURE_LARGE_BLOCKS, tx);
-	dmu_object_zapify(mos, dsobj, DMU_OT_DSL_DATASET, tx);
-
-	VERIFY0(zap_add(mos, dsobj, DS_FIELD_LARGE_BLOCKS,
-	    sizeof (zero), 1, &zero, tx));
-}
-
-static void
-dsl_dataset_activate_large_blocks_sync(void *arg, dmu_tx_t *tx)
-{
-	const char *dsname = arg;
-	dsl_dataset_t *ds;
-
-	VERIFY0(dsl_dataset_hold(dmu_tx_pool(tx), dsname, FTAG, &ds));
-
-	dsl_dataset_activate_large_blocks_sync_impl(ds->ds_object, tx);
-	ASSERT(!ds->ds_large_blocks);
-	ds->ds_large_blocks = B_TRUE;
-	dsl_dataset_rele(ds, FTAG);
-}
-
-int
-dsl_dataset_activate_large_blocks(const char *dsname)
-{
-	int error;
-
-	error = dsl_sync_task(dsname,
-	    dsl_dataset_activate_large_blocks_check,
-	    dsl_dataset_activate_large_blocks_sync, (void *)dsname,
-	    1, ZFS_SPACE_CHECK_RESERVED);
-
-	/*
-	 * EALREADY indicates that this dataset already supports large blocks.
-	 */
-	if (error == EALREADY)
-		error = 0;
-	return (error);
-}
-
 /*
  * Return TRUE if 'earlier' is an earlier snapshot in 'later's timeline.
  * For example, they could both be snapshots of the same filesystem, and
@@ -3344,7 +3349,6 @@ dsl_dataset_is_before(dsl_dataset_t *later, dsl_dataset_t *earlier,
 	return (ret);
 }
 
-
 void
 dsl_dataset_zapify(dsl_dataset_t *ds, dmu_tx_t *tx)
 {
diff --git a/uts/common/fs/zfs/dsl_destroy.c b/uts/common/fs/zfs/dsl_destroy.c
index 40e34cbf9deb..c7a623c8291b 100644
--- a/uts/common/fs/zfs/dsl_destroy.c
+++ b/uts/common/fs/zfs/dsl_destroy.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
  * Copyright (c) 2013 by Joyent, Inc. All rights reserved.
  */
@@ -267,9 +267,11 @@ dsl_destroy_snapshot_sync_impl(dsl_dataset_t *ds, boolean_t defer, dmu_tx_t *tx)
 
 	obj = ds->ds_object;
 
-	if (ds->ds_large_blocks) {
-		ASSERT0(zap_contains(mos, obj, DS_FIELD_LARGE_BLOCKS));
-		spa_feature_decr(dp->dp_spa, SPA_FEATURE_LARGE_BLOCKS, tx);
+	for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
+		if (ds->ds_feature_inuse[f]) {
+			dsl_dataset_deactivate_feature(obj, f, tx);
+			ds->ds_feature_inuse[f] = B_FALSE;
+		}
 	}
 	if (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) {
 		ASSERT3P(ds->ds_prev, ==, NULL);
@@ -736,13 +738,17 @@ dsl_destroy_head_sync_impl(dsl_dataset_t *ds, dmu_tx_t *tx)
 		ASSERT0(ds->ds_reserved);
 	}
 
-	if (ds->ds_large_blocks)
-		spa_feature_decr(dp->dp_spa, SPA_FEATURE_LARGE_BLOCKS, tx);
+	obj = ds->ds_object;
+
+	for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
+		if (ds->ds_feature_inuse[f]) {
+			dsl_dataset_deactivate_feature(obj, f, tx);
+			ds->ds_feature_inuse[f] = B_FALSE;
+		}
+	}
 
 	dsl_scan_ds_destroyed(ds, tx);
 
-	obj = ds->ds_object;
-
 	if (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) {
 		/* This is a clone */
 		ASSERT(ds->ds_prev != NULL);
diff --git a/uts/common/fs/zfs/sys/dsl_dataset.h b/uts/common/fs/zfs/sys/dsl_dataset.h
index 81757b5217d4..001bff566a88 100644
--- a/uts/common/fs/zfs/sys/dsl_dataset.h
+++ b/uts/common/fs/zfs/sys/dsl_dataset.h
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
@@ -38,6 +38,7 @@
 #include <sys/zfs_context.h>
 #include <sys/dsl_deadlist.h>
 #include <sys/refcount.h>
+#include <zfeature_common.h>
 
 #ifdef	__cplusplus
 extern "C" {
@@ -145,8 +146,6 @@ typedef struct dsl_dataset {
 	/* only used in syncing context, only valid for non-snapshots: */
 	struct dsl_dataset *ds_prev;
 	uint64_t ds_bookmarks;  /* DMU_OTN_ZAP_METADATA */
-	boolean_t ds_large_blocks;
-	boolean_t ds_need_large_blocks;
 
 	/* has internal locking: */
 	dsl_deadlist_t ds_deadlist;
@@ -185,6 +184,18 @@ typedef struct dsl_dataset {
 	kmutex_t ds_sendstream_lock;
 	list_t ds_sendstreams;
 
+	/*
+	 * For ZFEATURE_FLAG_PER_DATASET features, set if this dataset
+	 * uses this feature.
+	 */
+	uint8_t ds_feature_inuse[SPA_FEATURES];
+
+	/*
+	 * Set if we need to activate the feature on this dataset this txg
+	 * (used only in syncing context).
+	 */
+	uint8_t ds_feature_activation_needed[SPA_FEATURES];
+
 	/* Protected by ds_lock; keep at end of struct for better locality */
 	char ds_snapname[MAXNAMELEN];
 } dsl_dataset_t;
@@ -264,8 +275,6 @@ int dsl_dataset_space_written(dsl_dataset_t *oldsnap, dsl_dataset_t *new,
 int dsl_dataset_space_wouldfree(dsl_dataset_t *firstsnap, dsl_dataset_t *last,
     uint64_t *usedp, uint64_t *compp, uint64_t *uncompp);
 boolean_t dsl_dataset_is_dirty(dsl_dataset_t *ds);
-int dsl_dataset_activate_large_blocks(const char *dsname);
-void dsl_dataset_activate_large_blocks_sync_impl(uint64_t dsobj, dmu_tx_t *tx);
 
 int dsl_dsobj_to_dsname(char *pname, uint64_t obj, char *buf);
 
@@ -305,6 +314,9 @@ void dsl_dataset_set_refreservation_sync_impl(dsl_dataset_t *ds,
 void dsl_dataset_zapify(dsl_dataset_t *ds, dmu_tx_t *tx);
 int dsl_dataset_rollback(const char *fsname, void *owner, nvlist_t *result);
 
+void dsl_dataset_deactivate_feature(uint64_t dsobj,
+    spa_feature_t f, dmu_tx_t *tx);
+
 #ifdef ZFS_DEBUG
 #define	dprintf_ds(ds, fmt, ...) do { \
 	if (zfs_flags & ZFS_DEBUG_DPRINTF) { \
diff --git a/uts/common/fs/zfs/zfeature.c b/uts/common/fs/zfs/zfeature.c
index e69fcd3730ad..1833e1e27047 100644
--- a/uts/common/fs/zfs/zfeature.c
+++ b/uts/common/fs/zfs/zfeature.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -245,7 +245,7 @@ feature_get_refcount_from_disk(spa_t *spa, zfeature_info_t *feature,
 {
 	int err;
 	uint64_t refcount;
-	uint64_t zapobj = feature->fi_can_readonly ?
+	uint64_t zapobj = (feature->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ?
 	    spa->spa_feat_for_write_obj : spa->spa_feat_for_read_obj;
 
 	/*
@@ -296,7 +296,7 @@ feature_sync(spa_t *spa, zfeature_info_t *feature, uint64_t refcount,
     dmu_tx_t *tx)
 {
 	ASSERT(VALID_FEATURE_OR_NONE(feature->fi_feature));
-	uint64_t zapobj = feature->fi_can_readonly ?
+	uint64_t zapobj = (feature->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ?
 	    spa->spa_feat_for_write_obj : spa->spa_feat_for_read_obj;
 
 	VERIFY0(zap_update(spa->spa_meta_objset, zapobj, feature->fi_guid,
@@ -318,7 +318,7 @@ feature_sync(spa_t *spa, zfeature_info_t *feature, uint64_t refcount,
 
 	if (refcount == 0)
 		spa_deactivate_mos_feature(spa, feature->fi_guid);
-	else if (feature->fi_mos)
+	else if (feature->fi_flags & ZFEATURE_FLAG_MOS)
 		spa_activate_mos_feature(spa, feature->fi_guid, tx);
 }
 
@@ -329,8 +329,9 @@ feature_sync(spa_t *spa, zfeature_info_t *feature, uint64_t refcount,
 void
 feature_enable_sync(spa_t *spa, zfeature_info_t *feature, dmu_tx_t *tx)
 {
-	uint64_t initial_refcount = feature->fi_activate_on_enable ? 1 : 0;
-	uint64_t zapobj = feature->fi_can_readonly ?
+	uint64_t initial_refcount =
+	    (feature->fi_flags & ZFEATURE_FLAG_ACTIVATE_ON_ENABLE) ? 1 : 0;
+	uint64_t zapobj = (feature->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ?
 	    spa->spa_feat_for_write_obj : spa->spa_feat_for_read_obj;
 
 	ASSERT(0 != zapobj);
@@ -375,7 +376,7 @@ feature_do_action(spa_t *spa, spa_feature_t fid, feature_action_t action,
 {
 	uint64_t refcount;
 	zfeature_info_t *feature = &spa_feature_table[fid];
-	uint64_t zapobj = feature->fi_can_readonly ?
+	uint64_t zapobj = (feature->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ?
 	    spa->spa_feat_for_write_obj : spa->spa_feat_for_read_obj;
 
 	ASSERT(VALID_FEATURE_FID(fid));

From 4eafe9e22d1f24eb8ef1cadb75c94290e9bebf46 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Thu, 13 Aug 2015 00:07:23 +0000
Subject: [PATCH 119/380] 6093 zfsctl_shares_lookup should only VN_RELE() on
 zfs_zget() success

Reviewed by: Gordon Ross <gwr@nexenta.com>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: George Wilson <george.wilson@delphix.com>
Approved by: Robert Mustacchi <rm@joyent.com>
Author: Dan McDonald <danmcd@omniti.com>

illumos/illumos-gate@0f92170f1ec2737ee5a0e51b5f74093904811452
---
 uts/common/fs/zfs/zfs_ctldir.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/uts/common/fs/zfs/zfs_ctldir.c b/uts/common/fs/zfs/zfs_ctldir.c
index 2f174968383d..c1f5af28fc75 100644
--- a/uts/common/fs/zfs/zfs_ctldir.c
+++ b/uts/common/fs/zfs/zfs_ctldir.c
@@ -21,6 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright 2015, OmniTI Computer Consulting, Inc. All rights reserved.
  */
 
 /*
@@ -966,11 +967,12 @@ zfsctl_shares_lookup(vnode_t *dvp, char *nm, vnode_t **vpp, pathname_t *pnp,
 		ZFS_EXIT(zfsvfs);
 		return (SET_ERROR(ENOTSUP));
 	}
-	if ((error = zfs_zget(zfsvfs, zfsvfs->z_shares_dir, &dzp)) == 0)
+	if ((error = zfs_zget(zfsvfs, zfsvfs->z_shares_dir, &dzp)) == 0) {
 		error = VOP_LOOKUP(ZTOV(dzp), nm, vpp, pnp,
 		    flags, rdir, cr, ct, direntflags, realpnp);
+		VN_RELE(ZTOV(dzp));
+	}
 
-	VN_RELE(ZTOV(dzp));
 	ZFS_EXIT(zfsvfs);
 
 	return (error);

From 16c754ced59637007fe159949121850c22d8d0ed Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Thu, 13 Aug 2015 00:12:52 +0000
Subject: [PATCH 120/380] 6096 ZFS_SMB_ACL_RENAME needs to cleanup better

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Gordon Ross <gordon.w.ross@gmail.com>
Reviewed by: George Wilson <gwilson@zfsmail.com>
Approved by: Robert Mustacchi <rm@joyent.com>

illumos/illumos-gate@8f5190a540d64d2debee6664bbc740e4c38f5b7f
---
 uts/common/fs/zfs/zfs_ioctl.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/uts/common/fs/zfs/zfs_ioctl.c b/uts/common/fs/zfs/zfs_ioctl.c
index 1826c8aaacdf..5a56021c8cae 100644
--- a/uts/common/fs/zfs/zfs_ioctl.c
+++ b/uts/common/fs/zfs/zfs_ioctl.c
@@ -5060,6 +5060,7 @@ zfs_ioc_smb_acl(zfs_cmd_t *zc)
 		if ((error = get_nvlist(zc->zc_nvlist_src,
 		    zc->zc_nvlist_src_size, zc->zc_iflags, &nvlist)) != 0) {
 			VN_RELE(vp);
+			VN_RELE(ZTOV(sharedir));
 			ZFS_EXIT(zfsvfs);
 			return (error);
 		}

From 17e4b37064f813162b34deaf993438545da7cb4a Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Fri, 14 Aug 2015 09:37:54 +0000
Subject: [PATCH 121/380] 5817 change type of arcs_size from uint64_t to
 refcount_t

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Paul Dagnelie <paul.dagnelie@delphix.com>
Reviewed by: Adam Leventhal <ahl@delphix.com>
Reviewed by: Alex Reece <alex@delphix.com>
Reviewed by: Richard Elling <richard.elling@richardelling.com>
Approved by: Garrett D'Amore <garrett@damore.org>
Author: Prakash Surya <prakash.surya@delphix.com>

illumos/illumos-gate@2fd872a734cf486007a8dba532cec52bfb4d40e5

As a way to make it more difficult to introduce bugs into the ARC, and to
make it easier to diagnose issues when bugs do creep in, it would be
beneficial to change the type of the arc_state_t's arcs_size field to be
a refcount_t instead of a uint64_t. This would allow us to make stricter
checks when incrementing and decrementing the value with debugging enabled,
but still fallback to simple, fast atomic operations when debugging is
disabled.
---
 uts/common/fs/zfs/arc.c | 135 ++++++++++++++++++++++++++++++++--------
 1 file changed, 109 insertions(+), 26 deletions(-)

diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index 3efe13b1f34b..a6bb022dd0a9 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -21,7 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
- * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2014 by Saso Kiselkov. All rights reserved.
  * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
  */
@@ -273,7 +273,7 @@ typedef struct arc_state {
 	 * total amount of data in this state; this includes: evictable,
 	 * non-evictable, ARC_BUFC_DATA, and ARC_BUFC_METADATA.
 	 */
-	uint64_t arcs_size;
+	refcount_t arcs_size;
 } arc_state_t;
 
 /* The 6 states: */
@@ -1674,12 +1674,73 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
 		buf_hash_remove(hdr);
 
 	/* adjust state sizes (ignore arc_l2c_only) */
-	if (to_delta && new_state != arc_l2c_only)
-		atomic_add_64(&new_state->arcs_size, to_delta);
-	if (from_delta && old_state != arc_l2c_only) {
-		ASSERT3U(old_state->arcs_size, >=, from_delta);
-		atomic_add_64(&old_state->arcs_size, -from_delta);
+
+	if (to_delta && new_state != arc_l2c_only) {
+		ASSERT(HDR_HAS_L1HDR(hdr));
+		if (GHOST_STATE(new_state)) {
+			ASSERT0(datacnt);
+
+			/*
+			 * We moving a header to a ghost state, we first
+			 * remove all arc buffers. Thus, we'll have a
+			 * datacnt of zero, and no arc buffer to use for
+			 * the reference. As a result, we use the arc
+			 * header pointer for the reference.
+			 */
+			(void) refcount_add_many(&new_state->arcs_size,
+			    hdr->b_size, hdr);
+		} else {
+			ASSERT3U(datacnt, !=, 0);
+
+			/*
+			 * Each individual buffer holds a unique reference,
+			 * thus we must remove each of these references one
+			 * at a time.
+			 */
+			for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
+			    buf = buf->b_next) {
+				(void) refcount_add_many(&new_state->arcs_size,
+				    hdr->b_size, buf);
+			}
+		}
 	}
+
+	if (from_delta && old_state != arc_l2c_only) {
+		ASSERT(HDR_HAS_L1HDR(hdr));
+		if (GHOST_STATE(old_state)) {
+			/*
+			 * When moving a header off of a ghost state,
+			 * there's the possibility for datacnt to be
+			 * non-zero. This is because we first add the
+			 * arc buffer to the header prior to changing
+			 * the header's state. Since we used the header
+			 * for the reference when putting the header on
+			 * the ghost state, we must balance that and use
+			 * the header when removing off the ghost state
+			 * (even though datacnt is non zero).
+			 */
+
+			IMPLY(datacnt == 0, new_state == arc_anon ||
+			    new_state == arc_l2c_only);
+
+			(void) refcount_remove_many(&old_state->arcs_size,
+			    hdr->b_size, hdr);
+		} else {
+			ASSERT3P(datacnt, !=, 0);
+
+			/*
+			 * Each individual buffer holds a unique reference,
+			 * thus we must remove each of these references one
+			 * at a time.
+			 */
+			for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
+			    buf = buf->b_next) {
+				(void) refcount_remove_many(
+				    &old_state->arcs_size, hdr->b_size, buf);
+			}
+		}
+	}
+
 	if (HDR_HAS_L1HDR(hdr))
 		hdr->b_l1hdr.b_state = new_state;
 
@@ -2036,8 +2097,8 @@ arc_buf_destroy(arc_buf_t *buf, boolean_t remove)
 			ASSERT3U(*cnt, >=, size);
 			atomic_add_64(cnt, -size);
 		}
-		ASSERT3U(state->arcs_size, >=, size);
-		atomic_add_64(&state->arcs_size, -size);
+
+		(void) refcount_remove_many(&state->arcs_size, size, buf);
 		buf->b_data = NULL;
 
 		/*
@@ -2757,7 +2818,8 @@ arc_adjust_meta(void)
 	 * evict some from the MRU here, and some from the MFU below.
 	 */
 	target = MIN((int64_t)(arc_meta_used - arc_meta_limit),
-	    (int64_t)(arc_anon->arcs_size + arc_mru->arcs_size - arc_p));
+	    (int64_t)(refcount_count(&arc_anon->arcs_size) +
+	    refcount_count(&arc_mru->arcs_size) - arc_p));
 
 	total_evicted += arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
 
@@ -2767,7 +2829,7 @@ arc_adjust_meta(void)
 	 * space alloted to the MFU (which is defined as arc_c - arc_p).
 	 */
 	target = MIN((int64_t)(arc_meta_used - arc_meta_limit),
-	    (int64_t)(arc_mfu->arcs_size - (arc_c - arc_p)));
+	    (int64_t)(refcount_count(&arc_mfu->arcs_size) - (arc_c - arc_p)));
 
 	total_evicted += arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
 
@@ -2876,8 +2938,8 @@ arc_adjust(void)
 	 * arc_p here, and then evict more from the MFU below.
 	 */
 	target = MIN((int64_t)(arc_size - arc_c),
-	    (int64_t)(arc_anon->arcs_size + arc_mru->arcs_size + arc_meta_used -
-	    arc_p));
+	    (int64_t)(refcount_count(&arc_anon->arcs_size) +
+	    refcount_count(&arc_mru->arcs_size) + arc_meta_used - arc_p));
 
 	/*
 	 * If we're below arc_meta_min, always prefer to evict data.
@@ -2961,7 +3023,8 @@ arc_adjust(void)
 	 * cache. The following logic enforces these limits on the ghost
 	 * caches, and evicts from them as needed.
 	 */
-	target = arc_mru->arcs_size + arc_mru_ghost->arcs_size - arc_c;
+	target = refcount_count(&arc_mru->arcs_size) +
+	    refcount_count(&arc_mru_ghost->arcs_size) - arc_c;
 
 	bytes = arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_DATA);
 	total_evicted += bytes;
@@ -2979,7 +3042,8 @@ arc_adjust(void)
 	 *	mru + mfu + mru ghost + mfu ghost <= 2 * arc_c
 	 *		    mru ghost + mfu ghost <= arc_c
 	 */
-	target = arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size - arc_c;
+	target = refcount_count(&arc_mru_ghost->arcs_size) +
+	    refcount_count(&arc_mfu_ghost->arcs_size) - arc_c;
 
 	bytes = arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_DATA);
 	total_evicted += bytes;
@@ -3424,6 +3488,8 @@ arc_adapt(int bytes, arc_state_t *state)
 {
 	int mult;
 	uint64_t arc_p_min = (arc_c >> arc_p_min_shift);
+	int64_t mrug_size = refcount_count(&arc_mru_ghost->arcs_size);
+	int64_t mfug_size = refcount_count(&arc_mfu_ghost->arcs_size);
 
 	if (state == arc_l2c_only)
 		return;
@@ -3438,16 +3504,14 @@ arc_adapt(int bytes, arc_state_t *state)
 	 *	  target size of the MRU list.
 	 */
 	if (state == arc_mru_ghost) {
-		mult = ((arc_mru_ghost->arcs_size >= arc_mfu_ghost->arcs_size) ?
-		    1 : (arc_mfu_ghost->arcs_size/arc_mru_ghost->arcs_size));
+		mult = (mrug_size >= mfug_size) ? 1 : (mfug_size / mrug_size);
 		mult = MIN(mult, 10); /* avoid wild arc_p adjustment */
 
 		arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult);
 	} else if (state == arc_mfu_ghost) {
 		uint64_t delta;
 
-		mult = ((arc_mfu_ghost->arcs_size >= arc_mru_ghost->arcs_size) ?
-		    1 : (arc_mru_ghost->arcs_size/arc_mfu_ghost->arcs_size));
+		mult = (mfug_size >= mrug_size) ? 1 : (mrug_size / mfug_size);
 		mult = MIN(mult, 10);
 
 		delta = MIN(bytes * mult, arc_p);
@@ -3563,8 +3627,9 @@ arc_get_data_buf(arc_buf_t *buf)
 	 */
 	if (!GHOST_STATE(buf->b_hdr->b_l1hdr.b_state)) {
 		arc_buf_hdr_t *hdr = buf->b_hdr;
+		arc_state_t *state = hdr->b_l1hdr.b_state;
 
-		atomic_add_64(&hdr->b_l1hdr.b_state->arcs_size, size);
+		(void) refcount_add_many(&state->arcs_size, size, buf);
 
 		/*
 		 * If this is reached via arc_read, the link is
@@ -3585,7 +3650,8 @@ arc_get_data_buf(arc_buf_t *buf)
 		 * data, and we have outgrown arc_p, update arc_p
 		 */
 		if (arc_size < arc_c && hdr->b_l1hdr.b_state == arc_anon &&
-		    arc_anon->arcs_size + arc_mru->arcs_size > arc_p)
+		    (refcount_count(&arc_anon->arcs_size) +
+		    refcount_count(&arc_mru->arcs_size) > arc_p))
 			arc_p = MIN(arc_c, arc_p + size);
 	}
 }
@@ -4432,8 +4498,10 @@ arc_release(arc_buf_t *buf, void *tag)
 		buf->b_next = NULL;
 
 		ASSERT3P(state, !=, arc_l2c_only);
-		ASSERT3U(state->arcs_size, >=, hdr->b_size);
-		atomic_add_64(&state->arcs_size, -hdr->b_size);
+
+		(void) refcount_remove_many(
+		    &state->arcs_size, hdr->b_size, buf);
+
 		if (refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
 			ASSERT3P(state, !=, arc_l2c_only);
 			uint64_t *size = &state->arcs_lsize[type];
@@ -4474,7 +4542,7 @@ arc_release(arc_buf_t *buf, void *tag)
 		(void) refcount_add(&nhdr->b_l1hdr.b_refcnt, tag);
 		buf->b_hdr = nhdr;
 		mutex_exit(&buf->b_evict_lock);
-		atomic_add_64(&arc_anon->arcs_size, blksz);
+		(void) refcount_add_many(&arc_anon->arcs_size, blksz, buf);
 	} else {
 		mutex_exit(&buf->b_evict_lock);
 		ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
@@ -4739,7 +4807,8 @@ arc_tempreserve_space(uint64_t reserve, uint64_t txg)
 	 * network delays from blocking transactions that are ready to be
 	 * assigned to a txg.
 	 */
-	anon_size = MAX((int64_t)(arc_anon->arcs_size - arc_loaned_bytes), 0);
+	anon_size = MAX((int64_t)(refcount_count(&arc_anon->arcs_size) -
+	    arc_loaned_bytes), 0);
 
 	/*
 	 * Writes will, almost always, require additional memory allocations
@@ -4776,7 +4845,7 @@ static void
 arc_kstat_update_state(arc_state_t *state, kstat_named_t *size,
     kstat_named_t *evict_data, kstat_named_t *evict_metadata)
 {
-	size->value.ui64 = state->arcs_size;
+	size->value.ui64 = refcount_count(&state->arcs_size);
 	evict_data->value.ui64 = state->arcs_lsize[ARC_BUFC_DATA];
 	evict_metadata->value.ui64 = state->arcs_lsize[ARC_BUFC_METADATA];
 }
@@ -4993,6 +5062,13 @@ arc_init(void)
 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
 	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
 
+	refcount_create(&arc_anon->arcs_size);
+	refcount_create(&arc_mru->arcs_size);
+	refcount_create(&arc_mru_ghost->arcs_size);
+	refcount_create(&arc_mfu->arcs_size);
+	refcount_create(&arc_mfu_ghost->arcs_size);
+	refcount_create(&arc_l2c_only->arcs_size);
+
 	buf_init();
 
 	arc_reclaim_thread_exit = FALSE;
@@ -5078,6 +5154,13 @@ arc_fini(void)
 	mutex_destroy(&arc_user_evicts_lock);
 	cv_destroy(&arc_user_evicts_cv);
 
+	refcount_destroy(&arc_anon->arcs_size);
+	refcount_destroy(&arc_mru->arcs_size);
+	refcount_destroy(&arc_mru_ghost->arcs_size);
+	refcount_destroy(&arc_mfu->arcs_size);
+	refcount_destroy(&arc_mfu_ghost->arcs_size);
+	refcount_destroy(&arc_l2c_only->arcs_size);
+
 	multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
 	multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
 	multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);

From 8d852123a363994118de443abf63313c7c54b2bb Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Tue, 18 Aug 2015 14:10:04 +0000
Subject: [PATCH 122/380] 5692 expose the number of hole blocks in a file

illumos/illumos-gate@2bcf0248e992f292c7b814458bcdce2f004925d6

https://www.illumos.org/issues/5692
we would like to expose the number of hole (sparse) blocks in a file. this
can be useful to for example if you want to fill in the holes with some
data; knowing the number of holes in advances allows you to report progress
on hole filling. We could use SEEK_HOLE to do that but it would be O(n)
where n is the number of holes present in the file.

Author: Max Grossman <max.grossman@delphix.com>
Reviewed by: Adam Leventhal <ahl@delphix.com>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Boris Protopopov <bprotopopov@hotmail.com>
Approved by: Richard Lowe <richlowe@richlowe.net>
---
 uts/common/fs/zfs/dmu.c       | 51 ++++++++++++++++++++++++++---------
 uts/common/fs/zfs/sys/dmu.h   |  9 +++++++
 uts/common/fs/zfs/zfs_vnops.c | 47 ++++++++++++++++++++++++++++++++
 3 files changed, 94 insertions(+), 13 deletions(-)

diff --git a/uts/common/fs/zfs/dmu.c b/uts/common/fs/zfs/dmu.c
index 841651fa3bff..582542eab7a0 100644
--- a/uts/common/fs/zfs/dmu.c
+++ b/uts/common/fs/zfs/dmu.c
@@ -1830,25 +1830,20 @@ int
 dmu_offset_next(objset_t *os, uint64_t object, boolean_t hole, uint64_t *off)
 {
 	dnode_t *dn;
-	int i, err;
+	int err;
 
-	err = dnode_hold(os, object, FTAG, &dn);
-	if (err)
-		return (err);
 	/*
 	 * Sync any current changes before
 	 * we go trundling through the block pointers.
 	 */
-	for (i = 0; i < TXG_SIZE; i++) {
-		if (list_link_active(&dn->dn_dirty_link[i]))
-			break;
+	err = dmu_object_wait_synced(os, object);
+	if (err) {
+		return (err);
 	}
-	if (i != TXG_SIZE) {
-		dnode_rele(dn, FTAG);
-		txg_wait_synced(dmu_objset_pool(os), 0);
-		err = dnode_hold(os, object, FTAG, &dn);
-		if (err)
-			return (err);
+
+	err = dnode_hold(os, object, FTAG, &dn);
+	if (err) {
+		return (err);
 	}
 
 	err = dnode_next_offset(dn, (hole ? DNODE_FIND_HOLE : 0), off, 1, 1, 0);
@@ -1857,6 +1852,36 @@ dmu_offset_next(objset_t *os, uint64_t object, boolean_t hole, uint64_t *off)
 	return (err);
 }
 
+/*
+ * Given the ZFS object, if it contains any dirty nodes
+ * this function flushes all dirty blocks to disk. This
+ * ensures the DMU object info is updated. A more efficient
+ * future version might just find the TXG with the maximum
+ * ID and wait for that to be synced.
+ */
+int
+dmu_object_wait_synced(objset_t *os, uint64_t object) {
+	dnode_t *dn;
+	int error, i;
+
+	error = dnode_hold(os, object, FTAG, &dn);
+	if (error) {
+		return (error);
+	}
+
+	for (i = 0; i < TXG_SIZE; i++) {
+		if (list_link_active(&dn->dn_dirty_link[i])) {
+			break;
+		}
+	}
+	dnode_rele(dn, FTAG);
+	if (i != TXG_SIZE) {
+		txg_wait_synced(dmu_objset_pool(os), 0);
+	}
+
+	return (0);
+}
+
 void
 dmu_object_info_from_dnode(dnode_t *dn, dmu_object_info_t *doi)
 {
diff --git a/uts/common/fs/zfs/sys/dmu.h b/uts/common/fs/zfs/sys/dmu.h
index d7d4d6f519af..29386bee170a 100644
--- a/uts/common/fs/zfs/sys/dmu.h
+++ b/uts/common/fs/zfs/sys/dmu.h
@@ -905,6 +905,15 @@ int dmu_sync(struct zio *zio, uint64_t txg, dmu_sync_cb_t *done, zgd_t *zgd);
 int dmu_offset_next(objset_t *os, uint64_t object, boolean_t hole,
     uint64_t *off);
 
+/*
+ * Check if a DMU object has any dirty blocks. If so, sync out
+ * all pending transaction groups. Otherwise, this function
+ * does not alter DMU state. This could be improved to only sync
+ * out the necessary transaction groups for this particular
+ * object.
+ */
+int dmu_object_wait_synced(objset_t *os, uint64_t object);
+
 /*
  * Initial setup and final teardown.
  */
diff --git a/uts/common/fs/zfs/zfs_vnops.c b/uts/common/fs/zfs/zfs_vnops.c
index 518e276dff07..fe740a5d28c1 100644
--- a/uts/common/fs/zfs/zfs_vnops.c
+++ b/uts/common/fs/zfs/zfs_vnops.c
@@ -298,24 +298,31 @@ zfs_ioctl(vnode_t *vp, int com, intptr_t data, int flag, cred_t *cred,
     int *rvalp, caller_context_t *ct)
 {
 	offset_t off;
+	offset_t ndata;
+	dmu_object_info_t doi;
 	int error;
 	zfsvfs_t *zfsvfs;
 	znode_t *zp;
 
 	switch (com) {
 	case _FIOFFS:
+	{
 		return (zfs_sync(vp->v_vfsp, 0, cred));
 
 		/*
 		 * The following two ioctls are used by bfu.  Faking out,
 		 * necessary to avoid bfu errors.
 		 */
+	}
 	case _FIOGDIO:
 	case _FIOSDIO:
+	{
 		return (0);
+	}
 
 	case _FIO_SEEK_DATA:
 	case _FIO_SEEK_HOLE:
+	{
 		if (ddi_copyin((void *)data, &off, sizeof (off), flag))
 			return (SET_ERROR(EFAULT));
 
@@ -333,6 +340,46 @@ zfs_ioctl(vnode_t *vp, int com, intptr_t data, int flag, cred_t *cred,
 			return (SET_ERROR(EFAULT));
 		return (0);
 	}
+	case _FIO_COUNT_FILLED:
+	{
+		/*
+		 * _FIO_COUNT_FILLED adds a new ioctl command which
+		 * exposes the number of filled blocks in a
+		 * ZFS object.
+		 */
+		zp = VTOZ(vp);
+		zfsvfs = zp->z_zfsvfs;
+		ZFS_ENTER(zfsvfs);
+		ZFS_VERIFY_ZP(zp);
+
+		/*
+		 * Wait for all dirty blocks for this object
+		 * to get synced out to disk, and the DMU info
+		 * updated.
+		 */
+		error = dmu_object_wait_synced(zfsvfs->z_os, zp->z_id);
+		if (error) {
+			ZFS_EXIT(zfsvfs);
+			return (error);
+		}
+
+		/*
+		 * Retrieve fill count from DMU object.
+		 */
+		error = dmu_object_info(zfsvfs->z_os, zp->z_id, &doi);
+		if (error) {
+			ZFS_EXIT(zfsvfs);
+			return (error);
+		}
+
+		ndata = doi.doi_fill_count;
+
+		ZFS_EXIT(zfsvfs);
+		if (ddi_copyout(&ndata, (void *)data, sizeof (ndata), flag))
+			return (SET_ERROR(EFAULT));
+		return (0);
+	}
+	}
 	return (SET_ERROR(ENOTTY));
 }
 

From 41955a270ad902f30461c57a2c76953e60c33f8e Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Thu, 10 Sep 2015 16:00:04 +0000
Subject: [PATCH 123/380] 5997 FRU field not set during pool creation and never
 updated

illumos/illumos-gate@1437283407f89cab03860accf49408f94559bc34

https://www.illumos.org/issues/5997
  ZFS already supports storing the vdev FRU in a vdev property. There is code in
  libzfs to work with this property, and there is code in the zfs-retire FMA
  module that looks for that information. But there is no code actually setting
  or updating the FRU.
  To address this, ZFS is changed to send a handful of new events whenever a vdev
  is added, attached, cleared, or onlined, as well as when a pool is created or
  imported. The syseventd zfs module will handle these and update the FRU field
  when necessary.

Reviewed by: Dan Fields <dan.fields@nexenta.com>
Reviewed by: Josef Sipek <josef.sipek@nexenta.com>
Reviewed by: Richard Elling <richard.elling@gmail.com>
Reviewed by: George Wilson <george.wilson@delphix.com>
Approved by: Robert Mustacchi <rm@joyent.com>
Author:	Hans Rosenfeld <hans.rosenfeld@nexenta.com>
Porting notes:	only the kernel bits for the new events are imported
---
 uts/common/fs/zfs/spa.c             | 18 +++++++++++++-----
 uts/common/fs/zfs/vdev.c            |  9 +++++++++
 uts/common/sys/sysevent/eventdefs.h |  7 ++++++-
 3 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/uts/common/fs/zfs/spa.c b/uts/common/fs/zfs/spa.c
index 5e664d7d61ac..537c0945d15a 100644
--- a/uts/common/fs/zfs/spa.c
+++ b/uts/common/fs/zfs/spa.c
@@ -22,7 +22,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
- * Copyright (c) 2013, 2014, Nexenta Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015, Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
@@ -3689,6 +3689,7 @@ spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props,
 	txg_wait_synced(spa->spa_dsl_pool, txg);
 
 	spa_config_sync(spa, B_FALSE, B_TRUE);
+	spa_event_notify(spa, NULL, ESC_ZFS_POOL_CREATE);
 
 	spa_history_log_version(spa, "create");
 
@@ -3954,6 +3955,7 @@ spa_import(const char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags)
 			spa_configfile_set(spa, props, B_FALSE);
 
 		spa_config_sync(spa, B_FALSE, B_TRUE);
+		spa_event_notify(spa, NULL, ESC_ZFS_POOL_IMPORT);
 
 		mutex_exit(&spa_namespace_lock);
 		return (0);
@@ -4084,9 +4086,12 @@ spa_import(const char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags)
 	 */
 	spa_async_request(spa, SPA_ASYNC_AUTOEXPAND);
 
-	mutex_exit(&spa_namespace_lock);
 	spa_history_log_version(spa, "import");
 
+	spa_event_notify(spa, NULL, ESC_ZFS_POOL_IMPORT);
+
+	mutex_exit(&spa_namespace_lock);
+
 	return (0);
 }
 
@@ -4427,6 +4432,7 @@ spa_vdev_add(spa_t *spa, nvlist_t *nvroot)
 
 	mutex_enter(&spa_namespace_lock);
 	spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);
+	spa_event_notify(spa, NULL, ESC_ZFS_VDEV_ADD);
 	mutex_exit(&spa_namespace_lock);
 
 	return (0);
@@ -4621,6 +4627,11 @@ spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing)
 	 */
 	dsl_resilver_restart(spa->spa_dsl_pool, dtl_max_txg);
 
+	if (spa->spa_bootfs)
+		spa_event_notify(spa, newvd, ESC_ZFS_BOOTFS_VDEV_ATTACH);
+
+	spa_event_notify(spa, newvd, ESC_ZFS_VDEV_ATTACH);
+
 	/*
 	 * Commit the config
 	 */
@@ -4635,9 +4646,6 @@ spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing)
 	spa_strfree(oldvdpath);
 	spa_strfree(newvdpath);
 
-	if (spa->spa_bootfs)
-		spa_event_notify(spa, newvd, ESC_ZFS_BOOTFS_VDEV_ATTACH);
-
 	return (0);
 }
 
diff --git a/uts/common/fs/zfs/vdev.c b/uts/common/fs/zfs/vdev.c
index 3d53bf460b0f..df6f07e05c75 100644
--- a/uts/common/fs/zfs/vdev.c
+++ b/uts/common/fs/zfs/vdev.c
@@ -2357,6 +2357,7 @@ int
 vdev_online(spa_t *spa, uint64_t guid, uint64_t flags, vdev_state_t *newstate)
 {
 	vdev_t *vd, *tvd, *pvd, *rvd = spa->spa_root_vdev;
+	boolean_t postevent = B_FALSE;
 
 	spa_vdev_state_enter(spa, SCL_NONE);
 
@@ -2366,6 +2367,10 @@ vdev_online(spa_t *spa, uint64_t guid, uint64_t flags, vdev_state_t *newstate)
 	if (!vd->vdev_ops->vdev_op_leaf)
 		return (spa_vdev_state_exit(spa, NULL, ENOTSUP));
 
+	postevent =
+	    (vd->vdev_offline == B_TRUE || vd->vdev_tmpoffline == B_TRUE) ?
+	    B_TRUE : B_FALSE;
+
 	tvd = vd->vdev_top;
 	vd->vdev_offline = B_FALSE;
 	vd->vdev_tmpoffline = B_FALSE;
@@ -2401,6 +2406,10 @@ vdev_online(spa_t *spa, uint64_t guid, uint64_t flags, vdev_state_t *newstate)
 			return (spa_vdev_state_exit(spa, vd, ENOTSUP));
 		spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
 	}
+
+	if (postevent)
+		spa_event_notify(spa, vd, ESC_ZFS_VDEV_ONLINE);
+
 	return (spa_vdev_state_exit(spa, vd, 0));
 }
 
diff --git a/uts/common/sys/sysevent/eventdefs.h b/uts/common/sys/sysevent/eventdefs.h
index 5a75c5d84460..9c6907a08af6 100644
--- a/uts/common/sys/sysevent/eventdefs.h
+++ b/uts/common/sys/sysevent/eventdefs.h
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
  */
 
 #ifndef	_SYS_SYSEVENT_EVENTDEFS_H
@@ -249,9 +249,14 @@ extern "C" {
 #define	ESC_ZFS_RESILVER_START		"ESC_ZFS_resilver_start"
 #define	ESC_ZFS_RESILVER_FINISH		"ESC_ZFS_resilver_finish"
 #define	ESC_ZFS_VDEV_REMOVE		"ESC_ZFS_vdev_remove"
+#define	ESC_ZFS_POOL_CREATE		"ESC_ZFS_pool_create"
 #define	ESC_ZFS_POOL_DESTROY		"ESC_ZFS_pool_destroy"
+#define	ESC_ZFS_POOL_IMPORT		"ESC_ZFS_pool_import"
+#define	ESC_ZFS_VDEV_ADD		"ESC_ZFS_vdev_add"
+#define	ESC_ZFS_VDEV_ATTACH		"ESC_ZFS_vdev_attach"
 #define	ESC_ZFS_VDEV_CLEAR		"ESC_ZFS_vdev_clear"
 #define	ESC_ZFS_VDEV_CHECK		"ESC_ZFS_vdev_check"
+#define	ESC_ZFS_VDEV_ONLINE		"ESC_ZFS_vdev_online"
 #define	ESC_ZFS_CONFIG_SYNC		"ESC_ZFS_config_sync"
 #define	ESC_ZFS_SCRUB_START		"ESC_ZFS_scrub_start"
 #define	ESC_ZFS_SCRUB_FINISH		"ESC_ZFS_scrub_finish"

From f1beee0567a1ab71ed70a722a512ec161abe4327 Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Thu, 10 Sep 2015 16:13:44 +0000
Subject: [PATCH 124/380] 5987 zfs prefetch code needs work

illumos/illumos-gate@cf6106c8a0d6598b045811f9650d66e07eb332af

https://www.illumos.org/issues/5987
  The existing ZFS prefetch code (dmu_zfetch.c) has some problems:
  1. It's nearly impossible to understand. e.g. there are an abundance of kstats
  but it's hard to know what they mean (see below).
  2. For some workloads, it detects patterns that aren't really there (e.g.
  strided patterns, backwards scans), and generates needless i/os prefetching
  blocks that will never be referenced.
  3. It has lock contention issues. These are caused primarily by
  dmu_zfetch_colinear() calling dmu_zfetch_dofetch() (which can block waiting for
  i/o) with the zf_rwlock held for writer, thus blocking all other threads
  accessing this file.
  I suggest that we rewrite this code to detect only forward, sequential streams.
  [... truncated ...]

Reviewed by: Adam Leventhal <ahl@delphix.com>
Reviewed by: George Wilson <george.wilson@delphix.com>
Reviewed by: Paul Dagnelie <pcd@delphix.com>
Approved by: Gordon Ross <gordon.ross@nexenta.com>
Author: Matthew Ahrens <mahrens@delphix.com>
---
 uts/common/fs/zfs/arc.c            |  76 ++-
 uts/common/fs/zfs/dbuf.c           |  21 +-
 uts/common/fs/zfs/dmu.c            |  35 +-
 uts/common/fs/zfs/dmu_zfetch.c     | 722 ++++++-----------------------
 uts/common/fs/zfs/dnode.c          |   4 +-
 uts/common/fs/zfs/sys/arc.h        |  28 +-
 uts/common/fs/zfs/sys/dmu.h        |   5 +-
 uts/common/fs/zfs/sys/dmu_zfetch.h |  37 +-
 8 files changed, 270 insertions(+), 658 deletions(-)

diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index a6bb022dd0a9..889d4e528eff 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -495,6 +495,8 @@ typedef struct arc_stats {
 	kstat_named_t arcstat_meta_limit;
 	kstat_named_t arcstat_meta_max;
 	kstat_named_t arcstat_meta_min;
+	kstat_named_t arcstat_sync_wait_for_async;
+	kstat_named_t arcstat_demand_hit_predictive_prefetch;
 } arc_stats_t;
 
 static arc_stats_t arc_stats = {
@@ -580,7 +582,9 @@ static arc_stats_t arc_stats = {
 	{ "arc_meta_used",		KSTAT_DATA_UINT64 },
 	{ "arc_meta_limit",		KSTAT_DATA_UINT64 },
 	{ "arc_meta_max",		KSTAT_DATA_UINT64 },
-	{ "arc_meta_min",		KSTAT_DATA_UINT64 }
+	{ "arc_meta_min",		KSTAT_DATA_UINT64 },
+	{ "sync_wait_for_async",	KSTAT_DATA_UINT64 },
+	{ "demand_hit_predictive_prefetch", KSTAT_DATA_UINT64 },
 };
 
 #define	ARCSTAT(stat)	(arc_stats.stat.value.ui64)
@@ -4004,6 +4008,36 @@ top:
 
 		if (HDR_IO_IN_PROGRESS(hdr)) {
 
+			if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) &&
+			    priority == ZIO_PRIORITY_SYNC_READ) {
+				/*
+				 * This sync read must wait for an
+				 * in-progress async read (e.g. a predictive
+				 * prefetch).  Async reads are queued
+				 * separately at the vdev_queue layer, so
+				 * this is a form of priority inversion.
+				 * Ideally, we would "inherit" the demand
+				 * i/o's priority by moving the i/o from
+				 * the async queue to the synchronous queue,
+				 * but there is currently no mechanism to do
+				 * so.  Track this so that we can evaluate
+				 * the magnitude of this potential performance
+				 * problem.
+				 *
+				 * Note that if the prefetch i/o is already
+				 * active (has been issued to the device),
+				 * the prefetch improved performance, because
+				 * we issued it sooner than we would have
+				 * without the prefetch.
+				 */
+				DTRACE_PROBE1(arc__sync__wait__for__async,
+				    arc_buf_hdr_t *, hdr);
+				ARCSTAT_BUMP(arcstat_sync_wait_for_async);
+			}
+			if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) {
+				hdr->b_flags &= ~ARC_FLAG_PREDICTIVE_PREFETCH;
+			}
+
 			if (*arc_flags & ARC_FLAG_WAIT) {
 				cv_wait(&hdr->b_l1hdr.b_cv, hash_lock);
 				mutex_exit(hash_lock);
@@ -4012,7 +4046,7 @@ top:
 			ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
 
 			if (done) {
-				arc_callback_t	*acb = NULL;
+				arc_callback_t *acb = NULL;
 
 				acb = kmem_zalloc(sizeof (arc_callback_t),
 				    KM_SLEEP);
@@ -4037,6 +4071,19 @@ top:
 		    hdr->b_l1hdr.b_state == arc_mfu);
 
 		if (done) {
+			if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) {
+				/*
+				 * This is a demand read which does not have to
+				 * wait for i/o because we did a predictive
+				 * prefetch i/o for it, which has completed.
+				 */
+				DTRACE_PROBE1(
+				    arc__demand__hit__predictive__prefetch,
+				    arc_buf_hdr_t *, hdr);
+				ARCSTAT_BUMP(
+				    arcstat_demand_hit_predictive_prefetch);
+				hdr->b_flags &= ~ARC_FLAG_PREDICTIVE_PREFETCH;
+			}
 			add_reference(hdr, hash_lock, private);
 			/*
 			 * If this block is already in use, create a new
@@ -4099,12 +4146,16 @@ top:
 				goto top; /* restart the IO request */
 			}
 
-			/* if this is a prefetch, we don't have a reference */
-			if (*arc_flags & ARC_FLAG_PREFETCH) {
+			/*
+			 * If there is a callback, we pass our reference to
+			 * it; otherwise we remove our reference.
+			 */
+			if (done == NULL) {
 				(void) remove_reference(hdr, hash_lock,
 				    private);
-				hdr->b_flags |= ARC_FLAG_PREFETCH;
 			}
+			if (*arc_flags & ARC_FLAG_PREFETCH)
+				hdr->b_flags |= ARC_FLAG_PREFETCH;
 			if (*arc_flags & ARC_FLAG_L2CACHE)
 				hdr->b_flags |= ARC_FLAG_L2CACHE;
 			if (*arc_flags & ARC_FLAG_L2COMPRESS)
@@ -4127,11 +4178,13 @@ top:
 			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
 			ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
 
-			/* if this is a prefetch, we don't have a reference */
+			/*
+			 * If there is a callback, we pass a reference to it.
+			 */
+			if (done != NULL)
+				add_reference(hdr, hash_lock, private);
 			if (*arc_flags & ARC_FLAG_PREFETCH)
 				hdr->b_flags |= ARC_FLAG_PREFETCH;
-			else
-				add_reference(hdr, hash_lock, private);
 			if (*arc_flags & ARC_FLAG_L2CACHE)
 				hdr->b_flags |= ARC_FLAG_L2CACHE;
 			if (*arc_flags & ARC_FLAG_L2COMPRESS)
@@ -4149,6 +4202,8 @@ top:
 			arc_access(hdr, hash_lock);
 		}
 
+		if (*arc_flags & ARC_FLAG_PREDICTIVE_PREFETCH)
+			hdr->b_flags |= ARC_FLAG_PREDICTIVE_PREFETCH;
 		ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state));
 
 		acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
@@ -4188,6 +4243,11 @@ top:
 		    demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
 		    data, metadata, misses);
 
+		if (priority == ZIO_PRIORITY_ASYNC_READ)
+			hdr->b_flags |= ARC_FLAG_PRIO_ASYNC_READ;
+		else
+			hdr->b_flags &= ~ARC_FLAG_PRIO_ASYNC_READ;
+
 		if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) {
 			/*
 			 * Read from the L2ARC if the following are true:
diff --git a/uts/common/fs/zfs/dbuf.c b/uts/common/fs/zfs/dbuf.c
index e2ed92aac70c..e75bac094ecb 100644
--- a/uts/common/fs/zfs/dbuf.c
+++ b/uts/common/fs/zfs/dbuf.c
@@ -618,7 +618,7 @@ dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb)
 }
 
 static void
-dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags)
+dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
 {
 	dnode_t *dn;
 	zbookmark_phys_t zb;
@@ -664,7 +664,6 @@ dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags)
 		    db->db.db_size, db, type));
 		bzero(db->db.db_data, db->db.db_size);
 		db->db_state = DB_CACHED;
-		*flags |= DB_RF_CACHED;
 		mutex_exit(&db->db_mtx);
 		return;
 	}
@@ -687,10 +686,8 @@ dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags)
 
 	(void) arc_read(zio, db->db_objset->os_spa, db->db_blkptr,
 	    dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ,
-	    (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED,
+	    (flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED,
 	    &aflags, &zb);
-	if (aflags & ARC_FLAG_CACHED)
-		*flags |= DB_RF_CACHED;
 }
 
 int
@@ -723,8 +720,7 @@ dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
 	if (db->db_state == DB_CACHED) {
 		mutex_exit(&db->db_mtx);
 		if (prefetch)
-			dmu_zfetch(&dn->dn_zfetch, db->db.db_offset,
-			    db->db.db_size, TRUE);
+			dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1);
 		if ((flags & DB_RF_HAVESTRUCT) == 0)
 			rw_exit(&dn->dn_struct_rwlock);
 		DB_DNODE_EXIT(db);
@@ -733,13 +729,12 @@ dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
 
 		if (zio == NULL)
 			zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL);
-		dbuf_read_impl(db, zio, &flags);
+		dbuf_read_impl(db, zio, flags);
 
 		/* dbuf_read_impl has dropped db_mtx for us */
 
 		if (prefetch)
-			dmu_zfetch(&dn->dn_zfetch, db->db.db_offset,
-			    db->db.db_size, flags & DB_RF_CACHED);
+			dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1);
 
 		if ((flags & DB_RF_HAVESTRUCT) == 0)
 			rw_exit(&dn->dn_struct_rwlock);
@@ -758,8 +753,7 @@ dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
 		 */
 		mutex_exit(&db->db_mtx);
 		if (prefetch)
-			dmu_zfetch(&dn->dn_zfetch, db->db.db_offset,
-			    db->db.db_size, TRUE);
+			dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1);
 		if ((flags & DB_RF_HAVESTRUCT) == 0)
 			rw_exit(&dn->dn_struct_rwlock);
 		DB_DNODE_EXIT(db);
@@ -2059,6 +2053,9 @@ dbuf_prefetch(dnode_t *dn, int64_t level, uint64_t blkid, zio_priority_t prio,
 	ASSERT(blkid != DMU_BONUS_BLKID);
 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
 
+	if (blkid > dn->dn_maxblkid)
+		return;
+
 	if (dnode_block_freed(dn, blkid))
 		return;
 
diff --git a/uts/common/fs/zfs/dmu.c b/uts/common/fs/zfs/dmu.c
index 582542eab7a0..4b02f75d4f7a 100644
--- a/uts/common/fs/zfs/dmu.c
+++ b/uts/common/fs/zfs/dmu.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  */
 /* Copyright (c) 2013 by Saso Kiselkov. All rights reserved. */
 /* Copyright (c) 2013, Joyent, Inc. All rights reserved. */
@@ -386,7 +386,7 @@ dmu_spill_hold_by_bonus(dmu_buf_t *bonus, void *tag, dmu_buf_t **dbp)
  */
 static int
 dmu_buf_hold_array_by_dnode(dnode_t *dn, uint64_t offset, uint64_t length,
-    int read, void *tag, int *numbufsp, dmu_buf_t ***dbpp, uint32_t flags)
+    boolean_t read, void *tag, int *numbufsp, dmu_buf_t ***dbpp, uint32_t flags)
 {
 	dmu_buf_t **dbp;
 	uint64_t blkid, nblks, i;
@@ -396,15 +396,19 @@ dmu_buf_hold_array_by_dnode(dnode_t *dn, uint64_t offset, uint64_t length,
 
 	ASSERT(length <= DMU_MAX_ACCESS);
 
-	dbuf_flags = DB_RF_CANFAIL | DB_RF_NEVERWAIT | DB_RF_HAVESTRUCT;
-	if (flags & DMU_READ_NO_PREFETCH || length > zfetch_array_rd_sz)
-		dbuf_flags |= DB_RF_NOPREFETCH;
+	/*
+	 * Note: We directly notify the prefetch code of this read, so that
+	 * we can tell it about the multi-block read.  dbuf_read() only knows
+	 * about the one block it is accessing.
+	 */
+	dbuf_flags = DB_RF_CANFAIL | DB_RF_NEVERWAIT | DB_RF_HAVESTRUCT |
+	    DB_RF_NOPREFETCH;
 
 	rw_enter(&dn->dn_struct_rwlock, RW_READER);
 	if (dn->dn_datablkshift) {
 		int blkshift = dn->dn_datablkshift;
-		nblks = (P2ROUNDUP(offset+length, 1ULL<<blkshift) -
-		    P2ALIGN(offset, 1ULL<<blkshift)) >> blkshift;
+		nblks = (P2ROUNDUP(offset + length, 1ULL << blkshift) -
+		    P2ALIGN(offset, 1ULL << blkshift)) >> blkshift;
 	} else {
 		if (offset + length > dn->dn_datablksz) {
 			zfs_panic_recover("zfs: accessing past end of object "
@@ -423,19 +427,24 @@ dmu_buf_hold_array_by_dnode(dnode_t *dn, uint64_t offset, uint64_t length,
 	zio = zio_root(dn->dn_objset->os_spa, NULL, NULL, ZIO_FLAG_CANFAIL);
 	blkid = dbuf_whichblock(dn, 0, offset);
 	for (i = 0; i < nblks; i++) {
-		dmu_buf_impl_t *db = dbuf_hold(dn, blkid+i, tag);
+		dmu_buf_impl_t *db = dbuf_hold(dn, blkid + i, tag);
 		if (db == NULL) {
 			rw_exit(&dn->dn_struct_rwlock);
 			dmu_buf_rele_array(dbp, nblks, tag);
 			zio_nowait(zio);
 			return (SET_ERROR(EIO));
 		}
+
 		/* initiate async i/o */
-		if (read) {
+		if (read)
 			(void) dbuf_read(db, zio, dbuf_flags);
-		}
 		dbp[i] = &db->db;
 	}
+
+	if ((flags & DMU_READ_NO_PREFETCH) == 0 && read &&
+	    length < zfetch_array_rd_sz) {
+		dmu_zfetch(&dn->dn_zfetch, blkid, nblks);
+	}
 	rw_exit(&dn->dn_struct_rwlock);
 
 	/* wait for async i/o */
@@ -489,7 +498,8 @@ dmu_buf_hold_array(objset_t *os, uint64_t object, uint64_t offset,
 
 int
 dmu_buf_hold_array_by_bonus(dmu_buf_t *db_fake, uint64_t offset,
-    uint64_t length, int read, void *tag, int *numbufsp, dmu_buf_t ***dbpp)
+    uint64_t length, boolean_t read, void *tag, int *numbufsp,
+    dmu_buf_t ***dbpp)
 {
 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
 	dnode_t *dn;
@@ -537,9 +547,6 @@ dmu_prefetch(objset_t *os, uint64_t object, int64_t level, uint64_t offset,
 	uint64_t blkid;
 	int nblks, err;
 
-	if (zfs_prefetch_disable)
-		return;
-
 	if (len == 0) {  /* they're interested in the bonus buffer */
 		dn = DMU_META_DNODE(os);
 
diff --git a/uts/common/fs/zfs/dmu_zfetch.c b/uts/common/fs/zfs/dmu_zfetch.c
index 70930ae84587..b05e8f0cf2e4 100644
--- a/uts/common/fs/zfs/dmu_zfetch.c
+++ b/uts/common/fs/zfs/dmu_zfetch.c
@@ -24,7 +24,7 @@
  */
 
 /*
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -36,207 +36,42 @@
 #include <sys/kstat.h>
 
 /*
- * I'm against tune-ables, but these should probably exist as tweakable globals
- * until we can get this working the way we want it to.
+ * This tunable disables predictive prefetch.  Note that it leaves "prescient"
+ * prefetch (e.g. prefetch for zfs send) intact.  Unlike predictive prefetch,
+ * prescient prefetch never issues i/os that end up not being needed,
+ * so it can't hurt performance.
  */
-
-int zfs_prefetch_disable = 0;
+boolean_t zfs_prefetch_disable = B_FALSE;
 
 /* max # of streams per zfetch */
 uint32_t	zfetch_max_streams = 8;
 /* min time before stream reclaim */
 uint32_t	zfetch_min_sec_reap = 2;
-/* max number of blocks to fetch at a time */
-uint32_t	zfetch_block_cap = 256;
-/* number of bytes in a array_read at which we stop prefetching (1Mb) */
+/* max bytes to prefetch per stream (default 8MB) */
+uint32_t	zfetch_max_distance = 8 * 1024 * 1024;
+/* number of bytes in a array_read at which we stop prefetching (1MB) */
 uint64_t	zfetch_array_rd_sz = 1024 * 1024;
 
-/* forward decls for static routines */
-static boolean_t	dmu_zfetch_colinear(zfetch_t *, zstream_t *);
-static void		dmu_zfetch_dofetch(zfetch_t *, zstream_t *);
-static uint64_t		dmu_zfetch_fetch(dnode_t *, uint64_t, uint64_t);
-static uint64_t		dmu_zfetch_fetchsz(dnode_t *, uint64_t, uint64_t);
-static boolean_t	dmu_zfetch_find(zfetch_t *, zstream_t *, int);
-static int		dmu_zfetch_stream_insert(zfetch_t *, zstream_t *);
-static zstream_t	*dmu_zfetch_stream_reclaim(zfetch_t *);
-static void		dmu_zfetch_stream_remove(zfetch_t *, zstream_t *);
-static int		dmu_zfetch_streams_equal(zstream_t *, zstream_t *);
-
 typedef struct zfetch_stats {
 	kstat_named_t zfetchstat_hits;
 	kstat_named_t zfetchstat_misses;
-	kstat_named_t zfetchstat_colinear_hits;
-	kstat_named_t zfetchstat_colinear_misses;
-	kstat_named_t zfetchstat_stride_hits;
-	kstat_named_t zfetchstat_stride_misses;
-	kstat_named_t zfetchstat_reclaim_successes;
-	kstat_named_t zfetchstat_reclaim_failures;
-	kstat_named_t zfetchstat_stream_resets;
-	kstat_named_t zfetchstat_stream_noresets;
-	kstat_named_t zfetchstat_bogus_streams;
+	kstat_named_t zfetchstat_max_streams;
 } zfetch_stats_t;
 
 static zfetch_stats_t zfetch_stats = {
 	{ "hits",			KSTAT_DATA_UINT64 },
 	{ "misses",			KSTAT_DATA_UINT64 },
-	{ "colinear_hits",		KSTAT_DATA_UINT64 },
-	{ "colinear_misses",		KSTAT_DATA_UINT64 },
-	{ "stride_hits",		KSTAT_DATA_UINT64 },
-	{ "stride_misses",		KSTAT_DATA_UINT64 },
-	{ "reclaim_successes",		KSTAT_DATA_UINT64 },
-	{ "reclaim_failures",		KSTAT_DATA_UINT64 },
-	{ "streams_resets",		KSTAT_DATA_UINT64 },
-	{ "streams_noresets",		KSTAT_DATA_UINT64 },
-	{ "bogus_streams",		KSTAT_DATA_UINT64 },
+	{ "max_streams",		KSTAT_DATA_UINT64 },
 };
 
-#define	ZFETCHSTAT_INCR(stat, val) \
-	atomic_add_64(&zfetch_stats.stat.value.ui64, (val));
-
-#define	ZFETCHSTAT_BUMP(stat)		ZFETCHSTAT_INCR(stat, 1);
+#define	ZFETCHSTAT_BUMP(stat) \
+	atomic_inc_64(&zfetch_stats.stat.value.ui64);
 
 kstat_t		*zfetch_ksp;
 
-/*
- * Given a zfetch structure and a zstream structure, determine whether the
- * blocks to be read are part of a co-linear pair of existing prefetch
- * streams.  If a set is found, coalesce the streams, removing one, and
- * configure the prefetch so it looks for a strided access pattern.
- *
- * In other words: if we find two sequential access streams that are
- * the same length and distance N appart, and this read is N from the
- * last stream, then we are probably in a strided access pattern.  So
- * combine the two sequential streams into a single strided stream.
- *
- * Returns whether co-linear streams were found.
- */
-static boolean_t
-dmu_zfetch_colinear(zfetch_t *zf, zstream_t *zh)
-{
-	zstream_t	*z_walk;
-	zstream_t	*z_comp;
-
-	if (! rw_tryenter(&zf->zf_rwlock, RW_WRITER))
-		return (0);
-
-	if (zh == NULL) {
-		rw_exit(&zf->zf_rwlock);
-		return (0);
-	}
-
-	for (z_walk = list_head(&zf->zf_stream); z_walk;
-	    z_walk = list_next(&zf->zf_stream, z_walk)) {
-		for (z_comp = list_next(&zf->zf_stream, z_walk); z_comp;
-		    z_comp = list_next(&zf->zf_stream, z_comp)) {
-			int64_t		diff;
-
-			if (z_walk->zst_len != z_walk->zst_stride ||
-			    z_comp->zst_len != z_comp->zst_stride) {
-				continue;
-			}
-
-			diff = z_comp->zst_offset - z_walk->zst_offset;
-			if (z_comp->zst_offset + diff == zh->zst_offset) {
-				z_walk->zst_offset = zh->zst_offset;
-				z_walk->zst_direction = diff < 0 ? -1 : 1;
-				z_walk->zst_stride =
-				    diff * z_walk->zst_direction;
-				z_walk->zst_ph_offset =
-				    zh->zst_offset + z_walk->zst_stride;
-				dmu_zfetch_stream_remove(zf, z_comp);
-				mutex_destroy(&z_comp->zst_lock);
-				kmem_free(z_comp, sizeof (zstream_t));
-
-				dmu_zfetch_dofetch(zf, z_walk);
-
-				rw_exit(&zf->zf_rwlock);
-				return (1);
-			}
-
-			diff = z_walk->zst_offset - z_comp->zst_offset;
-			if (z_walk->zst_offset + diff == zh->zst_offset) {
-				z_walk->zst_offset = zh->zst_offset;
-				z_walk->zst_direction = diff < 0 ? -1 : 1;
-				z_walk->zst_stride =
-				    diff * z_walk->zst_direction;
-				z_walk->zst_ph_offset =
-				    zh->zst_offset + z_walk->zst_stride;
-				dmu_zfetch_stream_remove(zf, z_comp);
-				mutex_destroy(&z_comp->zst_lock);
-				kmem_free(z_comp, sizeof (zstream_t));
-
-				dmu_zfetch_dofetch(zf, z_walk);
-
-				rw_exit(&zf->zf_rwlock);
-				return (1);
-			}
-		}
-	}
-
-	rw_exit(&zf->zf_rwlock);
-	return (0);
-}
-
-/*
- * Given a zstream_t, determine the bounds of the prefetch.  Then call the
- * routine that actually prefetches the individual blocks.
- */
-static void
-dmu_zfetch_dofetch(zfetch_t *zf, zstream_t *zs)
-{
-	uint64_t	prefetch_tail;
-	uint64_t	prefetch_limit;
-	uint64_t	prefetch_ofst;
-	uint64_t	prefetch_len;
-	uint64_t	blocks_fetched;
-
-	zs->zst_stride = MAX((int64_t)zs->zst_stride, zs->zst_len);
-	zs->zst_cap = MIN(zfetch_block_cap, 2 * zs->zst_cap);
-
-	prefetch_tail = MAX((int64_t)zs->zst_ph_offset,
-	    (int64_t)(zs->zst_offset + zs->zst_stride));
-	/*
-	 * XXX: use a faster division method?
-	 */
-	prefetch_limit = zs->zst_offset + zs->zst_len +
-	    (zs->zst_cap * zs->zst_stride) / zs->zst_len;
-
-	while (prefetch_tail < prefetch_limit) {
-		prefetch_ofst = zs->zst_offset + zs->zst_direction *
-		    (prefetch_tail - zs->zst_offset);
-
-		prefetch_len = zs->zst_len;
-
-		/*
-		 * Don't prefetch beyond the end of the file, if working
-		 * backwards.
-		 */
-		if ((zs->zst_direction == ZFETCH_BACKWARD) &&
-		    (prefetch_ofst > prefetch_tail)) {
-			prefetch_len += prefetch_ofst;
-			prefetch_ofst = 0;
-		}
-
-		/* don't prefetch more than we're supposed to */
-		if (prefetch_len > zs->zst_len)
-			break;
-
-		blocks_fetched = dmu_zfetch_fetch(zf->zf_dnode,
-		    prefetch_ofst, zs->zst_len);
-
-		prefetch_tail += zs->zst_stride;
-		/* stop if we've run out of stuff to prefetch */
-		if (blocks_fetched < zs->zst_len)
-			break;
-	}
-	zs->zst_ph_offset = prefetch_tail;
-	zs->zst_last = ddi_get_lbolt();
-}
-
 void
 zfetch_init(void)
 {
-
 	zfetch_ksp = kstat_create("zfs", 0, "zfetchstats", "misc",
 	    KSTAT_TYPE_NAMED, sizeof (zfetch_stats) / sizeof (kstat_named_t),
 	    KSTAT_FLAG_VIRTUAL);
@@ -264,273 +99,41 @@ zfetch_fini(void)
 void
 dmu_zfetch_init(zfetch_t *zf, dnode_t *dno)
 {
-	if (zf == NULL) {
+	if (zf == NULL)
 		return;
-	}
 
 	zf->zf_dnode = dno;
-	zf->zf_stream_cnt = 0;
-	zf->zf_alloc_fail = 0;
 
 	list_create(&zf->zf_stream, sizeof (zstream_t),
-	    offsetof(zstream_t, zst_node));
+	    offsetof(zstream_t, zs_node));
 
 	rw_init(&zf->zf_rwlock, NULL, RW_DEFAULT, NULL);
 }
 
-/*
- * This function computes the actual size, in blocks, that can be prefetched,
- * and fetches it.
- */
-static uint64_t
-dmu_zfetch_fetch(dnode_t *dn, uint64_t blkid, uint64_t nblks)
+static void
+dmu_zfetch_stream_remove(zfetch_t *zf, zstream_t *zs)
 {
-	uint64_t	fetchsz;
-	uint64_t	i;
-
-	fetchsz = dmu_zfetch_fetchsz(dn, blkid, nblks);
-
-	for (i = 0; i < fetchsz; i++) {
-		dbuf_prefetch(dn, 0, blkid + i, ZIO_PRIORITY_ASYNC_READ,
-		    ARC_FLAG_PREFETCH);
-	}
-
-	return (fetchsz);
+	ASSERT(RW_WRITE_HELD(&zf->zf_rwlock));
+	list_remove(&zf->zf_stream, zs);
+	mutex_destroy(&zs->zs_lock);
+	kmem_free(zs, sizeof (*zs));
 }
 
 /*
- * this function returns the number of blocks that would be prefetched, based
- * upon the supplied dnode, blockid, and nblks.  This is used so that we can
- * update streams in place, and then prefetch with their old value after the
- * fact.  This way, we can delay the prefetch, but subsequent accesses to the
- * stream won't result in the same data being prefetched multiple times.
- */
-static uint64_t
-dmu_zfetch_fetchsz(dnode_t *dn, uint64_t blkid, uint64_t nblks)
-{
-	uint64_t	fetchsz;
-
-	if (blkid > dn->dn_maxblkid) {
-		return (0);
-	}
-
-	/* compute fetch size */
-	if (blkid + nblks + 1 > dn->dn_maxblkid) {
-		fetchsz = (dn->dn_maxblkid - blkid) + 1;
-		ASSERT(blkid + fetchsz - 1 <= dn->dn_maxblkid);
-	} else {
-		fetchsz = nblks;
-	}
-
-
-	return (fetchsz);
-}
-
-/*
- * given a zfetch and a zstream structure, see if there is an associated zstream
- * for this block read.  If so, it starts a prefetch for the stream it
- * located and returns true, otherwise it returns false
- */
-static boolean_t
-dmu_zfetch_find(zfetch_t *zf, zstream_t *zh, int prefetched)
-{
-	zstream_t	*zs;
-	int64_t		diff;
-	int		reset = !prefetched;
-	int		rc = 0;
-
-	if (zh == NULL)
-		return (0);
-
-	/*
-	 * XXX: This locking strategy is a bit coarse; however, it's impact has
-	 * yet to be tested.  If this turns out to be an issue, it can be
-	 * modified in a number of different ways.
-	 */
-
-	rw_enter(&zf->zf_rwlock, RW_READER);
-top:
-
-	for (zs = list_head(&zf->zf_stream); zs;
-	    zs = list_next(&zf->zf_stream, zs)) {
-
-		/*
-		 * XXX - should this be an assert?
-		 */
-		if (zs->zst_len == 0) {
-			/* bogus stream */
-			ZFETCHSTAT_BUMP(zfetchstat_bogus_streams);
-			continue;
-		}
-
-		/*
-		 * We hit this case when we are in a strided prefetch stream:
-		 * we will read "len" blocks before "striding".
-		 */
-		if (zh->zst_offset >= zs->zst_offset &&
-		    zh->zst_offset < zs->zst_offset + zs->zst_len) {
-			if (prefetched) {
-				/* already fetched */
-				ZFETCHSTAT_BUMP(zfetchstat_stride_hits);
-				rc = 1;
-				goto out;
-			} else {
-				ZFETCHSTAT_BUMP(zfetchstat_stride_misses);
-			}
-		}
-
-		/*
-		 * This is the forward sequential read case: we increment
-		 * len by one each time we hit here, so we will enter this
-		 * case on every read.
-		 */
-		if (zh->zst_offset == zs->zst_offset + zs->zst_len) {
-
-			reset = !prefetched && zs->zst_len > 1;
-
-			mutex_enter(&zs->zst_lock);
-
-			if (zh->zst_offset != zs->zst_offset + zs->zst_len) {
-				mutex_exit(&zs->zst_lock);
-				goto top;
-			}
-			zs->zst_len += zh->zst_len;
-			diff = zs->zst_len - zfetch_block_cap;
-			if (diff > 0) {
-				zs->zst_offset += diff;
-				zs->zst_len = zs->zst_len > diff ?
-				    zs->zst_len - diff : 0;
-			}
-			zs->zst_direction = ZFETCH_FORWARD;
-
-			break;
-
-		/*
-		 * Same as above, but reading backwards through the file.
-		 */
-		} else if (zh->zst_offset == zs->zst_offset - zh->zst_len) {
-			/* backwards sequential access */
-
-			reset = !prefetched && zs->zst_len > 1;
-
-			mutex_enter(&zs->zst_lock);
-
-			if (zh->zst_offset != zs->zst_offset - zh->zst_len) {
-				mutex_exit(&zs->zst_lock);
-				goto top;
-			}
-
-			zs->zst_offset = zs->zst_offset > zh->zst_len ?
-			    zs->zst_offset - zh->zst_len : 0;
-			zs->zst_ph_offset = zs->zst_ph_offset > zh->zst_len ?
-			    zs->zst_ph_offset - zh->zst_len : 0;
-			zs->zst_len += zh->zst_len;
-
-			diff = zs->zst_len - zfetch_block_cap;
-			if (diff > 0) {
-				zs->zst_ph_offset = zs->zst_ph_offset > diff ?
-				    zs->zst_ph_offset - diff : 0;
-				zs->zst_len = zs->zst_len > diff ?
-				    zs->zst_len - diff : zs->zst_len;
-			}
-			zs->zst_direction = ZFETCH_BACKWARD;
-
-			break;
-
-		} else if ((zh->zst_offset - zs->zst_offset - zs->zst_stride <
-		    zs->zst_len) && (zs->zst_len != zs->zst_stride)) {
-			/* strided forward access */
-
-			mutex_enter(&zs->zst_lock);
-
-			if ((zh->zst_offset - zs->zst_offset - zs->zst_stride >=
-			    zs->zst_len) || (zs->zst_len == zs->zst_stride)) {
-				mutex_exit(&zs->zst_lock);
-				goto top;
-			}
-
-			zs->zst_offset += zs->zst_stride;
-			zs->zst_direction = ZFETCH_FORWARD;
-
-			break;
-
-		} else if ((zh->zst_offset - zs->zst_offset + zs->zst_stride <
-		    zs->zst_len) && (zs->zst_len != zs->zst_stride)) {
-			/* strided reverse access */
-
-			mutex_enter(&zs->zst_lock);
-
-			if ((zh->zst_offset - zs->zst_offset + zs->zst_stride >=
-			    zs->zst_len) || (zs->zst_len == zs->zst_stride)) {
-				mutex_exit(&zs->zst_lock);
-				goto top;
-			}
-
-			zs->zst_offset = zs->zst_offset > zs->zst_stride ?
-			    zs->zst_offset - zs->zst_stride : 0;
-			zs->zst_ph_offset = (zs->zst_ph_offset >
-			    (2 * zs->zst_stride)) ?
-			    (zs->zst_ph_offset - (2 * zs->zst_stride)) : 0;
-			zs->zst_direction = ZFETCH_BACKWARD;
-
-			break;
-		}
-	}
-
-	if (zs) {
-		if (reset) {
-			zstream_t *remove = zs;
-
-			ZFETCHSTAT_BUMP(zfetchstat_stream_resets);
-			rc = 0;
-			mutex_exit(&zs->zst_lock);
-			rw_exit(&zf->zf_rwlock);
-			rw_enter(&zf->zf_rwlock, RW_WRITER);
-			/*
-			 * Relocate the stream, in case someone removes
-			 * it while we were acquiring the WRITER lock.
-			 */
-			for (zs = list_head(&zf->zf_stream); zs;
-			    zs = list_next(&zf->zf_stream, zs)) {
-				if (zs == remove) {
-					dmu_zfetch_stream_remove(zf, zs);
-					mutex_destroy(&zs->zst_lock);
-					kmem_free(zs, sizeof (zstream_t));
-					break;
-				}
-			}
-		} else {
-			ZFETCHSTAT_BUMP(zfetchstat_stream_noresets);
-			rc = 1;
-			dmu_zfetch_dofetch(zf, zs);
-			mutex_exit(&zs->zst_lock);
-		}
-	}
-out:
-	rw_exit(&zf->zf_rwlock);
-	return (rc);
-}
-
-/*
- * Clean-up state associated with a zfetch structure.  This frees allocated
- * structure members, empties the zf_stream tree, and generally makes things
- * nice.  This doesn't free the zfetch_t itself, that's left to the caller.
+ * Clean-up state associated with a zfetch structure (e.g. destroy the
+ * streams).  This doesn't free the zfetch_t itself, that's left to the caller.
  */
 void
-dmu_zfetch_rele(zfetch_t *zf)
+dmu_zfetch_fini(zfetch_t *zf)
 {
-	zstream_t	*zs;
-	zstream_t	*zs_next;
+	zstream_t *zs;
 
 	ASSERT(!RW_LOCK_HELD(&zf->zf_rwlock));
 
-	for (zs = list_head(&zf->zf_stream); zs; zs = zs_next) {
-		zs_next = list_next(&zf->zf_stream, zs);
-
-		list_remove(&zf->zf_stream, zs);
-		mutex_destroy(&zs->zst_lock);
-		kmem_free(zs, sizeof (zstream_t));
-	}
+	rw_enter(&zf->zf_rwlock, RW_WRITER);
+	while ((zs = list_head(&zf->zf_stream)) != NULL)
+		dmu_zfetch_stream_remove(zf, zs);
+	rw_exit(&zf->zf_rwlock);
 	list_destroy(&zf->zf_stream);
 	rw_destroy(&zf->zf_rwlock);
 
@@ -538,101 +141,55 @@ dmu_zfetch_rele(zfetch_t *zf)
 }
 
 /*
- * Given a zfetch and zstream structure, insert the zstream structure into the
- * AVL tree contained within the zfetch structure.  Peform the appropriate
- * book-keeping.  It is possible that another thread has inserted a stream which
- * matches one that we are about to insert, so we must be sure to check for this
- * case.  If one is found, return failure, and let the caller cleanup the
- * duplicates.
- */
-static int
-dmu_zfetch_stream_insert(zfetch_t *zf, zstream_t *zs)
-{
-	zstream_t	*zs_walk;
-	zstream_t	*zs_next;
-
-	ASSERT(RW_WRITE_HELD(&zf->zf_rwlock));
-
-	for (zs_walk = list_head(&zf->zf_stream); zs_walk; zs_walk = zs_next) {
-		zs_next = list_next(&zf->zf_stream, zs_walk);
-
-		if (dmu_zfetch_streams_equal(zs_walk, zs)) {
-			return (0);
-		}
-	}
-
-	list_insert_head(&zf->zf_stream, zs);
-	zf->zf_stream_cnt++;
-	return (1);
-}
-
-
-/*
- * Walk the list of zstreams in the given zfetch, find an old one (by time), and
- * reclaim it for use by the caller.
- */
-static zstream_t *
-dmu_zfetch_stream_reclaim(zfetch_t *zf)
-{
-	zstream_t	*zs;
-
-	if (! rw_tryenter(&zf->zf_rwlock, RW_WRITER))
-		return (0);
-
-	for (zs = list_head(&zf->zf_stream); zs;
-	    zs = list_next(&zf->zf_stream, zs)) {
-
-		if (((ddi_get_lbolt() - zs->zst_last)/hz) > zfetch_min_sec_reap)
-			break;
-	}
-
-	if (zs) {
-		dmu_zfetch_stream_remove(zf, zs);
-		mutex_destroy(&zs->zst_lock);
-		bzero(zs, sizeof (zstream_t));
-	} else {
-		zf->zf_alloc_fail++;
-	}
-	rw_exit(&zf->zf_rwlock);
-
-	return (zs);
-}
-
-/*
- * Given a zfetch and zstream structure, remove the zstream structure from its
- * container in the zfetch structure.  Perform the appropriate book-keeping.
+ * If there aren't too many streams already, create a new stream.
+ * The "blkid" argument is the next block that we expect this stream to access.
+ * While we're here, clean up old streams (which haven't been
+ * accessed for at least zfetch_min_sec_reap seconds).
  */
 static void
-dmu_zfetch_stream_remove(zfetch_t *zf, zstream_t *zs)
+dmu_zfetch_stream_create(zfetch_t *zf, uint64_t blkid)
 {
+	zstream_t *zs_next;
+	int numstreams = 0;
+
 	ASSERT(RW_WRITE_HELD(&zf->zf_rwlock));
 
-	list_remove(&zf->zf_stream, zs);
-	zf->zf_stream_cnt--;
-}
+	/*
+	 * Clean up old streams.
+	 */
+	for (zstream_t *zs = list_head(&zf->zf_stream);
+	    zs != NULL; zs = zs_next) {
+		zs_next = list_next(&zf->zf_stream, zs);
+		if (((gethrtime() - zs->zs_atime) / NANOSEC) >
+		    zfetch_min_sec_reap)
+			dmu_zfetch_stream_remove(zf, zs);
+		else
+			numstreams++;
+	}
 
-static int
-dmu_zfetch_streams_equal(zstream_t *zs1, zstream_t *zs2)
-{
-	if (zs1->zst_offset != zs2->zst_offset)
-		return (0);
+	/*
+	 * The maximum number of streams is normally zfetch_max_streams,
+	 * but for small files we lower it such that it's at least possible
+	 * for all the streams to be non-overlapping.
+	 *
+	 * If we are already at the maximum number of streams for this file,
+	 * even after removing old streams, then don't create this stream.
+	 */
+	uint32_t max_streams = MAX(1, MIN(zfetch_max_streams,
+	    zf->zf_dnode->dn_maxblkid * zf->zf_dnode->dn_datablksz /
+	    zfetch_max_distance));
+	if (numstreams >= max_streams) {
+		ZFETCHSTAT_BUMP(zfetchstat_max_streams);
+		return;
+	}
 
-	if (zs1->zst_len != zs2->zst_len)
-		return (0);
+	zstream_t *zs = kmem_zalloc(sizeof (*zs), KM_SLEEP);
+	zs->zs_blkid = blkid;
+	zs->zs_pf_blkid = blkid;
+	zs->zs_atime = gethrtime();
+	mutex_init(&zs->zs_lock, NULL, MUTEX_DEFAULT, NULL);
 
-	if (zs1->zst_stride != zs2->zst_stride)
-		return (0);
-
-	if (zs1->zst_ph_offset != zs2->zst_ph_offset)
-		return (0);
-
-	if (zs1->zst_cap != zs2->zst_cap)
-		return (0);
-
-	if (zs1->zst_direction != zs2->zst_direction)
-		return (0);
-
-	return (1);
+	list_insert_head(&zf->zf_stream, zs);
 }
 
 /*
@@ -640,91 +197,86 @@ dmu_zfetch_streams_equal(zstream_t *zs1, zstream_t *zs2)
  * routines to create, delete, find, or operate upon prefetch streams.
  */
 void
-dmu_zfetch(zfetch_t *zf, uint64_t offset, uint64_t size, int prefetched)
+dmu_zfetch(zfetch_t *zf, uint64_t blkid, uint64_t nblks)
 {
-	zstream_t	zst;
-	zstream_t	*newstream;
-	boolean_t	fetched;
-	int		inserted;
-	unsigned int	blkshft;
-	uint64_t	blksz;
+	zstream_t *zs;
 
 	if (zfs_prefetch_disable)
 		return;
 
-	/* files that aren't ln2 blocksz are only one block -- nothing to do */
-	if (!zf->zf_dnode->dn_datablkshift)
+	/*
+	 * As a fast path for small (single-block) files, ignore access
+	 * to the first block.
+	 */
+	if (blkid == 0)
 		return;
 
-	/* convert offset and size, into blockid and nblocks */
-	blkshft = zf->zf_dnode->dn_datablkshift;
-	blksz = (1 << blkshft);
+	rw_enter(&zf->zf_rwlock, RW_READER);
 
-	bzero(&zst, sizeof (zstream_t));
-	zst.zst_offset = offset >> blkshft;
-	zst.zst_len = (P2ROUNDUP(offset + size, blksz) -
-	    P2ALIGN(offset, blksz)) >> blkshft;
-
-	fetched = dmu_zfetch_find(zf, &zst, prefetched);
-	if (fetched) {
-		ZFETCHSTAT_BUMP(zfetchstat_hits);
-	} else {
-		ZFETCHSTAT_BUMP(zfetchstat_misses);
-		fetched = dmu_zfetch_colinear(zf, &zst);
-		if (fetched) {
-			ZFETCHSTAT_BUMP(zfetchstat_colinear_hits);
-		} else {
-			ZFETCHSTAT_BUMP(zfetchstat_colinear_misses);
+	for (zs = list_head(&zf->zf_stream); zs != NULL;
+	    zs = list_next(&zf->zf_stream, zs)) {
+		if (blkid == zs->zs_blkid) {
+			mutex_enter(&zs->zs_lock);
+			/*
+			 * zs_blkid could have changed before we
+			 * acquired zs_lock; re-check them here.
+			 */
+			if (blkid != zs->zs_blkid) {
+				mutex_exit(&zs->zs_lock);
+				continue;
+			}
+			break;
 		}
 	}
 
-	if (!fetched) {
-		newstream = dmu_zfetch_stream_reclaim(zf);
-
+	if (zs == NULL) {
 		/*
-		 * we still couldn't find a stream, drop the lock, and allocate
-		 * one if possible.  Otherwise, give up and go home.
+		 * This access is not part of any existing stream.  Create
+		 * a new stream for it.
 		 */
-		if (newstream) {
-			ZFETCHSTAT_BUMP(zfetchstat_reclaim_successes);
-		} else {
-			uint64_t	maxblocks;
-			uint32_t	max_streams;
-			uint32_t	cur_streams;
-
-			ZFETCHSTAT_BUMP(zfetchstat_reclaim_failures);
-			cur_streams = zf->zf_stream_cnt;
-			maxblocks = zf->zf_dnode->dn_maxblkid;
-
-			max_streams = MIN(zfetch_max_streams,
-			    (maxblocks / zfetch_block_cap));
-			if (max_streams == 0) {
-				max_streams++;
-			}
-
-			if (cur_streams >= max_streams) {
-				return;
-			}
-			newstream = kmem_zalloc(sizeof (zstream_t), KM_SLEEP);
-		}
-
-		newstream->zst_offset = zst.zst_offset;
-		newstream->zst_len = zst.zst_len;
-		newstream->zst_stride = zst.zst_len;
-		newstream->zst_ph_offset = zst.zst_len + zst.zst_offset;
-		newstream->zst_cap = zst.zst_len;
-		newstream->zst_direction = ZFETCH_FORWARD;
-		newstream->zst_last = ddi_get_lbolt();
-
-		mutex_init(&newstream->zst_lock, NULL, MUTEX_DEFAULT, NULL);
-
-		rw_enter(&zf->zf_rwlock, RW_WRITER);
-		inserted = dmu_zfetch_stream_insert(zf, newstream);
+		ZFETCHSTAT_BUMP(zfetchstat_misses);
+		if (rw_tryupgrade(&zf->zf_rwlock))
+			dmu_zfetch_stream_create(zf, blkid + nblks);
 		rw_exit(&zf->zf_rwlock);
-
-		if (!inserted) {
-			mutex_destroy(&newstream->zst_lock);
-			kmem_free(newstream, sizeof (zstream_t));
-		}
+		return;
 	}
+
+	/*
+	 * This access was to a block that we issued a prefetch for on
+	 * behalf of this stream. Issue further prefetches for this stream.
+	 *
+	 * Normally, we start prefetching where we stopped
+	 * prefetching last (zs_pf_blkid).  But when we get our first
+	 * hit on this stream, zs_pf_blkid == zs_blkid, we don't
+	 * want to prefetch to block we just accessed.  In this case,
+	 * start just after the block we just accessed.
+	 */
+	int64_t pf_start = MAX(zs->zs_pf_blkid, blkid + nblks);
+
+	/*
+	 * Double our amount of prefetched data, but don't let the
+	 * prefetch get further ahead than zfetch_max_distance.
+	 */
+	int pf_nblks =
+	    MIN((int64_t)zs->zs_pf_blkid - zs->zs_blkid + nblks,
+	    zs->zs_blkid + nblks +
+	    (zfetch_max_distance >> zf->zf_dnode->dn_datablkshift) - pf_start);
+
+	zs->zs_pf_blkid = pf_start + pf_nblks;
+	zs->zs_atime = gethrtime();
+	zs->zs_blkid = blkid + nblks;
+
+	/*
+	 * dbuf_prefetch() issues the prefetch i/o
+	 * asynchronously, but it may need to wait for an
+	 * indirect block to be read from disk.  Therefore
+	 * we do not want to hold any locks while we call it.
+	 */
+	mutex_exit(&zs->zs_lock);
+	rw_exit(&zf->zf_rwlock);
+	for (int i = 0; i < pf_nblks; i++) {
+		dbuf_prefetch(zf->zf_dnode, 0, pf_start + i,
+		    ZIO_PRIORITY_ASYNC_READ, ARC_FLAG_PREDICTIVE_PREFETCH);
+	}
+	ZFETCHSTAT_BUMP(zfetchstat_hits);
 }
diff --git a/uts/common/fs/zfs/dnode.c b/uts/common/fs/zfs/dnode.c
index 17edd95b287c..31ba4de04e32 100644
--- a/uts/common/fs/zfs/dnode.c
+++ b/uts/common/fs/zfs/dnode.c
@@ -523,7 +523,7 @@ dnode_destroy(dnode_t *dn)
 	dn->dn_id_flags = 0;
 	dn->dn_unlisted_l0_blkid = 0;
 
-	dmu_zfetch_rele(&dn->dn_zfetch);
+	dmu_zfetch_fini(&dn->dn_zfetch);
 	kmem_cache_free(dnode_cache, dn);
 	arc_space_return(sizeof (dnode_t), ARC_SPACE_OTHER);
 
@@ -771,8 +771,6 @@ dnode_move_impl(dnode_t *odn, dnode_t *ndn)
 	dmu_zfetch_init(&ndn->dn_zfetch, NULL);
 	list_move_tail(&ndn->dn_zfetch.zf_stream, &odn->dn_zfetch.zf_stream);
 	ndn->dn_zfetch.zf_dnode = odn->dn_zfetch.zf_dnode;
-	ndn->dn_zfetch.zf_stream_cnt = odn->dn_zfetch.zf_stream_cnt;
-	ndn->dn_zfetch.zf_alloc_fail = odn->dn_zfetch.zf_alloc_fail;
 
 	/*
 	 * Update back pointers. Updating the handle fixes the back pointer of
diff --git a/uts/common/fs/zfs/sys/arc.h b/uts/common/fs/zfs/sys/arc.h
index 843acfdab44d..f9b527008be0 100644
--- a/uts/common/fs/zfs/sys/arc.h
+++ b/uts/common/fs/zfs/sys/arc.h
@@ -64,27 +64,31 @@ typedef enum arc_flags
 	ARC_FLAG_CACHED			= 1 << 4,	/* I/O was in cache */
 	ARC_FLAG_L2CACHE		= 1 << 5,	/* cache in L2ARC */
 	ARC_FLAG_L2COMPRESS		= 1 << 6,	/* compress in L2ARC */
+	ARC_FLAG_PREDICTIVE_PREFETCH	= 1 << 7,	/* I/O from zfetch */
 
 	/*
 	 * Private ARC flags.  These flags are private ARC only flags that
 	 * will show up in b_flags in the arc_hdr_buf_t. These flags should
 	 * only be set by ARC code.
 	 */
-	ARC_FLAG_IN_HASH_TABLE		= 1 << 7,	/* buffer is hashed */
-	ARC_FLAG_IO_IN_PROGRESS		= 1 << 8,	/* I/O in progress */
-	ARC_FLAG_IO_ERROR		= 1 << 9,	/* I/O failed for buf */
-	ARC_FLAG_FREED_IN_READ		= 1 << 10,	/* freed during read */
-	ARC_FLAG_BUF_AVAILABLE		= 1 << 11,	/* block not in use */
-	ARC_FLAG_INDIRECT		= 1 << 12,	/* indirect block */
-	ARC_FLAG_L2_WRITING		= 1 << 13,	/* write in progress */
-	ARC_FLAG_L2_EVICTED		= 1 << 14,	/* evicted during I/O */
-	ARC_FLAG_L2_WRITE_HEAD		= 1 << 15,	/* head of write list */
+	ARC_FLAG_IN_HASH_TABLE		= 1 << 8,	/* buffer is hashed */
+	ARC_FLAG_IO_IN_PROGRESS		= 1 << 9,	/* I/O in progress */
+	ARC_FLAG_IO_ERROR		= 1 << 10,	/* I/O failed for buf */
+	ARC_FLAG_FREED_IN_READ		= 1 << 11,	/* freed during read */
+	ARC_FLAG_BUF_AVAILABLE		= 1 << 12,	/* block not in use */
+	ARC_FLAG_INDIRECT		= 1 << 13,	/* indirect block */
+	/* Indicates that block was read with ASYNC priority. */
+	ARC_FLAG_PRIO_ASYNC_READ	= 1 << 14,
+	ARC_FLAG_L2_WRITING		= 1 << 15,	/* write in progress */
+	ARC_FLAG_L2_EVICTED		= 1 << 16,	/* evicted during I/O */
+	ARC_FLAG_L2_WRITE_HEAD		= 1 << 17,	/* head of write list */
 	/* indicates that the buffer contains metadata (otherwise, data) */
-	ARC_FLAG_BUFC_METADATA		= 1 << 16,
+	ARC_FLAG_BUFC_METADATA		= 1 << 18,
 
 	/* Flags specifying whether optional hdr struct fields are defined */
-	ARC_FLAG_HAS_L1HDR		= 1 << 17,
-	ARC_FLAG_HAS_L2HDR		= 1 << 18,
+	ARC_FLAG_HAS_L1HDR		= 1 << 19,
+	ARC_FLAG_HAS_L2HDR		= 1 << 20,
+
 
 	/*
 	 * The arc buffer's compression mode is stored in the top 7 bits of the
diff --git a/uts/common/fs/zfs/sys/dmu.h b/uts/common/fs/zfs/sys/dmu.h
index 29386bee170a..149cb112f96a 100644
--- a/uts/common/fs/zfs/sys/dmu.h
+++ b/uts/common/fs/zfs/sys/dmu.h
@@ -490,7 +490,8 @@ uint64_t dmu_buf_refcount(dmu_buf_t *db);
  * individually with dmu_buf_rele.
  */
 int dmu_buf_hold_array_by_bonus(dmu_buf_t *db, uint64_t offset,
-    uint64_t length, int read, void *tag, int *numbufsp, dmu_buf_t ***dbpp);
+    uint64_t length, boolean_t read, void *tag,
+    int *numbufsp, dmu_buf_t ***dbpp);
 void dmu_buf_rele_array(dmu_buf_t **, int numbufs, void *tag);
 
 typedef void dmu_buf_evict_func_t(void *user_ptr);
@@ -734,7 +735,7 @@ void dmu_xuio_clear(struct xuio *uio, int i);
 void xuio_stat_wbuf_copied();
 void xuio_stat_wbuf_nocopy();
 
-extern int zfs_prefetch_disable;
+extern boolean_t zfs_prefetch_disable;
 extern int zfs_max_recordsize;
 
 /*
diff --git a/uts/common/fs/zfs/sys/dmu_zfetch.h b/uts/common/fs/zfs/sys/dmu_zfetch.h
index 78cadd2b1ee1..6f61198ebc45 100644
--- a/uts/common/fs/zfs/sys/dmu_zfetch.h
+++ b/uts/common/fs/zfs/sys/dmu_zfetch.h
@@ -23,8 +23,12 @@
  * Use is subject to license terms.
  */
 
-#ifndef	_DFETCH_H
-#define	_DFETCH_H
+/*
+ * Copyright (c) 2014 by Delphix. All rights reserved.
+ */
+
+#ifndef	_DMU_ZFETCH_H
+#define	_DMU_ZFETCH_H
 
 #include <sys/zfs_context.h>
 
@@ -36,41 +40,30 @@ extern uint64_t	zfetch_array_rd_sz;
 
 struct dnode;				/* so we can reference dnode */
 
-typedef enum zfetch_dirn {
-	ZFETCH_FORWARD = 1,		/* prefetch increasing block numbers */
-	ZFETCH_BACKWARD	= -1		/* prefetch decreasing block numbers */
-} zfetch_dirn_t;
-
 typedef struct zstream {
-	uint64_t	zst_offset;	/* offset of starting block in range */
-	uint64_t	zst_len;	/* length of range, in blocks */
-	zfetch_dirn_t	zst_direction;	/* direction of prefetch */
-	uint64_t	zst_stride;	/* length of stride, in blocks */
-	uint64_t	zst_ph_offset;	/* prefetch offset, in blocks */
-	uint64_t	zst_cap;	/* prefetch limit (cap), in blocks */
-	kmutex_t	zst_lock;	/* protects stream */
-	clock_t		zst_last;	/* lbolt of last prefetch */
-	avl_node_t	zst_node;	/* embed avl node here */
+	uint64_t	zs_blkid;	/* expect next access at this blkid */
+	uint64_t	zs_pf_blkid;	/* next block to prefetch */
+	kmutex_t	zs_lock;	/* protects stream */
+	hrtime_t	zs_atime;	/* time last prefetch issued */
+	list_node_t	zs_node;	/* link for zf_stream */
 } zstream_t;
 
 typedef struct zfetch {
 	krwlock_t	zf_rwlock;	/* protects zfetch structure */
-	list_t		zf_stream;	/* AVL tree of zstream_t's */
+	list_t		zf_stream;	/* list of zstream_t's */
 	struct dnode	*zf_dnode;	/* dnode that owns this zfetch */
-	uint32_t	zf_stream_cnt;	/* # of active streams */
-	uint64_t	zf_alloc_fail;	/* # of failed attempts to alloc strm */
 } zfetch_t;
 
 void		zfetch_init(void);
 void		zfetch_fini(void);
 
 void		dmu_zfetch_init(zfetch_t *, struct dnode *);
-void		dmu_zfetch_rele(zfetch_t *);
-void		dmu_zfetch(zfetch_t *, uint64_t, uint64_t, int);
+void		dmu_zfetch_fini(zfetch_t *);
+void		dmu_zfetch(zfetch_t *, uint64_t, uint64_t);
 
 
 #ifdef	__cplusplus
 }
 #endif
 
-#endif	/* _DFETCH_H */
+#endif	/* _DMU_ZFETCH_H */

From 39c17b240da7f350e7d613c29a4fe81d131ade4b Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Fri, 11 Sep 2015 20:43:14 +0000
Subject: [PATCH 125/380] 6091 avl_add doesn't assert on non-debug builds

illumos/illumos-gate@faa2b6be2fc102adf9ed584fc1a667b4ddf50d78

https://www.illumos.org/issues/6091
  Long story short, avl_add's use of ASSERT(0) can cause really strange looking
  crashes on non-debug builds of libavl.so because ASSERTs turn into no-ops.
  ...

Reviewed by: Andy Stormont <astormont@racktopsystems.com>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Steve Dougherty <steve@asksteved.com>
Approved by: Dan McDonald <danmcd@omniti.com>
Author: Josef 'Jeff' Sipek <josef.sipek@nexenta.com>
---
 common/avl/avl.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/common/avl/avl.c b/common/avl/avl.c
index 4223da49915c..ead2fca2ce5e 100644
--- a/common/avl/avl.c
+++ b/common/avl/avl.c
@@ -25,6 +25,7 @@
 
 /*
  * Copyright (c) 2014 by Delphix. All rights reserved.
+ * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
  */
 
 /*
@@ -635,14 +636,17 @@ avl_add(avl_tree_t *tree, void *new_node)
 	/*
 	 * This is unfortunate.  We want to call panic() here, even for
 	 * non-DEBUG kernels.  In userland, however, we can't depend on anything
-	 * in libc or else the rtld build process gets confused.  So, all we can
-	 * do in userland is resort to a normal ASSERT().
+	 * in libc or else the rtld build process gets confused.
+	 * Thankfully, rtld provides us with its own assfail() so we can use
+	 * that here.  We use assfail() directly to get a nice error message
+	 * in the core - much like what panic() does for crashdumps.
 	 */
 	if (avl_find(tree, new_node, &where) != NULL)
 #ifdef _KERNEL
 		panic("avl_find() succeeded inside avl_add()");
 #else
-		ASSERT(0);
+		(void) assfail("avl_find() succeeded inside avl_add()",
+		    __FILE__, __LINE__);
 #endif
 	avl_insert(tree, new_node, where);
 }

From 1a73b0039df5b88e7b257c398577817db9f43bdc Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Fri, 11 Sep 2015 23:49:41 +0000
Subject: [PATCH 126/380] 6214 zpools going south

illumos/illumos-gate@d4cd038c92c36fd0ae35945831a8fc2975b5272c

Reviewed by: Dan McDonald <danmcd@omniti.com>
Reviewed by: Igor Kozhukhov <ikozhukhov@gmail.com>
Reviewed by: George Wilson <george@delphix.com>
Reviewed by: Saso Kiselkov <saso.kiselkov@nexenta.com>
Approved by: Matthew Ahrens <mahrens@delphix.com>
Author: Arne Jansen <sensille@gmx.net>
---
 uts/common/fs/zfs/arc.c     | 33 ++++++++++++++-------------------
 uts/common/fs/zfs/sys/arc.h | 15 ---------------
 2 files changed, 14 insertions(+), 34 deletions(-)

diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index 889d4e528eff..b3afc4ff2b74 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -748,6 +748,7 @@ typedef struct l2arc_buf_hdr {
 	uint64_t		b_daddr;	/* disk address, offset byte */
 	/* real alloc'd buffer size depending on b_compress applied */
 	int32_t			b_asize;
+	uint8_t			b_compress;
 
 	list_node_t		b_l2node;
 } l2arc_buf_hdr_t;
@@ -807,15 +808,6 @@ static arc_buf_hdr_t arc_eviction_hdr;
 #define	HDR_HAS_L1HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L1HDR)
 #define	HDR_HAS_L2HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)
 
-/* For storing compression mode in b_flags */
-#define	HDR_COMPRESS_OFFSET	24
-#define	HDR_COMPRESS_NBITS	7
-
-#define	HDR_GET_COMPRESS(hdr)	((enum zio_compress)BF32_GET(hdr->b_flags, \
-	    HDR_COMPRESS_OFFSET, HDR_COMPRESS_NBITS))
-#define	HDR_SET_COMPRESS(hdr, cmp) BF32_SET(hdr->b_flags, \
-	    HDR_COMPRESS_OFFSET, HDR_COMPRESS_NBITS, (cmp))
-
 /*
  * Other sizes
  */
@@ -2040,7 +2032,7 @@ arc_buf_l2_cdata_free(arc_buf_hdr_t *hdr)
 	 * separately compressed buffer, so there's nothing to free (it
 	 * points to the same buffer as the arc_buf_t's b_data field).
 	 */
-	if (HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF) {
+	if (hdr->b_l2hdr.b_compress == ZIO_COMPRESS_OFF) {
 		hdr->b_l1hdr.b_tmp_cdata = NULL;
 		return;
 	}
@@ -2049,12 +2041,12 @@ arc_buf_l2_cdata_free(arc_buf_hdr_t *hdr)
 	 * There's nothing to free since the buffer was all zero's and
 	 * compressed to a zero length buffer.
 	 */
-	if (HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_EMPTY) {
+	if (hdr->b_l2hdr.b_compress == ZIO_COMPRESS_EMPTY) {
 		ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
 		return;
 	}
 
-	ASSERT(L2ARC_IS_VALID_COMPRESS(HDR_GET_COMPRESS(hdr)));
+	ASSERT(L2ARC_IS_VALID_COMPRESS(hdr->b_l2hdr.b_compress));
 
 	arc_buf_free_on_write(hdr->b_l1hdr.b_tmp_cdata,
 	    hdr->b_size, zio_data_buf_free);
@@ -4218,7 +4210,7 @@ top:
 		    (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) {
 			devw = hdr->b_l2hdr.b_dev->l2ad_writing;
 			addr = hdr->b_l2hdr.b_daddr;
-			b_compress = HDR_GET_COMPRESS(hdr);
+			b_compress = hdr->b_l2hdr.b_compress;
 			b_asize = hdr->b_l2hdr.b_asize;
 			/*
 			 * Lock out device removal.
@@ -5684,6 +5676,8 @@ l2arc_read_done(zio_t *zio)
 	if (cb->l2rcb_compress != ZIO_COMPRESS_OFF)
 		l2arc_decompress_zio(zio, hdr, cb->l2rcb_compress);
 	ASSERT(zio->io_data != NULL);
+	ASSERT3U(zio->io_size, ==, hdr->b_size);
+	ASSERT3U(BP_GET_LSIZE(&cb->l2rcb_bp), ==, hdr->b_size);
 
 	/*
 	 * Check this survived the L2ARC journey.
@@ -5720,7 +5714,7 @@ l2arc_read_done(zio_t *zio)
 			ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL);
 
 			zio_nowait(zio_read(pio, cb->l2rcb_spa, &cb->l2rcb_bp,
-			    buf->b_data, zio->io_size, arc_read_done, buf,
+			    buf->b_data, hdr->b_size, arc_read_done, buf,
 			    zio->io_priority, cb->l2rcb_flags, &cb->l2rcb_zb));
 		}
 	}
@@ -6018,7 +6012,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 			 * can't access without holding the ARC list locks
 			 * (which we want to avoid during compression/writing).
 			 */
-			HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_OFF);
+			hdr->b_l2hdr.b_compress = ZIO_COMPRESS_OFF;
 			hdr->b_l2hdr.b_asize = hdr->b_size;
 			hdr->b_l1hdr.b_tmp_cdata = hdr->b_l1hdr.b_buf->b_data;
 
@@ -6208,7 +6202,7 @@ l2arc_compress_buf(arc_buf_hdr_t *hdr)
 	l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
 
 	ASSERT(HDR_HAS_L1HDR(hdr));
-	ASSERT(HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF);
+	ASSERT3S(l2hdr->b_compress, ==, ZIO_COMPRESS_OFF);
 	ASSERT(hdr->b_l1hdr.b_tmp_cdata != NULL);
 
 	len = l2hdr->b_asize;
@@ -6226,7 +6220,7 @@ l2arc_compress_buf(arc_buf_hdr_t *hdr)
 	if (csize == 0) {
 		/* zero block, indicate that there's nothing to write */
 		zio_data_buf_free(cdata, len);
-		HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_EMPTY);
+		l2hdr->b_compress = ZIO_COMPRESS_EMPTY;
 		l2hdr->b_asize = 0;
 		hdr->b_l1hdr.b_tmp_cdata = NULL;
 		ARCSTAT_BUMP(arcstat_l2_compress_zeros);
@@ -6236,7 +6230,7 @@ l2arc_compress_buf(arc_buf_hdr_t *hdr)
 		 * Compression succeeded, we'll keep the cdata around for
 		 * writing and release it afterwards.
 		 */
-		HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_LZ4);
+		l2hdr->b_compress = ZIO_COMPRESS_LZ4;
 		l2hdr->b_asize = csize;
 		hdr->b_l1hdr.b_tmp_cdata = cdata;
 		ARCSTAT_BUMP(arcstat_l2_compress_successes);
@@ -6323,7 +6317,8 @@ l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr, enum zio_compress c)
 static void
 l2arc_release_cdata_buf(arc_buf_hdr_t *hdr)
 {
-	enum zio_compress comp = HDR_GET_COMPRESS(hdr);
+	ASSERT(HDR_HAS_L2HDR(hdr));
+	enum zio_compress comp = hdr->b_l2hdr.b_compress;
 
 	ASSERT(HDR_HAS_L1HDR(hdr));
 	ASSERT(comp == ZIO_COMPRESS_OFF || L2ARC_IS_VALID_COMPRESS(comp));
diff --git a/uts/common/fs/zfs/sys/arc.h b/uts/common/fs/zfs/sys/arc.h
index f9b527008be0..899b72114b99 100644
--- a/uts/common/fs/zfs/sys/arc.h
+++ b/uts/common/fs/zfs/sys/arc.h
@@ -88,21 +88,6 @@ typedef enum arc_flags
 	/* Flags specifying whether optional hdr struct fields are defined */
 	ARC_FLAG_HAS_L1HDR		= 1 << 19,
 	ARC_FLAG_HAS_L2HDR		= 1 << 20,
-
-
-	/*
-	 * The arc buffer's compression mode is stored in the top 7 bits of the
-	 * flags field, so these dummy flags are included so that MDB can
-	 * interpret the enum properly.
-	 */
-	ARC_FLAG_COMPRESS_0		= 1 << 24,
-	ARC_FLAG_COMPRESS_1		= 1 << 25,
-	ARC_FLAG_COMPRESS_2		= 1 << 26,
-	ARC_FLAG_COMPRESS_3		= 1 << 27,
-	ARC_FLAG_COMPRESS_4		= 1 << 28,
-	ARC_FLAG_COMPRESS_5		= 1 << 29,
-	ARC_FLAG_COMPRESS_6		= 1 << 30
-
 } arc_flags_t;
 
 struct arc_buf {

From 13fab2daa4ea8faa6597d8f933b783690e3ba810 Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Tue, 15 Sep 2015 09:59:13 +0000
Subject: [PATCH 127/380] 6220 memleak in l2arc on debug build

https://github.com/illumos/illumos-gate/commit/c546f36aa898d913ff77674fb5ff97f15b2e08b4
https://www.illumos.org/issues/6220
  5408 introduced a memleak in l2arc, namely the member b_thawed gets leaked when
  an arc_hdr is realloced from full to l2only.

Reviewed by: Saso Kiselkov <saso.kiselkov@nexenta.com>
Reviewed by: Simon Klinkert <simon.klinkert@gmail.com>
Reviewed by: George Wilson <george@delphix.com>
Approved by: Robert Mustacchi <rm@joyent.com>
Author: Arne Jansen <sensille@gmx.net>
---
 uts/common/fs/zfs/arc.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index b3afc4ff2b74..92e089e4b3d4 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -1305,6 +1305,13 @@ arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
 		VERIFY(!HDR_L2_WRITING(hdr));
 		VERIFY3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
 
+#ifdef ZFS_DEBUG
+		if (hdr->b_l1hdr.b_thawed != NULL) {
+			kmem_free(hdr->b_l1hdr.b_thawed, 1);
+			hdr->b_l1hdr.b_thawed = NULL;
+		}
+#endif
+
 		nhdr->b_flags &= ~ARC_FLAG_HAS_L1HDR;
 	}
 	/*

From fcd92b614f8abef484da68a0ecd2dc2bf052bc69 Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Mon, 21 Sep 2015 12:00:12 +0000
Subject: [PATCH 128/380] 6171 dsl_prop_unregister() slows down dataset
 eviction.

https://github.com/illumos/illumos-gate/commit/03bad06fbb261fd4a7151a70dfeff2f5041cce1f

https://www.illumos.org/issues/6171
  A change to a property on a dataset must be propagated to its descendants
  in case that property is inherited. For datasets whose information is
  not currently loaded into memory (e.g. a snapshot that isn't currently
  mounted), there is nothing to do; the property change will take effect
  the next time that dataset is loaded. To handle updates to datasets that
  are in-core, ZFS registers a callback entry for each property of each
  loaded dataset with the dsl directory that holds that dataset. There
  is a dsl directory associated with each live dataset that references
  both the live dataset and any snapshots of the live dataset. A property
  change is effected by doing a traversal of the tree of dsl directories
  for a pool, starting at the directory sourcing the change, and invoking
  these callbacks.
  The current implementation both registers and de-registers properties
  individually for each loaded dataset. While registration for a property is
  O(1) (insert into a list), de-registration is O(n) (search list and then
  remove). The 'n' for de-registration, however, is not limited to the size
  (number of snapshots + 1) of the dsl directory. The eviction portion
  of the life cycle for the in core state of datasets is asynchronous,
  which allows multiple copies of the dataset information to be in-core
  at once. Only one of these copies is active at any time with the rest
  going through tear down processing, but all copies contribute to the
  cost of performing a dsl_prop_unregister().
  ...
  The fix employed here is to make property de-registration O(1). With
  this change in place, it is hoped that a single thread is more than
  sufficient to handle eviction processing. If it isn't, the problem can
  be solved by increasing the number of threads devoted to the eviction
  taskq.

Author: Justin Gibbs <gibbs@scsiguy.com>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Prakash Surya <prakash.surya@delphix.com>
Approved by: Dan McDonald <danmcd@omniti.com>
---
 uts/common/fs/zfs/dmu_objset.c      |  36 +----
 uts/common/fs/zfs/dsl_dataset.c     |   4 +
 uts/common/fs/zfs/dsl_dir.c         |  12 +-
 uts/common/fs/zfs/dsl_prop.c        | 233 +++++++++++++++++-----------
 uts/common/fs/zfs/sys/dsl_dataset.h |   3 +
 uts/common/fs/zfs/sys/dsl_dir.h     |   2 +-
 uts/common/fs/zfs/sys/dsl_prop.h    |  16 +-
 uts/common/fs/zfs/zfs_vfsops.c      |  69 +-------
 8 files changed, 171 insertions(+), 204 deletions(-)

diff --git a/uts/common/fs/zfs/dmu_objset.c b/uts/common/fs/zfs/dmu_objset.c
index 9ad18f0d11fb..f84ff378c947 100644
--- a/uts/common/fs/zfs/dmu_objset.c
+++ b/uts/common/fs/zfs/dmu_objset.c
@@ -680,40 +680,8 @@ dmu_objset_evict(objset_t *os)
 	for (int t = 0; t < TXG_SIZE; t++)
 		ASSERT(!dmu_objset_is_dirty(os, t));
 
-	if (ds) {
-		if (!ds->ds_is_snapshot) {
-			VERIFY0(dsl_prop_unregister(ds,
-			    zfs_prop_to_name(ZFS_PROP_CHECKSUM),
-			    checksum_changed_cb, os));
-			VERIFY0(dsl_prop_unregister(ds,
-			    zfs_prop_to_name(ZFS_PROP_COMPRESSION),
-			    compression_changed_cb, os));
-			VERIFY0(dsl_prop_unregister(ds,
-			    zfs_prop_to_name(ZFS_PROP_COPIES),
-			    copies_changed_cb, os));
-			VERIFY0(dsl_prop_unregister(ds,
-			    zfs_prop_to_name(ZFS_PROP_DEDUP),
-			    dedup_changed_cb, os));
-			VERIFY0(dsl_prop_unregister(ds,
-			    zfs_prop_to_name(ZFS_PROP_LOGBIAS),
-			    logbias_changed_cb, os));
-			VERIFY0(dsl_prop_unregister(ds,
-			    zfs_prop_to_name(ZFS_PROP_SYNC),
-			    sync_changed_cb, os));
-			VERIFY0(dsl_prop_unregister(ds,
-			    zfs_prop_to_name(ZFS_PROP_REDUNDANT_METADATA),
-			    redundant_metadata_changed_cb, os));
-			VERIFY0(dsl_prop_unregister(ds,
-			    zfs_prop_to_name(ZFS_PROP_RECORDSIZE),
-			    recordsize_changed_cb, os));
-		}
-		VERIFY0(dsl_prop_unregister(ds,
-		    zfs_prop_to_name(ZFS_PROP_PRIMARYCACHE),
-		    primary_cache_changed_cb, os));
-		VERIFY0(dsl_prop_unregister(ds,
-		    zfs_prop_to_name(ZFS_PROP_SECONDARYCACHE),
-		    secondary_cache_changed_cb, os));
-	}
+	if (ds)
+		dsl_prop_unregister_all(ds, os);
 
 	if (os->os_sa)
 		sa_tear_down(os);
diff --git a/uts/common/fs/zfs/dsl_dataset.c b/uts/common/fs/zfs/dsl_dataset.c
index 7f63cd97bcde..6606d89da27e 100644
--- a/uts/common/fs/zfs/dsl_dataset.c
+++ b/uts/common/fs/zfs/dsl_dataset.c
@@ -282,6 +282,7 @@ dsl_dataset_evict(void *dbu)
 
 	ASSERT(!list_link_active(&ds->ds_synced_link));
 
+	list_destroy(&ds->ds_prop_cbs);
 	mutex_destroy(&ds->ds_lock);
 	mutex_destroy(&ds->ds_opening_lock);
 	mutex_destroy(&ds->ds_sendstream_lock);
@@ -424,6 +425,9 @@ dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
 		list_create(&ds->ds_sendstreams, sizeof (dmu_sendarg_t),
 		    offsetof(dmu_sendarg_t, dsa_link));
 
+		list_create(&ds->ds_prop_cbs, sizeof (dsl_prop_cb_record_t),
+		    offsetof(dsl_prop_cb_record_t, cbr_ds_node));
+
 		if (doi.doi_type == DMU_OTN_ZAP_METADATA) {
 			for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
 				if (!(spa_feature_table[f].fi_flags &
diff --git a/uts/common/fs/zfs/dsl_dir.c b/uts/common/fs/zfs/dsl_dir.c
index 6a20ab3ca2ad..7d86f72ad13d 100644
--- a/uts/common/fs/zfs/dsl_dir.c
+++ b/uts/common/fs/zfs/dsl_dir.c
@@ -147,11 +147,7 @@ dsl_dir_evict(void *dbu)
 
 	spa_async_close(dd->dd_pool->dp_spa, dd);
 
-	/*
-	 * The props callback list should have been cleaned up by
-	 * objset_evict().
-	 */
-	list_destroy(&dd->dd_prop_cbs);
+	dsl_prop_fini(dd);
 	mutex_destroy(&dd->dd_lock);
 	kmem_free(dd, sizeof (dsl_dir_t));
 }
@@ -186,9 +182,7 @@ dsl_dir_hold_obj(dsl_pool_t *dp, uint64_t ddobj,
 		dd->dd_dbuf = dbuf;
 		dd->dd_pool = dp;
 		mutex_init(&dd->dd_lock, NULL, MUTEX_DEFAULT, NULL);
-
-		list_create(&dd->dd_prop_cbs, sizeof (dsl_prop_cb_record_t),
-		    offsetof(dsl_prop_cb_record_t, cbr_node));
+		dsl_prop_init(dd);
 
 		dsl_dir_snap_cmtime_update(dd);
 
@@ -246,6 +240,7 @@ dsl_dir_hold_obj(dsl_pool_t *dp, uint64_t ddobj,
 		if (winner != NULL) {
 			if (dd->dd_parent)
 				dsl_dir_rele(dd->dd_parent, dd);
+			dsl_prop_fini(dd);
 			mutex_destroy(&dd->dd_lock);
 			kmem_free(dd, sizeof (dsl_dir_t));
 			dd = winner;
@@ -273,6 +268,7 @@ dsl_dir_hold_obj(dsl_pool_t *dp, uint64_t ddobj,
 errout:
 	if (dd->dd_parent)
 		dsl_dir_rele(dd->dd_parent, dd);
+	dsl_prop_fini(dd);
 	mutex_destroy(&dd->dd_lock);
 	kmem_free(dd, sizeof (dsl_dir_t));
 	dmu_buf_rele(dbuf, tag);
diff --git a/uts/common/fs/zfs/dsl_prop.c b/uts/common/fs/zfs/dsl_prop.c
index 09eec21604f4..353de42d1e9d 100644
--- a/uts/common/fs/zfs/dsl_prop.c
+++ b/uts/common/fs/zfs/dsl_prop.c
@@ -216,6 +216,58 @@ dsl_prop_get_ds(dsl_dataset_t *ds, const char *propname,
 	    intsz, numints, buf, setpoint, ds->ds_is_snapshot));
 }
 
+static dsl_prop_record_t *
+dsl_prop_record_find(dsl_dir_t *dd, const char *propname)
+{
+	dsl_prop_record_t *pr = NULL;
+
+	ASSERT(MUTEX_HELD(&dd->dd_lock));
+
+	for (pr = list_head(&dd->dd_props);
+	    pr != NULL; pr = list_next(&dd->dd_props, pr)) {
+		if (strcmp(pr->pr_propname, propname) == 0)
+			break;
+	}
+
+	return (pr);
+}
+
+static dsl_prop_record_t *
+dsl_prop_record_create(dsl_dir_t *dd, const char *propname)
+{
+	dsl_prop_record_t *pr;
+
+	ASSERT(MUTEX_HELD(&dd->dd_lock));
+
+	pr = kmem_alloc(sizeof (dsl_prop_record_t), KM_SLEEP);
+	pr->pr_propname = spa_strdup(propname);
+	list_create(&pr->pr_cbs, sizeof (dsl_prop_cb_record_t),
+	    offsetof(dsl_prop_cb_record_t, cbr_pr_node));
+	list_insert_head(&dd->dd_props, pr);
+
+	return (pr);
+}
+
+void
+dsl_prop_init(dsl_dir_t *dd)
+{
+	list_create(&dd->dd_props, sizeof (dsl_prop_record_t),
+	    offsetof(dsl_prop_record_t, pr_node));
+}
+
+void
+dsl_prop_fini(dsl_dir_t *dd)
+{
+	dsl_prop_record_t *pr;
+
+	while ((pr = list_remove_head(&dd->dd_props)) != NULL) {
+		list_destroy(&pr->pr_cbs);
+		strfree((char *)pr->pr_propname);
+		kmem_free(pr, sizeof (dsl_prop_record_t));
+	}
+	list_destroy(&dd->dd_props);
+}
+
 /*
  * Register interest in the named property.  We'll call the callback
  * once to notify it of the current property value, and again each time
@@ -230,6 +282,7 @@ dsl_prop_register(dsl_dataset_t *ds, const char *propname,
 	dsl_dir_t *dd = ds->ds_dir;
 	dsl_pool_t *dp = dd->dd_pool;
 	uint64_t value;
+	dsl_prop_record_t *pr;
 	dsl_prop_cb_record_t *cbr;
 	int err;
 
@@ -241,12 +294,16 @@ dsl_prop_register(dsl_dataset_t *ds, const char *propname,
 
 	cbr = kmem_alloc(sizeof (dsl_prop_cb_record_t), KM_SLEEP);
 	cbr->cbr_ds = ds;
-	cbr->cbr_propname = kmem_alloc(strlen(propname)+1, KM_SLEEP);
-	(void) strcpy((char *)cbr->cbr_propname, propname);
 	cbr->cbr_func = callback;
 	cbr->cbr_arg = cbarg;
+
 	mutex_enter(&dd->dd_lock);
-	list_insert_head(&dd->dd_prop_cbs, cbr);
+	pr = dsl_prop_record_find(dd, propname);
+	if (pr == NULL)
+		pr = dsl_prop_record_create(dd, propname);
+	cbr->cbr_pr = pr;
+	list_insert_head(&pr->pr_cbs, cbr);
+	list_insert_head(&ds->ds_prop_cbs, cbr);
 	mutex_exit(&dd->dd_lock);
 
 	cbr->cbr_func(cbr->cbr_arg, value);
@@ -377,56 +434,34 @@ dsl_prop_predict(dsl_dir_t *dd, const char *propname,
 }
 
 /*
- * Unregister this callback.  Return 0 on success, ENOENT if ddname is
- * invalid, or ENOMSG if no matching callback registered.
+ * Unregister all callbacks that are registered with the
+ * given callback argument.
  */
-int
-dsl_prop_unregister(dsl_dataset_t *ds, const char *propname,
-    dsl_prop_changed_cb_t *callback, void *cbarg)
+void
+dsl_prop_unregister_all(dsl_dataset_t *ds, void *cbarg)
 {
+	dsl_prop_cb_record_t *cbr, *next_cbr;
+
 	dsl_dir_t *dd = ds->ds_dir;
-	dsl_prop_cb_record_t *cbr;
 
 	mutex_enter(&dd->dd_lock);
-	for (cbr = list_head(&dd->dd_prop_cbs);
-	    cbr; cbr = list_next(&dd->dd_prop_cbs, cbr)) {
-		if (cbr->cbr_ds == ds &&
-		    cbr->cbr_func == callback &&
-		    cbr->cbr_arg == cbarg &&
-		    strcmp(cbr->cbr_propname, propname) == 0)
-			break;
+	next_cbr = list_head(&ds->ds_prop_cbs);
+	while (next_cbr != NULL) {
+		cbr = next_cbr;
+		next_cbr = list_next(&ds->ds_prop_cbs, cbr);
+		if (cbr->cbr_arg == cbarg) {
+			list_remove(&ds->ds_prop_cbs, cbr);
+			list_remove(&cbr->cbr_pr->pr_cbs, cbr);
+			kmem_free(cbr, sizeof (dsl_prop_cb_record_t));
+		}
 	}
-
-	if (cbr == NULL) {
-		mutex_exit(&dd->dd_lock);
-		return (SET_ERROR(ENOMSG));
-	}
-
-	list_remove(&dd->dd_prop_cbs, cbr);
 	mutex_exit(&dd->dd_lock);
-	kmem_free((void*)cbr->cbr_propname, strlen(cbr->cbr_propname)+1);
-	kmem_free(cbr, sizeof (dsl_prop_cb_record_t));
-
-	return (0);
 }
 
 boolean_t
 dsl_prop_hascb(dsl_dataset_t *ds)
 {
-	dsl_dir_t *dd = ds->ds_dir;
-	boolean_t rv = B_FALSE;
-	dsl_prop_cb_record_t *cbr;
-
-	mutex_enter(&dd->dd_lock);
-	for (cbr = list_head(&dd->dd_prop_cbs); cbr;
-	    cbr = list_next(&dd->dd_prop_cbs, cbr)) {
-		if (cbr->cbr_ds == ds) {
-			rv = B_TRUE;
-			break;
-		}
-	}
-	mutex_exit(&dd->dd_lock);
-	return (rv);
+	return (!list_is_empty(&ds->ds_prop_cbs));
 }
 
 /* ARGSUSED */
@@ -434,38 +469,50 @@ static int
 dsl_prop_notify_all_cb(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg)
 {
 	dsl_dir_t *dd = ds->ds_dir;
+	dsl_prop_record_t *pr;
 	dsl_prop_cb_record_t *cbr;
 
 	mutex_enter(&dd->dd_lock);
-	for (cbr = list_head(&dd->dd_prop_cbs); cbr;
-	    cbr = list_next(&dd->dd_prop_cbs, cbr)) {
-		uint64_t value;
+	for (pr = list_head(&dd->dd_props);
+	    pr; pr = list_next(&dd->dd_props, pr)) {
+		for (cbr = list_head(&pr->pr_cbs); cbr;
+		    cbr = list_next(&pr->pr_cbs, cbr)) {
+			uint64_t value;
 
-		/*
-		 * Callback entries do not have holds on their datasets
-		 * so that datasets with registered callbacks are still
-		 * eligible for eviction.  Unlike operations on callbacks
-		 * for a single dataset, we are performing a recursive
-		 * descent of related datasets and the calling context
-		 * for this iteration only has a dataset hold on the root.
-		 * Without a hold, the callback's pointer to the dataset
-		 * could be invalidated by eviction at any time.
-		 *
-		 * Use dsl_dataset_try_add_ref() to verify that the
-		 * dataset has not begun eviction processing and to
-		 * prevent eviction from occurring for the duration
-		 * of the callback.  If the hold attempt fails, this
-		 * object is already being evicted and the callback can
-		 * be safely ignored.
-		 */
-		if (!dsl_dataset_try_add_ref(dp, cbr->cbr_ds, FTAG))
-			continue;
+			/*
+			 * Callback entries do not have holds on their
+			 * datasets so that datasets with registered
+			 * callbacks are still eligible for eviction.
+			 * Unlike operations to update properties on a
+			 * single dataset, we are performing a recursive
+			 * descent of related head datasets.  The caller
+			 * of this function only has a dataset hold on
+			 * the passed in head dataset, not the snapshots
+			 * associated with this dataset.  Without a hold,
+			 * the dataset pointer within callback records
+			 * for snapshots can be invalidated by eviction
+			 * at any time.
+			 *
+			 * Use dsl_dataset_try_add_ref() to verify
+			 * that the dataset for a snapshot has not
+			 * begun eviction processing and to prevent
+			 * eviction from occurring for the duration of
+			 * the callback.  If the hold attempt fails,
+			 * this object is already being evicted and the
+			 * callback can be safely ignored.
+			 */
+			if (ds != cbr->cbr_ds &&
+			    !dsl_dataset_try_add_ref(dp, cbr->cbr_ds, FTAG))
+				continue;
 
-		if (dsl_prop_get_ds(cbr->cbr_ds, cbr->cbr_propname,
-		    sizeof (value), 1, &value, NULL) == 0)
-			cbr->cbr_func(cbr->cbr_arg, value);
+			if (dsl_prop_get_ds(cbr->cbr_ds,
+			    cbr->cbr_pr->pr_propname, sizeof (value), 1,
+			    &value, NULL) == 0)
+				cbr->cbr_func(cbr->cbr_arg, value);
 
-		dsl_dataset_rele(cbr->cbr_ds, FTAG);
+			if (ds != cbr->cbr_ds)
+				dsl_dataset_rele(cbr->cbr_ds, FTAG);
+		}
 	}
 	mutex_exit(&dd->dd_lock);
 
@@ -490,6 +537,7 @@ dsl_prop_changed_notify(dsl_pool_t *dp, uint64_t ddobj,
     const char *propname, uint64_t value, int first)
 {
 	dsl_dir_t *dd;
+	dsl_prop_record_t *pr;
 	dsl_prop_cb_record_t *cbr;
 	objset_t *mos = dp->dp_meta_objset;
 	zap_cursor_t zc;
@@ -516,30 +564,33 @@ dsl_prop_changed_notify(dsl_pool_t *dp, uint64_t ddobj,
 	}
 
 	mutex_enter(&dd->dd_lock);
-	for (cbr = list_head(&dd->dd_prop_cbs); cbr;
-	    cbr = list_next(&dd->dd_prop_cbs, cbr)) {
-		uint64_t propobj;
+	pr = dsl_prop_record_find(dd, propname);
+	if (pr != NULL) {
+		for (cbr = list_head(&pr->pr_cbs); cbr;
+		    cbr = list_next(&pr->pr_cbs, cbr)) {
+			uint64_t propobj;
 
-		/*
-		 * cbr->cbf_ds may be invalidated due to eviction,
-		 * requiring the use of dsl_dataset_try_add_ref().
-		 * See comment block in dsl_prop_notify_all_cb()
-		 * for details.
-		 */
-		if (strcmp(cbr->cbr_propname, propname) != 0 ||
-		    !dsl_dataset_try_add_ref(dp, cbr->cbr_ds, FTAG))
-			continue;
+			/*
+			 * cbr->cbr_ds may be invalidated due to eviction,
+			 * requiring the use of dsl_dataset_try_add_ref().
+			 * See comment block in dsl_prop_notify_all_cb()
+			 * for details.
+			 */
+			if (!dsl_dataset_try_add_ref(dp, cbr->cbr_ds, FTAG))
+				continue;
 
-		propobj = dsl_dataset_phys(cbr->cbr_ds)->ds_props_obj;
+			propobj = dsl_dataset_phys(cbr->cbr_ds)->ds_props_obj;
 
-		/*
-		 * If the property is not set on this ds, then it is
-		 * inherited here; call the callback.
-		 */
-		if (propobj == 0 || zap_contains(mos, propobj, propname) != 0)
-			cbr->cbr_func(cbr->cbr_arg, value);
+			/*
+			 * If the property is not set on this ds, then it is
+			 * inherited here; call the callback.
+			 */
+			if (propobj == 0 ||
+			    zap_contains(mos, propobj, propname) != 0)
+				cbr->cbr_func(cbr->cbr_arg, value);
 
-		dsl_dataset_rele(cbr->cbr_ds, FTAG);
+			dsl_dataset_rele(cbr->cbr_ds, FTAG);
+		}
 	}
 	mutex_exit(&dd->dd_lock);
 
@@ -679,10 +730,10 @@ dsl_prop_set_sync_impl(dsl_dataset_t *ds, const char *propname,
 			 * ds here.
 			 */
 			mutex_enter(&ds->ds_dir->dd_lock);
-			for (cbr = list_head(&ds->ds_dir->dd_prop_cbs); cbr;
-			    cbr = list_next(&ds->ds_dir->dd_prop_cbs, cbr)) {
-				if (cbr->cbr_ds == ds &&
-				    strcmp(cbr->cbr_propname, propname) == 0)
+			for (cbr = list_head(&ds->ds_prop_cbs); cbr;
+			    cbr = list_next(&ds->ds_prop_cbs, cbr)) {
+				if (strcmp(cbr->cbr_pr->pr_propname,
+				    propname) == 0)
 					cbr->cbr_func(cbr->cbr_arg, intval);
 			}
 			mutex_exit(&ds->ds_dir->dd_lock);
diff --git a/uts/common/fs/zfs/sys/dsl_dataset.h b/uts/common/fs/zfs/sys/dsl_dataset.h
index 001bff566a88..c9cd5890aa74 100644
--- a/uts/common/fs/zfs/sys/dsl_dataset.h
+++ b/uts/common/fs/zfs/sys/dsl_dataset.h
@@ -184,6 +184,9 @@ typedef struct dsl_dataset {
 	kmutex_t ds_sendstream_lock;
 	list_t ds_sendstreams;
 
+	/* Protected by our dsl_dir's dd_lock */
+	list_t ds_prop_cbs;
+
 	/*
 	 * For ZFEATURE_FLAG_PER_DATASET features, set if this dataset
 	 * uses this feature.
diff --git a/uts/common/fs/zfs/sys/dsl_dir.h b/uts/common/fs/zfs/sys/dsl_dir.h
index f50014d95a81..93fb73f37e6b 100644
--- a/uts/common/fs/zfs/sys/dsl_dir.h
+++ b/uts/common/fs/zfs/sys/dsl_dir.h
@@ -102,7 +102,7 @@ struct dsl_dir {
 
 	/* Protected by dd_lock */
 	kmutex_t dd_lock;
-	list_t dd_prop_cbs; /* list of dsl_prop_cb_record_t's */
+	list_t dd_props; /* list of dsl_prop_record_t's */
 	timestruc_t dd_snap_cmtime; /* last time snapshot namespace changed */
 	uint64_t dd_origin_txg;
 
diff --git a/uts/common/fs/zfs/sys/dsl_prop.h b/uts/common/fs/zfs/sys/dsl_prop.h
index 5fe18d6a7c55..21e6f4674be9 100644
--- a/uts/common/fs/zfs/sys/dsl_prop.h
+++ b/uts/common/fs/zfs/sys/dsl_prop.h
@@ -41,10 +41,17 @@ struct dsl_dir;
 /* The callback func may not call into the DMU or DSL! */
 typedef void (dsl_prop_changed_cb_t)(void *arg, uint64_t newval);
 
+typedef struct dsl_prop_record {
+	list_node_t pr_node; /* link on dd_props */
+	const char *pr_propname;
+	list_t pr_cbs;
+} dsl_prop_record_t;
+
 typedef struct dsl_prop_cb_record {
-	list_node_t cbr_node; /* link on dd_prop_cbs */
+	list_node_t cbr_pr_node; /* link on pr_cbs */
+	list_node_t cbr_ds_node; /* link on ds_prop_cbs */
+	dsl_prop_record_t *cbr_pr;
 	struct dsl_dataset *cbr_ds;
-	const char *cbr_propname;
 	dsl_prop_changed_cb_t *cbr_func;
 	void *cbr_arg;
 } dsl_prop_cb_record_t;
@@ -54,10 +61,11 @@ typedef struct dsl_props_arg {
 	zprop_source_t pa_source;
 } dsl_props_arg_t;
 
+void dsl_prop_init(dsl_dir_t *dd);
+void dsl_prop_fini(dsl_dir_t *dd);
 int dsl_prop_register(struct dsl_dataset *ds, const char *propname,
     dsl_prop_changed_cb_t *callback, void *cbarg);
-int dsl_prop_unregister(struct dsl_dataset *ds, const char *propname,
-    dsl_prop_changed_cb_t *callback, void *cbarg);
+void dsl_prop_unregister_all(struct dsl_dataset *ds, void *cbarg);
 void dsl_prop_notify_all(struct dsl_dir *dd);
 boolean_t dsl_prop_hascb(struct dsl_dataset *ds);
 
diff --git a/uts/common/fs/zfs/zfs_vfsops.c b/uts/common/fs/zfs/zfs_vfsops.c
index 3559b9b7b32d..712ac895a72c 100644
--- a/uts/common/fs/zfs/zfs_vfsops.c
+++ b/uts/common/fs/zfs/zfs_vfsops.c
@@ -561,33 +561,7 @@ zfs_register_callbacks(vfs_t *vfsp)
 	return (0);
 
 unregister:
-	/*
-	 * We may attempt to unregister some callbacks that are not
-	 * registered, but this is OK; it will simply return ENOMSG,
-	 * which we will ignore.
-	 */
-	(void) dsl_prop_unregister(ds, zfs_prop_to_name(ZFS_PROP_ATIME),
-	    atime_changed_cb, zfsvfs);
-	(void) dsl_prop_unregister(ds, zfs_prop_to_name(ZFS_PROP_XATTR),
-	    xattr_changed_cb, zfsvfs);
-	(void) dsl_prop_unregister(ds, zfs_prop_to_name(ZFS_PROP_RECORDSIZE),
-	    blksz_changed_cb, zfsvfs);
-	(void) dsl_prop_unregister(ds, zfs_prop_to_name(ZFS_PROP_READONLY),
-	    readonly_changed_cb, zfsvfs);
-	(void) dsl_prop_unregister(ds, zfs_prop_to_name(ZFS_PROP_DEVICES),
-	    devices_changed_cb, zfsvfs);
-	(void) dsl_prop_unregister(ds, zfs_prop_to_name(ZFS_PROP_SETUID),
-	    setuid_changed_cb, zfsvfs);
-	(void) dsl_prop_unregister(ds, zfs_prop_to_name(ZFS_PROP_EXEC),
-	    exec_changed_cb, zfsvfs);
-	(void) dsl_prop_unregister(ds, zfs_prop_to_name(ZFS_PROP_SNAPDIR),
-	    snapdir_changed_cb, zfsvfs);
-	(void) dsl_prop_unregister(ds, zfs_prop_to_name(ZFS_PROP_ACLMODE),
-	    acl_mode_changed_cb, zfsvfs);
-	(void) dsl_prop_unregister(ds, zfs_prop_to_name(ZFS_PROP_ACLINHERIT),
-	    acl_inherit_changed_cb, zfsvfs);
-	(void) dsl_prop_unregister(ds, zfs_prop_to_name(ZFS_PROP_VSCAN),
-	    vscan_changed_cb, zfsvfs);
+	dsl_prop_unregister_all(ds, zfsvfs);
 	return (error);
 }
 
@@ -1249,46 +1223,9 @@ void
 zfs_unregister_callbacks(zfsvfs_t *zfsvfs)
 {
 	objset_t *os = zfsvfs->z_os;
-	struct dsl_dataset *ds;
 
-	/*
-	 * Unregister properties.
-	 */
-	if (!dmu_objset_is_snapshot(os)) {
-		ds = dmu_objset_ds(os);
-		VERIFY(dsl_prop_unregister(ds, "atime", atime_changed_cb,
-		    zfsvfs) == 0);
-
-		VERIFY(dsl_prop_unregister(ds, "xattr", xattr_changed_cb,
-		    zfsvfs) == 0);
-
-		VERIFY(dsl_prop_unregister(ds, "recordsize", blksz_changed_cb,
-		    zfsvfs) == 0);
-
-		VERIFY(dsl_prop_unregister(ds, "readonly", readonly_changed_cb,
-		    zfsvfs) == 0);
-
-		VERIFY(dsl_prop_unregister(ds, "devices", devices_changed_cb,
-		    zfsvfs) == 0);
-
-		VERIFY(dsl_prop_unregister(ds, "setuid", setuid_changed_cb,
-		    zfsvfs) == 0);
-
-		VERIFY(dsl_prop_unregister(ds, "exec", exec_changed_cb,
-		    zfsvfs) == 0);
-
-		VERIFY(dsl_prop_unregister(ds, "snapdir", snapdir_changed_cb,
-		    zfsvfs) == 0);
-
-		VERIFY(dsl_prop_unregister(ds, "aclmode", acl_mode_changed_cb,
-		    zfsvfs) == 0);
-
-		VERIFY(dsl_prop_unregister(ds, "aclinherit",
-		    acl_inherit_changed_cb, zfsvfs) == 0);
-
-		VERIFY(dsl_prop_unregister(ds, "vscan",
-		    vscan_changed_cb, zfsvfs) == 0);
-	}
+	if (!dmu_objset_is_snapshot(os))
+		dsl_prop_unregister_all(dmu_objset_ds(os), zfsvfs);
 }
 
 /*

From d0e2e04f8a2bb1833033ebac12ff6673caaa2ca5 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Mon, 28 Sep 2015 00:20:14 +0000
Subject: [PATCH 129/380] Add ls -I testcases

---
 bin/ls/tests/ls_tests.sh | 45 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 44 insertions(+), 1 deletion(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index 5e791b73ffc3..e9d7725467ab 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -49,7 +49,6 @@ create_test_inputs()
 atf_test_case A_flag
 A_flag_head()
 {
-	atf_set "require.user" "unprivileged"
 }
 
 A_flag_body()
@@ -100,9 +99,53 @@ A_flag_implied_when_root_body()
 	atf_check_equal "$(cat $WITH_EXPLICIT)" "$(cat $WITH_IMPLIED)"
 }
 
+atf_test_case I_flag
+I_flag_head()
+{
+	atf_set "descr" "Verify that the output from ls -I is the same as ls for an unprivileged user"
+}
+
+I_flag_body()
+{
+	create_test_inputs
+
+	WITH_I=$PWD/../with_I.out
+	WITHOUT_I=$PWD/../without_I.out
+
+	atf_check -e empty -o save:$WITH_I -s exit:0 ls -I
+	atf_check -e empty -o save:$WITHOUT_I -s exit:0 ls
+
+	echo "Explicit -I usage"
+	cat $WITH_I
+	echo "No -I usage"
+	cat $WITHOUT_I
+
+	atf_check_equal "$(cat $WITH_I)" "$(cat $WITHOUT_I)"
+}
+
+atf_test_case I_flag_voids_A_flag_when_root
+I_flag_voids_A_flag_when_root_head()
+{
+	atf_set "descr" "Verify that -I voids out implied -A for root"
+	atf_set "require.user" "root"
+}
+
+I_flag_voids_A_flag_when_root_body()
+{
+	create_test_inputs
+
+	atf_check -o not-match:'\.f' -s exit:0 ls -I
+	atf_check -o not-match:'\.g' -s exit:0 ls -I
+
+	atf_check -o match:'\.f' -s exit:0 ls -A -I
+	atf_check -o match:'\.g' -s exit:0 ls -A -I
+}
+
 atf_init_test_cases()
 {
 
 	atf_add_test_case A_flag
 	atf_add_test_case A_flag_implied_when_root
+	atf_add_test_case I_flag
+	atf_add_test_case I_flag_voids_A_flag_when_root
 }

From 3498ea31d3989b8214dacf4d7b31c3a27382ee23 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Mon, 28 Sep 2015 01:17:31 +0000
Subject: [PATCH 130/380] Add -C and -1 testcases

---
 bin/ls/tests/ls_tests.sh | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index e9d7725467ab..46b523e7abcf 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -99,6 +99,19 @@ A_flag_implied_when_root_body()
 	atf_check_equal "$(cat $WITH_EXPLICIT)" "$(cat $WITH_IMPLIED)"
 }
 
+atf_test_case C_flag
+C_flag_head()
+{
+	atf_set "descr" "Verify that the output from ls -C"
+}
+
+C_flag_body()
+{
+	create_test_inputs
+
+	atf_check -e empty -o match:"$(printf 'a[[:space:]]+c[[:space:]]+d[[:space:]]+e[[:space:]]+h\n')" -s exit:0 ls -C
+}
+
 atf_test_case I_flag
 I_flag_head()
 {
@@ -141,11 +154,37 @@ I_flag_voids_A_flag_when_root_body()
 	atf_check -o match:'\.g' -s exit:0 ls -A -I
 }
 
+1_flag_head()
+{
+	atf_set "descr" "Verify that -1 prints out one item per line"
+}
+
+1_flag_body()
+{
+	create_test_inputs
+
+	WITH_1=$PWD/../with_1.out
+	WITHOUT_1=$PWD/../without_1.out
+
+	atf_check -e empty -o save:$WITH_1 -s exit:0 ls -1
+	atf_check -e empty -o save:$WITHOUT_1 -s exit:0 \
+		sh -c 'for i in $(ls); do echo $i; done'
+
+	echo "Explicit -1 usage"
+	cat $WITH_1
+	echo "No -1 usage"
+	cat $WITHOUT_1
+
+	atf_check_equal "$(cat $WITH_1)" "$(cat $WITHOUT_1)"
+}
+
 atf_init_test_cases()
 {
 
 	atf_add_test_case A_flag
 	atf_add_test_case A_flag_implied_when_root
+	atf_add_test_case C_flag
 	atf_add_test_case I_flag
 	atf_add_test_case I_flag_voids_A_flag_when_root
+	atf_add_test_case 1_flag
 }

From 1c621a11110e3f6593f99eb40279e17c3562316f Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Mon, 28 Sep 2015 01:58:29 +0000
Subject: [PATCH 131/380] Add testcases for -B and -l,

---
 bin/ls/tests/ls_tests.sh | 35 +++++++++++++++++++++++++++++++++--
 1 file changed, 33 insertions(+), 2 deletions(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index 46b523e7abcf..ebfd32d4bbf3 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -44,11 +44,13 @@ create_test_inputs()
 	atf_check -e empty -s exit:0 touch .f
 	atf_check -e empty -s exit:0 mkdir .g
 	atf_check -e empty -s exit:0 mkfifo h
+	atf_check -e ignore -s exit:0 dd if=/dev/zero of=i count=1000 bs=1
 }
 
 atf_test_case A_flag
 A_flag_head()
 {
+	atf_set "descr" "Verify -A support with unprivileged users"
 }
 
 A_flag_body()
@@ -99,17 +101,30 @@ A_flag_implied_when_root_body()
 	atf_check_equal "$(cat $WITH_EXPLICIT)" "$(cat $WITH_IMPLIED)"
 }
 
+atf_test_case B_flag
+B_flag_head()
+{
+	atf_set "descr" "Verify that the output from ls -B prints out non-printable characters"
+}
+
+B_flag_body()
+{
+
+	atf_check -e empty -o empty -s exit:0 touch "$(printf "y\013z")"
+	atf_check -e empty -o match:'y\\013z' -s exit:0 ls -B
+}
+
 atf_test_case C_flag
 C_flag_head()
 {
-	atf_set "descr" "Verify that the output from ls -C"
+	atf_set "descr" "Verify that the output from ls -C is multi-column"
 }
 
 C_flag_body()
 {
 	create_test_inputs
 
-	atf_check -e empty -o match:"$(printf 'a[[:space:]]+c[[:space:]]+d[[:space:]]+e[[:space:]]+h\n')" -s exit:0 ls -C
+	atf_check -e empty -o match:"$(printf 'a[[:space:]]+c[[:space:]]+d[[:space:]]+e[[:space:]]+h[[:space:]]i\n')" -s exit:0 ls -C
 }
 
 atf_test_case I_flag
@@ -154,6 +169,20 @@ I_flag_voids_A_flag_when_root_body()
 	atf_check -o match:'\.g' -s exit:0 ls -A -I
 }
 
+lcomma_flag_head()
+{
+	atf_set "descr" "Verify that -l, prints out the size with , delimiters"
+}
+
+lcomma_flag_body()
+{
+	create_test_inputs
+
+	atf_check \
+	    -o match:'\-rw\-r\-\-r\-\-[[:space:]]+.+[[:space:]]+1,000[[:space:]]+.+i' \
+	    env LC_ALL=en_US.ISO8859-1 ls -l, i
+}
+
 1_flag_head()
 {
 	atf_set "descr" "Verify that -1 prints out one item per line"
@@ -183,8 +212,10 @@ atf_init_test_cases()
 
 	atf_add_test_case A_flag
 	atf_add_test_case A_flag_implied_when_root
+	atf_add_test_case B_flag
 	atf_add_test_case C_flag
 	atf_add_test_case I_flag
 	atf_add_test_case I_flag_voids_A_flag_when_root
+	atf_add_test_case lcomma_flag
 	atf_add_test_case 1_flag
 }

From 3f2741c61d90edf3dd87da618c9bf7d02f73ca83 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Mon, 28 Sep 2015 02:51:10 +0000
Subject: [PATCH 132/380] - Separate out creating the temporary directory and
 creating   the temporary files - Create the temporary directory beforehand
 when testing out ls -a/-A   to ensure that "." and ".." are printed out with
 -a and not with -A - Create sockets with nc (for later with -l); check for
 its existence   in the Makefile - Add -a/-x testcases

---
 bin/ls/tests/Makefile    |   1 +
 bin/ls/tests/ls_tests.sh | 112 +++++++++++++++++++++++++++++++++++++--
 2 files changed, 109 insertions(+), 4 deletions(-)

diff --git a/bin/ls/tests/Makefile b/bin/ls/tests/Makefile
index 91a16505fe0d..dfac36300af0 100644
--- a/bin/ls/tests/Makefile
+++ b/bin/ls/tests/Makefile
@@ -4,5 +4,6 @@ ATF_TESTS_SH+=	ls_tests
 # This seems like overkill, but the idea in mind is that all of the testcases
 # should be runnable as !root
 TEST_METADATA.ls_tests+=	required_user="unprivileged"
+TEST_METADATA.ls_tests+=	required_files="/usr/bin/nc"
 
 .include <bsd.test.mk>
diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index ebfd32d4bbf3..6a79533281f1 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -27,9 +27,11 @@
 # $FreeBSD$
 #
 
-create_test_inputs()
+create_test_dir()
 {
-	ATF_TMPDIR=$(pwd)
+	[ -z "$ATF_TMPDIR" ] || return 0
+
+	export ATF_TMPDIR=$(pwd)
 
 	# XXX: need to nest this because of how kyua creates $TMPDIR; otherwise
 	# it will run into EPERM issues later
@@ -37,6 +39,12 @@ create_test_inputs()
 
 	atf_check -e empty -s exit:0 mkdir -m 0777 -p $TEST_INPUTS_DIR
 	cd $TEST_INPUTS_DIR
+}
+
+create_test_inputs()
+{
+	create_test_dir
+
 	atf_check -e empty -s exit:0 mkdir -m 0755 -p a/b
 	atf_check -e empty -s exit:0 ln -s a/b c
 	atf_check -e empty -s exit:0 touch d
@@ -45,6 +53,60 @@ create_test_inputs()
 	atf_check -e empty -s exit:0 mkdir .g
 	atf_check -e empty -s exit:0 mkfifo h
 	atf_check -e ignore -s exit:0 dd if=/dev/zero of=i count=1000 bs=1
+	atf_check -e empty -s exit:0 sh -c 'nc -lU j & sleep 0.5; kill %1'
+	atf_check -e empty -s exit:0 touch klmn
+	atf_check -e empty -s exit:0 touch opqr
+	atf_check -e empty -s exit:0 touch stuv
+	atf_check -e empty -s exit:0 touch wxyz
+	atf_check -e empty -s exit:0 touch 0b00000001
+	atf_check -e empty -s exit:0 touch 0b00000010
+	atf_check -e empty -s exit:0 touch 0b00000011
+	atf_check -e empty -s exit:0 touch 0b00000100
+	atf_check -e empty -s exit:0 touch 0b00000101
+	atf_check -e empty -s exit:0 touch 0b00000110
+	atf_check -e empty -s exit:0 touch 0b00000111
+	atf_check -e empty -s exit:0 touch 0b00001000
+	atf_check -e empty -s exit:0 touch 0b00001001
+	atf_check -e empty -s exit:0 touch 0b00001010
+	atf_check -e empty -s exit:0 touch 0b00001011
+	atf_check -e empty -s exit:0 touch 0b00001100
+	atf_check -e empty -s exit:0 touch 0b00001101
+	atf_check -e empty -s exit:0 touch 0b00001110
+	atf_check -e empty -s exit:0 touch 0b00001111
+}
+
+atf_test_case a_flag
+a_flag_head()
+{
+	atf_set "descr" "Verify -a support"
+}
+
+a_flag_body()
+{
+	create_test_dir
+
+	# Make sure "." and ".." show up with -a
+	atf_check -e empty -o match:'\.[[:space:]]+\.\.'  -s exit:0 ls -ax
+
+	create_test_inputs
+
+	WITH_a=$PWD/../with_a.out
+	WITHOUT_a=$PWD/../without_a.out
+
+	atf_check -e empty -o save:$WITH_a -s exit:0 ls -A
+	atf_check -e empty -o save:$WITHOUT_a -s exit:0 ls
+
+	echo "-A usage"
+	cat $WITH_a
+	echo "No -A usage"
+	cat $WITHOUT_a
+
+	for dot_path in '\.f' '\.g'; do
+		atf_check -e empty -o not-empty -s exit:0 grep "${dot_path}" \
+		    $WITH_a
+		atf_check -e empty -o empty -s not-exit:0 grep "${dot_path}" \
+		    $WITHOUT_a
+	done
 }
 
 atf_test_case A_flag
@@ -55,6 +117,10 @@ A_flag_head()
 
 A_flag_body()
 {
+	create_test_dir
+
+	atf_check -e empty -o empty -s exit:0 ls -A
+
 	create_test_inputs
 
 	WITH_A=$PWD/../with_A.out
@@ -85,6 +151,10 @@ A_flag_implied_when_root_head()
 
 A_flag_implied_when_root_body()
 {
+	create_test_dir
+
+	atf_check -e empty -o empty -s exit:0 ls -A
+
 	create_test_inputs
 
 	WITH_EXPLICIT=$PWD/../with_explicit_A.out
@@ -117,14 +187,24 @@ B_flag_body()
 atf_test_case C_flag
 C_flag_head()
 {
-	atf_set "descr" "Verify that the output from ls -C is multi-column"
+	atf_set "descr" "Verify that the output from ls -C is multi-column, sorted down"
 }
 
 C_flag_body()
 {
 	create_test_inputs
 
-	atf_check -e empty -o match:"$(printf 'a[[:space:]]+c[[:space:]]+d[[:space:]]+e[[:space:]]+h[[:space:]]i\n')" -s exit:0 ls -C
+	WITH_C=$PWD/../with_C.out
+
+	atf_check -e empty -o save:$WITH_C -s exit:0 ls -C
+
+	echo "With -C usage"
+	cat $WITH_C
+
+	atf_check -e ignore -o not-empty -s exit:0 \
+	    egrep "0b00000001[[:space:]]+0b00000111[[:space:]]+0b00001101[[:space:]]+e[[:space:]]+stuv" $WITH_C
+	atf_check -e ignore -o not-empty -s exit:0 \
+	    egrep "0b00000010[[:space:]]+0b00001000[[:space:]]+0b00001110[[:space:]]+h[[:space:]]+wxyz" $WITH_C
 }
 
 atf_test_case I_flag
@@ -183,6 +263,28 @@ lcomma_flag_body()
 	    env LC_ALL=en_US.ISO8859-1 ls -l, i
 }
 
+x_flag_head()
+{
+	atf_set "descr" "Verify that -x prints out one item per line"
+}
+
+x_flag_body()
+{
+	create_test_inputs
+
+	WITH_x=$PWD/../with_x.out
+
+	atf_check -e empty -o save:$WITH_x -s exit:0 ls -x
+
+	echo "With -x usage"
+	cat $WITH_x
+
+	atf_check -e ignore -o not-empty -s exit:0 \
+	    egrep "a[[:space:]]+c[[:space:]]+d[[:space:]]+e[[:space:]]+h" $WITH_x
+	atf_check -e ignore -o not-empty -s exit:0 \
+	    egrep "i[[:space:]]+j[[:space:]]+klmn[[:space:]]+opqr[[:space:]]+stuv" $WITH_x
+}
+
 1_flag_head()
 {
 	atf_set "descr" "Verify that -1 prints out one item per line"
@@ -210,6 +312,7 @@ lcomma_flag_body()
 atf_init_test_cases()
 {
 
+	atf_add_test_case a_flag
 	atf_add_test_case A_flag
 	atf_add_test_case A_flag_implied_when_root
 	atf_add_test_case B_flag
@@ -217,5 +320,6 @@ atf_init_test_cases()
 	atf_add_test_case I_flag
 	atf_add_test_case I_flag_voids_A_flag_when_root
 	atf_add_test_case lcomma_flag
+	atf_add_test_case x_flag
 	atf_add_test_case 1_flag
 }

From f93f8d5920bf570409b11a3d7ed99c39854cf355 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Mon, 28 Sep 2015 02:53:36 +0000
Subject: [PATCH 133/380] Sleep 2 seconds before trying to kill nc

---
 bin/ls/tests/ls_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index 6a79533281f1..1eec12cb2064 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -53,7 +53,7 @@ create_test_inputs()
 	atf_check -e empty -s exit:0 mkdir .g
 	atf_check -e empty -s exit:0 mkfifo h
 	atf_check -e ignore -s exit:0 dd if=/dev/zero of=i count=1000 bs=1
-	atf_check -e empty -s exit:0 sh -c 'nc -lU j & sleep 0.5; kill %1'
+	atf_check -e empty -s exit:0 sh -c 'nc -lU j & sleep 2; kill %1'
 	atf_check -e empty -s exit:0 touch klmn
 	atf_check -e empty -s exit:0 touch opqr
 	atf_check -e empty -s exit:0 touch stuv

From e944275ce7c891316395096fc4414eb95355b69c Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Mon, 28 Sep 2015 03:00:19 +0000
Subject: [PATCH 134/380] Use daemon to start up nc and kill nc instead via
 pkill

---
 bin/ls/tests/ls_tests.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index 1eec12cb2064..578afdd3696b 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -53,7 +53,8 @@ create_test_inputs()
 	atf_check -e empty -s exit:0 mkdir .g
 	atf_check -e empty -s exit:0 mkfifo h
 	atf_check -e ignore -s exit:0 dd if=/dev/zero of=i count=1000 bs=1
-	atf_check -e empty -s exit:0 sh -c 'nc -lU j & sleep 2; kill %1'
+	atf_check -e empty -s exit:0 \
+	    sh -c "pid=${ATF_TMPDIR}/nc.pid; daemon -p \$pid nc -lU j; sleep 2; pkill -F \$pid"
 	atf_check -e empty -s exit:0 touch klmn
 	atf_check -e empty -s exit:0 touch opqr
 	atf_check -e empty -s exit:0 touch stuv

From 74662e3e9ae06ea8ffca146b93d79bb5b96452f9 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Mon, 28 Sep 2015 03:14:24 +0000
Subject: [PATCH 135/380] - Rename I_flag_voids_A_flag_when_root to
 I_flag_voids_implied_A_flag_when_root - Add reminders for items that need to
 be tested in atf_init_test_cases

---
 bin/ls/tests/ls_tests.sh | 42 +++++++++++++++++++++++++++++++++++-----
 1 file changed, 37 insertions(+), 5 deletions(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index 578afdd3696b..360d44e7cb34 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -232,14 +232,14 @@ I_flag_body()
 	atf_check_equal "$(cat $WITH_I)" "$(cat $WITHOUT_I)"
 }
 
-atf_test_case I_flag_voids_A_flag_when_root
-I_flag_voids_A_flag_when_root_head()
+atf_test_case I_flag_voids_implied_A_flag_when_root
+I_flag_voids_implied_A_flag_when_root_head()
 {
 	atf_set "descr" "Verify that -I voids out implied -A for root"
 	atf_set "require.user" "root"
 }
 
-I_flag_voids_A_flag_when_root_body()
+I_flag_voids_implied_A_flag_when_root_body()
 {
 	create_test_inputs
 
@@ -313,14 +313,46 @@ x_flag_body()
 atf_init_test_cases()
 {
 
-	atf_add_test_case a_flag
 	atf_add_test_case A_flag
 	atf_add_test_case A_flag_implied_when_root
 	atf_add_test_case B_flag
 	atf_add_test_case C_flag
+	#atf_add_test_case D_flag
+	#atf_add_test_case F_flag
+	#atf_add_test_case G_flag
+	#atf_add_test_case H_flag
 	atf_add_test_case I_flag
-	atf_add_test_case I_flag_voids_A_flag_when_root
+	atf_add_test_case I_flag_voids_implied_A_flag_when_root
+	#atf_add_test_case L_flag
+	#atf_add_test_case P_flag
+	#atf_add_test_case R_flag
+	#atf_add_test_case S_flag
+	#atf_add_test_case T_flag
+	#atf_add_test_case U_flag
+	#atf_add_test_case W_flag
+	#atf_add_test_case Z_flag
+	#atf_add_test_case a_flag
+	#atf_add_test_case b_flag
+	#atf_add_test_case c_flag
+	#atf_add_test_case d_flag
+	#atf_add_test_case f_flag
+	#atf_add_test_case g_flag
+	#atf_add_test_case h_flag
+	#atf_add_test_case i_flag
+	#atf_add_test_case k_flag
+	#atf_add_test_case l_flag
 	atf_add_test_case lcomma_flag
+	#atf_add_test_case m_flag
+	#atf_add_test_case n_flag
+	#atf_add_test_case o_flag
+	#atf_add_test_case p_flag
+	#atf_add_test_case q_flag
+	#atf_add_test_case r_flag
+	#atf_add_test_case s_flag
+	#atf_add_test_case t_flag
+	#atf_add_test_case u_flag
+	#atf_add_test_case w_flag
 	atf_add_test_case x_flag
+	#atf_add_test_case y_flag
 	atf_add_test_case 1_flag
 }

From dc898f519b6e38bf49f6ea8e577d50c9a09afeaa Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Mon, 28 Sep 2015 03:15:38 +0000
Subject: [PATCH 136/380] Uncomment a_flag testcase again

---
 bin/ls/tests/ls_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index 360d44e7cb34..3d0d1e917c91 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -331,7 +331,7 @@ atf_init_test_cases()
 	#atf_add_test_case U_flag
 	#atf_add_test_case W_flag
 	#atf_add_test_case Z_flag
-	#atf_add_test_case a_flag
+	atf_add_test_case a_flag
 	#atf_add_test_case b_flag
 	#atf_add_test_case c_flag
 	#atf_add_test_case d_flag

From eb55c40fb9d7416d2fb4d8ae05ab7783ea39ad5c Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Mon, 28 Sep 2015 03:31:01 +0000
Subject: [PATCH 137/380] MFhead@r288328

---
 sys/dev/otus/if_otus.c          | 39 +++++++++++++++++++++++++++++-
 sys/dev/otus/if_otusreg.h       |  4 ++++
 sys/dev/usb/wlan/if_rsu.c       |  2 ++
 sys/net80211/ieee80211_ht.c     |  7 +++---
 sys/net80211/ieee80211_ht.h     | 12 ++++++++++
 sys/net80211/ieee80211_node.h   |  3 +++
 sys/net80211/ieee80211_output.c | 21 +++++++++++++++++
 sys/net80211/ieee80211_superg.c | 42 ++++++++++++++++++++-------------
 8 files changed, 109 insertions(+), 21 deletions(-)

diff --git a/sys/dev/otus/if_otus.c b/sys/dev/otus/if_otus.c
index 2343953598c1..e60157adda9d 100644
--- a/sys/dev/otus/if_otus.c
+++ b/sys/dev/otus/if_otus.c
@@ -24,6 +24,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_wlan.h"
+
 #include <sys/param.h>
 #include <sys/endian.h>
 #include <sys/sockio.h>
@@ -60,6 +62,9 @@ __FBSDID("$FreeBSD$");
 #include <net80211/ieee80211_radiotap.h>
 #include <net80211/ieee80211_ratectl.h>
 #include <net80211/ieee80211_input.h>
+#ifdef	IEEE80211_SUPPORT_SUPERG
+#include <net80211/ieee80211_superg.h>
+#endif
 
 #include <dev/usb/usb.h>
 #include <dev/usb/usbdi.h>
@@ -340,6 +345,7 @@ otus_detach(device_t self)
 	taskqueue_drain(taskqueue_thread, &sc->tx_task);
 	taskqueue_drain(taskqueue_thread, &sc->wme_update_task);
 
+	otus_close_pipes(sc);
 #if 0
 	/* Wait for all queued asynchronous commands to complete. */
 	usb_rem_wait_task(sc->sc_udev, &sc->sc_task);
@@ -348,7 +354,6 @@ otus_detach(device_t self)
 #endif
 
 	ieee80211_ifdetach(ic);
-	otus_close_pipes(sc);
 	mtx_destroy(&sc->sc_mtx);
 	return 0;
 }
@@ -949,9 +954,12 @@ fail:	otus_close_pipes(sc);
 void
 otus_close_pipes(struct otus_softc *sc)
 {
+
+	OTUS_LOCK(sc);
 	otus_free_tx_cmd_list(sc);
 	otus_free_tx_list(sc);
 	otus_free_rx_list(sc);
+	OTUS_UNLOCK(sc);
 
 	usbd_transfer_unsetup(sc->sc_xfer, OTUS_N_XFER);
 }
@@ -1838,6 +1846,9 @@ tr_setup:
 			} else
 				(void)ieee80211_input_mimo_all(ic, m, NULL);
 		}
+#ifdef	IEEE80211_SUPPORT_SUPERG
+		ieee80211_ff_age_all(ic, 100);
+#endif
 		OTUS_LOCK(sc);
 		break;
 	default:
@@ -1866,6 +1877,14 @@ otus_txeof(struct usb_xfer *xfer, struct otus_data *data)
 
 	OTUS_LOCK_ASSERT(sc);
 
+	if (sc->sc_tx_n_active == 0) {
+		device_printf(sc->sc_dev,
+		    "%s: completed but tx_active=0\n",
+		    __func__);
+	} else {
+		sc->sc_tx_n_active--;
+	}
+
 	if (data->m) {
 		/* XXX status? */
 		/* XXX we get TX status via the RX path.. */
@@ -1926,6 +1945,7 @@ tr_setup:
 		if (data == NULL) {
 			OTUS_DPRINTF(sc, OTUS_DEBUG_XMIT,
 			    "%s: empty pending queue sc %p\n", __func__, sc);
+			sc->sc_tx_n_active = 0;
 			goto finish;
 		}
 		STAILQ_REMOVE_HEAD(&sc->sc_tx_pending[which], next);
@@ -1934,6 +1954,7 @@ tr_setup:
 		OTUS_DPRINTF(sc, OTUS_DEBUG_XMIT,
 		    "%s: submitting transfer %p\n", __func__, data);
 		usbd_transfer_submit(xfer);
+		sc->sc_tx_n_active++;
 		break;
 	default:
 		data = STAILQ_FIRST(&sc->sc_tx_active[which]);
@@ -1952,6 +1973,22 @@ tr_setup:
 	}
 
 finish:
+#ifdef	IEEE80211_SUPPORT_SUPERG
+	/*
+	 * If the TX active queue drops below a certain
+	 * threshold, ensure we age fast-frames out so they're
+	 * transmitted.
+	 */
+	if (sc->sc_tx_n_active < 2) {
+		/* XXX ew - net80211 should defer this for us! */
+		OTUS_UNLOCK(sc);
+		ieee80211_ff_flush(ic, WME_AC_VO);
+		ieee80211_ff_flush(ic, WME_AC_VI);
+		ieee80211_ff_flush(ic, WME_AC_BE);
+		ieee80211_ff_flush(ic, WME_AC_BK);
+		OTUS_LOCK(sc);
+	}
+#endif
 	/* Kick TX */
 	otus_tx_start(sc);
 }
diff --git a/sys/dev/otus/if_otusreg.h b/sys/dev/otus/if_otusreg.h
index 35924529c554..4f11b8dfd772 100644
--- a/sys/dev/otus/if_otusreg.h
+++ b/sys/dev/otus/if_otusreg.h
@@ -984,6 +984,10 @@ struct otus_softc {
 	/* current noisefloor, from SET_FREQUENCY */
 	int				sc_nf[OTUS_NUM_CHAINS];
 
+	/* How many pending, active transmit frames */
+	int				sc_tx_n_pending;
+	int				sc_tx_n_active;
+
 	const uint32_t			*phy_vals;
 
 	struct {
diff --git a/sys/dev/usb/wlan/if_rsu.c b/sys/dev/usb/wlan/if_rsu.c
index e935dedcb777..ee650364b604 100644
--- a/sys/dev/usb/wlan/if_rsu.c
+++ b/sys/dev/usb/wlan/if_rsu.c
@@ -28,6 +28,8 @@ __FBSDID("$FreeBSD$");
  *   o power-save operation
  */
 
+#include "opt_wlan.h"
+
 #include <sys/param.h>
 #include <sys/endian.h>
 #include <sys/sockio.h>
diff --git a/sys/net80211/ieee80211_ht.c b/sys/net80211/ieee80211_ht.c
index b43bc075472f..a850a334ea20 100644
--- a/sys/net80211/ieee80211_ht.c
+++ b/sys/net80211/ieee80211_ht.c
@@ -1081,7 +1081,7 @@ ieee80211_ht_node_init(struct ieee80211_node *ni)
 		tap = &ni->ni_tx_ampdu[tid];
 		tap->txa_tid = tid;
 		tap->txa_ni = ni;
-		tap->txa_lastsample = ticks;
+		ieee80211_txampdu_init_pps(tap);
 		/* NB: further initialization deferred */
 	}
 	ni->ni_flags |= IEEE80211_NODE_HT | IEEE80211_NODE_AMPDU;
@@ -1251,7 +1251,7 @@ ieee80211_ht_wds_init(struct ieee80211_node *ni)
 	for (tid = 0; tid < WME_NUM_TID; tid++) {
 		tap = &ni->ni_tx_ampdu[tid];
 		tap->txa_tid = tid;
-		tap->txa_lastsample = ticks;
+		ieee80211_txampdu_init_pps(tap);
 	}
 	/* NB: AMPDU tx/rx governed by IEEE80211_FHT_AMPDU_{TX,RX} */
 	ni->ni_flags |= IEEE80211_NODE_HT | IEEE80211_NODE_AMPDU;
@@ -1752,8 +1752,7 @@ ampdu_tx_stop(struct ieee80211_tx_ampdu *tap)
 	/*
 	 * Reset packet estimate.
 	 */
-	tap->txa_lastsample = ticks;
-	tap->txa_avgpps = 0;
+	ieee80211_txampdu_init_pps(tap);
 
 	/* NB: clearing NAK means we may re-send ADDBA */ 
 	tap->txa_flags &= ~(IEEE80211_AGGR_SETUP | IEEE80211_AGGR_NAK);
diff --git a/sys/net80211/ieee80211_ht.h b/sys/net80211/ieee80211_ht.h
index f099ffe77464..e2b331854dff 100644
--- a/sys/net80211/ieee80211_ht.h
+++ b/sys/net80211/ieee80211_ht.h
@@ -83,9 +83,20 @@ struct ieee80211_tx_ampdu {
  * available and more streams are requested than available.
  */
 
+static __inline void
+ieee80211_txampdu_init_pps(struct ieee80211_tx_ampdu *tap)
+{
+	/*
+	 * Reset packet estimate.
+	 */
+	tap->txa_lastsample = ticks;
+	tap->txa_avgpps = 0;
+}
+
 static __inline void
 ieee80211_txampdu_update_pps(struct ieee80211_tx_ampdu *tap)
 {
+
 	/* NB: scale factor of 2 was picked heuristically */
 	tap->txa_avgpps = ((tap->txa_avgpps << 2) -
 	     tap->txa_avgpps + tap->txa_pkts) >> 2;
@@ -97,6 +108,7 @@ ieee80211_txampdu_update_pps(struct ieee80211_tx_ampdu *tap)
 static __inline void
 ieee80211_txampdu_count_packet(struct ieee80211_tx_ampdu *tap)
 {
+
 	/* XXX bound loop/do more crude estimate? */
 	while (ticks - tap->txa_lastsample >= hz) {
 		ieee80211_txampdu_update_pps(tap);
diff --git a/sys/net80211/ieee80211_node.h b/sys/net80211/ieee80211_node.h
index bb31136892d6..e9b73933f66e 100644
--- a/sys/net80211/ieee80211_node.h
+++ b/sys/net80211/ieee80211_node.h
@@ -219,6 +219,9 @@ struct ieee80211_node {
 	struct ieee80211_tx_ampdu ni_tx_ampdu[WME_NUM_TID];
 	struct ieee80211_rx_ampdu ni_rx_ampdu[WME_NUM_TID];
 
+	/* fast-frames state */
+	struct mbuf *		ni_tx_superg[WME_NUM_TID];
+
 	/* others */
 	short			ni_inact;	/* inactivity mark count */
 	short			ni_inact_reload;/* inactivity reload value */
diff --git a/sys/net80211/ieee80211_output.c b/sys/net80211/ieee80211_output.c
index 4e061d80ffb0..27fe929ac288 100644
--- a/sys/net80211/ieee80211_output.c
+++ b/sys/net80211/ieee80211_output.c
@@ -213,6 +213,22 @@ ieee80211_vap_pkt_send_dest(struct ieee80211vap *vap, struct mbuf *m,
 		}
 	}
 
+	/*
+	 * XXX If we aren't doing AMPDU TX then we /could/ do
+	 * fast-frames encapsulation, however right now this
+	 * output logic doesn't handle that case.
+	 *
+	 * So we'll be limited to "fast-frames" xmit for non-11n STA
+	 * and "no fast frames" xmit for 11n STAs.
+	 * It'd be nice to eventually test fast-frames out by
+	 * gracefully falling from failing A-MPDU transmission
+	 * (driver says no, fail to negotiate it with peer) to
+	 * using fast-frames.
+	 *
+	 * Note: we can actually put A-MSDU's inside an A-MPDU,
+	 * so hopefully we can figure out how to make that particular
+	 * combination work right.
+	 */
 #ifdef IEEE80211_SUPPORT_SUPERG
 	else if (IEEE80211_ATH_CAP(vap, ni, IEEE80211_NODE_FF)) {
 		m = ieee80211_ff_check(ni, m);
@@ -230,6 +246,11 @@ ieee80211_vap_pkt_send_dest(struct ieee80211vap *vap, struct mbuf *m,
 	 */
 	IEEE80211_TX_LOCK(ic);
 
+	/*
+	 * XXX make the encap and transmit code a separate function
+	 * so things like the FF (and later A-MSDU) path can just call
+	 * it for flushed frames.
+	 */
 	if (__predict_true((vap->iv_caps & IEEE80211_C_8023ENCAP) == 0)) {
 		/*
 		 * Encapsulate the packet in prep for transmission.
diff --git a/sys/net80211/ieee80211_superg.c b/sys/net80211/ieee80211_superg.c
index 2e2bbd18be84..173d4e729c06 100644
--- a/sys/net80211/ieee80211_superg.c
+++ b/sys/net80211/ieee80211_superg.c
@@ -530,7 +530,6 @@ ieee80211_ff_age(struct ieee80211com *ic, struct ieee80211_stageq *sq,
 {
 	struct mbuf *m, *head;
 	struct ieee80211_node *ni;
-	struct ieee80211_tx_ampdu *tap;
 
 #if 0
 	KASSERT(sq->head != NULL, ("stageq empty"));
@@ -541,11 +540,10 @@ ieee80211_ff_age(struct ieee80211com *ic, struct ieee80211_stageq *sq,
 	while ((m = sq->head) != NULL && M_AGE_GET(m) < quanta) {
 		int tid = WME_AC_TO_TID(M_WME_GETAC(m));
 
-		/* clear tap ref to frame */
+		/* clear staging ref to frame */
 		ni = (struct ieee80211_node *) m->m_pkthdr.rcvif;
-		tap = &ni->ni_tx_ampdu[tid];
-		KASSERT(tap->txa_private == m, ("staging queue empty"));
-		tap->txa_private = NULL;
+		KASSERT(ni->ni_tx_superg[tid] == m, ("staging queue empty"));
+		ni->ni_tx_superg[tid] = NULL;
 
 		sq->head = m->m_nextpkt;
 		sq->depth--;
@@ -658,7 +656,12 @@ ieee80211_ff_check(struct ieee80211_node *ni, struct mbuf *m)
 	 */
 	IEEE80211_LOCK(ic);
 	tap = &ni->ni_tx_ampdu[WME_AC_TO_TID(pri)];
-	mstaged = tap->txa_private;		/* NB: we reuse AMPDU state */
+	mstaged = ni->ni_tx_superg[WME_AC_TO_TID(pri)];
+	/* XXX NOTE: reusing packet counter state from A-MPDU */
+	/*
+	 * XXX NOTE: this means we're double-counting; it should just
+	 * be done in ieee80211_output.c once for both superg and A-MPDU.
+	 */
 	ieee80211_txampdu_count_packet(tap);
 
 	/*
@@ -676,6 +679,8 @@ ieee80211_ff_check(struct ieee80211_node *ni, struct mbuf *m)
 	 * If there is no frame to combine with and the pps is
 	 * too low; then do not attempt to aggregate this frame.
 	 */
+	IEEE80211_DPRINTF(vap, IEEE80211_MSG_SUPERG,
+	    "%s: staged: %p; pps: %d\n", __func__, mstaged, ieee80211_txampdu_getpps(tap));
 	if (mstaged == NULL &&
 	    ieee80211_txampdu_getpps(tap) < ieee80211_ffppsmin) {
 		IEEE80211_UNLOCK(ic);
@@ -698,7 +703,7 @@ ieee80211_ff_check(struct ieee80211_node *ni, struct mbuf *m)
 		    "%s: txtime %u exceeds txop limit %u\n",
 		    __func__, txtime, limit);
 
-		tap->txa_private = NULL;
+		ni->ni_tx_superg[WME_AC_TO_TID(pri)] = NULL;
 		if (mstaged != NULL)
 			stageq_remove(ic, sq, mstaged);
 		IEEE80211_UNLOCK(ic);
@@ -721,7 +726,7 @@ ieee80211_ff_check(struct ieee80211_node *ni, struct mbuf *m)
 	 * hold their node reference.
 	 */
 	if (mstaged != NULL) {
-		tap->txa_private = NULL;
+		ni->ni_tx_superg[WME_AC_TO_TID(pri)] = NULL;
 		stageq_remove(ic, sq, mstaged);
 		IEEE80211_UNLOCK(ic);
 
@@ -739,9 +744,10 @@ ieee80211_ff_check(struct ieee80211_node *ni, struct mbuf *m)
 		mstaged->m_nextpkt = m;
 		mstaged->m_flags |= M_FF; /* NB: mark for encap work */
 	} else {
-		KASSERT(tap->txa_private == NULL,
-		    ("txa_private %p", tap->txa_private));
-		tap->txa_private = m;
+		KASSERT(ni->ni_tx_superg[WME_AC_TO_TID(pri)]== NULL,
+		    ("ni_tx_superg[]: %p",
+		    ni->ni_tx_superg[WME_AC_TO_TID(pri)]));
+		ni->ni_tx_superg[WME_AC_TO_TID(pri)] = m;
 
 		stageq_add(ic, sq, m);
 		IEEE80211_UNLOCK(ic);
@@ -769,7 +775,6 @@ ieee80211_ff_node_cleanup(struct ieee80211_node *ni)
 {
 	struct ieee80211com *ic = ni->ni_ic;
 	struct ieee80211_superg *sg = ic->ic_superg;
-	struct ieee80211_tx_ampdu *tap;
 	struct mbuf *m, *next_m, *head;
 	int tid;
 
@@ -777,11 +782,16 @@ ieee80211_ff_node_cleanup(struct ieee80211_node *ni)
 	head = NULL;
 	for (tid = 0; tid < WME_NUM_TID; tid++) {
 		int ac = TID_TO_WME_AC(tid);
-
-		tap = &ni->ni_tx_ampdu[tid];
-		m = tap->txa_private;
+		/*
+		 * XXX Initialise the packet counter.
+		 *
+		 * This may be double-work for 11n stations;
+		 * but without it we never setup things.
+		 */
+		ieee80211_txampdu_init_pps(&ni->ni_tx_ampdu[tid]);
+		m = ni->ni_tx_superg[tid];
 		if (m != NULL) {
-			tap->txa_private = NULL;
+			ni->ni_tx_superg[tid] = NULL;
 			stageq_remove(ic, &sg->ff_stageq[ac], m);
 			m->m_nextpkt = head;
 			head = m;

From f32990ea9973fd92d6f5a46e1519f58d68466c81 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Mon, 28 Sep 2015 03:41:18 +0000
Subject: [PATCH 138/380] Delete

---
 bin/ls/tests/Makefile    |   9 -
 bin/ls/tests/ls_tests.sh | 358 ---------------------------------------
 2 files changed, 367 deletions(-)
 delete mode 100644 bin/ls/tests/Makefile
 delete mode 100755 bin/ls/tests/ls_tests.sh

diff --git a/bin/ls/tests/Makefile b/bin/ls/tests/Makefile
deleted file mode 100644
index dfac36300af0..000000000000
--- a/bin/ls/tests/Makefile
+++ /dev/null
@@ -1,9 +0,0 @@
-TESTSDIR=	${TESTSBASE}/bin/ls
-
-ATF_TESTS_SH+=	ls_tests
-# This seems like overkill, but the idea in mind is that all of the testcases
-# should be runnable as !root
-TEST_METADATA.ls_tests+=	required_user="unprivileged"
-TEST_METADATA.ls_tests+=	required_files="/usr/bin/nc"
-
-.include <bsd.test.mk>
diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
deleted file mode 100755
index 3d0d1e917c91..000000000000
--- a/bin/ls/tests/ls_tests.sh
+++ /dev/null
@@ -1,358 +0,0 @@
-#
-# Copyright 2015 EMC Corp.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-# * Redistributions of source code must retain the above copyright
-#   notice, this list of conditions and the following disclaimer.
-# * Redistributions in binary form must reproduce the above copyright
-#   notice, this list of conditions and the following disclaimer in the
-#   documentation and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
-# $FreeBSD$
-#
-
-create_test_dir()
-{
-	[ -z "$ATF_TMPDIR" ] || return 0
-
-	export ATF_TMPDIR=$(pwd)
-
-	# XXX: need to nest this because of how kyua creates $TMPDIR; otherwise
-	# it will run into EPERM issues later
-	TEST_INPUTS_DIR="${ATF_TMPDIR}/test/inputs"
-
-	atf_check -e empty -s exit:0 mkdir -m 0777 -p $TEST_INPUTS_DIR
-	cd $TEST_INPUTS_DIR
-}
-
-create_test_inputs()
-{
-	create_test_dir
-
-	atf_check -e empty -s exit:0 mkdir -m 0755 -p a/b
-	atf_check -e empty -s exit:0 ln -s a/b c
-	atf_check -e empty -s exit:0 touch d
-	atf_check -e empty -s exit:0 ln d e
-	atf_check -e empty -s exit:0 touch .f
-	atf_check -e empty -s exit:0 mkdir .g
-	atf_check -e empty -s exit:0 mkfifo h
-	atf_check -e ignore -s exit:0 dd if=/dev/zero of=i count=1000 bs=1
-	atf_check -e empty -s exit:0 \
-	    sh -c "pid=${ATF_TMPDIR}/nc.pid; daemon -p \$pid nc -lU j; sleep 2; pkill -F \$pid"
-	atf_check -e empty -s exit:0 touch klmn
-	atf_check -e empty -s exit:0 touch opqr
-	atf_check -e empty -s exit:0 touch stuv
-	atf_check -e empty -s exit:0 touch wxyz
-	atf_check -e empty -s exit:0 touch 0b00000001
-	atf_check -e empty -s exit:0 touch 0b00000010
-	atf_check -e empty -s exit:0 touch 0b00000011
-	atf_check -e empty -s exit:0 touch 0b00000100
-	atf_check -e empty -s exit:0 touch 0b00000101
-	atf_check -e empty -s exit:0 touch 0b00000110
-	atf_check -e empty -s exit:0 touch 0b00000111
-	atf_check -e empty -s exit:0 touch 0b00001000
-	atf_check -e empty -s exit:0 touch 0b00001001
-	atf_check -e empty -s exit:0 touch 0b00001010
-	atf_check -e empty -s exit:0 touch 0b00001011
-	atf_check -e empty -s exit:0 touch 0b00001100
-	atf_check -e empty -s exit:0 touch 0b00001101
-	atf_check -e empty -s exit:0 touch 0b00001110
-	atf_check -e empty -s exit:0 touch 0b00001111
-}
-
-atf_test_case a_flag
-a_flag_head()
-{
-	atf_set "descr" "Verify -a support"
-}
-
-a_flag_body()
-{
-	create_test_dir
-
-	# Make sure "." and ".." show up with -a
-	atf_check -e empty -o match:'\.[[:space:]]+\.\.'  -s exit:0 ls -ax
-
-	create_test_inputs
-
-	WITH_a=$PWD/../with_a.out
-	WITHOUT_a=$PWD/../without_a.out
-
-	atf_check -e empty -o save:$WITH_a -s exit:0 ls -A
-	atf_check -e empty -o save:$WITHOUT_a -s exit:0 ls
-
-	echo "-A usage"
-	cat $WITH_a
-	echo "No -A usage"
-	cat $WITHOUT_a
-
-	for dot_path in '\.f' '\.g'; do
-		atf_check -e empty -o not-empty -s exit:0 grep "${dot_path}" \
-		    $WITH_a
-		atf_check -e empty -o empty -s not-exit:0 grep "${dot_path}" \
-		    $WITHOUT_a
-	done
-}
-
-atf_test_case A_flag
-A_flag_head()
-{
-	atf_set "descr" "Verify -A support with unprivileged users"
-}
-
-A_flag_body()
-{
-	create_test_dir
-
-	atf_check -e empty -o empty -s exit:0 ls -A
-
-	create_test_inputs
-
-	WITH_A=$PWD/../with_A.out
-	WITHOUT_A=$PWD/../without_A.out
-
-	atf_check -e empty -o save:$WITH_A -s exit:0 ls -A
-	atf_check -e empty -o save:$WITHOUT_A -s exit:0 ls
-
-	echo "-A usage"
-	cat $WITH_A
-	echo "No -A usage"
-	cat $WITHOUT_A
-
-	for dot_path in '\.f' '\.g'; do
-		atf_check -e empty -o not-empty -s exit:0 grep "${dot_path}" \
-		    $WITH_A
-		atf_check -e empty -o empty -s not-exit:0 grep "${dot_path}" \
-		    $WITHOUT_A
-	done
-}
-
-atf_test_case A_flag_implied_when_root
-A_flag_implied_when_root_head()
-{
-	atf_set "descr" "Verify that -A is implied for root"
-	atf_set "require.user" "root"
-}
-
-A_flag_implied_when_root_body()
-{
-	create_test_dir
-
-	atf_check -e empty -o empty -s exit:0 ls -A
-
-	create_test_inputs
-
-	WITH_EXPLICIT=$PWD/../with_explicit_A.out
-	WITH_IMPLIED=$PWD/../with_implied_A.out
-
-	atf_check -e empty -o save:$WITH_EXPLICIT -s exit:0 ls -A
-	atf_check -e empty -o save:$WITH_IMPLIED -s exit:0 ls
-
-	echo "Explicit -A usage"
-	cat $WITH_EXPLICIT
-	echo "Implicit -A usage"
-	cat $WITH_IMPLIED
-
-	atf_check_equal "$(cat $WITH_EXPLICIT)" "$(cat $WITH_IMPLIED)"
-}
-
-atf_test_case B_flag
-B_flag_head()
-{
-	atf_set "descr" "Verify that the output from ls -B prints out non-printable characters"
-}
-
-B_flag_body()
-{
-
-	atf_check -e empty -o empty -s exit:0 touch "$(printf "y\013z")"
-	atf_check -e empty -o match:'y\\013z' -s exit:0 ls -B
-}
-
-atf_test_case C_flag
-C_flag_head()
-{
-	atf_set "descr" "Verify that the output from ls -C is multi-column, sorted down"
-}
-
-C_flag_body()
-{
-	create_test_inputs
-
-	WITH_C=$PWD/../with_C.out
-
-	atf_check -e empty -o save:$WITH_C -s exit:0 ls -C
-
-	echo "With -C usage"
-	cat $WITH_C
-
-	atf_check -e ignore -o not-empty -s exit:0 \
-	    egrep "0b00000001[[:space:]]+0b00000111[[:space:]]+0b00001101[[:space:]]+e[[:space:]]+stuv" $WITH_C
-	atf_check -e ignore -o not-empty -s exit:0 \
-	    egrep "0b00000010[[:space:]]+0b00001000[[:space:]]+0b00001110[[:space:]]+h[[:space:]]+wxyz" $WITH_C
-}
-
-atf_test_case I_flag
-I_flag_head()
-{
-	atf_set "descr" "Verify that the output from ls -I is the same as ls for an unprivileged user"
-}
-
-I_flag_body()
-{
-	create_test_inputs
-
-	WITH_I=$PWD/../with_I.out
-	WITHOUT_I=$PWD/../without_I.out
-
-	atf_check -e empty -o save:$WITH_I -s exit:0 ls -I
-	atf_check -e empty -o save:$WITHOUT_I -s exit:0 ls
-
-	echo "Explicit -I usage"
-	cat $WITH_I
-	echo "No -I usage"
-	cat $WITHOUT_I
-
-	atf_check_equal "$(cat $WITH_I)" "$(cat $WITHOUT_I)"
-}
-
-atf_test_case I_flag_voids_implied_A_flag_when_root
-I_flag_voids_implied_A_flag_when_root_head()
-{
-	atf_set "descr" "Verify that -I voids out implied -A for root"
-	atf_set "require.user" "root"
-}
-
-I_flag_voids_implied_A_flag_when_root_body()
-{
-	create_test_inputs
-
-	atf_check -o not-match:'\.f' -s exit:0 ls -I
-	atf_check -o not-match:'\.g' -s exit:0 ls -I
-
-	atf_check -o match:'\.f' -s exit:0 ls -A -I
-	atf_check -o match:'\.g' -s exit:0 ls -A -I
-}
-
-lcomma_flag_head()
-{
-	atf_set "descr" "Verify that -l, prints out the size with , delimiters"
-}
-
-lcomma_flag_body()
-{
-	create_test_inputs
-
-	atf_check \
-	    -o match:'\-rw\-r\-\-r\-\-[[:space:]]+.+[[:space:]]+1,000[[:space:]]+.+i' \
-	    env LC_ALL=en_US.ISO8859-1 ls -l, i
-}
-
-x_flag_head()
-{
-	atf_set "descr" "Verify that -x prints out one item per line"
-}
-
-x_flag_body()
-{
-	create_test_inputs
-
-	WITH_x=$PWD/../with_x.out
-
-	atf_check -e empty -o save:$WITH_x -s exit:0 ls -x
-
-	echo "With -x usage"
-	cat $WITH_x
-
-	atf_check -e ignore -o not-empty -s exit:0 \
-	    egrep "a[[:space:]]+c[[:space:]]+d[[:space:]]+e[[:space:]]+h" $WITH_x
-	atf_check -e ignore -o not-empty -s exit:0 \
-	    egrep "i[[:space:]]+j[[:space:]]+klmn[[:space:]]+opqr[[:space:]]+stuv" $WITH_x
-}
-
-1_flag_head()
-{
-	atf_set "descr" "Verify that -1 prints out one item per line"
-}
-
-1_flag_body()
-{
-	create_test_inputs
-
-	WITH_1=$PWD/../with_1.out
-	WITHOUT_1=$PWD/../without_1.out
-
-	atf_check -e empty -o save:$WITH_1 -s exit:0 ls -1
-	atf_check -e empty -o save:$WITHOUT_1 -s exit:0 \
-		sh -c 'for i in $(ls); do echo $i; done'
-
-	echo "Explicit -1 usage"
-	cat $WITH_1
-	echo "No -1 usage"
-	cat $WITHOUT_1
-
-	atf_check_equal "$(cat $WITH_1)" "$(cat $WITHOUT_1)"
-}
-
-atf_init_test_cases()
-{
-
-	atf_add_test_case A_flag
-	atf_add_test_case A_flag_implied_when_root
-	atf_add_test_case B_flag
-	atf_add_test_case C_flag
-	#atf_add_test_case D_flag
-	#atf_add_test_case F_flag
-	#atf_add_test_case G_flag
-	#atf_add_test_case H_flag
-	atf_add_test_case I_flag
-	atf_add_test_case I_flag_voids_implied_A_flag_when_root
-	#atf_add_test_case L_flag
-	#atf_add_test_case P_flag
-	#atf_add_test_case R_flag
-	#atf_add_test_case S_flag
-	#atf_add_test_case T_flag
-	#atf_add_test_case U_flag
-	#atf_add_test_case W_flag
-	#atf_add_test_case Z_flag
-	atf_add_test_case a_flag
-	#atf_add_test_case b_flag
-	#atf_add_test_case c_flag
-	#atf_add_test_case d_flag
-	#atf_add_test_case f_flag
-	#atf_add_test_case g_flag
-	#atf_add_test_case h_flag
-	#atf_add_test_case i_flag
-	#atf_add_test_case k_flag
-	#atf_add_test_case l_flag
-	atf_add_test_case lcomma_flag
-	#atf_add_test_case m_flag
-	#atf_add_test_case n_flag
-	#atf_add_test_case o_flag
-	#atf_add_test_case p_flag
-	#atf_add_test_case q_flag
-	#atf_add_test_case r_flag
-	#atf_add_test_case s_flag
-	#atf_add_test_case t_flag
-	#atf_add_test_case u_flag
-	#atf_add_test_case w_flag
-	atf_add_test_case x_flag
-	#atf_add_test_case y_flag
-	atf_add_test_case 1_flag
-}

From c646a13be046bfc481c30b35b66a32e006649f9b Mon Sep 17 00:00:00 2001
From: markj <markj@FreeBSD.org>
Date: Wed, 30 Sep 2015 03:30:24 +0000
Subject: [PATCH 139/380] 6266 harden dtrace_difo_chunksize() with respect to
 malicious DIF

illumos/illumos-gate@395c7a3dcfc66b8b671dc4b3c4a2f0ca26449922

Reviewed by: Alex Wilson <alex.wilson@joyent.com>
Reviewed by: Dan McDonald <danmcd@omniti.com>
Approved by: Garrett D'Amore <garrett@damore.org>
Author: Bryan Cantrill <bryan@joyent.com>
---
 uts/common/dtrace/dtrace.c   | 67 +++++++++++++++++++++++++++++++-----
 uts/common/sys/dtrace_impl.h | 19 +++++-----
 2 files changed, 70 insertions(+), 16 deletions(-)

diff --git a/uts/common/dtrace/dtrace.c b/uts/common/dtrace/dtrace.c
index aa4b1d67e07d..f2c3fec010dd 100644
--- a/uts/common/dtrace/dtrace.c
+++ b/uts/common/dtrace/dtrace.c
@@ -118,7 +118,7 @@ int		dtrace_destructive_disallow = 0;
 dtrace_optval_t	dtrace_nonroot_maxsize = (16 * 1024 * 1024);
 size_t		dtrace_difo_maxsize = (256 * 1024);
 dtrace_optval_t	dtrace_dof_maxsize = (8 * 1024 * 1024);
-size_t		dtrace_global_maxsize = (16 * 1024);
+size_t		dtrace_statvar_maxsize = (16 * 1024);
 size_t		dtrace_actions_max = (16 * 1024);
 size_t		dtrace_retain_max = 1024;
 dtrace_optval_t	dtrace_helper_actions_max = 1024;
@@ -592,13 +592,33 @@ dtrace_canstore_statvar(uint64_t addr, size_t sz,
     dtrace_statvar_t **svars, int nsvars)
 {
 	int i;
+	size_t maxglobalsize, maxlocalsize;
+
+	if (nsvars == 0)
+		return (0);
+
+	maxglobalsize = dtrace_statvar_maxsize;
+	maxlocalsize = (maxglobalsize + sizeof (uint64_t)) * NCPU;
 
 	for (i = 0; i < nsvars; i++) {
 		dtrace_statvar_t *svar = svars[i];
+		uint8_t scope;
+		size_t size;
 
-		if (svar == NULL || svar->dtsv_size == 0)
+		if (svar == NULL || (size = svar->dtsv_size) == 0)
 			continue;
 
+		scope = svar->dtsv_var.dtdv_scope;
+
+		/*
+		 * We verify that our size is valid in the spirit of providing
+		 * defense in depth:  we want to prevent attackers from using
+		 * DTrace to escalate an orthogonal kernel heap corruption bug
+		 * into the ability to store to arbitrary locations in memory.
+		 */
+		VERIFY((scope == DIFV_SCOPE_GLOBAL && size < maxglobalsize) ||
+		    (scope == DIFV_SCOPE_LOCAL && size < maxlocalsize));
+
 		if (DTRACE_INRANGE(addr, sz, svar->dtsv_data, svar->dtsv_size))
 			return (1);
 	}
@@ -4165,7 +4185,8 @@ dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs,
 
 		if (!dtrace_destructive_disallow &&
 		    dtrace_priv_proc_control(state, mstate) &&
-		    !dtrace_istoxic(kaddr, size)) {
+		    !dtrace_istoxic(kaddr, size) &&
+		    dtrace_canload(kaddr, size, mstate, vstate)) {
 			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
 			dtrace_copyout(kaddr, uaddr, size, flags);
 			DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
@@ -4180,7 +4201,8 @@ dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs,
 
 		if (!dtrace_destructive_disallow &&
 		    dtrace_priv_proc_control(state, mstate) &&
-		    !dtrace_istoxic(kaddr, size)) {
+		    !dtrace_istoxic(kaddr, size) &&
+		    dtrace_strcanload(kaddr, size, mstate, vstate)) {
 			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
 			dtrace_copyoutstr(kaddr, uaddr, size, flags);
 			DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
@@ -6078,6 +6100,11 @@ dtrace_dif_emulate(dtrace_difo_t *difo, dtrace_mstate_t *mstate,
 				    regs[r2] ? regs[r2] :
 				    dtrace_strsize_default) + 1;
 			} else {
+				if (regs[r2] > LONG_MAX) {
+					*flags |= CPU_DTRACE_ILLOP;
+					break;
+				}
+
 				tupregs[ttop].dttk_size = regs[r2];
 			}
 
@@ -9344,9 +9371,10 @@ dtrace_difo_validate(dtrace_difo_t *dp, dtrace_vstate_t *vstate, uint_t nregs,
 				break;
 			}
 
-			if (v->dtdv_scope == DIFV_SCOPE_GLOBAL &&
-			    vt->dtdt_size > dtrace_global_maxsize) {
-				err += efunc(i, "oversized by-ref global\n");
+			if ((v->dtdv_scope == DIFV_SCOPE_GLOBAL ||
+			    v->dtdv_scope == DIFV_SCOPE_LOCAL) &&
+			    vt->dtdt_size > dtrace_statvar_maxsize) {
+				err += efunc(i, "oversized by-ref static\n");
 				break;
 			}
 		}
@@ -9683,6 +9711,9 @@ dtrace_difo_chunksize(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
 				if (srd == 0)
 					return;
 
+				if (sval > LONG_MAX)
+					return;
+
 				tupregs[ttop++].dttk_size = sval;
 			}
 
@@ -9744,6 +9775,19 @@ dtrace_difo_chunksize(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
 		 */
 		size = P2ROUNDUP(size, sizeof (uint64_t));
 
+		/*
+		 * Before setting the chunk size, check that we're not going
+		 * to set it to a negative value...
+		 */
+		if (size > LONG_MAX)
+			return;
+
+		/*
+		 * ...and make certain that we didn't badly overflow.
+		 */
+		if (size < ksize || size < sizeof (dtrace_dynvar_t))
+			return;
+
 		if (size > vstate->dtvs_dynvars.dtds_chunksize)
 			vstate->dtvs_dynvars.dtds_chunksize = size;
 	}
@@ -13184,6 +13228,8 @@ dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size)
 	if ((dstate->dtds_chunksize = chunksize) == 0)
 		dstate->dtds_chunksize = DTRACE_DYNVAR_CHUNKSIZE;
 
+	VERIFY(dstate->dtds_chunksize < LONG_MAX);
+
 	if (size < (min = dstate->dtds_chunksize + sizeof (dtrace_dynhash_t)))
 		size = min;
 
@@ -13224,6 +13270,9 @@ dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size)
 	    ((uintptr_t)base + hashsize * sizeof (dtrace_dynhash_t));
 	limit = (uintptr_t)base + size;
 
+	VERIFY((uintptr_t)start < limit);
+	VERIFY((uintptr_t)start >= (uintptr_t)base);
+
 	maxper = (limit - (uintptr_t)start) / NCPU;
 	maxper = (maxper / dstate->dtds_chunksize) * dstate->dtds_chunksize;
 
@@ -13245,7 +13294,7 @@ dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size)
 			start = (dtrace_dynvar_t *)limit;
 		}
 
-		ASSERT(limit <= (uintptr_t)base + size);
+		VERIFY(limit <= (uintptr_t)base + size);
 
 		for (;;) {
 			next = (dtrace_dynvar_t *)((uintptr_t)dvar +
@@ -13254,6 +13303,8 @@ dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size)
 			if ((uintptr_t)next + dstate->dtds_chunksize >= limit)
 				break;
 
+			VERIFY((uintptr_t)dvar >= (uintptr_t)base &&
+			    (uintptr_t)dvar <= (uintptr_t)base + size);
 			dvar->dtdv_next = next;
 			dvar = next;
 		}
diff --git a/uts/common/sys/dtrace_impl.h b/uts/common/sys/dtrace_impl.h
index f79bf1e42ef8..3f4fbb0c2262 100644
--- a/uts/common/sys/dtrace_impl.h
+++ b/uts/common/sys/dtrace_impl.h
@@ -1290,16 +1290,19 @@ extern void dtrace_copystr(uintptr_t, uintptr_t, size_t, volatile uint16_t *);
 /*
  * DTrace Assertions
  *
- * DTrace calls ASSERT from probe context.  To assure that a failed ASSERT
- * does not induce a markedly more catastrophic failure (e.g., one from which
- * a dump cannot be gleaned), DTrace must define its own ASSERT to be one that
- * may safely be called from probe context.  This header file must thus be
- * included by any DTrace component that calls ASSERT from probe context, and
- * _only_ by those components.  (The only exception to this is kernel
- * debugging infrastructure at user-level that doesn't depend on calling
- * ASSERT.)
+ * DTrace calls ASSERT and VERIFY from probe context.  To assure that a failed
+ * ASSERT or VERIFY does not induce a markedly more catastrophic failure (e.g.,
+ * one from which a dump cannot be gleaned), DTrace must define its own ASSERT
+ * and VERIFY macros to be ones that may safely be called from probe context.
+ * This header file must thus be included by any DTrace component that calls
+ * ASSERT and/or VERIFY from probe context, and _only_ by those components.
+ * (The only exception to this is kernel debugging infrastructure at user-level
+ * that doesn't depend on calling ASSERT.)
  */
 #undef ASSERT
+#undef VERIFY
+#define	VERIFY(EX)	((void)((EX) || \
+			dtrace_assfail(#EX, __FILE__, __LINE__)))
 #ifdef DEBUG
 #define	ASSERT(EX)	((void)((EX) || \
 			dtrace_assfail(#EX, __FILE__, __LINE__)))

From 704f96b428e0d7ce98f612285b507713e53c0f8f Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Wed, 30 Sep 2015 17:40:09 +0000
Subject: [PATCH 140/380] Stop blowing up freebsd-current@ because kyua
 report-jenkins doesn't properly escape non-printable chars

---
 bin/ls/tests/ls_tests.sh | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index 3d0d1e917c91..79258fac9449 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -180,6 +180,7 @@ B_flag_head()
 
 B_flag_body()
 {
+	atf_skip "kyua report-jenkins doesn't properly escape non-printable chars: https://github.com/jmmv/kyua/issues/136"
 
 	atf_check -e empty -o empty -s exit:0 touch "$(printf "y\013z")"
 	atf_check -e empty -o match:'y\\013z' -s exit:0 ls -B
@@ -264,9 +265,29 @@ lcomma_flag_body()
 	    env LC_ALL=en_US.ISO8859-1 ls -l, i
 }
 
+L_flag_and_P_flag_head()
+{
+	atf_set "descr" "Verify that -L prints out the symbolic link and conversely -P prints out the target for the symbolic link"
+}
+
+L_flag_and_P_flag_body()
+{
+	atf_expect_fail "XXX: this is currently broken"
+
+	atf_check -e empty -o empty -s exit:0 mkdir a
+	atf_check -e empty -o empty -s exit:0 ln -s a/ b
+	atf_check -e empty -o match:b ls -L b
+	atf_check -e empty -o match:a ls -P b
+
+	atf_check -e empty -o empty -s exit:0 touch c
+	atf_check -e empty -o empty -s exit:0 ln -s c d
+	atf_check -e empty -o match:d ls -L d
+	atf_check -e empty -o match:c ls -P d
+}
+
 x_flag_head()
 {
-	atf_set "descr" "Verify that -x prints out one item per line"
+	atf_set "descr" "Verify that the output from ls -x is multi-column, sorted across"
 }
 
 x_flag_body()
@@ -323,8 +344,7 @@ atf_init_test_cases()
 	#atf_add_test_case H_flag
 	atf_add_test_case I_flag
 	atf_add_test_case I_flag_voids_implied_A_flag_when_root
-	#atf_add_test_case L_flag
-	#atf_add_test_case P_flag
+	atf_add_test_case L_flag_and_P_flag
 	#atf_add_test_case R_flag
 	#atf_add_test_case S_flag
 	#atf_add_test_case T_flag

From 8b9bcd3554f9699746ea59acf1da1510da7efeff Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Fri, 2 Oct 2015 06:34:34 +0000
Subject: [PATCH 141/380] Add testcases for ls -q, ls -w, and ls -y

---
 bin/ls/tests/ls_tests.sh | 143 +++++++++++++++++++++++++--------------
 1 file changed, 92 insertions(+), 51 deletions(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index 79258fac9449..da977abf975d 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -76,40 +76,6 @@ create_test_inputs()
 	atf_check -e empty -s exit:0 touch 0b00001111
 }
 
-atf_test_case a_flag
-a_flag_head()
-{
-	atf_set "descr" "Verify -a support"
-}
-
-a_flag_body()
-{
-	create_test_dir
-
-	# Make sure "." and ".." show up with -a
-	atf_check -e empty -o match:'\.[[:space:]]+\.\.'  -s exit:0 ls -ax
-
-	create_test_inputs
-
-	WITH_a=$PWD/../with_a.out
-	WITHOUT_a=$PWD/../without_a.out
-
-	atf_check -e empty -o save:$WITH_a -s exit:0 ls -A
-	atf_check -e empty -o save:$WITHOUT_a -s exit:0 ls
-
-	echo "-A usage"
-	cat $WITH_a
-	echo "No -A usage"
-	cat $WITHOUT_a
-
-	for dot_path in '\.f' '\.g'; do
-		atf_check -e empty -o not-empty -s exit:0 grep "${dot_path}" \
-		    $WITH_a
-		atf_check -e empty -o empty -s not-exit:0 grep "${dot_path}" \
-		    $WITHOUT_a
-	done
-}
-
 atf_test_case A_flag
 A_flag_head()
 {
@@ -251,20 +217,6 @@ I_flag_voids_implied_A_flag_when_root_body()
 	atf_check -o match:'\.g' -s exit:0 ls -A -I
 }
 
-lcomma_flag_head()
-{
-	atf_set "descr" "Verify that -l, prints out the size with , delimiters"
-}
-
-lcomma_flag_body()
-{
-	create_test_inputs
-
-	atf_check \
-	    -o match:'\-rw\-r\-\-r\-\-[[:space:]]+.+[[:space:]]+1,000[[:space:]]+.+i' \
-	    env LC_ALL=en_US.ISO8859-1 ls -l, i
-}
-
 L_flag_and_P_flag_head()
 {
 	atf_set "descr" "Verify that -L prints out the symbolic link and conversely -P prints out the target for the symbolic link"
@@ -285,6 +237,73 @@ L_flag_and_P_flag_body()
 	atf_check -e empty -o match:c ls -P d
 }
 
+atf_test_case a_flag
+a_flag_head()
+{
+	atf_set "descr" "Verify -a support"
+}
+
+a_flag_body()
+{
+	create_test_dir
+
+	# Make sure "." and ".." show up with -a
+	atf_check -e empty -o match:'\.[[:space:]]+\.\.'  -s exit:0 ls -ax
+
+	create_test_inputs
+
+	WITH_a=$PWD/../with_a.out
+	WITHOUT_a=$PWD/../without_a.out
+
+	atf_check -e empty -o save:$WITH_a -s exit:0 ls -a
+	atf_check -e empty -o save:$WITHOUT_a -s exit:0 ls
+
+	echo "-a usage"
+	cat $WITH_a
+	echo "No -a usage"
+	cat $WITHOUT_a
+
+	for dot_path in '\.f' '\.g'; do
+		atf_check -e empty -o not-empty -s exit:0 grep "${dot_path}" \
+		    $WITH_a
+		atf_check -e empty -o empty -s not-exit:0 grep "${dot_path}" \
+		    $WITHOUT_a
+	done
+}
+
+lcomma_flag_head()
+{
+	atf_set "descr" "Verify that -l, prints out the size with , delimiters"
+}
+
+lcomma_flag_body()
+{
+	create_test_inputs
+
+	atf_check \
+	    -o match:'\-rw\-r\-\-r\-\-[[:space:]]+.+[[:space:]]+1,000[[:space:]]+.+i' \
+	    env LC_ALL=en_US.ISO8859-1 ls -l, i
+}
+
+q_flag_and_w_flag_head()
+{
+	atf_set "descr" "Verify that the output from ls -q prints out '?' for ESC and ls -w prints out the escape character"
+}
+
+q_flag_and_w_flag_body()
+{
+	atf_skip "kyua report-jenkins doesn't properly escape non-printable chars: https://github.com/jmmv/kyua/issues/136"
+
+	create_test_dir
+
+	test_file="$(printf "y\01z")"
+
+	atf_check -e empty -o empty -s exit:0 touch "$test_file"
+
+	atf_check -e empty -o match:'y\?z' -s exit:0 ls -q "$test_file"
+	atf_check -e empty -o match:"$test_file" -s exit:0 ls -w "$test_file"
+}
+
 x_flag_head()
 {
 	atf_set "descr" "Verify that the output from ls -x is multi-column, sorted across"
@@ -307,6 +326,29 @@ x_flag_body()
 	    egrep "i[[:space:]]+j[[:space:]]+klmn[[:space:]]+opqr[[:space:]]+stuv" $WITH_x
 }
 
+y_flag_head()
+{
+	atf_set "descr" "Verify that the output from ls -y sorts the same way as sort(1)"
+}
+
+y_flag_body()
+{
+	create_test_inputs
+
+	WITH_sort=$PWD/../with_sort.out
+	WITH_y=$PWD/../with_y.out
+
+	atf_check -e empty -o save:$WITH_sort -s exit:0 sh -c 'ls -1 | sort'
+	atf_check -e empty -o save:$WITH_y -s exit:0 ls -1y
+
+	echo "Sorted with sort(1)"
+	cat $WITH_sort
+	echo "Sorted with -y"
+	cat $WITH_y
+
+	atf_check_equal "$(cat $WITH_sort)" "$(cat $WITH_y)"
+}
+
 1_flag_head()
 {
 	atf_set "descr" "Verify that -1 prints out one item per line"
@@ -366,13 +408,12 @@ atf_init_test_cases()
 	#atf_add_test_case n_flag
 	#atf_add_test_case o_flag
 	#atf_add_test_case p_flag
-	#atf_add_test_case q_flag
+	atf_add_test_case q_flag_and_w_flag
 	#atf_add_test_case r_flag
 	#atf_add_test_case s_flag
 	#atf_add_test_case t_flag
 	#atf_add_test_case u_flag
-	#atf_add_test_case w_flag
 	atf_add_test_case x_flag
-	#atf_add_test_case y_flag
+	atf_add_test_case y_flag
 	atf_add_test_case 1_flag
 }

From 5c5a968f419b1a0be6432a3d2cd4daa73bc14655 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Fri, 2 Oct 2015 07:06:26 +0000
Subject: [PATCH 142/380] Add testcases for ls -u

---
 bin/ls/tests/ls_tests.sh | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index da977abf975d..05addde77669 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -304,6 +304,32 @@ q_flag_and_w_flag_body()
 	atf_check -e empty -o match:"$test_file" -s exit:0 ls -w "$test_file"
 }
 
+u_flag_head()
+{
+	atf_set "descr" "Verify that the output from ls -u sorts by last access with -l and -t"
+}
+
+u_flag_body()
+{
+	create_test_dir
+
+	atf_check -e empty -o empty -s exit:0 touch a.file
+	sleep 0.3
+	atf_check -e empty -o empty -s exit:0 touch b.file
+	sleep 0.3
+
+	atf_check -e empty -o match:'b\.file' -s exit:0 sh -c 'ls -lu | tail -n 1'
+	atf_check -e empty -o match:'a\.file.*b\.file' -s exit:0 ls -Cu
+
+	atf_check -e empty -o empty -s exit:0 sh -c 'echo "i am a" > a.file'
+	sleep 0.3
+	atf_check -e empty -o match:'i am a' -s exit:0 cat a.file
+	sleep 0.3
+
+	atf_check -e empty -o match:'b\.file' -s exit:0 sh -c 'ls -lu | tail -n 1'
+	atf_check -e empty -o match:'a\.file.*b\.file' -s exit:0 ls -Cu
+}
+
 x_flag_head()
 {
 	atf_set "descr" "Verify that the output from ls -x is multi-column, sorted across"
@@ -412,7 +438,7 @@ atf_init_test_cases()
 	#atf_add_test_case r_flag
 	#atf_add_test_case s_flag
 	#atf_add_test_case t_flag
-	#atf_add_test_case u_flag
+	atf_add_test_case u_flag
 	atf_add_test_case x_flag
 	atf_add_test_case y_flag
 	atf_add_test_case 1_flag

From 319ea26a92f4d306a7d77b20eb23c446b63b8884 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Fri, 2 Oct 2015 07:26:24 +0000
Subject: [PATCH 143/380] Add ls -t testcases

Use sync instead of sleep
---
 bin/ls/tests/ls_tests.sh | 37 +++++++++++++++++++++++++++++++------
 1 file changed, 31 insertions(+), 6 deletions(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index 05addde77669..d8e07982c30f 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -304,9 +304,34 @@ q_flag_and_w_flag_body()
 	atf_check -e empty -o match:"$test_file" -s exit:0 ls -w "$test_file"
 }
 
+t_flag_head()
+{
+	atf_set "descr" "Verify that the output from ls -u sorts by last access with -t"
+}
+
+t_flag_body()
+{
+	create_test_dir
+
+	atf_check -e empty -o empty -s exit:0 touch a.file
+	sync
+	atf_check -e empty -o empty -s exit:0 touch b.file
+	sync
+
+	atf_check -e empty -o match:'a\.file' -s exit:0 sh -c 'ls -lt | tail -n 1'
+	atf_check -e empty -o match:'b\.file.*a\.file' -s exit:0 ls -Ct
+
+	atf_check -e empty -o empty -s exit:0 rm a.file
+	atf_check -e empty -o empty -s exit:0 sh -c 'echo "i am a" > a.file'
+	sync
+
+	atf_check -e empty -o match:'b\.file' -s exit:0 sh -c 'ls -lt | tail -n 1'
+	atf_check -e empty -o match:'a\.file.*b\.file' -s exit:0 ls -Ct
+}
+
 u_flag_head()
 {
-	atf_set "descr" "Verify that the output from ls -u sorts by last access with -l and -t"
+	atf_set "descr" "Verify that the output from ls -u sorts by last access"
 }
 
 u_flag_body()
@@ -314,17 +339,17 @@ u_flag_body()
 	create_test_dir
 
 	atf_check -e empty -o empty -s exit:0 touch a.file
-	sleep 0.3
+	sync
 	atf_check -e empty -o empty -s exit:0 touch b.file
-	sleep 0.3
+	sync
 
 	atf_check -e empty -o match:'b\.file' -s exit:0 sh -c 'ls -lu | tail -n 1'
 	atf_check -e empty -o match:'a\.file.*b\.file' -s exit:0 ls -Cu
 
 	atf_check -e empty -o empty -s exit:0 sh -c 'echo "i am a" > a.file'
-	sleep 0.3
+	sync
 	atf_check -e empty -o match:'i am a' -s exit:0 cat a.file
-	sleep 0.3
+	sync
 
 	atf_check -e empty -o match:'b\.file' -s exit:0 sh -c 'ls -lu | tail -n 1'
 	atf_check -e empty -o match:'a\.file.*b\.file' -s exit:0 ls -Cu
@@ -437,7 +462,7 @@ atf_init_test_cases()
 	atf_add_test_case q_flag_and_w_flag
 	#atf_add_test_case r_flag
 	#atf_add_test_case s_flag
-	#atf_add_test_case t_flag
+	atf_add_test_case t_flag
 	atf_add_test_case u_flag
 	atf_add_test_case x_flag
 	atf_add_test_case y_flag

From 50142a0097b649cf0f8084677a8a7fa8aab42ce6 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Fri, 2 Oct 2015 08:13:45 +0000
Subject: [PATCH 144/380] Add a testcase to test ls -s; remove an unnecessary
 sync

---
 bin/ls/tests/ls_tests.sh | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index d8e07982c30f..c3cf9dd64765 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -304,9 +304,31 @@ q_flag_and_w_flag_body()
 	atf_check -e empty -o match:"$test_file" -s exit:0 ls -w "$test_file"
 }
 
+s_flag_head()
+{
+	atf_set "descr" "Verify that the output from ls -s matches the output from stat(1)"
+}
+
+s_flag_body()
+{
+	create_test_dir
+
+	for filesize in 512 1 2048 10240 $(( 512 * 1024 )); do
+		atf_check -e ignore -o empty -s exit:0 \
+		    dd if=/dev/zero of=${filesize}.file bs=${filesize} count=1
+		files="${files} ${filesize}.file"
+	done
+
+	for file in $files; do
+		atf_check -e empty \
+		    -o match:"$(stat -f "%b" $file)[[:space:]]+$file" ls -s $file
+		stat -f "%b" $file
+	done
+}
+
 t_flag_head()
 {
-	atf_set "descr" "Verify that the output from ls -u sorts by last access with -t"
+	atf_set "descr" "Verify that the output from ls -t sorts by modification time"
 }
 
 t_flag_body()
@@ -314,7 +336,6 @@ t_flag_body()
 	create_test_dir
 
 	atf_check -e empty -o empty -s exit:0 touch a.file
-	sync
 	atf_check -e empty -o empty -s exit:0 touch b.file
 	sync
 
@@ -461,7 +482,7 @@ atf_init_test_cases()
 	#atf_add_test_case p_flag
 	atf_add_test_case q_flag_and_w_flag
 	#atf_add_test_case r_flag
-	#atf_add_test_case s_flag
+	atf_add_test_case s_flag
 	atf_add_test_case t_flag
 	atf_add_test_case u_flag
 	atf_add_test_case x_flag

From 9a4d28fcaaf1bef74b36ba8d9052d576a69908a6 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Fri, 2 Oct 2015 08:33:06 +0000
Subject: [PATCH 145/380] Add a testcase for ls -p

---
 bin/ls/tests/ls_tests.sh | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index c3cf9dd64765..7efd227920e5 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -285,6 +285,26 @@ lcomma_flag_body()
 	    env LC_ALL=en_US.ISO8859-1 ls -l, i
 }
 
+p_flag_head()
+{
+	atf_set "descr" "Verify that the output from ls -p prints out '/' after directories"
+}
+
+p_flag_body()
+{
+	create_test_inputs
+
+	for path in $(find -L .); do
+		suffix=
+		# If path is not a symlink and is a directory, then the suffix
+		# must be "/".
+		if [ ! -L "${path}" -a -d "$path" ]; then
+			suffix=/
+		fi
+		atf_check -e empty -o match:"$path${suffix}" ls -dp $path
+	done
+}
+
 q_flag_and_w_flag_head()
 {
 	atf_set "descr" "Verify that the output from ls -q prints out '?' for ESC and ls -w prints out the escape character"
@@ -479,7 +499,7 @@ atf_init_test_cases()
 	#atf_add_test_case m_flag
 	#atf_add_test_case n_flag
 	#atf_add_test_case o_flag
-	#atf_add_test_case p_flag
+	atf_add_test_case p_flag
 	atf_add_test_case q_flag_and_w_flag
 	#atf_add_test_case r_flag
 	atf_add_test_case s_flag

From 0d7ba2a8614988b2e10946de979360496c2f9361 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Fri, 2 Oct 2015 09:03:28 +0000
Subject: [PATCH 146/380] - Add a testcase for ls -n - Detect errors with find
 when grabbing the paths for testing ls -p - Be explicit with testing for an
 exit code of 0 when running ls -dp

---
 bin/ls/tests/ls_tests.sh | 30 +++++++++++++++++++++++++++---
 1 file changed, 27 insertions(+), 3 deletions(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index 7efd227920e5..f20b225b98d4 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -285,6 +285,26 @@ lcomma_flag_body()
 	    env LC_ALL=en_US.ISO8859-1 ls -l, i
 }
 
+n_flag_head()
+{
+	atf_set "descr" "Verify that the output from ls -p prints out numeric GIDs/UIDs instead of symbolic GIDs/UIDs"
+	atf_set "require.user" "root"
+}
+
+n_flag_body()
+{
+	daemon_gid=$(id -g daemon) || atf_skip "could not resolve gid for daemon (!)"
+	nobody_uid=$(id -u nobody) || atf_skip "could not resolve uid for nobody (!)"
+
+	atf_check -e empty -o empty -s exit:0 touch a.file
+	atf_check -e empty -o empty -s exit:0 chown $nobody_uid:$daemon_gid a.file
+
+	atf_check -e empty \
+	    -o match:'\-rw\-r\-\-r\-\-[[:space:]]+1[[:space:]]+'"$nobody_uid[[:space:]]+$daemon_gid"'[[:space:]]+.+a\.file' \
+	    ls -ln a.file
+
+}
+
 p_flag_head()
 {
 	atf_set "descr" "Verify that the output from ls -p prints out '/' after directories"
@@ -294,14 +314,18 @@ p_flag_body()
 {
 	create_test_inputs
 
-	for path in $(find -L .); do
+	paths=$(find -L .)
+	[ -n "$paths" ] || atf_skip 'Could not find any paths to iterate over (!)'
+
+	for path in $paths; do
 		suffix=
 		# If path is not a symlink and is a directory, then the suffix
 		# must be "/".
 		if [ ! -L "${path}" -a -d "$path" ]; then
 			suffix=/
 		fi
-		atf_check -e empty -o match:"$path${suffix}" ls -dp $path
+		atf_check -e empty -o match:"$path${suffix}" -s exit:0 \
+		    ls -dp $path
 	done
 }
 
@@ -497,7 +521,7 @@ atf_init_test_cases()
 	#atf_add_test_case l_flag
 	atf_add_test_case lcomma_flag
 	#atf_add_test_case m_flag
-	#atf_add_test_case n_flag
+	atf_add_test_case n_flag
 	#atf_add_test_case o_flag
 	atf_add_test_case p_flag
 	atf_add_test_case q_flag_and_w_flag

From 9c75161e0c0346b2512986bebab6415532c19098 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sat, 3 Oct 2015 09:47:29 +0000
Subject: [PATCH 147/380] Add testcases for ls -D, ls -F, ls -H, ls -k, and ls
 -r; separate out ls -L and ls -P testcases, but enable the -L testcases only

---
 bin/ls/tests/ls_tests.sh | 126 ++++++++++++++++++++++++++++++++-------
 1 file changed, 103 insertions(+), 23 deletions(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index f20b225b98d4..0d5dcce625be 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -45,7 +45,7 @@ create_test_inputs()
 {
 	create_test_dir
 
-	atf_check -e empty -s exit:0 mkdir -m 0755 -p a/b
+	atf_check -e empty -s exit:0 mkdir -m 0755 -p a/b/1
 	atf_check -e empty -s exit:0 ln -s a/b c
 	atf_check -e empty -s exit:0 touch d
 	atf_check -e empty -s exit:0 ln d e
@@ -58,7 +58,7 @@ create_test_inputs()
 	atf_check -e empty -s exit:0 touch klmn
 	atf_check -e empty -s exit:0 touch opqr
 	atf_check -e empty -s exit:0 touch stuv
-	atf_check -e empty -s exit:0 touch wxyz
+	atf_check -e empty -s exit:0 install -m 0755 /dev/null wxyz
 	atf_check -e empty -s exit:0 touch 0b00000001
 	atf_check -e empty -s exit:0 touch 0b00000010
 	atf_check -e empty -s exit:0 touch 0b00000011
@@ -175,6 +175,50 @@ C_flag_body()
 	    egrep "0b00000010[[:space:]]+0b00001000[[:space:]]+0b00001110[[:space:]]+h[[:space:]]+wxyz" $WITH_C
 }
 
+atf_test_case D_flag
+D_flag_head()
+{
+	atf_set "descr" "Verify that the output from ls -D modifies the time format used with ls -l"
+}
+
+D_flag_body()
+{
+	atf_check -e empty -o empty -s exit:0 touch a.file
+	atf_check -e empty -o match:"$(stat -f '%c[[:space:]]+%N' a.file)" \
+	    -s exit:0 ls -lD '%s'
+}
+
+atf_test_case F_flag
+F_flag_head()
+{
+	atf_set "descr" "Verify that the output from ls -F prints out appropriate symbols after files"
+}
+
+F_flag_body()
+{
+	create_test_inputs
+
+	atf_check -e empty -o match:'a/' -s exit:0 ls -F
+	atf_check -e empty -o match:'c@' -s exit:0 ls -F
+	atf_check -e empty -o match:'h\|' -s exit:0 ls -F
+	atf_check -e empty -o match:'j=' -s exit:0 ls -F
+	#atf_check -e empty -o match:'<whiteout-file>%' -s exit:0 ls -F
+	atf_check -e empty -o match:'wxyz\*' -s exit:0 ls -F
+}
+
+atf_test_case H_flag
+H_flag_head()
+{
+	atf_set "descr" "Verify that ls -H follows symlinks"
+}
+
+H_flag_body()
+{
+	create_test_inputs
+
+	atf_check -e empty -o match:'1' -s exit:0 ls -H c
+}
+
 atf_test_case I_flag
 I_flag_head()
 {
@@ -217,24 +261,16 @@ I_flag_voids_implied_A_flag_when_root_body()
 	atf_check -o match:'\.g' -s exit:0 ls -A -I
 }
 
-L_flag_and_P_flag_head()
+L_flag_head()
 {
 	atf_set "descr" "Verify that -L prints out the symbolic link and conversely -P prints out the target for the symbolic link"
 }
 
-L_flag_and_P_flag_body()
+L_flag_body()
 {
-	atf_expect_fail "XXX: this is currently broken"
-
-	atf_check -e empty -o empty -s exit:0 mkdir a
-	atf_check -e empty -o empty -s exit:0 ln -s a/ b
-	atf_check -e empty -o match:b ls -L b
-	atf_check -e empty -o match:a ls -P b
-
-	atf_check -e empty -o empty -s exit:0 touch c
-	atf_check -e empty -o empty -s exit:0 ln -s c d
-	atf_check -e empty -o match:d ls -L d
-	atf_check -e empty -o match:c ls -P d
+	atf_check -e empty -o empty -s exit:0 ln -s target1/target2 link1
+	atf_check -e empty -o match:link1 -s exit:0 ls -L
+	atf_check -e empty -o not-match:target1/target2 -s exit:0 ls -L
 }
 
 atf_test_case a_flag
@@ -271,9 +307,30 @@ a_flag_body()
 	done
 }
 
+k_flag_head()
+{
+	atf_set "descr" "Verify that -k prints out the size with a block size of 1kB"
+}
+
+k_flag_body()
+{
+	create_test_dir
+
+	for filesize in 512 1 2048 10240 $(( 512 * 1024 )); do
+		atf_check -e ignore -o empty -s exit:0 \
+		    dd if=/dev/zero of=${filesize}.file bs=${filesize} count=1
+		files="${files} ${filesize}.file"
+	done
+
+	for file in $files; do
+		atf_check -e empty \
+		    -o match:"[[:space:]]+$(stat -f "%z" $file)[[:space:]]+.+[[:space:]]+$file" ls -lk $file
+	done
+}
+
 lcomma_flag_head()
 {
-	atf_set "descr" "Verify that -l, prints out the size with , delimiters"
+	atf_set "descr" "Verify that -l, prints out the size with ',' delimiters"
 }
 
 lcomma_flag_body()
@@ -348,6 +405,29 @@ q_flag_and_w_flag_body()
 	atf_check -e empty -o match:"$test_file" -s exit:0 ls -w "$test_file"
 }
 
+r_flag_head()
+{
+	atf_set "descr" "Verify that the output from ls -r sorts the same way as reverse sorting with sort(1)"
+}
+
+r_flag_body()
+{
+	create_test_inputs
+
+	WITH_r=$PWD/../with_r.out
+	WITH_sort=$PWD/../with_sort.out
+
+	atf_check -e empty -o save:$WITH_r -s exit:0 ls -1r
+	atf_check -e empty -o save:$WITH_sort -s exit:0 sh -c 'ls -1 | sort -r'
+
+	echo "Sorted with -r"
+	cat $WITH_r
+	echo "Reverse sorted with sort(1)"
+	cat $WITH_sort
+
+	atf_check_equal "$(cat $WITH_r)" "$(cat $WITH_sort)"
+}
+
 s_flag_head()
 {
 	atf_set "descr" "Verify that the output from ls -s matches the output from stat(1)"
@@ -366,7 +446,6 @@ s_flag_body()
 	for file in $files; do
 		atf_check -e empty \
 		    -o match:"$(stat -f "%b" $file)[[:space:]]+$file" ls -s $file
-		stat -f "%b" $file
 	done
 }
 
@@ -496,13 +575,14 @@ atf_init_test_cases()
 	atf_add_test_case A_flag_implied_when_root
 	atf_add_test_case B_flag
 	atf_add_test_case C_flag
-	#atf_add_test_case D_flag
-	#atf_add_test_case F_flag
+	atf_add_test_case D_flag
+	atf_add_test_case F_flag
 	#atf_add_test_case G_flag
-	#atf_add_test_case H_flag
+	atf_add_test_case H_flag
 	atf_add_test_case I_flag
 	atf_add_test_case I_flag_voids_implied_A_flag_when_root
-	atf_add_test_case L_flag_and_P_flag
+	atf_add_test_case L_flag
+	#atf_add_test_case P_flag
 	#atf_add_test_case R_flag
 	#atf_add_test_case S_flag
 	#atf_add_test_case T_flag
@@ -517,7 +597,7 @@ atf_init_test_cases()
 	#atf_add_test_case g_flag
 	#atf_add_test_case h_flag
 	#atf_add_test_case i_flag
-	#atf_add_test_case k_flag
+	atf_add_test_case k_flag
 	#atf_add_test_case l_flag
 	atf_add_test_case lcomma_flag
 	#atf_add_test_case m_flag
@@ -525,7 +605,7 @@ atf_init_test_cases()
 	#atf_add_test_case o_flag
 	atf_add_test_case p_flag
 	atf_add_test_case q_flag_and_w_flag
-	#atf_add_test_case r_flag
+	atf_add_test_case r_flag
 	atf_add_test_case s_flag
 	atf_add_test_case t_flag
 	atf_add_test_case u_flag

From 5d6d6c0747a31957b7fd08f01592a4780256b37f Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sun, 4 Oct 2015 06:22:23 +0000
Subject: [PATCH 148/380] Add testcases for ls -d, ls -g, ls -h, ls -i, and ls
 -o; refactor creating files of known sizes into create_test_inputs2

---
 bin/ls/tests/ls_tests.sh | 177 ++++++++++++++++++++++++++++++++++-----
 1 file changed, 155 insertions(+), 22 deletions(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index 0d5dcce625be..d2d17749c88f 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -76,6 +76,32 @@ create_test_inputs()
 	atf_check -e empty -s exit:0 touch 0b00001111
 }
 
+KB=1024
+MB=$(( 1024 * $KB ))
+GB=$(( 1024 * $MB ))
+TB=$(( 1024 * $GB ))
+PB=$(( 1024 * $TB ))
+
+create_test_inputs2()
+{
+	create_test_dir
+
+	for filesize in 1 512 $(( 2 * $KB )) $(( 10 * $KB )) $(( 512 * $KB )); \
+	do
+		atf_check -e ignore -o empty -s exit:0 \
+		    dd if=/dev/zero of=${filesize}.file bs=1 \
+		    count=1 oseek=${filesize} conv=sparse
+		files="${files} ${filesize}.file"
+	done
+
+	for filesize in $MB $GB $TB; do
+		atf_check -e ignore -o empty -s exit:0 \
+		    dd if=/dev/zero of=${filesize}.file bs=$MB \
+		    count=1 oseek=$(( $filesize / $MB )) conv=sparse
+		files="${files} ${filesize}.file"
+	done
+}
+
 atf_test_case A_flag
 A_flag_head()
 {
@@ -307,6 +333,103 @@ a_flag_body()
 	done
 }
 
+d_flag_head()
+{
+	atf_set "descr" "Verify that -d doesn't descend down directories"
+}
+
+d_flag_body()
+{
+	create_test_dir
+
+	output=$PWD/../output
+
+	atf_check -e empty -o empty -s exit:0 mkdir -p a/b
+
+	for path in . $PWD a; do
+		atf_check -e empty -o save:$output -s exit:0 ls -d $path
+		atf_check_equal "$(cat $output)" "$path"
+	done
+}
+
+g_flag_head()
+{
+	atf_set "descr" "Verify that -g does nothing (compatibility flag)"
+}
+
+g_flag_body()
+{
+	create_test_inputs2
+	for file in $files; do
+		atf_check -e empty -o match:"$(ls -a $file)" -s exit:0 \
+		    ls -ag $file
+		atf_check -e empty -o match:"$(ls -la $file)" -s exit:0 \
+		    ls -alg $file
+	done
+}
+
+h_flag_head()
+{
+	atf_set "descr" "Verify that -h prints out the humanized units for file sizes with ls -l"
+}
+
+h_flag_body()
+{
+	# XXX: this test doesn't currently show how 999 bytes will be 999B,
+	# but 1000 bytes will be 1.0K, due to how humanize_number(3) works.
+	create_test_inputs2
+	for file in $files; do
+		file_size=$(stat -f '%z' "$file") || \
+		    atf_fail "stat'ing $file failed"
+		scale=2
+		if [ $file_size -lt $KB ]; then
+			divisor=1
+			scale=0
+			suffix=B
+		elif [ $file_size -lt $MB ]; then
+			divisor=$KB
+			suffix=K
+		elif [ $file_size -lt $GB ]; then
+			divisor=$MB
+			suffix=M
+		elif [ $file_size -lt $TB ]; then
+			divisor=$GB
+			suffix=G
+		elif [ $file_size -lt $PB ]; then
+			divisor=$TB
+			suffix=T
+		else
+			divisor=$PB
+			suffix=P
+		fi
+
+		bc_expr="$(printf "scale=%s\n%s/%s\nquit" $scale $file_size $divisor)"
+		size_humanized=$(bc -e "$bc_expr" | tr '.' '\.' | sed -e 's,\.00,,')
+
+		atf_check -e empty -o match:"$size_humanized.+$file" \
+		    -s exit:0 ls -hl $file
+	done
+}
+
+i_flag_head()
+{
+	atf_set "descr" "Verify that -i prints out the inode for files"
+}
+
+i_flag_body()
+{
+	create_test_inputs
+
+	paths=$(find -L . -mindepth 1)
+	[ -n "$paths" ] || atf_skip 'Could not find any paths to iterate over (!)'
+
+	for path in $paths; do
+		atf_check -e empty \
+		    -o match:"$(stat -f '[[:space:]]*%i[[:space:]]+%N' $path)" \
+		    -s exit:0 ls -d1i $path
+	done
+}
+
 k_flag_head()
 {
 	atf_set "descr" "Verify that -k prints out the size with a block size of 1kB"
@@ -314,14 +437,7 @@ k_flag_head()
 
 k_flag_body()
 {
-	create_test_dir
-
-	for filesize in 512 1 2048 10240 $(( 512 * 1024 )); do
-		atf_check -e ignore -o empty -s exit:0 \
-		    dd if=/dev/zero of=${filesize}.file bs=${filesize} count=1
-		files="${files} ${filesize}.file"
-	done
-
+	create_test_inputs2
 	for file in $files; do
 		atf_check -e empty \
 		    -o match:"[[:space:]]+$(stat -f "%z" $file)[[:space:]]+.+[[:space:]]+$file" ls -lk $file
@@ -344,7 +460,7 @@ lcomma_flag_body()
 
 n_flag_head()
 {
-	atf_set "descr" "Verify that the output from ls -p prints out numeric GIDs/UIDs instead of symbolic GIDs/UIDs"
+	atf_set "descr" "Verify that the output from ls -n prints out numeric GIDs/UIDs instead of symbolic GIDs/UIDs"
 	atf_set "require.user" "root"
 }
 
@@ -362,6 +478,30 @@ n_flag_body()
 
 }
 
+o_flag_head()
+{
+	atf_set "descr" "Verify that the output from ls -o prints out the chflag values or '-' if none are set"
+	atf_set "require.user" "root"
+}
+
+o_flag_body()
+{
+	local size=12345
+
+	create_test_dir
+
+	atf_check -e ignore -o empty -s exit:0 dd if=/dev/zero of=a.file \
+	    bs=$size count=1
+	atf_check -e ignore -o empty -s exit:0 dd if=/dev/zero of=b.file \
+	    bs=$size count=1
+	atf_check -e empty -o empty -s exit:0 chflags uarch a.file
+
+	atf_check -e empty -o match:"[[:space:]]+uarch[[:space:]]$size+.+a\\.file" \
+	    -s exit:0 ls -lo a.file
+	atf_check -e empty -o match:"[[:space:]]+\\-[[:space:]]$size+.+b\\.file" \
+	    -s exit:0 ls -lo b.file
+}
+
 p_flag_head()
 {
 	atf_set "descr" "Verify that the output from ls -p prints out '/' after directories"
@@ -435,14 +575,7 @@ s_flag_head()
 
 s_flag_body()
 {
-	create_test_dir
-
-	for filesize in 512 1 2048 10240 $(( 512 * 1024 )); do
-		atf_check -e ignore -o empty -s exit:0 \
-		    dd if=/dev/zero of=${filesize}.file bs=${filesize} count=1
-		files="${files} ${filesize}.file"
-	done
-
+	create_test_inputs2
 	for file in $files; do
 		atf_check -e empty \
 		    -o match:"$(stat -f "%b" $file)[[:space:]]+$file" ls -s $file
@@ -592,17 +725,17 @@ atf_init_test_cases()
 	atf_add_test_case a_flag
 	#atf_add_test_case b_flag
 	#atf_add_test_case c_flag
-	#atf_add_test_case d_flag
+	atf_add_test_case d_flag
 	#atf_add_test_case f_flag
-	#atf_add_test_case g_flag
-	#atf_add_test_case h_flag
-	#atf_add_test_case i_flag
+	atf_add_test_case g_flag
+	atf_add_test_case h_flag
+	atf_add_test_case i_flag
 	atf_add_test_case k_flag
 	#atf_add_test_case l_flag
 	atf_add_test_case lcomma_flag
 	#atf_add_test_case m_flag
 	atf_add_test_case n_flag
-	#atf_add_test_case o_flag
+	atf_add_test_case o_flag
 	atf_add_test_case p_flag
 	atf_add_test_case q_flag_and_w_flag
 	atf_add_test_case r_flag

From 0341619b91675a8337cb3b836bed4354ef7bc060 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sun, 4 Oct 2015 06:31:30 +0000
Subject: [PATCH 149/380] Add ls -m testcases and check for /usr/bin/bc in the
 ls -h testcases

---
 bin/ls/tests/ls_tests.sh | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index d2d17749c88f..8cbb8c273599 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -371,6 +371,7 @@ g_flag_body()
 h_flag_head()
 {
 	atf_set "descr" "Verify that -h prints out the humanized units for file sizes with ls -l"
+	atf_set "require.files" "/usr/bin/bc"
 }
 
 h_flag_body()
@@ -458,6 +459,24 @@ lcomma_flag_body()
 	    env LC_ALL=en_US.ISO8859-1 ls -l, i
 }
 
+m_flag_head()
+{
+	atf_set "descr" "Verify that the output from ls -m is comma-separated"
+}
+
+m_flag_body()
+{
+	create_test_dir
+
+	output=$PWD/../output
+
+	atf_check -e empty -o empty -s exit:0 touch ,, "a,b " c d e
+
+	atf_check -e empty -o save:$output -s exit:0 ls -m
+
+	atf_check_equal "$(cat $output)" ",,, a,b , c, d, e"
+}
+
 n_flag_head()
 {
 	atf_set "descr" "Verify that the output from ls -n prints out numeric GIDs/UIDs instead of symbolic GIDs/UIDs"
@@ -733,7 +752,7 @@ atf_init_test_cases()
 	atf_add_test_case k_flag
 	#atf_add_test_case l_flag
 	atf_add_test_case lcomma_flag
-	#atf_add_test_case m_flag
+	atf_add_test_case m_flag
 	atf_add_test_case n_flag
 	atf_add_test_case o_flag
 	atf_add_test_case p_flag

From d5282dd217cad3ac893319e9b8dd200d408b15a4 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sun, 4 Oct 2015 08:18:24 +0000
Subject: [PATCH 150/380] Look for awk/sort explicitly

---
 bin/ls/tests/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/ls/tests/Makefile b/bin/ls/tests/Makefile
index 38770e3a0521..ba8038c1ce93 100644
--- a/bin/ls/tests/Makefile
+++ b/bin/ls/tests/Makefile
@@ -6,6 +6,6 @@ ATF_TESTS_SH+=	ls_tests
 # This seems like overkill, but the idea in mind is that all of the testcases
 # should be runnable as !root
 TEST_METADATA.ls_tests+=	required_user="unprivileged"
-TEST_METADATA.ls_tests+=	required_files="/usr/bin/nc"
+TEST_METADATA.ls_tests+=	required_files="/usr/bin/awk /usr/bin/nc /usr/bin/sort"
 
 .include <bsd.test.mk>

From 5d2df4d9ab000b25716077acc7db1b9ff1aaf87f Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sun, 4 Oct 2015 08:18:37 +0000
Subject: [PATCH 151/380] Add a ls -S testcase

---
 bin/ls/tests/ls_tests.sh | 47 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 46 insertions(+), 1 deletion(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index 8cbb8c273599..181c46647de7 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -299,6 +299,51 @@ L_flag_body()
 	atf_check -e empty -o not-match:target1/target2 -s exit:0 ls -L
 }
 
+S_flag_head()
+{
+	atf_set "descr" "Verify that -S sorts by file size, then by filename lexicographically"
+}
+
+S_flag_body()
+{
+	create_test_dir
+
+	file_list_dir=$PWD/../files
+
+	atf_check -e empty -o empty -s exit:0 mkdir -p $file_list_dir
+
+	create_test_inputs
+	create_test_inputs2
+
+	WITH_S=$PWD/../with_S.out
+	WITHOUT_S=$PWD/../without_S.out
+
+	atf_check -e empty -o save:$WITH_S ls -D '%s' -lS
+	atf_check -e empty -o save:$WITHOUT_S ls -D '%s' -l
+
+	#echo "-lS usage"
+	#cat $WITH_S
+	#echo "-l usage"
+	#cat $WITHOUT_S
+
+	WITH_S_parsed=$(awk '! /^total/ { print $7 }' $WITH_S)
+	set -- $(awk '! /^total/ { print $5, $7 }' $WITHOUT_S)
+	while [ $# -gt 0 ]; do
+		size=$1; shift
+		filename=$1; shift
+		echo $filename >> $file_list_dir/${size}
+	done
+	file_lists=$(find $file_list_dir -type f -exec basename {} \; | sort -nr)
+	WITHOUT_S_parsed=$(for file_list in $file_lists; do sort < $file_list_dir/$file_list; done)
+
+	echo "-lS usage (parsed)"
+	echo "$WITH_S_parsed"
+	echo "-l usage (parsed)"
+	echo "$WITHOUT_S_parsed"
+
+	atf_check_equal "$WITHOUT_S_parsed" "$WITH_S_parsed"
+}
+
 atf_test_case a_flag
 a_flag_head()
 {
@@ -736,7 +781,7 @@ atf_init_test_cases()
 	atf_add_test_case L_flag
 	#atf_add_test_case P_flag
 	#atf_add_test_case R_flag
-	#atf_add_test_case S_flag
+	atf_add_test_case S_flag
 	#atf_add_test_case T_flag
 	#atf_add_test_case U_flag
 	#atf_add_test_case W_flag

From fa7f61440f22d7fe1fcc57fca1b767dac8fa2991 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sun, 4 Oct 2015 08:36:28 +0000
Subject: [PATCH 152/380] Add atf_test_case before all _head function
 definitions

---
 bin/ls/tests/ls_tests.sh | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index 181c46647de7..d0e7897fffab 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -287,6 +287,7 @@ I_flag_voids_implied_A_flag_when_root_body()
 	atf_check -o match:'\.g' -s exit:0 ls -A -I
 }
 
+atf_test_case L_flag
 L_flag_head()
 {
 	atf_set "descr" "Verify that -L prints out the symbolic link and conversely -P prints out the target for the symbolic link"
@@ -299,6 +300,7 @@ L_flag_body()
 	atf_check -e empty -o not-match:target1/target2 -s exit:0 ls -L
 }
 
+atf_test_case S_flag
 S_flag_head()
 {
 	atf_set "descr" "Verify that -S sorts by file size, then by filename lexicographically"
@@ -378,6 +380,7 @@ a_flag_body()
 	done
 }
 
+atf_test_case d_flag
 d_flag_head()
 {
 	atf_set "descr" "Verify that -d doesn't descend down directories"
@@ -397,6 +400,7 @@ d_flag_body()
 	done
 }
 
+atf_test_case g_flag
 g_flag_head()
 {
 	atf_set "descr" "Verify that -g does nothing (compatibility flag)"
@@ -413,6 +417,7 @@ g_flag_body()
 	done
 }
 
+atf_test_case h_flag
 h_flag_head()
 {
 	atf_set "descr" "Verify that -h prints out the humanized units for file sizes with ls -l"
@@ -457,6 +462,7 @@ h_flag_body()
 	done
 }
 
+atf_test_case i_flag
 i_flag_head()
 {
 	atf_set "descr" "Verify that -i prints out the inode for files"
@@ -476,6 +482,7 @@ i_flag_body()
 	done
 }
 
+atf_test_case k_flag
 k_flag_head()
 {
 	atf_set "descr" "Verify that -k prints out the size with a block size of 1kB"
@@ -490,6 +497,7 @@ k_flag_body()
 	done
 }
 
+atf_test_case lcomma_flag
 lcomma_flag_head()
 {
 	atf_set "descr" "Verify that -l, prints out the size with ',' delimiters"
@@ -504,6 +512,7 @@ lcomma_flag_body()
 	    env LC_ALL=en_US.ISO8859-1 ls -l, i
 }
 
+atf_test_case m_flag
 m_flag_head()
 {
 	atf_set "descr" "Verify that the output from ls -m is comma-separated"
@@ -522,6 +531,7 @@ m_flag_body()
 	atf_check_equal "$(cat $output)" ",,, a,b , c, d, e"
 }
 
+atf_test_case n_flag
 n_flag_head()
 {
 	atf_set "descr" "Verify that the output from ls -n prints out numeric GIDs/UIDs instead of symbolic GIDs/UIDs"
@@ -542,6 +552,7 @@ n_flag_body()
 
 }
 
+atf_test_case o_flag
 o_flag_head()
 {
 	atf_set "descr" "Verify that the output from ls -o prints out the chflag values or '-' if none are set"
@@ -566,6 +577,7 @@ o_flag_body()
 	    -s exit:0 ls -lo b.file
 }
 
+atf_test_case p_flag
 p_flag_head()
 {
 	atf_set "descr" "Verify that the output from ls -p prints out '/' after directories"
@@ -590,6 +602,7 @@ p_flag_body()
 	done
 }
 
+atf_test_case q_flag_and_w_flag
 q_flag_and_w_flag_head()
 {
 	atf_set "descr" "Verify that the output from ls -q prints out '?' for ESC and ls -w prints out the escape character"
@@ -609,6 +622,7 @@ q_flag_and_w_flag_body()
 	atf_check -e empty -o match:"$test_file" -s exit:0 ls -w "$test_file"
 }
 
+atf_test_case r_flag
 r_flag_head()
 {
 	atf_set "descr" "Verify that the output from ls -r sorts the same way as reverse sorting with sort(1)"
@@ -632,6 +646,7 @@ r_flag_body()
 	atf_check_equal "$(cat $WITH_r)" "$(cat $WITH_sort)"
 }
 
+atf_test_case s_flag
 s_flag_head()
 {
 	atf_set "descr" "Verify that the output from ls -s matches the output from stat(1)"
@@ -646,6 +661,7 @@ s_flag_body()
 	done
 }
 
+atf_test_case t_flag
 t_flag_head()
 {
 	atf_set "descr" "Verify that the output from ls -t sorts by modification time"
@@ -670,6 +686,7 @@ t_flag_body()
 	atf_check -e empty -o match:'a\.file.*b\.file' -s exit:0 ls -Ct
 }
 
+atf_test_case u_flag
 u_flag_head()
 {
 	atf_set "descr" "Verify that the output from ls -u sorts by last access"
@@ -696,6 +713,7 @@ u_flag_body()
 	atf_check -e empty -o match:'a\.file.*b\.file' -s exit:0 ls -Cu
 }
 
+atf_test_case x_flag
 x_flag_head()
 {
 	atf_set "descr" "Verify that the output from ls -x is multi-column, sorted across"
@@ -718,6 +736,7 @@ x_flag_body()
 	    egrep "i[[:space:]]+j[[:space:]]+klmn[[:space:]]+opqr[[:space:]]+stuv" $WITH_x
 }
 
+atf_test_case y_flag
 y_flag_head()
 {
 	atf_set "descr" "Verify that the output from ls -y sorts the same way as sort(1)"
@@ -741,6 +760,7 @@ y_flag_body()
 	atf_check_equal "$(cat $WITH_sort)" "$(cat $WITH_y)"
 }
 
+atf_test_case 1_flag
 1_flag_head()
 {
 	atf_set "descr" "Verify that -1 prints out one item per line"

From 5621453fc98c155ffdb6ac92006073d285b60318 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sun, 4 Oct 2015 09:16:27 +0000
Subject: [PATCH 153/380] Add testcases that exercise ls -b and ls -T

---
 bin/ls/tests/ls_tests.sh | 37 +++++++++++++++++++++++++++++++++++--
 1 file changed, 35 insertions(+), 2 deletions(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index d0e7897fffab..6f3520295cf0 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -346,6 +346,25 @@ S_flag_body()
 	atf_check_equal "$WITHOUT_S_parsed" "$WITH_S_parsed"
 }
 
+atf_test_case T_flag
+T_flag_head()
+{
+	atf_set "descr" "Verify -T support"
+}
+
+T_flag_body()
+{
+	create_test_dir
+
+	atf_check -e empty -o empty -s exit:0 touch a.file
+
+	birthtime_in_secs=$(stat -f %B -t %s a.file)
+	birthtime=$(date -j -f %s $birthtime_in_secs +"[[:space:]]+%b[[:space:]]+%e[[:space:]]+%H:%M:%S[[:space:]]+%Y")
+
+	atf_check -e empty -o match:"$birthtime"'[[:space:]]+a\.file' \
+	    -s exit:0 ls -lT a.file
+}
+
 atf_test_case a_flag
 a_flag_head()
 {
@@ -380,6 +399,20 @@ a_flag_body()
 	done
 }
 
+atf_test_case b_flag
+b_flag_head()
+{
+	atf_set "descr" "Verify that the output from ls -b prints out non-printable characters"
+}
+
+b_flag_body()
+{
+	atf_skip "kyua report-jenkins doesn't properly escape non-printable chars: https://github.com/jmmv/kyua/issues/136"
+
+	atf_check -e empty -o empty -s exit:0 touch "$(printf "y\013z")"
+	atf_check -e empty -o match:'y\\vz' -s exit:0 ls -b
+}
+
 atf_test_case d_flag
 d_flag_head()
 {
@@ -802,12 +835,12 @@ atf_init_test_cases()
 	#atf_add_test_case P_flag
 	#atf_add_test_case R_flag
 	atf_add_test_case S_flag
-	#atf_add_test_case T_flag
+	atf_add_test_case T_flag
 	#atf_add_test_case U_flag
 	#atf_add_test_case W_flag
 	#atf_add_test_case Z_flag
 	atf_add_test_case a_flag
-	#atf_add_test_case b_flag
+	atf_add_test_case b_flag
 	#atf_add_test_case c_flag
 	atf_add_test_case d_flag
 	#atf_add_test_case f_flag

From ed9ddbeaef185da3438b35de6567425cac10efc3 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sun, 4 Oct 2015 09:25:57 +0000
Subject: [PATCH 154/380] Add another case for ls -F (regular file)

---
 bin/ls/tests/ls_tests.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index 6f3520295cf0..cecd85626e69 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -229,6 +229,7 @@ F_flag_body()
 	atf_check -e empty -o match:'h\|' -s exit:0 ls -F
 	atf_check -e empty -o match:'j=' -s exit:0 ls -F
 	#atf_check -e empty -o match:'<whiteout-file>%' -s exit:0 ls -F
+	atf_check -e empty -o match:'stuv' -s exit:0 ls -F
 	atf_check -e empty -o match:'wxyz\*' -s exit:0 ls -F
 }
 

From 5e5a08d5b5e10abb6f9ec770c142e516084a3a48 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sun, 4 Oct 2015 10:26:58 +0000
Subject: [PATCH 155/380] Add a ls -R testcase; remove some debugging in the ls
 -S testcase

---
 bin/ls/tests/ls_tests.sh | 38 ++++++++++++++++++++++++++++++++------
 1 file changed, 32 insertions(+), 6 deletions(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index cecd85626e69..224e0d425f63 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -301,6 +301,37 @@ L_flag_body()
 	atf_check -e empty -o not-match:target1/target2 -s exit:0 ls -L
 }
 
+atf_test_case R_flag
+R_flag_head()
+{
+	atf_set "descr" "Verify that the output from ls -R prints out the directory contents recursively"
+}
+
+R_flag_body()
+{
+	create_test_inputs
+
+	WITH_R=$PWD/../with_R.out
+	WITH_R_expected_output=$PWD/../with_R_expected.out
+
+	atf_check -e empty -o save:$WITH_R -s exit:0 ls -R
+
+	set -- . $(find -s . \! -name '.*' -type d)
+	while [ $# -gt 0 ]; do
+		dir=$1; shift
+		[ "$dir" != "." ] && echo "$dir:"
+		(cd $dir && ls -1A | sed -e '/^\./d')
+		[ $# -ne 0 ] && echo
+	done > $WITH_R_expected_output
+
+	echo "-R usage"
+	cat $WITH_R
+	echo "-R expected output"
+	cat $WITH_R_expected_output
+
+	atf_check_equal "$(cat $WITH_R)" "$(cat $WITH_R_expected_output)"
+}
+
 atf_test_case S_flag
 S_flag_head()
 {
@@ -324,11 +355,6 @@ S_flag_body()
 	atf_check -e empty -o save:$WITH_S ls -D '%s' -lS
 	atf_check -e empty -o save:$WITHOUT_S ls -D '%s' -l
 
-	#echo "-lS usage"
-	#cat $WITH_S
-	#echo "-l usage"
-	#cat $WITHOUT_S
-
 	WITH_S_parsed=$(awk '! /^total/ { print $7 }' $WITH_S)
 	set -- $(awk '! /^total/ { print $5, $7 }' $WITHOUT_S)
 	while [ $# -gt 0 ]; do
@@ -834,7 +860,7 @@ atf_init_test_cases()
 	atf_add_test_case I_flag_voids_implied_A_flag_when_root
 	atf_add_test_case L_flag
 	#atf_add_test_case P_flag
-	#atf_add_test_case R_flag
+	atf_add_test_case R_flag
 	atf_add_test_case S_flag
 	atf_add_test_case T_flag
 	#atf_add_test_case U_flag

From 84b774971382ed389da151d591529ae774f9ce1d Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sun, 4 Oct 2015 23:38:35 +0000
Subject: [PATCH 156/380] Add ls -l testcase

---
 bin/ls/tests/ls_tests.sh | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index 224e0d425f63..78d4652d1663 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -557,6 +557,25 @@ k_flag_body()
 	done
 }
 
+atf_test_case l_flag
+l_flag_head()
+{
+	atf_set "descr" "Verify that -l prints out the output in long format"
+}
+
+l_flag_body()
+{
+
+	atf_check -e empty -o empty -s exit:0 touch a.file
+
+	birthtime_in_secs=$(stat -f "%B" -t "%s" a.file)
+	birthtime=$(date -j -f "%s" $birthtime_in_secs +"[[:space:]]+%b[[:space:]]+%e[[:space:]]+%H:%M")
+
+	expected_output=$(stat -f "%Sp[[:space:]]+%i%Su[[:space:]]+%Sg[[:space:]]+%z[[:space:]]+$birthtime[[:space:]]+a\\.file")
+
+	atf_check -e empty -o match:"$expected_output" -s exit:0 ls -l a.file
+}
+
 atf_test_case lcomma_flag
 lcomma_flag_head()
 {
@@ -875,7 +894,7 @@ atf_init_test_cases()
 	atf_add_test_case h_flag
 	atf_add_test_case i_flag
 	atf_add_test_case k_flag
-	#atf_add_test_case l_flag
+	atf_add_test_case l_flag
 	atf_add_test_case lcomma_flag
 	atf_add_test_case m_flag
 	atf_add_test_case n_flag

From 4a8e4ed8ca2fe626c07c7b0d4bf5183bb96e14ca Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sun, 4 Oct 2015 23:41:08 +0000
Subject: [PATCH 157/380] - Speed up the testing process by moving socket
 creation   with nc to the -F testcase - Make the -C testcase more robust to
 test input change - Adjust some input to adjust to changes in creation of  
 the socket file

---
 bin/ls/tests/ls_tests.sh | 47 ++++++++++++++++++++++++++++++++++------
 1 file changed, 40 insertions(+), 7 deletions(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index 78d4652d1663..55b1a02d6077 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -53,8 +53,6 @@ create_test_inputs()
 	atf_check -e empty -s exit:0 mkdir .g
 	atf_check -e empty -s exit:0 mkfifo h
 	atf_check -e ignore -s exit:0 dd if=/dev/zero of=i count=1000 bs=1
-	atf_check -e empty -s exit:0 \
-	    sh -c "pid=${ATF_TMPDIR}/nc.pid; daemon -p \$pid nc -lU j; sleep 2; pkill -F \$pid"
 	atf_check -e empty -s exit:0 touch klmn
 	atf_check -e empty -s exit:0 touch opqr
 	atf_check -e empty -s exit:0 touch stuv
@@ -184,21 +182,53 @@ C_flag_head()
 	atf_set "descr" "Verify that the output from ls -C is multi-column, sorted down"
 }
 
+print_index()
+{
+	local i=1
+	local wanted_index=$1; shift
+
+	while [ $i -le $wanted_index ]; do
+		if [ $i -eq $wanted_index ]; then
+			echo $1
+			return
+		fi
+		shift
+		: $(( i += 1 ))
+	done
+}
+
 C_flag_body()
 {
 	create_test_inputs
 
 	WITH_C=$PWD/../with_C.out
 
+	export COLUMNS=40
 	atf_check -e empty -o save:$WITH_C -s exit:0 ls -C
 
 	echo "With -C usage"
 	cat $WITH_C
 
-	atf_check -e ignore -o not-empty -s exit:0 \
-	    egrep "0b00000001[[:space:]]+0b00000111[[:space:]]+0b00001101[[:space:]]+e[[:space:]]+stuv" $WITH_C
-	atf_check -e ignore -o not-empty -s exit:0 \
-	    egrep "0b00000010[[:space:]]+0b00001000[[:space:]]+0b00001110[[:space:]]+h[[:space:]]+wxyz" $WITH_C
+	paths=$(find -s . -mindepth 1 -maxdepth 1 \! -name '.*' -exec basename {} \; )
+	set -- $paths
+	num_paths=$#
+	num_columns=2
+
+	max_num_paths_per_column=$(( $(( $num_paths + 1 )) / $num_columns ))
+
+	local i=1
+	while [ $i -le $max_num_paths_per_column ]; do
+		column_1=$(print_index $i $paths)
+		column_2=$(print_index $(( $i + $max_num_paths_per_column )) $paths)
+		#echo "paths[$(( $i + $max_num_paths_per_column ))] = $column_2"
+		expected_expr="$column_1"
+		if [ -n "$column_2" ]; then
+			expected_expr="$expected_expr[[:space:]]+$column_2"
+		fi
+		atf_check -e ignore -o not-empty -s exit:0 \
+		    egrep "$expected_expr" $WITH_C
+		: $(( i += 1 ))
+	done
 }
 
 atf_test_case D_flag
@@ -224,6 +254,9 @@ F_flag_body()
 {
 	create_test_inputs
 
+	atf_check -e empty -s exit:0 \
+	    sh -c "pid=${ATF_TMPDIR}/nc.pid; daemon -p \$pid nc -lU j; sleep 2; pkill -F \$pid"
+
 	atf_check -e empty -o match:'a/' -s exit:0 ls -F
 	atf_check -e empty -o match:'c@' -s exit:0 ls -F
 	atf_check -e empty -o match:'h\|' -s exit:0 ls -F
@@ -812,7 +845,7 @@ x_flag_body()
 	atf_check -e ignore -o not-empty -s exit:0 \
 	    egrep "a[[:space:]]+c[[:space:]]+d[[:space:]]+e[[:space:]]+h" $WITH_x
 	atf_check -e ignore -o not-empty -s exit:0 \
-	    egrep "i[[:space:]]+j[[:space:]]+klmn[[:space:]]+opqr[[:space:]]+stuv" $WITH_x
+	    egrep "i[[:space:]]+klmn[[:space:]]+opqr[[:space:]]+stuv[[:space:]]+wxyz" $WITH_x
 }
 
 atf_test_case y_flag

From 0da1be5a9079f2e366a4285fdf311fe93d351860 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sun, 4 Oct 2015 23:50:02 +0000
Subject: [PATCH 158/380] Fix the ls -l testcase

---
 bin/ls/tests/ls_tests.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index 55b1a02d6077..a5ea45e01b02 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -602,9 +602,9 @@ l_flag_body()
 	atf_check -e empty -o empty -s exit:0 touch a.file
 
 	birthtime_in_secs=$(stat -f "%B" -t "%s" a.file)
-	birthtime=$(date -j -f "%s" $birthtime_in_secs +"[[:space:]]+%b[[:space:]]+%e[[:space:]]+%H:%M")
+	birthtime=$(date -j -f "%s" $birthtime_in_secs +"%b[[:space:]]+%e[[:space:]]+%H:%M")
 
-	expected_output=$(stat -f "%Sp[[:space:]]+%i%Su[[:space:]]+%Sg[[:space:]]+%z[[:space:]]+$birthtime[[:space:]]+a\\.file")
+	expected_output=$(stat -f "%Sp[[:space:]]+%l[[:space:]]+%Su[[:space:]]+%Sg[[:space:]]+%z[[:space:]]+$birthtime[[:space:]]+a\\.file" a.file)
 
 	atf_check -e empty -o match:"$expected_output" -s exit:0 ls -l a.file
 }

From ca6cf0ba737487849f2ee102afce74545600850c Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Mon, 5 Oct 2015 00:02:58 +0000
Subject: [PATCH 159/380] Add a testcase for ls -f

---
 bin/ls/tests/ls_tests.sh | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/bin/ls/tests/ls_tests.sh b/bin/ls/tests/ls_tests.sh
index a5ea45e01b02..da597e98bb97 100755
--- a/bin/ls/tests/ls_tests.sh
+++ b/bin/ls/tests/ls_tests.sh
@@ -493,6 +493,31 @@ d_flag_body()
 	done
 }
 
+atf_test_case f_flag
+f_flag_head()
+{
+	atf_set "descr" "Verify that -f prints out the contents of a directory unsorted"
+}
+
+f_flag_body()
+{
+	create_test_inputs
+
+	output=$PWD/../output
+
+	# XXX: I don't have enough understanding of how the algorithm works yet
+	# to determine more than the fact that all the entries printed out
+	# exist
+	paths=$(find -s . -mindepth 1 -maxdepth 1 \! -name '.*' -exec basename {} \; )
+
+	atf_check -e empty -o save:$output -s exit:0 ls -f
+
+	for path in $paths; do
+		atf_check -e ignore -o not-empty -s exit:0 \
+		    egrep "^$path$" $output
+	done
+}
+
 atf_test_case g_flag
 g_flag_head()
 {
@@ -922,7 +947,7 @@ atf_init_test_cases()
 	atf_add_test_case b_flag
 	#atf_add_test_case c_flag
 	atf_add_test_case d_flag
-	#atf_add_test_case f_flag
+	atf_add_test_case f_flag
 	atf_add_test_case g_flag
 	atf_add_test_case h_flag
 	atf_add_test_case i_flag

From 51803bd7651c0a4179c4c6c89cf7376a9dc63faf Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Mon, 5 Oct 2015 00:51:41 +0000
Subject: [PATCH 160/380] Fix compilation with -Werror + -Wsigned enabled

---
 tests/sys/vm/mmap_test.c | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/tests/sys/vm/mmap_test.c b/tests/sys/vm/mmap_test.c
index 9df1da33ed98..1d119c2f1aa0 100644
--- a/tests/sys/vm/mmap_test.c
+++ b/tests/sys/vm/mmap_test.c
@@ -177,8 +177,7 @@ ATF_TC_WITHOUT_HEAD(mmap__dev_zero_private);
 ATF_TC_BODY(mmap__dev_zero_private, tc)
 {
 	char *p1, *p2, *p3;
-	size_t i;
-	int fd;
+	int fd, i;
 
 	ATF_REQUIRE((fd = open("/dev/zero", O_RDONLY)) >= 0);
 
@@ -191,7 +190,7 @@ ATF_TC_BODY(mmap__dev_zero_private, tc)
 	ATF_REQUIRE(p2 != MAP_FAILED);
 
 	for (i = 0; i < getpagesize(); i++)
-		ATF_REQUIRE_EQ_MSG(0, p1[i], "byte at p1[%zu] is %x", i, p1[i]);
+		ATF_REQUIRE_EQ_MSG(0, p1[i], "byte at p1[%d] is %x", i, p1[i]);
 
 	ATF_REQUIRE(memcmp(p1, p2, getpagesize()) == 0);
 
@@ -214,8 +213,7 @@ ATF_TC_WITHOUT_HEAD(mmap__dev_zero_shared);
 ATF_TC_BODY(mmap__dev_zero_shared, tc)
 {
 	char *p1, *p2, *p3;
-	size_t i;
-	int fd;
+	int fd, i;
 
 	ATF_REQUIRE((fd = open("/dev/zero", O_RDWR)) >= 0);
 
@@ -228,7 +226,7 @@ ATF_TC_BODY(mmap__dev_zero_shared, tc)
 	ATF_REQUIRE(p2 != MAP_FAILED);
 
 	for (i = 0; i < getpagesize(); i++)
-		ATF_REQUIRE_EQ_MSG(0, p1[i], "byte at p1[%zu] is %x", i, p1[i]);
+		ATF_REQUIRE_EQ_MSG(0, p1[i], "byte at p1[%d] is %x", i, p1[i]);
 
 	ATF_REQUIRE(memcmp(p1, p2, getpagesize()) == 0);
 

From 7f1b3a0b6809a66c4a0742fbdd1c8fe2db90852f Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Mon, 5 Oct 2015 00:55:16 +0000
Subject: [PATCH 161/380] Integrate tools/regression/acltools into
 tests/sys/acl

Apply patches I've been running for months on my GitHub project
---
 etc/mtree/BSD.tests.dist                      |  2 ++
 tests/sys/Makefile                            |  1 +
 .../acltools/00.t => tests/sys/acl/00.sh      | 19 ++++++++++-------
 .../acltools/01.t => tests/sys/acl/01.sh      | 21 ++++++++++---------
 .../acltools/02.t => tests/sys/acl/02.sh      | 19 ++++++++++-------
 .../acltools/03.t => tests/sys/acl/03.sh      | 17 ++++++++++-----
 .../acltools/04.t => tests/sys/acl/04.sh      | 14 ++++++++-----
 .../acltools => tests/sys/acl}/aclfuzzer.sh   |  0
 .../acltools => tests/sys/acl}/mktrivial.sh   |  0
 .../regression/acltools => tests/sys/acl}/run |  0
 .../sys/acl}/tools-crossfs.test               |  0
 .../sys/acl}/tools-nfs4-psarc.test            |  0
 .../sys/acl}/tools-nfs4-trivial.test          |  0
 .../sys/acl}/tools-nfs4.test                  |  0
 .../sys/acl}/tools-posix.test                 |  0
 15 files changed, 57 insertions(+), 36 deletions(-)
 rename tools/regression/acltools/00.t => tests/sys/acl/00.sh (89%)
 rename tools/regression/acltools/01.t => tests/sys/acl/01.sh (89%)
 rename tools/regression/acltools/02.t => tests/sys/acl/02.sh (89%)
 rename tools/regression/acltools/03.t => tests/sys/acl/03.sh (91%)
 rename tools/regression/acltools/04.t => tests/sys/acl/04.sh (91%)
 rename {tools/regression/acltools => tests/sys/acl}/aclfuzzer.sh (100%)
 rename {tools/regression/acltools => tests/sys/acl}/mktrivial.sh (100%)
 rename {tools/regression/acltools => tests/sys/acl}/run (100%)
 rename {tools/regression/acltools => tests/sys/acl}/tools-crossfs.test (100%)
 rename {tools/regression/acltools => tests/sys/acl}/tools-nfs4-psarc.test (100%)
 rename {tools/regression/acltools => tests/sys/acl}/tools-nfs4-trivial.test (100%)
 rename {tools/regression/acltools => tests/sys/acl}/tools-nfs4.test (100%)
 rename {tools/regression/acltools => tests/sys/acl}/tools-posix.test (100%)

diff --git a/etc/mtree/BSD.tests.dist b/etc/mtree/BSD.tests.dist
index 57c85c014cd1..db4cb6f79160 100644
--- a/etc/mtree/BSD.tests.dist
+++ b/etc/mtree/BSD.tests.dist
@@ -356,6 +356,8 @@
         ..
     ..
     sys
+        acl
+        ..
         aio
         ..
         fifo
diff --git a/tests/sys/Makefile b/tests/sys/Makefile
index 8f3749019be8..217a8ec83a8e 100644
--- a/tests/sys/Makefile
+++ b/tests/sys/Makefile
@@ -4,6 +4,7 @@
 
 TESTSDIR= ${TESTSBASE}/sys
 
+TESTS_SUBDIRS+=		acl
 TESTS_SUBDIRS+=		aio
 TESTS_SUBDIRS+=		fifo
 TESTS_SUBDIRS+=		file
diff --git a/tools/regression/acltools/00.t b/tests/sys/acl/00.sh
similarity index 89%
rename from tools/regression/acltools/00.t
rename to tests/sys/acl/00.sh
index 88990887773b..61e8115525ff 100644
--- a/tools/regression/acltools/00.t
+++ b/tests/sys/acl/00.sh
@@ -37,12 +37,16 @@
 #
 # Output should be obvious.
 
-echo "1..4"
-
-if [ `whoami` != "root" ]; then
-	echo "not ok 1 - you need to be root to run this test."
-	exit 1
+if [ $(sysctl -n kern.features.ufs_acl 2>/dev/null || echo 0) -eq 0 ]; then
+	echo "1..0 # SKIP system does not have UFS ACL support"
+	exit 0
 fi
+if [ $(id -u) -ne 0 ]; then
+	echo "1..0 # SKIP you must be root"
+	exit 0
+fi
+
+echo "1..4"
 
 TESTDIR=$(dirname $(realpath $0))
 
@@ -50,9 +54,11 @@ TESTDIR=$(dirname $(realpath $0))
 MD=`mdconfig -at swap -s 10m`
 MNT=`mktemp -dt acltools`
 newfs /dev/$MD > /dev/null
+trap "cd /; umount -f $MNT; rmdir $MNT; mdconfig -d -u $MD" EXIT
 mount -o acls /dev/$MD $MNT
 if [ $? -ne 0 ]; then
 	echo "not ok 1 - mount failed."
+	echo 'Bail out!'
 	exit 1
 fi
 
@@ -78,8 +84,5 @@ else
 fi
 
 cd /
-umount -f $MNT
-rmdir $MNT
-mdconfig -du $MD
 
 echo "ok 4"
diff --git a/tools/regression/acltools/01.t b/tests/sys/acl/01.sh
similarity index 89%
rename from tools/regression/acltools/01.t
rename to tests/sys/acl/01.sh
index f84e0a98b463..93487a77263f 100644
--- a/tools/regression/acltools/01.t
+++ b/tests/sys/acl/01.sh
@@ -39,21 +39,27 @@
 #
 # Output should be obvious.
 
-echo "1..4"
-
-if [ `whoami` != "root" ]; then
-	echo "not ok 1 - you need to be root to run this test."
-	exit 1
+if ! sysctl vfs.zfs.version.spa >/dev/null 2>&1; then
+	echo "1..0 # SKIP system doesn't have ZFS loaded"
+	exit 0
 fi
+if [ $(id -u) -ne 0 ]; then
+	echo "1..0 # SKIP you must be root"
+	exit 0
+fi
+
+echo "1..4"
 
 TESTDIR=$(dirname $(realpath $0))
 
 # Set up the test filesystem.
 MD=`mdconfig -at swap -s 64m`
 MNT=`mktemp -dt acltools`
+trap "cd /; zpool destroy -f acltools; rmdir $MNT; mdconfig -d -u $MD" EXIT
 zpool create -m $MNT acltools /dev/$MD
 if [ $? -ne 0 ]; then
 	echo "not ok 1 - 'zpool create' failed."
+	echo 'Bail out!'
 	exit 1
 fi
 
@@ -78,9 +84,4 @@ else
 	echo "not ok 3"
 fi
 
-cd /
-zpool destroy -f acltools
-rmdir $MNT
-mdconfig -du $MD
-
 echo "ok 4"
diff --git a/tools/regression/acltools/02.t b/tests/sys/acl/02.sh
similarity index 89%
rename from tools/regression/acltools/02.t
rename to tests/sys/acl/02.sh
index 9a82c78d1456..8fbb39de4d6a 100644
--- a/tools/regression/acltools/02.t
+++ b/tests/sys/acl/02.sh
@@ -37,12 +37,16 @@
 #
 # Output should be obvious.
 
-echo "1..4"
-
-if [ `whoami` != "root" ]; then
-	echo "not ok 1 - you need to be root to run this test."
-	exit 1
+if [ $(sysctl -n kern.features.ufs_acl 2>/dev/null || echo 0) -eq 0 ]; then
+	echo "1..0 # SKIP system does not have UFS ACL support"
+	exit 0
 fi
+if [ $(id -u) -ne 0 ]; then
+	echo "1..0 # SKIP you must be root"
+	exit 0
+fi
+
+echo "1..4"
 
 TESTDIR=$(dirname $(realpath $0))
 
@@ -50,9 +54,11 @@ TESTDIR=$(dirname $(realpath $0))
 MD=`mdconfig -at swap -s 10m`
 MNT=`mktemp -dt acltools`
 newfs /dev/$MD > /dev/null
+trap "cd /; umount -f $MNT; rmdir $MNT; mdconfig -d -u $MD" EXIT
 mount -o nfsv4acls /dev/$MD $MNT
 if [ $? -ne 0 ]; then
 	echo "not ok 1 - mount failed."
+	echo 'Bail out!'
 	exit 1
 fi
 
@@ -82,9 +88,6 @@ else
 fi
 
 cd /
-umount -f $MNT
-rmdir $MNT
-mdconfig -du $MD
 
 echo "ok 4"
 
diff --git a/tools/regression/acltools/03.t b/tests/sys/acl/03.sh
similarity index 91%
rename from tools/regression/acltools/03.t
rename to tests/sys/acl/03.sh
index a0b4199c2bf2..2b81b2662306 100644
--- a/tools/regression/acltools/03.t
+++ b/tests/sys/acl/03.sh
@@ -34,12 +34,16 @@
 #
 # Output should be obvious.
 
-echo "1..5"
-
-if [ `whoami` != "root" ]; then
-	echo "not ok 1 - you need to be root to run this test."
-	exit 1
+if ! sysctl vfs.zfs.version.spa >/dev/null 2>&1; then
+	echo "1..0 # SKIP system doesn't have ZFS loaded"
+	exit 0
 fi
+if [ $(id -u) -ne 0 ]; then
+	echo "1..0 # SKIP you must be root"
+	exit 0
+fi
+
+echo "1..5"
 
 TESTDIR=$(dirname $(realpath $0))
 MNTROOT=`mktemp -dt acltools`
@@ -51,6 +55,7 @@ mkdir $MNT1
 zpool create -m $MNT1 acltools /dev/$MD1
 if [ $? -ne 0 ]; then
 	echo "not ok 1 - 'zpool create' failed."
+	echo 'Bail out!'
 	exit 1
 fi
 
@@ -63,6 +68,7 @@ newfs /dev/$MD2 > /dev/null
 mount -o acls /dev/$MD2 $MNT2
 if [ $? -ne 0 ]; then
 	echo "not ok 2 - mount failed."
+	echo 'Bail out!'
 	exit 1
 fi
 
@@ -75,6 +81,7 @@ newfs /dev/$MD3 > /dev/null
 mount /dev/$MD3 $MNT3
 if [ $? -ne 0 ]; then
 	echo "not ok 3 - mount failed."
+	echo 'Bail out!'
 	exit 1
 fi
 
diff --git a/tools/regression/acltools/04.t b/tests/sys/acl/04.sh
similarity index 91%
rename from tools/regression/acltools/04.t
rename to tests/sys/acl/04.sh
index 1a13183475fe..5669c5a48d24 100644
--- a/tools/regression/acltools/04.t
+++ b/tests/sys/acl/04.sh
@@ -31,12 +31,16 @@
 #
 # WARNING: It uses hardcoded ZFS pool name "acltools"
 
-echo "1..3"
-
-if [ `whoami` != "root" ]; then
-	echo "not ok 1 - you need to be root to run this test."
-	exit 1
+if ! sysctl vfs.zfs.version.spa >/dev/null 2>&1; then
+	echo "1..0 # SKIP system doesn't have ZFS loaded"
+	exit 0
 fi
+if [ $(id -u) -ne 0 ]; then
+	echo "1..0 # SKIP you must be root"
+	exit 0
+fi
+
+echo "1..3"
 
 TESTDIR=$(dirname $(realpath $0))
 
diff --git a/tools/regression/acltools/aclfuzzer.sh b/tests/sys/acl/aclfuzzer.sh
similarity index 100%
rename from tools/regression/acltools/aclfuzzer.sh
rename to tests/sys/acl/aclfuzzer.sh
diff --git a/tools/regression/acltools/mktrivial.sh b/tests/sys/acl/mktrivial.sh
similarity index 100%
rename from tools/regression/acltools/mktrivial.sh
rename to tests/sys/acl/mktrivial.sh
diff --git a/tools/regression/acltools/run b/tests/sys/acl/run
similarity index 100%
rename from tools/regression/acltools/run
rename to tests/sys/acl/run
diff --git a/tools/regression/acltools/tools-crossfs.test b/tests/sys/acl/tools-crossfs.test
similarity index 100%
rename from tools/regression/acltools/tools-crossfs.test
rename to tests/sys/acl/tools-crossfs.test
diff --git a/tools/regression/acltools/tools-nfs4-psarc.test b/tests/sys/acl/tools-nfs4-psarc.test
similarity index 100%
rename from tools/regression/acltools/tools-nfs4-psarc.test
rename to tests/sys/acl/tools-nfs4-psarc.test
diff --git a/tools/regression/acltools/tools-nfs4-trivial.test b/tests/sys/acl/tools-nfs4-trivial.test
similarity index 100%
rename from tools/regression/acltools/tools-nfs4-trivial.test
rename to tests/sys/acl/tools-nfs4-trivial.test
diff --git a/tools/regression/acltools/tools-nfs4.test b/tests/sys/acl/tools-nfs4.test
similarity index 100%
rename from tools/regression/acltools/tools-nfs4.test
rename to tests/sys/acl/tools-nfs4.test
diff --git a/tools/regression/acltools/tools-posix.test b/tests/sys/acl/tools-posix.test
similarity index 100%
rename from tools/regression/acltools/tools-posix.test
rename to tests/sys/acl/tools-posix.test

From 52414b14f1c6336e8ba31620578b58901a92b409 Mon Sep 17 00:00:00 2001
From: markj <markj@FreeBSD.org>
Date: Thu, 8 Oct 2015 04:29:39 +0000
Subject: [PATCH 162/380] 6271 dtrace caused excessive fork time

Author: Bryan Cantrill <bryan@joyent.com>
Reviewed by: Adam Leventhal <ahl@delphix.com>
Reviewed by: Dan McDonald <danmcd@omniti.com>
Reviewed by: Richard Lowe <richlowe@richlowe.net>
Approved by: Gordon Ross <gwr@nexenta.com>

illumos/illumos-gate@7bd3c1d12d0c764e1517c3aca62c634409356764
---
 uts/common/dtrace/dtrace.c   | 11 +++++++++--
 uts/common/dtrace/fasttrap.c | 28 ++++++++++++----------------
 uts/common/sys/dtrace.h      | 18 ++++++++++++------
 3 files changed, 33 insertions(+), 24 deletions(-)

diff --git a/uts/common/dtrace/dtrace.c b/uts/common/dtrace/dtrace.c
index f2c3fec010dd..533ce81ab41c 100644
--- a/uts/common/dtrace/dtrace.c
+++ b/uts/common/dtrace/dtrace.c
@@ -14809,8 +14809,8 @@ dtrace_helper_provider_add(dof_helper_t *dofhp, int gen)
 	 * Check to make sure this isn't a duplicate.
 	 */
 	for (i = 0; i < help->dthps_nprovs; i++) {
-		if (dofhp->dofhp_dof ==
-		    help->dthps_provs[i]->dthp_prov.dofhp_dof)
+		if (dofhp->dofhp_addr ==
+		    help->dthps_provs[i]->dthp_prov.dofhp_addr)
 			return (EALREADY);
 	}
 
@@ -15162,7 +15162,14 @@ dtrace_helper_slurp(dof_hdr_t *dof, dof_helper_t *dhp)
 	dtrace_enabling_destroy(enab);
 
 	if (dhp != NULL && nprovs > 0) {
+		/*
+		 * Now that this is in-kernel, we change the sense of the
+		 * members:  dofhp_dof denotes the in-kernel copy of the DOF
+		 * and dofhp_addr denotes the address at user-level.
+		 */
+		dhp->dofhp_addr = dhp->dofhp_dof;
 		dhp->dofhp_dof = (uint64_t)(uintptr_t)dof;
+
 		if (dtrace_helper_provider_add(dhp, gen) == 0) {
 			mutex_exit(&dtrace_lock);
 			dtrace_helper_provider_register(curproc, help, dhp);
diff --git a/uts/common/dtrace/fasttrap.c b/uts/common/dtrace/fasttrap.c
index 215b1a411142..3948dd258eab 100644
--- a/uts/common/dtrace/fasttrap.c
+++ b/uts/common/dtrace/fasttrap.c
@@ -1784,6 +1784,18 @@ fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
 	return (provider);
 }
 
+/*
+ * We know a few things about our context here:  we know that the probe being
+ * created doesn't already exist (DTrace won't load DOF at the same address
+ * twice, even if explicitly told to do so) and we know that we are
+ * single-threaded with respect to the meta provider machinery. Knowing that
+ * this is a new probe and that there is no way for us to race with another
+ * operation on this provider allows us an important optimization: we need not
+ * lookup a probe before adding it.  Saving this lookup is important because
+ * this code is in the fork path for processes with USDT probes, and lookups
+ * here are potentially very expensive because of long hash conflicts on
+ * module, function and name (DTrace doesn't hash on provider name).
+ */
 /*ARGSUSED*/
 static void
 fasttrap_meta_create_probe(void *arg, void *parg,
@@ -1820,19 +1832,6 @@ fasttrap_meta_create_probe(void *arg, void *parg,
 			return;
 	}
 
-	/*
-	 * Grab the creation lock to ensure consistency between calls to
-	 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
-	 * other threads creating probes.
-	 */
-	mutex_enter(&provider->ftp_cmtx);
-
-	if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod,
-	    dhpb->dthpb_func, dhpb->dthpb_name) != 0) {
-		mutex_exit(&provider->ftp_cmtx);
-		return;
-	}
-
 	ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
 	ASSERT(ntps > 0);
 
@@ -1840,7 +1839,6 @@ fasttrap_meta_create_probe(void *arg, void *parg,
 
 	if (fasttrap_total > fasttrap_max) {
 		atomic_add_32(&fasttrap_total, -ntps);
-		mutex_exit(&provider->ftp_cmtx);
 		return;
 	}
 
@@ -1904,8 +1902,6 @@ fasttrap_meta_create_probe(void *arg, void *parg,
 	 */
 	pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
 	    dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
-
-	mutex_exit(&provider->ftp_cmtx);
 }
 
 /*ARGSUSED*/
diff --git a/uts/common/sys/dtrace.h b/uts/common/sys/dtrace.h
index 3b1869a4ff89..b1bb6d8dd2c6 100644
--- a/uts/common/sys/dtrace.h
+++ b/uts/common/sys/dtrace.h
@@ -2131,12 +2131,18 @@ extern void dtrace_probe(dtrace_id_t, uintptr_t arg0, uintptr_t arg1,
  *
  * 1.2.4  Caller's context
  *
- *   dtms_create_probe() is called from either ioctl() or module load context.
- *   The DTrace framework is locked in such a way that meta providers may not
- *   register or unregister. This means that the meta provider cannot call
- *   dtrace_meta_register() or dtrace_meta_unregister(). However, the context is
- *   such that the provider may (and is expected to) call provider-related
- *   DTrace provider APIs including dtrace_probe_create().
+ *   dtms_create_probe() is called from either ioctl() or module load context
+ *   in the context of a newly-created provider (that is, a provider that
+ *   is a result of a call to dtms_provide_pid()). The DTrace framework is
+ *   locked in such a way that meta providers may not register or unregister,
+ *   such that no other thread can call into a meta provider operation and that
+ *   atomicity is assured with respect to meta provider operations across
+ *   dtms_provide_pid() and subsequent calls to dtms_create_probe().
+ *   The context is thus effectively single-threaded with respect to the meta
+ *   provider, and that the meta provider cannot call dtrace_meta_register()
+ *   or dtrace_meta_unregister(). However, the context is such that the
+ *   provider may (and is expected to) call provider-related DTrace provider
+ *   APIs including dtrace_probe_create().
  *
  * 1.3  void *dtms_provide_pid(void *arg, dtrace_meta_provider_t *mprov,
  *	      pid_t pid)

From 1d717c724dfb40459d2574df4280907eb993e3bf Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Mon, 12 Oct 2015 14:23:00 +0000
Subject: [PATCH 163/380] 6250 zvol_dump_init() can hold txg open

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Prakash Surya <prakash.surya@delphix.com>
Reviewed by: Albert Lee <trisk@omniti.com>
Reviewed by: Xin Li <delphij@freebsd.org>
Approved by: Garrett D'Amore <garrett@damore.org>
Author: George Wilson <george.wilson@delphix.com>

illumos/illumos-gate@b10bba72460aeaa53119c76ff5e647fd5585bece
---
 uts/common/fs/zfs/zvol.c | 107 +++++++++++++++++++++++----------------
 1 file changed, 63 insertions(+), 44 deletions(-)

diff --git a/uts/common/fs/zfs/zvol.c b/uts/common/fs/zfs/zvol.c
index 101566a1ea55..585500bbac86 100644
--- a/uts/common/fs/zfs/zvol.c
+++ b/uts/common/fs/zfs/zvol.c
@@ -1893,13 +1893,15 @@ zvol_dump_init(zvol_state_t *zv, boolean_t resize)
 	vdev_t *vd = spa->spa_root_vdev;
 	nvlist_t *nv = NULL;
 	uint64_t version = spa_version(spa);
-	enum zio_checksum checksum;
+	uint64_t checksum, compress, refresrv, vbs, dedup;
 
 	ASSERT(MUTEX_HELD(&zfsdev_state_lock));
 	ASSERT(vd->vdev_ops == &vdev_root_ops);
 
 	error = dmu_free_long_range(zv->zv_objset, ZVOL_OBJ, 0,
 	    DMU_OBJECT_END);
+	if (error != 0)
+		return (error);
 	/* wait for dmu_free_long_range to actually free the blocks */
 	txg_wait_synced(dmu_objset_pool(zv->zv_objset), 0);
 
@@ -1923,23 +1925,41 @@ zvol_dump_init(zvol_state_t *zv, boolean_t resize)
 		    2, ZFS_SPACE_CHECK_RESERVED);
 	}
 
+	if (!resize) {
+		error = dsl_prop_get_integer(zv->zv_name,
+		    zfs_prop_to_name(ZFS_PROP_COMPRESSION), &compress, NULL);
+		if (error == 0) {
+			error = dsl_prop_get_integer(zv->zv_name,
+			    zfs_prop_to_name(ZFS_PROP_CHECKSUM), &checksum,
+			    NULL);
+		}
+		if (error == 0) {
+			error = dsl_prop_get_integer(zv->zv_name,
+			    zfs_prop_to_name(ZFS_PROP_REFRESERVATION),
+			    &refresrv, NULL);
+		}
+		if (error == 0) {
+			error = dsl_prop_get_integer(zv->zv_name,
+			    zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE), &vbs,
+			    NULL);
+		}
+		if (version >= SPA_VERSION_DEDUP && error == 0) {
+			error = dsl_prop_get_integer(zv->zv_name,
+			    zfs_prop_to_name(ZFS_PROP_DEDUP), &dedup, NULL);
+		}
+	}
+	if (error != 0)
+		return (error);
+
 	tx = dmu_tx_create(os);
 	dmu_tx_hold_zap(tx, ZVOL_ZAP_OBJ, TRUE, NULL);
 	dmu_tx_hold_bonus(tx, ZVOL_OBJ);
 	error = dmu_tx_assign(tx, TXG_WAIT);
-	if (error) {
+	if (error != 0) {
 		dmu_tx_abort(tx);
 		return (error);
 	}
 
-	/*
-	 * If MULTI_VDEV_CRASH_DUMP is active, use the NOPARITY checksum
-	 * function.  Otherwise, use the old default -- OFF.
-	 */
-	checksum = spa_feature_is_active(spa,
-	    SPA_FEATURE_MULTI_VDEV_CRASH_DUMP) ? ZIO_CHECKSUM_NOPARITY :
-	    ZIO_CHECKSUM_OFF;
-
 	/*
 	 * If we are resizing the dump device then we only need to
 	 * update the refreservation to match the newly updated
@@ -1951,37 +1971,30 @@ zvol_dump_init(zvol_state_t *zv, boolean_t resize)
 		    zfs_prop_to_name(ZFS_PROP_REFRESERVATION), 8, 1,
 		    &zv->zv_volsize, tx);
 	} else {
-		uint64_t checksum, compress, refresrv, vbs, dedup;
-
-		error = dsl_prop_get_integer(zv->zv_name,
-		    zfs_prop_to_name(ZFS_PROP_COMPRESSION), &compress, NULL);
-		error = error ? error : dsl_prop_get_integer(zv->zv_name,
-		    zfs_prop_to_name(ZFS_PROP_CHECKSUM), &checksum, NULL);
-		error = error ? error : dsl_prop_get_integer(zv->zv_name,
-		    zfs_prop_to_name(ZFS_PROP_REFRESERVATION), &refresrv, NULL);
-		error = error ? error : dsl_prop_get_integer(zv->zv_name,
-		    zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE), &vbs, NULL);
-		if (version >= SPA_VERSION_DEDUP) {
-			error = error ? error :
-			    dsl_prop_get_integer(zv->zv_name,
-			    zfs_prop_to_name(ZFS_PROP_DEDUP), &dedup, NULL);
-		}
-
-		error = error ? error : zap_update(os, ZVOL_ZAP_OBJ,
+		error = zap_update(os, ZVOL_ZAP_OBJ,
 		    zfs_prop_to_name(ZFS_PROP_COMPRESSION), 8, 1,
 		    &compress, tx);
-		error = error ? error : zap_update(os, ZVOL_ZAP_OBJ,
-		    zfs_prop_to_name(ZFS_PROP_CHECKSUM), 8, 1, &checksum, tx);
-		error = error ? error : zap_update(os, ZVOL_ZAP_OBJ,
-		    zfs_prop_to_name(ZFS_PROP_REFRESERVATION), 8, 1,
-		    &refresrv, tx);
-		error = error ? error : zap_update(os, ZVOL_ZAP_OBJ,
-		    zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE), 8, 1,
-		    &vbs, tx);
-		error = error ? error : dmu_object_set_blocksize(
-		    os, ZVOL_OBJ, SPA_OLD_MAXBLOCKSIZE, 0, tx);
-		if (version >= SPA_VERSION_DEDUP) {
-			error = error ? error : zap_update(os, ZVOL_ZAP_OBJ,
+		if (error == 0) {
+			error = zap_update(os, ZVOL_ZAP_OBJ,
+			    zfs_prop_to_name(ZFS_PROP_CHECKSUM), 8, 1,
+			    &checksum, tx);
+		}
+		if (error == 0) {
+			error = zap_update(os, ZVOL_ZAP_OBJ,
+			    zfs_prop_to_name(ZFS_PROP_REFRESERVATION), 8, 1,
+			    &refresrv, tx);
+		}
+		if (error == 0) {
+			error = zap_update(os, ZVOL_ZAP_OBJ,
+			    zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE), 8, 1,
+			    &vbs, tx);
+		}
+		if (error == 0) {
+			error = dmu_object_set_blocksize(
+			    os, ZVOL_OBJ, SPA_OLD_MAXBLOCKSIZE, 0, tx);
+		}
+		if (version >= SPA_VERSION_DEDUP && error == 0) {
+			error = zap_update(os, ZVOL_ZAP_OBJ,
 			    zfs_prop_to_name(ZFS_PROP_DEDUP), 8, 1,
 			    &dedup, tx);
 		}
@@ -1994,7 +2007,15 @@ zvol_dump_init(zvol_state_t *zv, boolean_t resize)
 	 * We only need update the zvol's property if we are initializing
 	 * the dump area for the first time.
 	 */
-	if (!resize) {
+	if (error == 0 && !resize) {
+		/*
+		 * If MULTI_VDEV_CRASH_DUMP is active, use the NOPARITY checksum
+		 * function.  Otherwise, use the old default -- OFF.
+		 */
+		checksum = spa_feature_is_active(spa,
+		    SPA_FEATURE_MULTI_VDEV_CRASH_DUMP) ? ZIO_CHECKSUM_NOPARITY :
+		    ZIO_CHECKSUM_OFF;
+
 		VERIFY(nvlist_alloc(&nv, NV_UNIQUE_NAME, KM_SLEEP) == 0);
 		VERIFY(nvlist_add_uint64(nv,
 		    zfs_prop_to_name(ZFS_PROP_REFRESERVATION), 0) == 0);
@@ -2013,13 +2034,11 @@ zvol_dump_init(zvol_state_t *zv, boolean_t resize)
 		error = zfs_set_prop_nvlist(zv->zv_name, ZPROP_SRC_LOCAL,
 		    nv, NULL);
 		nvlist_free(nv);
-
-		if (error)
-			return (error);
 	}
 
 	/* Allocate the space for the dump */
-	error = zvol_prealloc(zv);
+	if (error == 0)
+		error = zvol_prealloc(zv);
 	return (error);
 }
 

From 5c05c9c3b93403487a78839bc6e4c38eb43359a1 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Mon, 12 Oct 2015 14:24:11 +0000
Subject: [PATCH 164/380] 6251 add tunable to disable free_bpobj processing

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Prakash Surya <prakash.surya@delphix.com>
Reviewed by: Simon Klinkert <simon.klinkert@gmail.com>
Reviewed by: Richard Elling <Richard.Elling@RichardElling.com>
Reviewed by: Albert Lee <trisk@omniti.com>
Reviewed by: Xin Li <delphij@freebsd.org>
Approved by: Garrett D'Amore <garrett@damore.org>
Author: George Wilson <george.wilson@delphix.com>

illumos/illumos-gate@139510fb6efa97dbe5f5479594b308d940cab8d1
---
 uts/common/fs/zfs/dsl_scan.c | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/uts/common/fs/zfs/dsl_scan.c b/uts/common/fs/zfs/dsl_scan.c
index ba397b1bb0cb..6ba5cb6a1c83 100644
--- a/uts/common/fs/zfs/dsl_scan.c
+++ b/uts/common/fs/zfs/dsl_scan.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  */
 
 #include <sys/dsl_scan.h>
@@ -78,6 +78,11 @@ uint64_t zfs_free_max_blocks = UINT64_MAX;
 
 extern int zfs_txg_timeout;
 
+/*
+ * Enable/disable the processing of the free_bpobj object.
+ */
+boolean_t zfs_free_bpobj_enabled = B_TRUE;
+
 /* the order has to match pool_scan_type */
 static scan_cb_t *scan_funcs[POOL_SCAN_FUNCS] = {
 	NULL,
@@ -1426,7 +1431,8 @@ dsl_scan_sync(dsl_pool_t *dp, dmu_tx_t *tx)
 	 * have to worry about traversing it.  It is also faster to free the
 	 * blocks than to scrub them.
 	 */
-	if (spa_version(dp->dp_spa) >= SPA_VERSION_DEADLISTS) {
+	if (zfs_free_bpobj_enabled &&
+	    spa_version(dp->dp_spa) >= SPA_VERSION_DEADLISTS) {
 		scn->scn_is_bptree = B_FALSE;
 		scn->scn_zio_root = zio_root(dp->dp_spa, NULL,
 		    NULL, ZIO_FLAG_MUSTSUCCEED);

From 45f574e1893ed21876eb04beaae8ac0ee7aff33d Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Mon, 12 Oct 2015 14:27:27 +0000
Subject: [PATCH 165/380] 6281 prefetching should apply to 1MB reads

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Paul Dagnelie <pcd@delphix.com>
Reviewed by: Alexander Motin <mav@freebsd.org>
Reviewed by: Dan McDonald <danmcd@omniti.com>
Reviewed by: Justin Gibbs <gibbs@scsiguy.com>
Reviewed by: Xin Li <delphij@freebsd.org>
Approved by: Gordon Ross <gordon.ross@nexenta.com>
Author: George Wilson <george.wilson@delphix.com>

illumos/illumos-gate@632802744ef6d17e06d6980a95f631615c3b060f
---
 uts/common/fs/zfs/dmu.c        | 2 +-
 uts/common/fs/zfs/dmu_zfetch.c | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/uts/common/fs/zfs/dmu.c b/uts/common/fs/zfs/dmu.c
index 4b02f75d4f7a..6ebd95cd3ef2 100644
--- a/uts/common/fs/zfs/dmu.c
+++ b/uts/common/fs/zfs/dmu.c
@@ -442,7 +442,7 @@ dmu_buf_hold_array_by_dnode(dnode_t *dn, uint64_t offset, uint64_t length,
 	}
 
 	if ((flags & DMU_READ_NO_PREFETCH) == 0 && read &&
-	    length < zfetch_array_rd_sz) {
+	    length <= zfetch_array_rd_sz) {
 		dmu_zfetch(&dn->dn_zfetch, blkid, nblks);
 	}
 	rw_exit(&dn->dn_struct_rwlock);
diff --git a/uts/common/fs/zfs/dmu_zfetch.c b/uts/common/fs/zfs/dmu_zfetch.c
index b05e8f0cf2e4..f2cdf863d83c 100644
--- a/uts/common/fs/zfs/dmu_zfetch.c
+++ b/uts/common/fs/zfs/dmu_zfetch.c
@@ -24,7 +24,7 @@
  */
 
 /*
- * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2013, 2015 by Delphix. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -49,7 +49,7 @@ uint32_t	zfetch_max_streams = 8;
 uint32_t	zfetch_min_sec_reap = 2;
 /* max bytes to prefetch per stream (default 8MB) */
 uint32_t	zfetch_max_distance = 8 * 1024 * 1024;
-/* number of bytes in a array_read at which we stop prefetching (1MB) */
+/* max number of bytes in an array_read in which we allow prefetching (1MB) */
 uint64_t	zfetch_array_rd_sz = 1024 * 1024;
 
 typedef struct zfetch_stats {

From 2c46ba7a42c08be87ceaa99985a950cd19b140e0 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Tue, 13 Oct 2015 16:50:12 +0000
Subject: [PATCH 166/380] Integrate tools/test/posixshm and
 tools/regression/posixshm into the FreeBSD test suite as tests/sys/posixshm

---
 etc/mtree/BSD.tests.dist                            |  2 ++
 tests/sys/Makefile                                  |  1 +
 tests/sys/posixshm/Makefile                         | 11 +++++++++++
 {tools/regression => tests/sys}/posixshm/posixshm.c |  0
 {tools/test => tests/sys}/posixshm/shm_test.c       |  7 ++++++-
 {tools/regression => tests/sys}/posixshm/test.c     |  0
 {tools/regression => tests/sys}/posixshm/test.h     |  0
 tools/regression/posixshm/Makefile                  |  9 ---------
 tools/regression/posixshm/posixshm.t                |  5 -----
 tools/test/posixshm/README                          |  4 ----
 10 files changed, 20 insertions(+), 19 deletions(-)
 create mode 100644 tests/sys/posixshm/Makefile
 rename {tools/regression => tests/sys}/posixshm/posixshm.c (100%)
 rename {tools/test => tests/sys}/posixshm/shm_test.c (94%)
 rename {tools/regression => tests/sys}/posixshm/test.c (100%)
 rename {tools/regression => tests/sys}/posixshm/test.h (100%)
 delete mode 100644 tools/regression/posixshm/Makefile
 delete mode 100644 tools/regression/posixshm/posixshm.t
 delete mode 100644 tools/test/posixshm/README

diff --git a/etc/mtree/BSD.tests.dist b/etc/mtree/BSD.tests.dist
index 1fb33654ac5b..e6f40655b8b6 100644
--- a/etc/mtree/BSD.tests.dist
+++ b/etc/mtree/BSD.tests.dist
@@ -410,6 +410,8 @@
             unlink
             ..
         ..
+        posixshm
+        ..
         vm
         ..
     ..
diff --git a/tests/sys/Makefile b/tests/sys/Makefile
index 066c918188b4..3ed85be370ae 100644
--- a/tests/sys/Makefile
+++ b/tests/sys/Makefile
@@ -12,6 +12,7 @@ TESTS_SUBDIRS+=		kqueue
 TESTS_SUBDIRS+=		mqueue
 TESTS_SUBDIRS+=		netinet
 TESTS_SUBDIRS+=		opencrypto
+TESTS_SUBDIRS+=		posixshm
 TESTS_SUBDIRS+=		vm
 
 # Items not integrated into kyua runs by default
diff --git a/tests/sys/posixshm/Makefile b/tests/sys/posixshm/Makefile
new file mode 100644
index 000000000000..7ddd7f0e87fd
--- /dev/null
+++ b/tests/sys/posixshm/Makefile
@@ -0,0 +1,11 @@
+# $FreeBSD$
+
+TESTSDIR=	${TESTSBASE}/sys/posixshm
+
+TAP_TESTS_C+=	posixshm_test
+PLAIN_TESTS_C+=	posixshm_test2
+
+SRCS.posixshm_test=	posixshm.c test.c
+SRCS.posixshm_test2=	shm_test.c
+
+.include <bsd.test.mk>
diff --git a/tools/regression/posixshm/posixshm.c b/tests/sys/posixshm/posixshm.c
similarity index 100%
rename from tools/regression/posixshm/posixshm.c
rename to tests/sys/posixshm/posixshm.c
diff --git a/tools/test/posixshm/shm_test.c b/tests/sys/posixshm/shm_test.c
similarity index 94%
rename from tools/test/posixshm/shm_test.c
rename to tests/sys/posixshm/shm_test.c
index 0ec890a85781..a46fc0ba1e26 100644
--- a/tools/test/posixshm/shm_test.c
+++ b/tests/sys/posixshm/shm_test.c
@@ -79,7 +79,12 @@ main(int argc, char **argv)
 			/*
 			 * Can't use mkstemp for obvious reasons...
 			 */
-			strcpy(buf, "/tmp/shmtest.XXXXXXXXXXXX");
+			char *tmpdir = getenv("TMPDIR");
+			if (tmpdir == NULL)
+				tmpdir = "/tmp";
+			snprintf(buf, sizeof(buf) - 1,
+			    "%s/shmtest.XXXXXXXXXXXX", tmpdir);
+			buf[sizeof(buf) - 1] = '\0';
 			mktemp(buf);
 			desc = shm_open(buf, O_EXCL | O_CREAT | O_RDWR, 0600);
 		} while (desc < 0 && errno == EEXIST);
diff --git a/tools/regression/posixshm/test.c b/tests/sys/posixshm/test.c
similarity index 100%
rename from tools/regression/posixshm/test.c
rename to tests/sys/posixshm/test.c
diff --git a/tools/regression/posixshm/test.h b/tests/sys/posixshm/test.h
similarity index 100%
rename from tools/regression/posixshm/test.h
rename to tests/sys/posixshm/test.h
diff --git a/tools/regression/posixshm/Makefile b/tools/regression/posixshm/Makefile
deleted file mode 100644
index 5ef70c93bbdb..000000000000
--- a/tools/regression/posixshm/Makefile
+++ /dev/null
@@ -1,9 +0,0 @@
-# $FreeBSD$
-
-PROG=	posixshm
-SRCS=	posixshm.c test.c
-MAN=
-
-WARNS?=	3
-
-.include <bsd.prog.mk>
diff --git a/tools/regression/posixshm/posixshm.t b/tools/regression/posixshm/posixshm.t
deleted file mode 100644
index a2159aca05a3..000000000000
--- a/tools/regression/posixshm/posixshm.t
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-
-./posixshm
diff --git a/tools/test/posixshm/README b/tools/test/posixshm/README
deleted file mode 100644
index 514e18b893a7..000000000000
--- a/tools/test/posixshm/README
+++ /dev/null
@@ -1,4 +0,0 @@
-$FreeBSD$
-
-This is a simple program to test/demonstrate the POSIX Shared Memory
-Objects feature set.  `make shm_test' to build.

From d3814e98fdfad8518b88ea31bcdec25c7d98e113 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Tue, 13 Oct 2015 16:51:12 +0000
Subject: [PATCH 167/380] Integrate tools/regression/vfs into the FreeBSD test
 suite as tests/sys/vfs

---
 etc/mtree/BSD.tests.dist                                        | 2 ++
 tests/sys/Makefile                                              | 1 +
 .../vfs/trailing_slash.t => tests/sys/vfs/trailing_slash.sh     | 0
 3 files changed, 3 insertions(+)
 rename tools/regression/vfs/trailing_slash.t => tests/sys/vfs/trailing_slash.sh (100%)

diff --git a/etc/mtree/BSD.tests.dist b/etc/mtree/BSD.tests.dist
index e6f40655b8b6..1ca3d4d6c7d8 100644
--- a/etc/mtree/BSD.tests.dist
+++ b/etc/mtree/BSD.tests.dist
@@ -412,6 +412,8 @@
         ..
         posixshm
         ..
+        vfs
+        ..
         vm
         ..
     ..
diff --git a/tests/sys/Makefile b/tests/sys/Makefile
index 3ed85be370ae..a22214a1444d 100644
--- a/tests/sys/Makefile
+++ b/tests/sys/Makefile
@@ -13,6 +13,7 @@ TESTS_SUBDIRS+=		mqueue
 TESTS_SUBDIRS+=		netinet
 TESTS_SUBDIRS+=		opencrypto
 TESTS_SUBDIRS+=		posixshm
+TESTS_SUBDIRS+=		vfs
 TESTS_SUBDIRS+=		vm
 
 # Items not integrated into kyua runs by default
diff --git a/tools/regression/vfs/trailing_slash.t b/tests/sys/vfs/trailing_slash.sh
similarity index 100%
rename from tools/regression/vfs/trailing_slash.t
rename to tests/sys/vfs/trailing_slash.sh

From 8dbf348e51b0f1f6d3f065bea26be5982fd07784 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Tue, 13 Oct 2015 17:05:07 +0000
Subject: [PATCH 168/380] Replace references to /dev/acd0 with /dev/cd0

atapicd(4) was removed in r249083 (it was replaced by cd(4))
---
 sbin/mount/mount.8          | 2 +-
 share/man/man5/devfs.conf.5 | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/sbin/mount/mount.8 b/sbin/mount/mount.8
index 2b1ac527ba93..9b9497f73b4d 100644
--- a/sbin/mount/mount.8
+++ b/sbin/mount/mount.8
@@ -189,7 +189,7 @@ to use the specified program to mount the file system, instead of calling
 directly.
 For example:
 .Bd -literal
-mount -t foofs -o mountprog=/mydir/fooprog /dev/acd0 /mnt
+mount -t foofs -o mountprog=/mydir/fooprog /dev/cd0 /mnt
 .Ed
 .It Cm multilabel
 Enable multi-label Mandatory Access Control, or MAC, on the specified file
diff --git a/share/man/man5/devfs.conf.5 b/share/man/man5/devfs.conf.5
index 834d6e768e96..26796c1b812b 100644
--- a/share/man/man5/devfs.conf.5
+++ b/share/man/man5/devfs.conf.5
@@ -109,9 +109,9 @@ link	cd0	cdrom
 .Pp
 Similarly, to link
 .Pa /dev/cdrom
-to the first ATAPI CD-ROM device, the following action may be used:
+to the first SCSI CD-ROM device, the following action may be used:
 .Bd -literal -offset indent
-link	acd0	cdrom
+link	cd0	cdrom
 .Ed
 .Pp
 To set the owner of a device, the

From 017eec70b4e060c5c0fc967352f9771f2a48846d Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Tue, 13 Oct 2015 17:06:41 +0000
Subject: [PATCH 169/380] Bump .Dd

---
 sbin/mount/mount.8          | 2 +-
 share/man/man5/devfs.conf.5 | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/sbin/mount/mount.8 b/sbin/mount/mount.8
index 9b9497f73b4d..4c07ab284b26 100644
--- a/sbin/mount/mount.8
+++ b/sbin/mount/mount.8
@@ -28,7 +28,7 @@
 .\"     @(#)mount.8	8.8 (Berkeley) 6/16/94
 .\" $FreeBSD$
 .\"
-.Dd December 3, 2014
+.Dd October 13, 2015
 .Dt MOUNT 8
 .Os
 .Sh NAME
diff --git a/share/man/man5/devfs.conf.5 b/share/man/man5/devfs.conf.5
index 26796c1b812b..955abe20dd5d 100644
--- a/share/man/man5/devfs.conf.5
+++ b/share/man/man5/devfs.conf.5
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd May 17, 2005
+.Dd October 13, 2015
 .Dt DEVFS.CONF 5
 .Os
 .Sh NAME

From 488dfdbecee1e221f973b6702cee0108cc62739b Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Tue, 13 Oct 2015 17:09:07 +0000
Subject: [PATCH 170/380] Replace /dev/acd0 with /dev/cd1 as atapicd(4) has
 been removed since r249083.

Update mount.conf(8) to reflect the change in behavior
---
 sbin/mount/mount.conf.8  | 6 +++---
 sys/kern/vfs_mountroot.c | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/sbin/mount/mount.conf.8 b/sbin/mount/mount.conf.8
index 45b8257ebae9..f87af9ac7e55 100644
--- a/sbin/mount/mount.conf.8
+++ b/sbin/mount/mount.conf.8
@@ -26,7 +26,7 @@
 .\" $FreeBSD$
 .\"
 .\"
-.Dd July 7, 2013
+.Dd October 13, 2013
 .Dt MOUNT.CONF 8
 .Os
 .Sh NAME
@@ -154,7 +154,7 @@ will direct the kernel to try mounting the root file system
 first as an ISO CD9660 file system on
 .Pa /dev/cd0 ,
 then if that does not work, as an ISO CD9660 file system on
-.Pa /dev/acd0 ,
+.Pa /dev/cd1 ,
 and then if that does not work, as a UFS file system on
 .Pa /dev/ada0s1a .
 If that does not work, a
@@ -167,7 +167,7 @@ Finally if that does not work, the kernel will panic.
 .Li .timeout 3
 cd9660:/dev/cd0 ro
 .Li .timeout 0
-cd9660:/dev/acd0 ro
+cd9660:/dev/cd1 ro
 .Li .timeout 3
 ufs:/dev/ada0s1a
 .Li .ask
diff --git a/sys/kern/vfs_mountroot.c b/sys/kern/vfs_mountroot.c
index 73ed5b652b2e..aecddbda2a63 100644
--- a/sys/kern/vfs_mountroot.c
+++ b/sys/kern/vfs_mountroot.c
@@ -468,9 +468,9 @@ parse_dir_ask(char **conf)
 	printf("\n");
 	printf("    eg. ufs:/dev/da0s1a\n");
 	printf("        zfs:tank\n");
-	printf("        cd9660:/dev/acd0 ro\n");
+	printf("        cd9660:/dev/cd0 ro\n");
 	printf("          (which is equivalent to: ");
-	printf("mount -t cd9660 -o ro /dev/acd0 /)\n");
+	printf("mount -t cd9660 -o ro /dev/cd0 /)\n");
 	printf("\n");
 	printf("  ?               List valid disk boot devices\n");
 	printf("  .               Yield 1 second (for background tasks)\n");
@@ -837,7 +837,7 @@ vfs_mountroot_conf0(struct sbuf *sb)
 	if (boothowto & RB_CDROM) {
 		sbuf_printf(sb, "cd9660:/dev/cd0 ro\n");
 		sbuf_printf(sb, ".timeout 0\n");
-		sbuf_printf(sb, "cd9660:/dev/acd0 ro\n");
+		sbuf_printf(sb, "cd9660:/dev/cd1 ro\n");
 		sbuf_printf(sb, ".timeout %d\n", root_mount_timeout);
 	}
 	s = kern_getenv("vfs.root.mountfrom");

From 5d430003c819997e598b5958c8c0838f897a015d Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Tue, 13 Oct 2015 17:18:26 +0000
Subject: [PATCH 171/380] Add Makefile forgotten in r289224

---
 tests/sys/vfs/Makefile | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 tests/sys/vfs/Makefile

diff --git a/tests/sys/vfs/Makefile b/tests/sys/vfs/Makefile
new file mode 100644
index 000000000000..48f522634bfa
--- /dev/null
+++ b/tests/sys/vfs/Makefile
@@ -0,0 +1,7 @@
+# $FreeBSD$
+
+TESTSDIR=		${TESTSBASE}/sys/vfs
+
+PLAIN_TESTS_SH+=	trailing_slash
+
+.include <bsd.test.mk>

From f704bed62801656e7361875052f75afba0bd97bb Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Tue, 13 Oct 2015 22:55:17 +0000
Subject: [PATCH 172/380] Use chroot(2) instead of using prefixes for files.

Previously, the code prefixes the chroot path to actual file paths to
simulate the effect.  This, however, will not work for tzset(3) which
expects the current system have a working set of timezone data files,
and that is not always the case.

This changeset simplifies the handling of paths and use an actual
chroot(2) call to implement the effect.

PR:		bin/197313
MFC after:	2 weeks
---
 usr.sbin/tzsetup/tzsetup.c | 26 +++++++++++---------------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/usr.sbin/tzsetup/tzsetup.c b/usr.sbin/tzsetup/tzsetup.c
index fc803646c23a..7d2a9360ebb4 100644
--- a/usr.sbin/tzsetup/tzsetup.c
+++ b/usr.sbin/tzsetup/tzsetup.c
@@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$");
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sysexits.h>
 #include <time.h>
 #include <unistd.h>
 
@@ -944,23 +945,18 @@ main(int argc, char **argv)
 	if (argc - optind > 1)
 		usage();
 
-	if (chrootenv == NULL) {
-		strcpy(path_zonetab, _PATH_ZONETAB);
-		strcpy(path_iso3166, _PATH_ISO3166);
-		strcpy(path_zoneinfo, _PATH_ZONEINFO);
-		strcpy(path_localtime, _PATH_LOCALTIME);
-		strcpy(path_db, _PATH_DB);
-		strcpy(path_wall_cmos_clock, _PATH_WALL_CMOS_CLOCK);
-	} else {
-		sprintf(path_zonetab, "%s/%s", chrootenv, _PATH_ZONETAB);
-		sprintf(path_iso3166, "%s/%s", chrootenv, _PATH_ISO3166);
-		sprintf(path_zoneinfo, "%s/%s", chrootenv, _PATH_ZONEINFO);
-		sprintf(path_localtime, "%s/%s", chrootenv, _PATH_LOCALTIME);
-		sprintf(path_db, "%s/%s", chrootenv, _PATH_DB);
-		sprintf(path_wall_cmos_clock, "%s/%s", chrootenv,
-		    _PATH_WALL_CMOS_CLOCK);
+	if (chrootenv != NULL) {
+		rv = chroot(chrootenv);
+		if (rv != 0)
+			err(EX_OSERR, "chroot to %s", chrootenv);
 	}
 
+	strcpy(path_zonetab, _PATH_ZONETAB);
+	strcpy(path_iso3166, _PATH_ISO3166);
+	strcpy(path_zoneinfo, _PATH_ZONEINFO);
+	strcpy(path_localtime, _PATH_LOCALTIME);
+	strcpy(path_db, _PATH_DB);
+	strcpy(path_wall_cmos_clock, _PATH_WALL_CMOS_CLOCK);
 
 	/* Override the user-supplied umask. */
 	(void)umask(S_IWGRP | S_IWOTH);

From bc762f4426d89a794efd0f55bffc42691d6b1b85 Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Tue, 13 Oct 2015 23:41:06 +0000
Subject: [PATCH 173/380] NTB: MFV c529aa30: Xeon Doorbell errata workaround

Modifications to the 14th bit of the B2BDOORBELL register will not be
mirrored to the remote system due to a hardware issue.  To get around
the issue, shrink the number of available doorbell bits by 1.  The max
number of doorbells was being used as a way to referencing the Link
Doorbell bit.  Since this would no longer work, the driver must now
explicitly reference that bit.

This does not affect the xeon_errata_workaround case, as it is not using
the b2bdoorbell register.

Authored by:	Jon Mason
Obtained from:	Linux (Dual BSD/GPL driver)
Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/ntb_hw/ntb_hw.c   | 17 +++++++++++++++--
 sys/dev/ntb/ntb_hw/ntb_regs.h |  1 +
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c
index 66d0d7443096..5bd0025e87ed 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.c
@@ -479,7 +479,7 @@ ntb_setup_interrupts(struct ntb_softc *ntb)
 		ntb_reg_write(8, ntb->reg_ofs.ldb_mask, ~0);
 	else
 		ntb_reg_write(2, ntb->reg_ofs.ldb_mask,
-		    ~(1 << ntb->limits.max_db_bits));
+		    (uint16_t) ~(1 << XEON_LINK_DB));
 
 	num_vectors = MIN(pci_msix_count(ntb->device),
 	    ntb->limits.max_db_bits);
@@ -616,7 +616,7 @@ handle_xeon_event_irq(void *arg)
 		device_printf(ntb->device, "Error determining link status\n");
 
 	/* bit 15 is always the link bit */
-	ntb_reg_write(2, ntb->reg_ofs.ldb, 1 << ntb->limits.max_db_bits);
+	ntb_reg_write(2, ntb->reg_ofs.ldb, 1 << XEON_LINK_DB);
 }
 
 static void
@@ -784,6 +784,19 @@ ntb_setup_xeon(struct ntb_softc *ntb)
 	ntb->limits.msix_cnt	 = XEON_MSIX_CNT;
 	ntb->bits_per_vector	 = XEON_DB_BITS_PER_VEC;
 
+	/*
+	 * HW Errata on bit 14 of b2bdoorbell register.  Writes will not be
+	 * mirrored to the remote system.  Shrink the number of bits by one,
+	 * since bit 14 is the last bit.
+	 *
+	 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register
+	 * anyway.  Nor for non-B2B connection types.
+	 */
+	if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14) &&
+	    !HAS_FEATURE(NTB_REGS_THRU_MW) &&
+	    connection_type == NTB_CONN_B2B)
+		ntb->limits.max_db_bits = XEON_MAX_DB_BITS - 1;
+
 	configure_xeon_secondary_side_bars(ntb);
 
 	/* Enable Bus Master and Memory Space on the secondary side */
diff --git a/sys/dev/ntb/ntb_hw/ntb_regs.h b/sys/dev/ntb/ntb_hw/ntb_regs.h
index 2567f830e931..d97d504b7db7 100644
--- a/sys/dev/ntb/ntb_hw/ntb_regs.h
+++ b/sys/dev/ntb/ntb_hw/ntb_regs.h
@@ -38,6 +38,7 @@
 #define XEON_MAX_COMPAT_SPADS	16
 /* Reserve the uppermost bit for link interrupt */
 #define XEON_MAX_DB_BITS	15
+#define XEON_LINK_DB		15
 #define XEON_DB_BITS_PER_VEC	5
 
 #define XEON_DB_HW_LINK		0x8000

From 704a018d1211bd82121b9b82a2626089c351c5a7 Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Tue, 13 Oct 2015 23:41:40 +0000
Subject: [PATCH 174/380] NTB: MFV 9fec60c4: Fix NTB-RP Link Up

The Xeon NTB-RP setup, the transparent side does not get a link up/down
interrupt.  Since the presence of a NTB device on the transparent side
means that we have a NTB link up, we can work around the lack of an
interrupt by simply calling the link up function to notify the upper
layers.

Authored by:	Jon Mason
Obtained from:	Linux (Dual BSD/GPL driver)
Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/ntb_hw/ntb_hw.c | 50 +++++++++++++++++++++++++++++--------
 1 file changed, 40 insertions(+), 10 deletions(-)

diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c
index 5bd0025e87ed..7f7fa5f82a8b 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.c
@@ -207,13 +207,15 @@ static void ntb_handle_legacy_interrupt(void *arg);
 static int ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors);
 static void ntb_free_callbacks(struct ntb_softc *ntb);
 static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id);
-static int ntb_initialize_hw(struct ntb_softc *ntb);
 static int ntb_setup_xeon(struct ntb_softc *ntb);
 static int ntb_setup_soc(struct ntb_softc *ntb);
+static void ntb_teardown_xeon(struct ntb_softc *ntb);
 static void configure_soc_secondary_side_bars(struct ntb_softc *ntb);
 static void configure_xeon_secondary_side_bars(struct ntb_softc *ntb);
 static void ntb_handle_heartbeat(void *arg);
 static void ntb_handle_link_event(struct ntb_softc *ntb, int link_state);
+static void ntb_hw_link_down(struct ntb_softc *ntb);
+static void ntb_hw_link_up(struct ntb_softc *ntb);
 static void recover_soc_link(void *arg);
 static int ntb_check_link_status(struct ntb_softc *ntb);
 static void save_bar_parameters(struct ntb_pci_bar_info *bar);
@@ -301,7 +303,10 @@ ntb_attach(device_t device)
 	error = ntb_map_pci_bars(ntb);
 	if (error)
 		goto out;
-	error = ntb_initialize_hw(ntb);
+	if (ntb->type == NTB_SOC)
+		error = ntb_setup_soc(ntb);
+	else
+		error = ntb_setup_xeon(ntb);
 	if (error)
 		goto out;
 	error = ntb_setup_interrupts(ntb);
@@ -324,6 +329,8 @@ ntb_detach(device_t device)
 	ntb = DEVICE2SOFTC(device);
 	callout_drain(&ntb->heartbeat_timer);
 	callout_drain(&ntb->lr_timer);
+	if (ntb->type == NTB_XEON)
+		ntb_teardown_xeon(ntb);
 	ntb_teardown_interrupts(ntb);
 	ntb_unmap_pci_bar(ntb);
 
@@ -691,14 +698,11 @@ ntb_get_device_info(uint32_t device_id)
 	return (NULL);
 }
 
-static int
-ntb_initialize_hw(struct ntb_softc *ntb)
+static void
+ntb_teardown_xeon(struct ntb_softc *ntb)
 {
 
-	if (ntb->type == NTB_SOC)
-		return (ntb_setup_soc(ntb));
-	else
-		return (ntb_setup_xeon(ntb));
+	ntb_hw_link_down(ntb);
 }
 
 static int
@@ -805,8 +809,7 @@ ntb_setup_xeon(struct ntb_softc *ntb)
 		    PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
 
 	/* Enable link training */
-	ntb_reg_write(4, ntb->reg_ofs.lnk_cntl,
-	    NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP);
+	ntb_hw_link_up(ntb);
 
 	return (0);
 }
@@ -1039,6 +1042,33 @@ ntb_handle_link_event(struct ntb_softc *ntb, int link_state)
 		ntb->event_cb(ntb->ntb_transport, event);
 }
 
+static void
+ntb_hw_link_up(struct ntb_softc *ntb)
+{
+
+	if (ntb->conn_type == NTB_CONN_TRANSPARENT)
+		ntb_handle_link_event(ntb, NTB_LINK_UP);
+	else
+		ntb_reg_write(4, ntb->reg_ofs.lnk_cntl,
+		    NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP);
+}
+
+static void
+ntb_hw_link_down(struct ntb_softc *ntb)
+{
+	uint32_t cntl;
+
+	if (ntb->conn_type == NTB_CONN_TRANSPARENT) {
+		ntb_handle_link_event(ntb, NTB_LINK_DOWN);
+		return;
+	}
+
+	cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl);
+	cntl &= ~(NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP);
+	cntl |= NTB_CNTL_LINK_DISABLE;
+	ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, cntl);
+}
+
 static void
 recover_soc_link(void *arg)
 {

From 7e386ea836692da341989f00d89c867acbc990c3 Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Tue, 13 Oct 2015 23:42:13 +0000
Subject: [PATCH 175/380] NTB: MFV fca4d518: Fix ntb_transport link down race

A WARN_ON is being hit in ntb_qp_link_work due to the NTB transport link
being down while the ntb qp link is still active.  This is caused by the
transport link being brought down prior to the qp link worker thread
being terminated.  To correct this, shutdown the qp's prior to bringing
the transport link down.  Also, only call the qp worker thread if it is
in interrupt context, otherwise call the function directly.

Authored by:	Jon Mason
Obtained from:	Linux (Dual BSD/GPL driver)
Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/if_ntb/if_ntb.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/sys/dev/ntb/if_ntb/if_ntb.c b/sys/dev/ntb/if_ntb/if_ntb.c
index 81ebe73a4208..02bab46f8b6a 100644
--- a/sys/dev/ntb/if_ntb/if_ntb.c
+++ b/sys/dev/ntb/if_ntb/if_ntb.c
@@ -519,7 +519,7 @@ ntb_transport_free(void *transport)
 	struct ntb_softc *ntb = nt->ntb;
 	int i;
 
-	nt->transport_link = NTB_LINK_DOWN;
+	ntb_transport_link_cleanup(nt);
 
 	callout_drain(&nt->link_work);
 
@@ -1257,16 +1257,16 @@ ntb_transport_link_cleanup(struct ntb_netdev *nt)
 {
 	int i;
 
-	if (nt->transport_link == NTB_LINK_DOWN)
-		callout_drain(&nt->link_work);
-	else
-		nt->transport_link = NTB_LINK_DOWN;
-
 	/* Pass along the info to any clients */
 	for (i = 0; i < nt->max_qps; i++)
 		if (!test_bit(i, &nt->qp_bitmap))
 			ntb_qp_link_down(&nt->qps[i]);
 
+	if (nt->transport_link == NTB_LINK_DOWN)
+		callout_drain(&nt->link_work);
+	else
+		nt->transport_link = NTB_LINK_DOWN;
+
 	/* 
 	 * The scratchpad registers keep the values if the remote side
 	 * goes down, blast them now to give them a sane value the next

From 66946565079046433adc73a6528f8698c4eb249c Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Tue, 13 Oct 2015 23:43:06 +0000
Subject: [PATCH 176/380] NTB: MFV 58b88920: Document HW errata

Add a comment describing the necessary ordering of modifications to the
NTB Limit and Base registers.

Authored by:	Jon Mason
Obtained from:	Linux (Dual BSD/GPL driver)
Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/ntb_hw/ntb_hw.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c
index 7f7fa5f82a8b..b45e8258f3c0 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.c
@@ -764,6 +764,11 @@ ntb_setup_xeon(struct ntb_softc *ntb)
 	 * which may hang the system.  To workaround this use the second memory
 	 * window to access the interrupt and scratch pad registers on the
 	 * remote system.
+	 *
+	 * There is another HW errata on the limit registers -- they can only
+	 * be written when the base register is (?)4GB aligned and < 32-bit.
+	 * This should already be the case based on the driver defaults, but
+	 * write the limit registers first just in case.
 	 */
 	if (HAS_FEATURE(NTB_REGS_THRU_MW))
 		/*

From 891ebb5d1c21174bf8603121556ff1f9ef13d49a Mon Sep 17 00:00:00 2001
From: emaste <emaste@FreeBSD.org>
Date: Wed, 14 Oct 2015 00:23:31 +0000
Subject: [PATCH 177/380] Add WITHOUT_LLDB for src.conf(5)

It will be enabled by default on certain architectures.
---
 tools/build/options/WITHOUT_LLDB | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 tools/build/options/WITHOUT_LLDB

diff --git a/tools/build/options/WITHOUT_LLDB b/tools/build/options/WITHOUT_LLDB
new file mode 100644
index 000000000000..7cc45d91efdc
--- /dev/null
+++ b/tools/build/options/WITHOUT_LLDB
@@ -0,0 +1,2 @@
+.\" $FreeBSD$
+Set to not build the LLDB debugger.

From 0d123061888740d00bc717c489bb0efd082eb2aa Mon Sep 17 00:00:00 2001
From: hiren <hiren@FreeBSD.org>
Date: Wed, 14 Oct 2015 00:35:37 +0000
Subject: [PATCH 178/380] There are times when it would be really nice to have
 a record of the last few packets and/or state transitions from each TCP
 socket. That would help with narrowing down certain problems we see in the
 field that are hard to reproduce without understanding the history of how we
 got into a certain state. This change provides just that.

It saves copies of the last N packets in a list in the tcpcb. When the tcpcb is
destroyed, the list is freed. I thought this was likely to be more
performance-friendly than saving copies of the tcpcb. Plus, with the packets,
you should be able to reverse-engineer what happened to the tcpcb.

To enable the feature, you will need to compile a kernel with the TCPPCAP
option. Even then, the feature defaults to being deactivated. You can activate
it by setting a positive value for the number of captured packets. You can do
that on either a global basis or on a per-socket basis (via a setsockopt call).

There is no way to get the packets out of the kernel other than using kmem or
getting a coredump. I thought that would help some of the legal/privacy concerns
regarding such a feature. However, it should be possible to add a future effort
to export them in PCAP format.

I tested this at low scale, and found that there were no mbuf leaks and the peak
mbuf usage appeared to be unchanged with and without the feature.

The main performance concern I can envision is the number of mbufs that would be
used on systems with a large number of sockets. If you save five packets per
direction per socket and have 3,000 sockets, that will consume at least 30,000
mbufs just to keep these packets. I tried to reduce the concerns associated with
this by limiting the number of clusters (not mbufs) that could be used for this
feature. Again, in my testing, that appears to work correctly.

Differential Revision:	D3100
Submitted by:		Jonathan Looney <jlooney at juniper dot net>
Reviewed by:		gnn, hiren
---
 sys/conf/NOTES           |   4 +
 sys/conf/files           |   1 +
 sys/conf/options         |   1 +
 sys/kern/uipc_mbuf.c     |   2 +-
 sys/netinet/tcp.h        |   2 +
 sys/netinet/tcp_input.c  |   8 +
 sys/netinet/tcp_output.c |  13 ++
 sys/netinet/tcp_pcap.c   | 437 +++++++++++++++++++++++++++++++++++++++
 sys/netinet/tcp_pcap.h   |  39 ++++
 sys/netinet/tcp_subr.c   |  18 ++
 sys/netinet/tcp_usrreq.c |  31 +++
 sys/netinet/tcp_var.h    |  13 +-
 sys/sys/mbuf.h           |   1 +
 13 files changed, 568 insertions(+), 2 deletions(-)
 create mode 100644 sys/netinet/tcp_pcap.c
 create mode 100644 sys/netinet/tcp_pcap.h

diff --git a/sys/conf/NOTES b/sys/conf/NOTES
index c6478f9bb78a..c1210e240191 100644
--- a/sys/conf/NOTES
+++ b/sys/conf/NOTES
@@ -960,6 +960,9 @@ device		lagg
 # for sockets with the SO_DEBUG option set, which can then be examined
 # using the trpt(8) utility.
 #
+# TCPPCAP enables code which keeps the last n packets sent and received
+# on a TCP socket.
+#
 # RADIX_MPATH provides support for equal-cost multi-path routing.
 #
 options 	MROUTING		# Multicast routing
@@ -976,6 +979,7 @@ options 	IPFILTER_DEFAULT_BLOCK	#block all packets by default
 options 	IPSTEALTH		#support for stealth forwarding
 options 	PF_DEFAULT_TO_DROP	#drop everything by default
 options 	TCPDEBUG
+options 	TCPPCAP
 options 	RADIX_MPATH
 
 # The MBUF_STRESS_TEST option enables options which create
diff --git a/sys/conf/files b/sys/conf/files
index 3ddf41287a09..5e7bdf384bbe 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -3682,6 +3682,7 @@ netinet/tcp_input.c		optional inet | inet6
 netinet/tcp_lro.c		optional inet | inet6
 netinet/tcp_output.c		optional inet | inet6
 netinet/tcp_offload.c		optional tcp_offload inet | tcp_offload inet6
+netinet/tcp_pcap.c		optional tcppcap
 netinet/tcp_reass.c		optional inet | inet6
 netinet/tcp_sack.c		optional inet | inet6
 netinet/tcp_subr.c		optional inet | inet6
diff --git a/sys/conf/options b/sys/conf/options
index ccbbd800c3cf..dcea435a3c97 100644
--- a/sys/conf/options
+++ b/sys/conf/options
@@ -436,6 +436,7 @@ ROUTETABLES		opt_route.h
 RSS			opt_rss.h
 SLIP_IFF_OPTS		opt_slip.h
 TCPDEBUG
+TCPPCAP		opt_global.h
 SIFTR
 TCP_OFFLOAD		opt_inet.h # Enable code to dispatch TCP offloading
 TCP_SIGNATURE		opt_inet.h
diff --git a/sys/kern/uipc_mbuf.c b/sys/kern/uipc_mbuf.c
index 7f994fef1ef5..de6b313f1b43 100644
--- a/sys/kern/uipc_mbuf.c
+++ b/sys/kern/uipc_mbuf.c
@@ -395,7 +395,7 @@ mb_free_ext(struct mbuf *m)
  * Attach the cluster from *m to *n, set up m_ext in *n
  * and bump the refcount of the cluster.
  */
-static void
+void
 mb_dupcl(struct mbuf *n, const struct mbuf *m)
 {
 
diff --git a/sys/netinet/tcp.h b/sys/netinet/tcp.h
index fb2f8108d4f6..37608605367f 100644
--- a/sys/netinet/tcp.h
+++ b/sys/netinet/tcp.h
@@ -165,6 +165,8 @@ struct tcphdr {
 #define	TCP_KEEPIDLE	256	/* L,N,X start keeplives after this period */
 #define	TCP_KEEPINTVL	512	/* L,N interval between keepalives */
 #define	TCP_KEEPCNT	1024	/* L,N number of keepalives before close */
+#define	TCP_PCAP_OUT	2048	/* number of output packets to keep */
+#define	TCP_PCAP_IN	4096	/* number of input packets to keep */
 
 /* Start of reserved space for third-party user-settable options. */
 #define	TCP_VENDOR	SO_VENDOR
diff --git a/sys/netinet/tcp_input.c b/sys/netinet/tcp_input.c
index b6ea859de310..64971f3da293 100644
--- a/sys/netinet/tcp_input.c
+++ b/sys/netinet/tcp_input.c
@@ -104,6 +104,9 @@ __FBSDID("$FreeBSD$");
 #include <netinet/tcp_var.h>
 #include <netinet6/tcp6_var.h>
 #include <netinet/tcpip.h>
+#ifdef TCPPCAP
+#include <netinet/tcp_pcap.h>
+#endif
 #include <netinet/tcp_syncache.h>
 #ifdef TCPDEBUG
 #include <netinet/tcp_debug.h>
@@ -1524,6 +1527,11 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 	KASSERT(tp->t_state != TCPS_TIME_WAIT, ("%s: TCPS_TIME_WAIT",
 	    __func__));
 
+#ifdef TCPPCAP
+	/* Save segment, if requested. */
+	tcp_pcap_add(th, m, &(tp->t_inpkts));
+#endif
+
 	/*
 	 * Segment received on connection.
 	 * Reset idle time and keep-alive timer.
diff --git a/sys/netinet/tcp_output.c b/sys/netinet/tcp_output.c
index fc90b069b9d4..d295dcfbcd6b 100644
--- a/sys/netinet/tcp_output.c
+++ b/sys/netinet/tcp_output.c
@@ -74,6 +74,9 @@ __FBSDID("$FreeBSD$");
 #include <netinet/tcp_timer.h>
 #include <netinet/tcp_var.h>
 #include <netinet/tcpip.h>
+#ifdef TCPPCAP
+#include <netinet/tcp_pcap.h>
+#endif
 #ifdef TCPDEBUG
 #include <netinet/tcp_debug.h>
 #endif
@@ -1305,6 +1308,11 @@ send:
 
 		TCP_PROBE5(send, NULL, tp, ip6, tp, th);
 
+#ifdef TCPPCAP
+		/* Save packet, if requested. */
+		tcp_pcap_add(th, m, &(tp->t_outpkts));
+#endif
+
 		/* TODO: IPv6 IP6TOS_ECT bit on */
 		error = ip6_output(m, tp->t_inpcb->in6p_outputopts, &ro,
 		    ((so->so_options & SO_DONTROUTE) ?  IP_ROUTETOIF : 0),
@@ -1348,6 +1356,11 @@ send:
 
 	TCP_PROBE5(send, NULL, tp, ip, tp, th);
 
+#ifdef TCPPCAP
+	/* Save packet, if requested. */
+	tcp_pcap_add(th, m, &(tp->t_outpkts));
+#endif
+
 	error = ip_output(m, tp->t_inpcb->inp_options, &ro,
 	    ((so->so_options & SO_DONTROUTE) ? IP_ROUTETOIF : 0), 0,
 	    tp->t_inpcb);
diff --git a/sys/netinet/tcp_pcap.c b/sys/netinet/tcp_pcap.c
new file mode 100644
index 000000000000..7a3514f9646d
--- /dev/null
+++ b/sys/netinet/tcp_pcap.c
@@ -0,0 +1,437 @@
+/*-
+ * Copyright (c) 2015
+ *	Jonathan Looney. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/queue.h>
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/socketvar.h>
+#include <sys/sysctl.h>
+#include <sys/systm.h>
+#include <sys/mbuf.h>
+#include <sys/eventhandler.h>
+#include <machine/atomic.h>
+#include <netinet/tcp_var.h>
+#include <netinet/tcp_pcap.h>
+
+#define M_LEADINGSPACE_NOWRITE(m)					\
+	((m)->m_data - M_START(m))
+
+static int tcp_pcap_clusters_referenced_cur = 0;
+static int tcp_pcap_clusters_referenced_max = 0;
+
+SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_pcap_clusters_referenced_cur,
+	CTLFLAG_RD, &tcp_pcap_clusters_referenced_cur, 0,
+	"Number of clusters currently referenced on TCP PCAP queues");
+SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_pcap_clusters_referenced_max,
+	CTLFLAG_RW, &tcp_pcap_clusters_referenced_max, 0,
+	"Maximum number of clusters allowed to be referenced on TCP PCAP "
+	"queues");
+
+static int tcp_pcap_alloc_reuse_ext = 0;
+static int tcp_pcap_alloc_reuse_mbuf = 0;
+static int tcp_pcap_alloc_new_mbuf = 0;
+SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_pcap_alloc_reuse_ext,
+	CTLFLAG_RD, &tcp_pcap_alloc_reuse_ext, 0,
+	"Number of mbufs with external storage reused for the TCP PCAP "
+	"functionality");
+SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_pcap_alloc_reuse_mbuf,
+	CTLFLAG_RD, &tcp_pcap_alloc_reuse_mbuf, 0,
+	"Number of mbufs with internal storage reused for the TCP PCAP "
+	"functionality");
+SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_pcap_alloc_new_mbuf,
+	CTLFLAG_RD, &tcp_pcap_alloc_new_mbuf, 0,
+	"Number of new mbufs allocated for the TCP PCAP functionality");
+
+VNET_DEFINE(int, tcp_pcap_packets) = 0;
+#define V_tcp_pcap_packets	VNET(tcp_pcap_packets)
+SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_pcap_packets, CTLFLAG_RW,
+	&V_tcp_pcap_packets, 0, "Default number of packets saved per direction "
+	"per TCPCB");
+
+/* Initialize the values. */
+static void
+tcp_pcap_max_set() {
+	tcp_pcap_clusters_referenced_max = nmbclusters / 4;
+}
+
+void
+tcp_pcap_init() {
+	tcp_pcap_max_set();
+	EVENTHANDLER_REGISTER(nmbclusters_change, tcp_pcap_max_set,
+		NULL, EVENTHANDLER_PRI_ANY);
+}
+
+/*
+ * If we are below the maximum allowed cluster references,
+ * increment the reference count and return TRUE. Otherwise,
+ * leave the reference count alone and return FALSE.
+ */
+static __inline bool
+tcp_pcap_take_cluster_reference(void)
+{
+	if (atomic_fetchadd_int(&tcp_pcap_clusters_referenced_cur, 1) >=
+		tcp_pcap_clusters_referenced_max) {
+		atomic_add_int(&tcp_pcap_clusters_referenced_cur, -1);
+		return FALSE;
+	}
+	return TRUE;
+}
+
+/*
+ * For all the external entries in m, apply the given adjustment.
+ * This can be used to adjust the counter when an mbuf chain is
+ * copied or freed.
+ */
+static __inline void
+tcp_pcap_adj_cluster_reference(struct mbuf *m, int adj)
+{
+	while (m) {
+		if (m->m_flags & M_EXT)
+			atomic_add_int(&tcp_pcap_clusters_referenced_cur, adj);
+
+		m = m->m_next;
+	}
+}
+
+/*
+ * Free all mbufs in a chain, decrementing the reference count as
+ * necessary.
+ *
+ * Functions in this file should use this instead of m_freem() when
+ * they are freeing mbuf chains that may contain clusters that were
+ * already included in tcp_pcap_clusters_referenced_cur.
+ */
+static void
+tcp_pcap_m_freem(struct mbuf *mb)
+{
+	while (mb != NULL) {
+		if (mb->m_flags & M_EXT)
+			atomic_subtract_int(&tcp_pcap_clusters_referenced_cur,
+			    1);
+		mb = m_free(mb);
+	}
+}
+
+/*
+ * Copy data from m to n, where n cannot fit all the data we might
+ * want from m.
+ *
+ * Prioritize data like this:
+ * 1. TCP header
+ * 2. IP header
+ * 3. Data
+ */
+static void
+tcp_pcap_copy_bestfit(struct tcphdr *th, struct mbuf *m, struct mbuf *n)
+{
+	struct mbuf *m_cur = m;
+	int bytes_to_copy=0, trailing_data, skip=0, tcp_off;
+
+	/* Below, we assume these will be non-NULL. */
+	KASSERT(th, ("%s: called with th == NULL", __func__));
+	KASSERT(m, ("%s: called with m == NULL", __func__));
+	KASSERT(n, ("%s: called with n == NULL", __func__));
+
+	/* We assume this initialization occurred elsewhere. */
+	KASSERT(n->m_len == 0, ("%s: called with n->m_len=%d (expected 0)",
+		__func__, n->m_len));
+	KASSERT(n->m_data == M_START(n),
+		("%s: called with n->m_data != M_START(n)", __func__));
+
+	/*
+	 * Calculate the size of the TCP header. We use this often
+	 * enough that it is worth just calculating at the start.
+	 */
+	tcp_off = th->th_off << 2;
+
+	/* Trim off leading empty mbufs. */
+	while (m && m->m_len == 0)
+		m = m->m_next;
+
+	if (m) {
+		m_cur = m;
+	}
+	else {
+		/*
+		 * No data? Highly unusual. We would expect to at
+		 * least see a TCP header in the mbuf.
+		 * As we have a pointer to the TCP header, I guess
+		 * we should just copy that. (???)
+		 */
+fallback:
+		bytes_to_copy = tcp_off;
+		if (bytes_to_copy > M_SIZE(n))
+			bytes_to_copy = M_SIZE(n);
+		bcopy(th, n->m_data, bytes_to_copy);
+		n->m_len = bytes_to_copy;
+		return;
+	}
+
+	/*
+	 * Find TCP header. Record the total number of bytes up to,
+	 * and including, the TCP header.
+	 */
+	while (m_cur) {
+		if ((caddr_t) th >= (caddr_t) m_cur->m_data &&
+			(caddr_t) th < (caddr_t) (m_cur->m_data + m_cur->m_len))
+			break;
+		bytes_to_copy += m_cur->m_len;
+		m_cur = m_cur->m_next;
+	}
+	if (m_cur)
+		bytes_to_copy += (caddr_t) th - (caddr_t) m_cur->m_data;
+	else
+		goto fallback;
+	bytes_to_copy += tcp_off;
+
+	/*
+	 * If we already want to copy more bytes than we can hold
+	 * in the destination mbuf, skip leading bytes and copy
+	 * what we can.
+	 *
+	 * Otherwise, consider trailing data.
+	 */
+	if (bytes_to_copy > M_SIZE(n)) {
+		skip  = bytes_to_copy - M_SIZE(n);
+		bytes_to_copy = M_SIZE(n);
+	}
+	else {
+		/*
+		 * Determine how much trailing data is in the chain.
+		 * We start with the length of this mbuf (the one
+		 * containing th) and subtract the size of the TCP
+		 * header (tcp_off) and the size of the data prior
+		 * to th (th - m_cur->m_data).
+		 *
+		 * This *should not* be negative, as the TCP code
+		 * should put the whole TCP header in a single
+		 * mbuf. But, it isn't a problem if it is. We will
+		 * simple work off our negative balance as we look
+		 * at subsequent mbufs.
+		 */
+		trailing_data = m_cur->m_len - tcp_off;
+		trailing_data -= (caddr_t) th - (caddr_t) m_cur->m_data;
+		m_cur = m_cur->m_next;
+		while (m_cur) {
+			trailing_data += m_cur->m_len;
+			m_cur = m_cur->m_next;
+		}
+		if ((bytes_to_copy + trailing_data) > M_SIZE(n))
+			bytes_to_copy = M_SIZE(n);
+		else
+			bytes_to_copy += trailing_data;
+	}
+
+	m_copydata(m, skip, bytes_to_copy, n->m_data);
+	n->m_len = bytes_to_copy;
+}
+
+void
+tcp_pcap_add(struct tcphdr *th, struct mbuf *m, struct mbufq *queue)
+{
+	struct mbuf *n = NULL, *mhead;
+
+	KASSERT(th, ("%s: called with th == NULL", __func__));
+	KASSERT(m, ("%s: called with m == NULL", __func__));
+	KASSERT(queue, ("%s: called with queue == NULL", __func__));
+
+	/* We only care about data packets. */
+	while (m && m->m_type != MT_DATA)
+		m = m->m_next;
+
+	/* We only need to do something if we still have an mbuf. */
+	if (!m)
+		return;
+
+	/* If we are not saving mbufs, return now. */
+	if (queue->mq_maxlen == 0)
+		return;
+
+	/*
+	 * Check to see if we will need to recycle mbufs.
+	 *
+	 * If we need to get rid of mbufs to stay below
+	 * our packet count, try to reuse the mbuf. Once
+	 * we already have a new mbuf (n), then we can
+	 * simply free subsequent mbufs.
+	 *
+	 * Note that most of the logic in here is to deal
+	 * with the reuse. If we are fine with constant
+	 * mbuf allocs/deallocs, we could ditch this logic.
+	 * But, it only seems to make sense to reuse
+	 * mbufs we already have.
+	 */
+	while (mbufq_full(queue)) {
+		mhead = mbufq_dequeue(queue);
+
+		if (n) {
+			tcp_pcap_m_freem(mhead);
+		}
+		else {
+			/*
+			 * If this held an external cluster, try to
+			 * detach the cluster. But, if we held the
+			 * last reference, go through the normal
+			 * free-ing process.
+			 */
+			if (mhead->m_flags & M_EXT) {
+				switch (mhead->m_ext.ext_type) {
+				case EXT_SFBUF:
+					/* Don't mess around with these. */
+					tcp_pcap_m_freem(mhead);
+					continue;
+				default:
+					if (atomic_fetchadd_int(
+						mhead->m_ext.ext_cnt, -1) == 1)
+					{
+						/*
+						 * We held the last reference
+						 * on this cluster. Restore
+						 * the reference count and put
+						 * it back in the pool.
+				 		 */
+						*(mhead->m_ext.ext_cnt) = 1;
+						tcp_pcap_m_freem(mhead);
+						continue;
+					}
+					/*
+					 * We were able to cleanly free the
+					 * reference.
+				 	 */
+					atomic_subtract_int(
+					    &tcp_pcap_clusters_referenced_cur,
+					    1);
+					tcp_pcap_alloc_reuse_ext++;
+					break;
+				}
+			}
+			else {
+				tcp_pcap_alloc_reuse_mbuf++;
+			}
+
+			n = mhead;
+			tcp_pcap_m_freem(n->m_next);
+			m_init(n, NULL, 0, M_NOWAIT, MT_DATA, 0);
+		}
+	}
+
+	/* Check to see if we need to get a new mbuf. */
+	if (!n) {
+		if (!(n = m_get(M_NOWAIT, MT_DATA)))
+			return;
+		tcp_pcap_alloc_new_mbuf++;
+	}
+
+	/*
+	 * What are we dealing with? If a cluster, attach it. Otherwise,
+	 * try to copy the data from the beginning of the mbuf to the
+	 * end of data. (There may be data between the start of the data
+	 * area and the current data pointer. We want to get this, because
+	 * it may contain header information that is useful.)
+	 * In cases where that isn't possible, settle for what we can
+	 * get.
+	 */
+	if ((m->m_flags & M_EXT) && tcp_pcap_take_cluster_reference()) {
+		n->m_data = m->m_data;
+		n->m_len = m->m_len;
+		mb_dupcl(n, m);
+	}
+	else if (((m->m_data + m->m_len) - M_START(m)) <= M_SIZE(n)) {
+		/*
+		 * At this point, n is guaranteed to be a normal mbuf
+		 * with no cluster and no packet header. Because the
+		 * logic in this code block requires this, the assert
+		 * is here to catch any instances where someone
+		 * changes the logic to invalidate that assumption.
+		 */
+		KASSERT((n->m_flags & (M_EXT | M_PKTHDR)) == 0,
+			("%s: Unexpected flags (%#x) for mbuf",
+			__func__, n->m_flags));
+		n->m_data = n->m_dat + M_LEADINGSPACE_NOWRITE(m);
+		n->m_len = m->m_len;
+		bcopy(M_START(m), n->m_dat,
+			m->m_len + M_LEADINGSPACE_NOWRITE(m));
+	}
+	else {
+		/*
+		 * This is the case where we need to "settle for what
+		 * we can get". The most probable way to this code
+		 * path is that we've already taken references to the
+		 * maximum number of mbuf clusters we can, and the data
+		 * is too long to fit in an mbuf's internal storage.
+		 * Try for a "best fit".
+		 */
+		tcp_pcap_copy_bestfit(th, m, n);
+
+		/* Don't try to get additional data. */
+		goto add_to_queue;
+	}
+
+	if (m->m_next) {
+		n->m_next = m_copym(m->m_next, 0, M_COPYALL, M_NOWAIT);
+		tcp_pcap_adj_cluster_reference(n->m_next, 1);
+	}
+
+add_to_queue:
+	/* Add the new mbuf to the list. */
+	if (mbufq_enqueue(queue, n)) {
+		/* This shouldn't happen. If INVARIANTS is defined, panic. */
+		KASSERT(0, ("%s: mbufq was unexpectedly full!", __func__));
+		tcp_pcap_m_freem(n);
+	}
+}
+
+void
+tcp_pcap_drain(struct mbufq *queue)
+{
+	struct mbuf *m;
+	while ((m = mbufq_dequeue(queue)))
+		tcp_pcap_m_freem(m);
+}
+
+void
+tcp_pcap_tcpcb_init(struct tcpcb *tp)
+{
+	mbufq_init(&(tp->t_inpkts), V_tcp_pcap_packets);
+	mbufq_init(&(tp->t_outpkts), V_tcp_pcap_packets);
+}
+
+void
+tcp_pcap_set_sock_max(struct mbufq *queue, int newval)
+{
+	queue->mq_maxlen = newval;
+	while (queue->mq_len > queue->mq_maxlen)
+		tcp_pcap_m_freem(mbufq_dequeue(queue));
+}
+
+int
+tcp_pcap_get_sock_max(struct mbufq *queue)
+{
+	return queue->mq_maxlen;
+}
diff --git a/sys/netinet/tcp_pcap.h b/sys/netinet/tcp_pcap.h
new file mode 100644
index 000000000000..ec8fdde03837
--- /dev/null
+++ b/sys/netinet/tcp_pcap.h
@@ -0,0 +1,39 @@
+/*-
+ * Copyright (c) 2015
+ *	Jonathan Looney. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _NETINET_TCP_PCAP_H_
+#define _NETINET_TCP_PCAP_H_
+
+void tcp_pcap_init(void);
+void tcp_pcap_add(struct tcphdr *th, struct mbuf *m, struct mbufq *queue);
+void tcp_pcap_drain(struct mbufq *queue);
+void tcp_pcap_tcpcb_init(struct tcpcb *tp);
+void tcp_pcap_set_sock_max(struct mbufq *queue, int newval);
+int tcp_pcap_get_sock_max(struct mbufq *queue);
+
+#endif /* _NETINET_TCP_PCAP_H_ */
diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c
index a20bd8109316..e3f5b1324ce8 100644
--- a/sys/netinet/tcp_subr.c
+++ b/sys/netinet/tcp_subr.c
@@ -92,6 +92,9 @@ __FBSDID("$FreeBSD$");
 #include <netinet6/tcp6_var.h>
 #endif
 #include <netinet/tcpip.h>
+#ifdef TCPPCAP
+#include <netinet/tcp_pcap.h>
+#endif
 #ifdef TCPDEBUG
 #include <netinet/tcp_debug.h>
 #endif
@@ -427,6 +430,9 @@ tcp_init(void)
 		SHUTDOWN_PRI_DEFAULT);
 	EVENTHANDLER_REGISTER(maxsockets_change, tcp_zone_change, NULL,
 		EVENTHANDLER_PRI_ANY);
+#ifdef TCPPCAP
+	tcp_pcap_init();
+#endif
 }
 
 #ifdef VIMAGE
@@ -832,6 +838,12 @@ tcp_newtcpcb(struct inpcb *inp)
 	 */
 	inp->inp_ip_ttl = V_ip_defttl;
 	inp->inp_ppcb = tp;
+#ifdef TCPPCAP
+	/*
+	 * Init the TCP PCAP queues.
+	 */
+	tcp_pcap_tcpcb_init(tp);
+#endif
 	return (tp);		/* XXX */
 }
 
@@ -1016,6 +1028,12 @@ tcp_discardcb(struct tcpcb *tp)
 		
 	tcp_free_sackholes(tp);
 
+#ifdef TCPPCAP
+	/* Free the TCP PCAP queues. */
+	tcp_pcap_drain(&(tp->t_inpkts));
+	tcp_pcap_drain(&(tp->t_outpkts));
+#endif
+
 	/* Allow the CC algorithm to clean up after itself. */
 	if (CC_ALGO(tp)->cb_destroy != NULL)
 		CC_ALGO(tp)->cb_destroy(tp->ccv);
diff --git a/sys/netinet/tcp_usrreq.c b/sys/netinet/tcp_usrreq.c
index 7d1ec6a0f2a9..a1f8a0c07fa2 100644
--- a/sys/netinet/tcp_usrreq.c
+++ b/sys/netinet/tcp_usrreq.c
@@ -86,6 +86,9 @@ __FBSDID("$FreeBSD$");
 #include <netinet/tcp_timer.h>
 #include <netinet/tcp_var.h>
 #include <netinet/tcpip.h>
+#ifdef TCPPCAP
+#include <netinet/tcp_pcap.h>
+#endif
 #ifdef TCPDEBUG
 #include <netinet/tcp_debug.h>
 #endif
@@ -1577,6 +1580,25 @@ unlock_and_done:
 				    TP_MAXIDLE(tp));
 			goto unlock_and_done;
 
+#ifdef TCPPCAP
+		case TCP_PCAP_OUT:
+		case TCP_PCAP_IN:
+			INP_WUNLOCK(inp);
+			error = sooptcopyin(sopt, &optval, sizeof optval,
+			    sizeof optval);
+			if (error)
+				return (error);
+
+			INP_WLOCK_RECHECK(inp);
+			if (optval >= 0)
+				tcp_pcap_set_sock_max(TCP_PCAP_OUT ?
+					&(tp->t_outpkts) : &(tp->t_inpkts),
+					optval);
+			else
+				error = EINVAL;
+			goto unlock_and_done;
+#endif
+
 		default:
 			INP_WUNLOCK(inp);
 			error = ENOPROTOOPT;
@@ -1647,6 +1669,15 @@ unlock_and_done:
 			INP_WUNLOCK(inp);
 			error = sooptcopyout(sopt, &ui, sizeof(ui));
 			break;
+#ifdef TCPPCAP
+		case TCP_PCAP_OUT:
+		case TCP_PCAP_IN:
+			optval = tcp_pcap_get_sock_max(TCP_PCAP_OUT ?
+					&(tp->t_outpkts) : &(tp->t_inpkts));
+			INP_WUNLOCK(inp);
+			error = sooptcopyout(sopt, &optval, sizeof optval);
+			break;
+#endif
 		default:
 			INP_WUNLOCK(inp);
 			error = ENOPROTOOPT;
diff --git a/sys/netinet/tcp_var.h b/sys/netinet/tcp_var.h
index 61fc41903fd6..07a28feb4956 100644
--- a/sys/netinet/tcp_var.h
+++ b/sys/netinet/tcp_var.h
@@ -37,6 +37,7 @@
 
 #ifdef _KERNEL
 #include <net/vnet.h>
+#include <sys/mbuf.h>
 
 /*
  * Kernel variables for tcp.
@@ -204,7 +205,17 @@ struct tcpcb {
 
 	uint32_t t_ispare[8];		/* 5 UTO, 3 TBD */
 	void	*t_pspare2[4];		/* 1 TCP_SIGNATURE, 3 TBD */
-	uint64_t _pad[6];		/* 6 TBD (1-2 CC/RTT?) */
+#if defined(_KERNEL) && defined(TCPPCAP)
+	struct mbufq t_inpkts;		/* List of saved input packets. */
+	struct mbufq t_outpkts;		/* List of saved output packets. */
+#ifdef _LP64
+	uint64_t _pad[0];		/* all used! */
+#else
+	uint64_t _pad[2];		/* 2 are available */
+#endif /* _LP64 */
+#else
+	uint64_t _pad[6];
+#endif /* defined(_KERNEL) && defined(TCPPCAP) */
 };
 
 /*
diff --git a/sys/sys/mbuf.h b/sys/sys/mbuf.h
index 8d335507456a..119bb6dd5ce1 100644
--- a/sys/sys/mbuf.h
+++ b/sys/sys/mbuf.h
@@ -524,6 +524,7 @@ extern uma_zone_t	zone_jumbo9;
 extern uma_zone_t	zone_jumbo16;
 extern uma_zone_t	zone_ext_refcnt;
 
+void		 mb_dupcl(struct mbuf *, const struct mbuf *);
 void		 mb_free_ext(struct mbuf *);
 int		 m_pkthdr_init(struct mbuf *, int);
 

From db1849f980f196d06da4ecad4fa943543e6b4f27 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Wed, 14 Oct 2015 00:36:33 +0000
Subject: [PATCH 179/380] Add a note about the mysterious files/includes/config
 block.

This originated from r96668.
---
 share/mk/bsd.subdir.mk | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/share/mk/bsd.subdir.mk b/share/mk/bsd.subdir.mk
index 293b017e2259..8afe40a693b0 100644
--- a/share/mk/bsd.subdir.mk
+++ b/share/mk/bsd.subdir.mk
@@ -123,6 +123,9 @@ _sub.${__target}: _SUBDIR
 .endif
 .endfor
 
+# This is to support 'make includes' calling 'make buildincludes' and
+# 'make installincludes' in the proper order, and to support these
+# targets as SUBDIR_TARGETS.
 .for __target in files includes config
 .for __stage in build install
 ${__stage}${__target}:

From c54a8cc39cd2b64e58b64ca5e162671b9fcca9a7 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Wed, 14 Oct 2015 00:43:29 +0000
Subject: [PATCH 180/380] Correct a comment in bsd.incs.mk forgotten in r274662
 and copied into bsd.confs.mk.

The bsd.confs.mk may be wrong but for now fix it.

Sponsored by:	EMC / Isilon Storage Division
---
 share/mk/bsd.confs.mk | 2 +-
 share/mk/bsd.incs.mk  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/share/mk/bsd.confs.mk b/share/mk/bsd.confs.mk
index 4e219e828bcf..0750a4e4fa57 100644
--- a/share/mk/bsd.confs.mk
+++ b/share/mk/bsd.confs.mk
@@ -84,4 +84,4 @@ STAGE_TARGETS+= stage_config
 .endif
 .endif
 
-.endif # ${MK_TOOLCHAIN} != "no"
+.endif # ${MK_INCLUDES} != "no"
diff --git a/share/mk/bsd.incs.mk b/share/mk/bsd.incs.mk
index 3f9680a6ef23..e1398c95dcd1 100644
--- a/share/mk/bsd.incs.mk
+++ b/share/mk/bsd.incs.mk
@@ -99,4 +99,4 @@ STAGE_SYMLINKS.INCS= ${INCSLINKS}
 .endif
 .endif
 
-.endif # ${MK_TOOLCHAIN} != "no"
+.endif # ${MK_INCLUDES} != "no"

From 4402204d479cb7e4d3842d36bab68c9bd3d3a4cb Mon Sep 17 00:00:00 2001
From: jeff <jeff@FreeBSD.org>
Date: Wed, 14 Oct 2015 02:10:07 +0000
Subject: [PATCH 181/380] Parallelize the buffer cache and rewrite getnewbuf().
  This results in a 8x performance improvement in a micro benchmark on a 4
 socket machine.

 - Get buffer headers from a per-cpu uma cache that sits in from of the
   free queue.
 - Use a per-cpu quantum cache in vmem to eliminate contention for kva.
 - Use multiple clean queues according to buffer cache size to eliminate
   clean queue lock contention.
 - Introduce a bufspace daemon that attempts to prevent getnewbuf() callers
   from blocking or doing direct recycling.
 - Close some bufspace allocation races that could lead to endless
   recycling.
 - Further the transition to a more modern style of small functions grouped
   by prefix in order to improve growing complexity.

Sponsored by:	EMC / Isilon
Reviewed by:	kib
Tested by:	pho
---
 sys/kern/vfs_bio.c | 1292 ++++++++++++++++++++++++--------------------
 sys/vm/vm_init.c   |    6 +-
 2 files changed, 719 insertions(+), 579 deletions(-)

diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c
index 70dc565fae39..4cc9a4ed1889 100644
--- a/sys/kern/vfs_bio.c
+++ b/sys/kern/vfs_bio.c
@@ -63,6 +63,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/proc.h>
 #include <sys/resourcevar.h>
 #include <sys/rwlock.h>
+#include <sys/smp.h>
 #include <sys/sysctl.h>
 #include <sys/sysproto.h>
 #include <sys/vmem.h>
@@ -100,6 +101,7 @@ caddr_t unmapped_buf;
 
 /* Used below and for softdep flushing threads in ufs/ffs/ffs_softdep.c */
 struct proc *bufdaemonproc;
+struct proc *bufspacedaemonproc;
 
 static int inmem(struct vnode *vp, daddr_t blkno);
 static void vm_hold_free_pages(struct buf *bp, int newbsize);
@@ -116,11 +118,18 @@ static void vfs_vmio_extend(struct buf *bp, int npages, int size);
 static int vfs_bio_clcheck(struct vnode *vp, int size,
 		daddr_t lblkno, daddr_t blkno);
 static int buf_flush(struct vnode *vp, int);
+static int buf_recycle(bool);
+static int buf_scan(bool);
 static int flushbufqueues(struct vnode *, int, int);
 static void buf_daemon(void);
 static void bremfreel(struct buf *bp);
 static __inline void bd_wakeup(void);
 static int sysctl_runningspace(SYSCTL_HANDLER_ARGS);
+static void bufkva_reclaim(vmem_t *, int);
+static void bufkva_free(struct buf *);
+static int buf_import(void *, void **, int, int);
+static void buf_release(void *, void **, int);
+
 #if defined(COMPAT_FREEBSD4) || defined(COMPAT_FREEBSD5) || \
     defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD7)
 static int sysctl_bufspace(SYSCTL_HANDLER_ARGS);
@@ -145,23 +154,23 @@ static long bufkvaspace;
 SYSCTL_LONG(_vfs, OID_AUTO, bufkvaspace, CTLFLAG_RD, &bufkvaspace, 0,
     "Kernel virtual memory used for buffers");
 static long maxbufspace;
-SYSCTL_LONG(_vfs, OID_AUTO, maxbufspace, CTLFLAG_RD, &maxbufspace, 0,
-    "Maximum allowed value of bufspace (including buf_daemon)");
+SYSCTL_LONG(_vfs, OID_AUTO, maxbufspace, CTLFLAG_RW, &maxbufspace, 0,
+    "Maximum allowed value of bufspace (including metadata)");
 static long bufmallocspace;
 SYSCTL_LONG(_vfs, OID_AUTO, bufmallocspace, CTLFLAG_RD, &bufmallocspace, 0,
     "Amount of malloced memory for buffers");
 static long maxbufmallocspace;
-SYSCTL_LONG(_vfs, OID_AUTO, maxmallocbufspace, CTLFLAG_RW, &maxbufmallocspace, 0,
-    "Maximum amount of malloced memory for buffers");
+SYSCTL_LONG(_vfs, OID_AUTO, maxmallocbufspace, CTLFLAG_RW, &maxbufmallocspace,
+    0, "Maximum amount of malloced memory for buffers");
 static long lobufspace;
-SYSCTL_LONG(_vfs, OID_AUTO, lobufspace, CTLFLAG_RD, &lobufspace, 0,
+SYSCTL_LONG(_vfs, OID_AUTO, lobufspace, CTLFLAG_RW, &lobufspace, 0,
     "Minimum amount of buffers we want to have");
 long hibufspace;
-SYSCTL_LONG(_vfs, OID_AUTO, hibufspace, CTLFLAG_RD, &hibufspace, 0,
-    "Maximum allowed value of bufspace (excluding buf_daemon)");
-static int bufreusecnt;
-SYSCTL_INT(_vfs, OID_AUTO, bufreusecnt, CTLFLAG_RW, &bufreusecnt, 0,
-    "Number of times we have reused a buffer");
+SYSCTL_LONG(_vfs, OID_AUTO, hibufspace, CTLFLAG_RW, &hibufspace, 0,
+    "Maximum allowed value of bufspace (excluding metadata)");
+long bufspacethresh;
+SYSCTL_LONG(_vfs, OID_AUTO, bufspacethresh, CTLFLAG_RW, &bufspacethresh,
+    0, "Bufspace consumed before waking the daemon to free some");
 static int buffreekvacnt;
 SYSCTL_INT(_vfs, OID_AUTO, buffreekvacnt, CTLFLAG_RW, &buffreekvacnt, 0,
     "Number of times we have freed the KVA space from some buffer");
@@ -205,10 +214,10 @@ SYSCTL_INT(_vfs, OID_AUTO, numfreebuffers, CTLFLAG_RD, &numfreebuffers, 0,
     "Number of free buffers");
 static int lofreebuffers;
 SYSCTL_INT(_vfs, OID_AUTO, lofreebuffers, CTLFLAG_RW, &lofreebuffers, 0,
-   "XXX Unused");
+   "Target number of free buffers");
 static int hifreebuffers;
 SYSCTL_INT(_vfs, OID_AUTO, hifreebuffers, CTLFLAG_RW, &hifreebuffers, 0,
-   "XXX Complicatedly unused");
+   "Threshold for clean buffer recycling");
 static int getnewbufcalls;
 SYSCTL_INT(_vfs, OID_AUTO, getnewbufcalls, CTLFLAG_RW, &getnewbufcalls, 0,
    "Number of calls to getnewbuf");
@@ -219,6 +228,9 @@ static int mappingrestarts;
 SYSCTL_INT(_vfs, OID_AUTO, mappingrestarts, CTLFLAG_RW, &mappingrestarts, 0,
     "Number of times getblk has had to restart a buffer mapping for "
     "unmapped buffer");
+static int numbufallocfails;
+SYSCTL_INT(_vfs, OID_AUTO, numbufallocfails, CTLFLAG_RW, &numbufallocfails, 0,
+    "Number of times buffer allocations failed");
 static int flushbufqtarget = 100;
 SYSCTL_INT(_vfs, OID_AUTO, flushbufqtarget, CTLFLAG_RW, &flushbufqtarget, 0,
     "Amount of work to do in flushbufqueues when helping bufdaemon");
@@ -232,16 +244,6 @@ SYSCTL_INT(_vfs, OID_AUTO, unmapped_buf_allowed, CTLFLAG_RD,
     &unmapped_buf_allowed, 0,
     "Permit the use of the unmapped i/o");
 
-/*
- * Lock for the non-dirty bufqueues
- */
-static struct mtx_padalign bqclean;
-
-/*
- * Lock for the dirty queue.
- */
-static struct mtx_padalign bqdirty;
-
 /*
  * This lock synchronizes access to bd_request.
  */
@@ -270,6 +272,11 @@ static struct mtx_padalign bdirtylock;
  */
 static int bd_request;
 
+/*
+ * Request/wakeup point for the bufspace daemon.
+ */
+static int bufspace_request;
+
 /*
  * Request for the buf daemon to write more buffers than is indicated by
  * lodirtybuf.  This may be necessary to push out excess dependencies or
@@ -298,7 +305,7 @@ static int runningbufreq;
  * Synchronization (sleep/wakeup) variable for buffer requests.
  * Can contain the VFS_BIO_NEED flags defined below; setting/clearing is done
  * by and/or.
- * Used in numdirtywakeup(), bufspacewakeup(), bufcountadd(), bwillwrite(),
+ * Used in numdirtywakeup(), bufspace_wakeup(), bwillwrite(),
  * getnewbuf(), and getblk().
  */
 static volatile int needsbuffer;
@@ -311,30 +318,43 @@ static int bdirtywait;
 /*
  * Definitions for the buffer free lists.
  */
-#define BUFFER_QUEUES	4	/* number of free buffer queues */
-
 #define QUEUE_NONE	0	/* on no queue */
-#define QUEUE_CLEAN	1	/* non-B_DELWRI buffers */
+#define QUEUE_EMPTY	1	/* empty buffer headers */
 #define QUEUE_DIRTY	2	/* B_DELWRI buffers */
-#define QUEUE_EMPTY	3	/* empty buffer headers */
+#define QUEUE_CLEAN	3	/* non-B_DELWRI buffers */
 #define QUEUE_SENTINEL	1024	/* not an queue index, but mark for sentinel */
 
+/* Maximum number of clean buffer queues. */
+#define	CLEAN_QUEUES	16
+
+/* Configured number of clean queues. */
+static int clean_queues;
+
+/* Maximum number of buffer queues. */
+#define BUFFER_QUEUES	(QUEUE_CLEAN + CLEAN_QUEUES)
+
 /* Queues for free buffers with various properties */
 static TAILQ_HEAD(bqueues, buf) bufqueues[BUFFER_QUEUES] = { { 0 } };
 #ifdef INVARIANTS
 static int bq_len[BUFFER_QUEUES];
 #endif
 
+/*
+ * Lock for each bufqueue
+ */
+static struct mtx_padalign bqlocks[BUFFER_QUEUES];
+
+/*
+ * per-cpu empty buffer cache.
+ */
+uma_zone_t buf_zone;
+
 /*
  * Single global constant for BUF_WMESG, to avoid getting multiple references.
  * buf_wmesg is referred from macros.
  */
 const char *buf_wmesg = BUF_WMESG;
 
-#define VFS_BIO_NEED_ANY	0x01	/* any freeable buffer */
-#define VFS_BIO_NEED_FREE	0x04	/* wait for free bufs, hi hysteresis */
-#define VFS_BIO_NEED_BUFSPACE	0x08	/* wait for buf space, lo hysteresis */
-
 static int
 sysctl_runningspace(SYSCTL_HANDLER_ARGS)
 {
@@ -382,6 +402,21 @@ sysctl_bufspace(SYSCTL_HANDLER_ARGS)
 }
 #endif
 
+static int
+bqcleanq(void)
+{
+	static int nextq;
+
+	return ((atomic_fetchadd_int(&nextq, 1) % clean_queues) + QUEUE_CLEAN);
+}
+
+static int
+bqisclean(int qindex)
+{
+
+	return (qindex >= QUEUE_CLEAN && qindex < QUEUE_CLEAN + CLEAN_QUEUES);
+}
+
 /*
  *	bqlock:
  *
@@ -391,9 +426,7 @@ static inline struct mtx *
 bqlock(int qindex)
 {
 
-	if (qindex == QUEUE_DIRTY)
-		return (struct mtx *)(&bqdirty);
-	return (struct mtx *)(&bqclean);
+	return (struct mtx *)&bqlocks[qindex];
 }
 
 /*
@@ -447,61 +480,254 @@ bdirtyadd(void)
 }
 
 /*
- *	bufspacewakeup:
+ *	bufspace_wakeup:
  *
  *	Called when buffer space is potentially available for recovery.
  *	getnewbuf() will block on this flag when it is unable to free 
  *	sufficient buffer space.  Buffer space becomes recoverable when 
  *	bp's get placed back in the queues.
  */
-static __inline void
-bufspacewakeup(void)
+static void
+bufspace_wakeup(void)
 {
-	int need_wakeup, on;
 
 	/*
-	 * If someone is waiting for bufspace, wake them up.  Even
-	 * though we may not have freed the kva space yet, the waiting
-	 * process will be able to now.
+	 * If someone is waiting for bufspace, wake them up.
+	 *
+	 * Since needsbuffer is set prior to doing an additional queue
+	 * scan it is safe to check for the flag prior to acquiring the
+	 * lock.  The thread that is preparing to scan again before
+	 * blocking would discover the buf we released.
 	 */
-	rw_rlock(&nblock);
-	for (;;) {
-		need_wakeup = 0;
-		on = needsbuffer;
-		if ((on & VFS_BIO_NEED_BUFSPACE) == 0)
-			break;
-		need_wakeup = 1;
-		if (atomic_cmpset_rel_int(&needsbuffer, on,
-		    on & ~VFS_BIO_NEED_BUFSPACE))
-			break;
+	if (needsbuffer) {
+		rw_rlock(&nblock);
+		if (atomic_cmpset_int(&needsbuffer, 1, 0) == 1)
+			wakeup(__DEVOLATILE(void *, &needsbuffer));
+		rw_runlock(&nblock);
+	}
+}
+
+/*
+ *	bufspace_daemonwakeup:
+ *
+ *	Wakeup the daemon responsible for freeing clean bufs.
+ */
+static void
+bufspace_daemonwakeup(void)
+{
+	rw_rlock(&nblock);
+	if (bufspace_request == 0) {
+		bufspace_request = 1;
+		wakeup(&bufspace_request);
 	}
-	if (need_wakeup)
-		wakeup(__DEVOLATILE(void *, &needsbuffer));
 	rw_runlock(&nblock);
 }
 
 /*
- *	bufspaceadjust:
+ *	bufspace_adjust:
  *
  *	Adjust the reported bufspace for a KVA managed buffer, possibly
  * 	waking any waiters.
  */
 static void
-bufspaceadjust(struct buf *bp, int bufsize)
+bufspace_adjust(struct buf *bp, int bufsize)
 {
+	long space;
 	int diff;
 
 	KASSERT((bp->b_flags & B_MALLOC) == 0,
-	    ("bufspaceadjust: malloc buf %p", bp));
+	    ("bufspace_adjust: malloc buf %p", bp));
 	diff = bufsize - bp->b_bufsize;
 	if (diff < 0) {
 		atomic_subtract_long(&bufspace, -diff);
-		bufspacewakeup();
-	} else
-		atomic_add_long(&bufspace, diff);
+		bufspace_wakeup();
+	} else {
+		space = atomic_fetchadd_long(&bufspace, diff);
+		/* Wake up the daemon on the transition. */
+		if (space < bufspacethresh && space + diff >= bufspacethresh)
+			bufspace_daemonwakeup();
+	}
 	bp->b_bufsize = bufsize;
 }
 
+/*
+ *	bufspace_reserve:
+ *
+ *	Reserve bufspace before calling allocbuf().  metadata has a
+ *	different space limit than data.
+ */
+static int
+bufspace_reserve(int size, bool metadata)
+{
+	long limit;
+	long space;
+
+	if (metadata)
+		limit = maxbufspace;
+	else
+		limit = hibufspace;
+	do {
+		space = bufspace;
+		if (space + size > limit)
+			return (ENOSPC);
+	} while (atomic_cmpset_long(&bufspace, space, space + size) == 0);
+
+	/* Wake up the daemon on the transition. */
+	if (space < bufspacethresh && space + size >= bufspacethresh)
+		bufspace_daemonwakeup();
+
+	return (0);
+}
+
+/*
+ *	bufspace_release:
+ *
+ *	Release reserved bufspace after bufspace_adjust() has consumed it.
+ */
+static void
+bufspace_release(int size)
+{
+	atomic_subtract_long(&bufspace, size);
+	bufspace_wakeup();
+}
+
+/*
+ *	bufspace_wait:
+ *
+ *	Wait for bufspace, acting as the buf daemon if a locked vnode is
+ *	supplied.  needsbuffer must be set in a safe fashion prior to
+ *	polling for space.  The operation must be re-tried on return.
+ */
+static void
+bufspace_wait(struct vnode *vp, int gbflags, int slpflag, int slptimeo)
+{
+	struct thread *td;
+	int error, fl, norunbuf;
+
+	if ((gbflags & GB_NOWAIT_BD) != 0)
+		return;
+
+	td = curthread;
+	rw_wlock(&nblock);
+	while (needsbuffer != 0) {
+		if (vp != NULL && vp->v_type != VCHR &&
+		    (td->td_pflags & TDP_BUFNEED) == 0) {
+			rw_wunlock(&nblock);
+			/*
+			 * getblk() is called with a vnode locked, and
+			 * some majority of the dirty buffers may as
+			 * well belong to the vnode.  Flushing the
+			 * buffers there would make a progress that
+			 * cannot be achieved by the buf_daemon, that
+			 * cannot lock the vnode.
+			 */
+			norunbuf = ~(TDP_BUFNEED | TDP_NORUNNINGBUF) |
+			    (td->td_pflags & TDP_NORUNNINGBUF);
+
+			/*
+			 * Play bufdaemon.  The getnewbuf() function
+			 * may be called while the thread owns lock
+			 * for another dirty buffer for the same
+			 * vnode, which makes it impossible to use
+			 * VOP_FSYNC() there, due to the buffer lock
+			 * recursion.
+			 */
+			td->td_pflags |= TDP_BUFNEED | TDP_NORUNNINGBUF;
+			fl = buf_flush(vp, flushbufqtarget);
+			td->td_pflags &= norunbuf;
+			rw_wlock(&nblock);
+			if (fl != 0)
+				continue;
+			if (needsbuffer == 0)
+				break;
+		}
+		error = rw_sleep(__DEVOLATILE(void *, &needsbuffer), &nblock,
+		    (PRIBIO + 4) | slpflag, "newbuf", slptimeo);
+		if (error != 0)
+			break;
+	}
+	rw_wunlock(&nblock);
+}
+
+
+/*
+ *	bufspace_daemon:
+ *
+ *	buffer space management daemon.  Tries to maintain some marginal
+ *	amount of free buffer space so that requesting processes neither
+ *	block nor work to reclaim buffers.
+ */
+static void
+bufspace_daemon(void)
+{
+	for (;;) {
+		kproc_suspend_check(bufspacedaemonproc);
+
+		/*
+		 * Free buffers from the clean queue until we meet our
+		 * targets.
+		 *
+		 * Theory of operation:  The buffer cache is most efficient
+		 * when some free buffer headers and space are always
+		 * available to getnewbuf().  This daemon attempts to prevent
+		 * the excessive blocking and synchronization associated
+		 * with shortfall.  It goes through three phases according
+		 * demand:
+		 *
+		 * 1)	The daemon wakes up voluntarily once per-second
+		 *	during idle periods when the counters are below
+		 *	the wakeup thresholds (bufspacethresh, lofreebuffers).
+		 *
+		 * 2)	The daemon wakes up as we cross the thresholds
+		 *	ahead of any potential blocking.  This may bounce
+		 *	slightly according to the rate of consumption and
+		 *	release.
+		 *
+		 * 3)	The daemon and consumers are starved for working
+		 *	clean buffers.  This is the 'bufspace' sleep below
+		 *	which will inefficiently trade bufs with bqrelse
+		 *	until we return to condition 2.
+		 */
+		while (bufspace > lobufspace ||
+		    numfreebuffers < hifreebuffers) {
+			if (buf_recycle(false) != 0) {
+				atomic_set_int(&needsbuffer, 1);
+				if (buf_recycle(false) != 0) {
+					rw_wlock(&nblock);
+					if (needsbuffer)
+						rw_sleep(__DEVOLATILE(void *,
+						    &needsbuffer), &nblock,
+						    PRIBIO|PDROP, "bufspace",
+						    hz/10);
+					else
+						rw_wunlock(&nblock);
+				}
+			}
+			maybe_yield();
+		}
+
+		/*
+		 * Re-check our limits under the exclusive nblock.
+		 */
+		rw_wlock(&nblock);
+		if (bufspace < bufspacethresh &&
+		    numfreebuffers > lofreebuffers) {
+			bufspace_request = 0;
+			rw_sleep(&bufspace_request, &nblock, PRIBIO|PDROP,
+			    "-", hz);
+		} else
+			rw_wunlock(&nblock);
+	}
+}
+
+static struct kproc_desc bufspace_kp = {
+	"bufspacedaemon",
+	bufspace_daemon,
+	&bufspacedaemonproc
+};
+SYSINIT(bufspacedaemon, SI_SUB_KTHREAD_BUF, SI_ORDER_FIRST, kproc_start,
+    &bufspace_kp);
+
 /*
  *	bufmallocadjust:
  *
@@ -516,10 +742,9 @@ bufmallocadjust(struct buf *bp, int bufsize)
 	KASSERT((bp->b_flags & B_MALLOC) != 0,
 	    ("bufmallocadjust: non-malloc buf %p", bp));
 	diff = bufsize - bp->b_bufsize;
-	if (diff < 0) {
+	if (diff < 0)
 		atomic_subtract_long(&bufmallocspace, -diff);
-		bufspacewakeup();
-	} else
+	else
 		atomic_add_long(&bufmallocspace, diff);
 	bp->b_bufsize = bufsize;
 }
@@ -570,67 +795,6 @@ runningbufwakeup(struct buf *bp)
 	runningwakeup();
 }
 
-/*
- *	bufcountadd:
- *
- *	Called when a buffer has been added to one of the free queues to
- *	account for the buffer and to wakeup anyone waiting for free buffers.
- *	This typically occurs when large amounts of metadata are being handled
- *	by the buffer cache ( else buffer space runs out first, usually ).
- */
-static __inline void
-bufcountadd(struct buf *bp)
-{
-	int mask, need_wakeup, old, on;
-
-	KASSERT((bp->b_flags & B_INFREECNT) == 0,
-	    ("buf %p already counted as free", bp));
-	bp->b_flags |= B_INFREECNT;
-	old = atomic_fetchadd_int(&numfreebuffers, 1);
-	KASSERT(old >= 0 && old < nbuf,
-	    ("numfreebuffers climbed to %d", old + 1));
-	mask = VFS_BIO_NEED_ANY;
-	if (numfreebuffers >= hifreebuffers)
-		mask |= VFS_BIO_NEED_FREE;
-	rw_rlock(&nblock);
-	for (;;) {
-		need_wakeup = 0;
-		on = needsbuffer;
-		if (on == 0)
-			break;
-		need_wakeup = 1;
-		if (atomic_cmpset_rel_int(&needsbuffer, on, on & ~mask))
-			break;
-	}
-	if (need_wakeup)
-		wakeup(__DEVOLATILE(void *, &needsbuffer));
-	rw_runlock(&nblock);
-}
-
-/*
- *	bufcountsub:
- *
- *	Decrement the numfreebuffers count as needed.
- */
-static void
-bufcountsub(struct buf *bp)
-{
-	int old;
-
-	/*
-	 * Fixup numfreebuffers count.  If the buffer is invalid or not
-	 * delayed-write, the buffer was free and we must decrement
-	 * numfreebuffers.
-	 */
-	if ((bp->b_flags & B_INVAL) || (bp->b_flags & B_DELWRI) == 0) {
-		KASSERT((bp->b_flags & B_INFREECNT) != 0,
-		    ("buf %p not counted in numfreebuffers", bp));
-		bp->b_flags &= ~B_INFREECNT;
-		old = atomic_fetchadd_int(&numfreebuffers, -1);
-		KASSERT(old > 0, ("numfreebuffers dropped to %d", old - 1));
-	}
-}
-
 /*
  *	waitrunningbufspace()
  *
@@ -847,8 +1011,10 @@ bufinit(void)
 	int i;
 
 	CTASSERT(MAXBCACHEBUF >= MAXBSIZE);
-	mtx_init(&bqclean, "bufq clean lock", NULL, MTX_DEF);
-	mtx_init(&bqdirty, "bufq dirty lock", NULL, MTX_DEF);
+	mtx_init(&bqlocks[QUEUE_DIRTY], "bufq dirty lock", NULL, MTX_DEF);
+	mtx_init(&bqlocks[QUEUE_EMPTY], "bufq empty lock", NULL, MTX_DEF);
+	for (i = QUEUE_CLEAN; i < QUEUE_CLEAN + CLEAN_QUEUES; i++)
+		mtx_init(&bqlocks[i], "bufq clean lock", NULL, MTX_DEF);
 	mtx_init(&rbreqlock, "runningbufspace lock", NULL, MTX_DEF);
 	rw_init(&nblock, "needsbuffer lock");
 	mtx_init(&bdlock, "buffer daemon lock", NULL, MTX_DEF);
@@ -864,7 +1030,7 @@ bufinit(void)
 	for (i = 0; i < nbuf; i++) {
 		bp = &buf[i];
 		bzero(bp, sizeof *bp);
-		bp->b_flags = B_INVAL | B_INFREECNT;
+		bp->b_flags = B_INVAL;
 		bp->b_rcred = NOCRED;
 		bp->b_wcred = NOCRED;
 		bp->b_qindex = QUEUE_EMPTY;
@@ -881,18 +1047,19 @@ bufinit(void)
 	/*
 	 * maxbufspace is the absolute maximum amount of buffer space we are 
 	 * allowed to reserve in KVM and in real terms.  The absolute maximum
-	 * is nominally used by buf_daemon.  hibufspace is the nominal maximum
-	 * used by most other processes.  The differential is required to 
-	 * ensure that buf_daemon is able to run when other processes might 
-	 * be blocked waiting for buffer space.
+	 * is nominally used by metadata.  hibufspace is the nominal maximum
+	 * used by most other requests.  The differential is required to 
+	 * ensure that metadata deadlocks don't occur.
 	 *
 	 * maxbufspace is based on BKVASIZE.  Allocating buffers larger then
 	 * this may result in KVM fragmentation which is not handled optimally
-	 * by the system.
+	 * by the system. XXX This is less true with vmem.  We could use
+	 * PAGE_SIZE.
 	 */
 	maxbufspace = (long)nbuf * BKVASIZE;
 	hibufspace = lmax(3 * maxbufspace / 4, maxbufspace - MAXBCACHEBUF * 10);
-	lobufspace = hibufspace - MAXBCACHEBUF;
+	lobufspace = (hibufspace / 20) * 19; /* 95% */
+	bufspacethresh = lobufspace + (hibufspace - lobufspace) / 2;
 
 	/*
 	 * Note: The 16 MiB upper limit for hirunningspace was chosen
@@ -906,44 +1073,61 @@ bufinit(void)
 	    16 * 1024 * 1024), 1024 * 1024);
 	lorunningspace = roundup((hirunningspace * 2) / 3, MAXBCACHEBUF);
 
-/*
- * Limit the amount of malloc memory since it is wired permanently into
- * the kernel space.  Even though this is accounted for in the buffer
- * allocation, we don't want the malloced region to grow uncontrolled.
- * The malloc scheme improves memory utilization significantly on average
- * (small) directories.
- */
+	/*
+	 * Limit the amount of malloc memory since it is wired permanently into
+	 * the kernel space.  Even though this is accounted for in the buffer
+	 * allocation, we don't want the malloced region to grow uncontrolled.
+	 * The malloc scheme improves memory utilization significantly on
+	 * average (small) directories.
+	 */
 	maxbufmallocspace = hibufspace / 20;
 
-/*
- * Reduce the chance of a deadlock occuring by limiting the number
- * of delayed-write dirty buffers we allow to stack up.
- */
+	/*
+	 * Reduce the chance of a deadlock occuring by limiting the number
+	 * of delayed-write dirty buffers we allow to stack up.
+	 */
 	hidirtybuffers = nbuf / 4 + 20;
 	dirtybufthresh = hidirtybuffers * 9 / 10;
 	numdirtybuffers = 0;
-/*
- * To support extreme low-memory systems, make sure hidirtybuffers cannot
- * eat up all available buffer space.  This occurs when our minimum cannot
- * be met.  We try to size hidirtybuffers to 3/4 our buffer space assuming
- * BKVASIZE'd buffers.
- */
+	/*
+	 * To support extreme low-memory systems, make sure hidirtybuffers
+	 * cannot eat up all available buffer space.  This occurs when our
+	 * minimum cannot be met.  We try to size hidirtybuffers to 3/4 our
+	 * buffer space assuming BKVASIZE'd buffers.
+	 */
 	while ((long)hidirtybuffers * BKVASIZE > 3 * hibufspace / 4) {
 		hidirtybuffers >>= 1;
 	}
 	lodirtybuffers = hidirtybuffers / 2;
 
-/*
- * Try to keep the number of free buffers in the specified range,
- * and give special processes (e.g. like buf_daemon) access to an 
- * emergency reserve.
- */
-	lofreebuffers = nbuf / 18 + 5;
-	hifreebuffers = 2 * lofreebuffers;
+	/*
+	 * lofreebuffers should be sufficient to avoid stalling waiting on
+	 * buf headers under heavy utilization.  The bufs in per-cpu caches
+	 * are counted as free but will be unavailable to threads executing
+	 * on other cpus.
+	 *
+	 * hifreebuffers is the free target for the bufspace daemon.  This
+	 * should be set appropriately to limit work per-iteration.
+	 */
+	lofreebuffers = MIN((nbuf / 25) + (20 * mp_ncpus), 128 * mp_ncpus);
+	hifreebuffers = (3 * lofreebuffers) / 2;
 	numfreebuffers = nbuf;
 
 	bogus_page = vm_page_alloc(NULL, 0, VM_ALLOC_NOOBJ |
 	    VM_ALLOC_NORMAL | VM_ALLOC_WIRED);
+
+	/* Setup the kva and free list allocators. */
+	vmem_set_reclaim(buffer_arena, bufkva_reclaim);
+	buf_zone = uma_zcache_create("buf free cache", sizeof(struct buf),
+	    NULL, NULL, NULL, NULL, buf_import, buf_release, NULL, 0);
+
+	/*
+	 * Size the clean queue according to the amount of buffer space.
+	 * One queue per-256mb up to the max.  More queues gives better
+	 * concurrency but less accurate LRU.
+	 */
+	clean_queues = MIN(howmany(maxbufspace, 256*1024*1024), CLEAN_QUEUES);
+
 }
 
 #ifdef INVARIANTS
@@ -1129,10 +1313,25 @@ binsfree(struct buf *bp, int qindex)
 {
 	struct mtx *olock, *nlock;
 
-	BUF_ASSERT_XLOCKED(bp);
+	if (qindex != QUEUE_EMPTY) {
+		BUF_ASSERT_XLOCKED(bp);
+	}
 
+	/*
+	 * Stick to the same clean queue for the lifetime of the buf to
+	 * limit locking below.  Otherwise pick ont sequentially.
+	 */
+	if (qindex == QUEUE_CLEAN) {
+		if (bqisclean(bp->b_qindex))
+			qindex = bp->b_qindex;
+		else
+			qindex = bqcleanq();
+	}
+
+	/*
+	 * Handle delayed bremfree() processing.
+	 */
 	nlock = bqlock(qindex);
-	/* Handle delayed bremfree() processing. */
 	if (bp->b_flags & B_REMFREE) {
 		olock = bqlock(bp->b_qindex);
 		mtx_lock(olock);
@@ -1156,15 +1355,263 @@ binsfree(struct buf *bp, int qindex)
 	bq_len[bp->b_qindex]++;
 #endif
 	mtx_unlock(nlock);
+}
+
+/*
+ * buf_free:
+ *
+ *	Free a buffer to the buf zone once it no longer has valid contents.
+ */
+static void
+buf_free(struct buf *bp)
+{
+
+	if (bp->b_flags & B_REMFREE)
+		bremfreef(bp);
+	if (bp->b_vflags & BV_BKGRDINPROG)
+		panic("losing buffer 1");
+	if (bp->b_rcred != NOCRED) {
+		crfree(bp->b_rcred);
+		bp->b_rcred = NOCRED;
+	}
+	if (bp->b_wcred != NOCRED) {
+		crfree(bp->b_wcred);
+		bp->b_wcred = NOCRED;
+	}
+	if (!LIST_EMPTY(&bp->b_dep))
+		buf_deallocate(bp);
+	bufkva_free(bp);
+	BUF_UNLOCK(bp);
+	uma_zfree(buf_zone, bp);
+	atomic_add_int(&numfreebuffers, 1);
+	bufspace_wakeup();
+}
+
+/*
+ * buf_import:
+ *
+ *	Import bufs into the uma cache from the buf list.  The system still
+ *	expects a static array of bufs and much of the synchronization
+ *	around bufs assumes type stable storage.  As a result, UMA is used
+ *	only as a per-cpu cache of bufs still maintained on a global list.
+ */
+static int
+buf_import(void *arg, void **store, int cnt, int flags)
+{
+	struct buf *bp;
+	int i;
+
+	mtx_lock(&bqlocks[QUEUE_EMPTY]);
+	for (i = 0; i < cnt; i++) {
+		bp = TAILQ_FIRST(&bufqueues[QUEUE_EMPTY]);
+		if (bp == NULL)
+			break;
+		bremfreel(bp);
+		store[i] = bp;
+	}
+	mtx_unlock(&bqlocks[QUEUE_EMPTY]);
+
+	return (i);
+}
+
+/*
+ * buf_release:
+ *
+ *	Release bufs from the uma cache back to the buffer queues.
+ */
+static void
+buf_release(void *arg, void **store, int cnt)
+{
+        int i;
+
+        for (i = 0; i < cnt; i++)
+		binsfree(store[i], QUEUE_EMPTY);
+}
+
+/*
+ * buf_alloc:
+ *
+ *	Allocate an empty buffer header.
+ */
+static struct buf *
+buf_alloc(void)
+{
+	struct buf *bp;
+
+	bp = uma_zalloc(buf_zone, M_NOWAIT);
+	if (bp == NULL) {
+		bufspace_daemonwakeup();
+		atomic_add_int(&numbufallocfails, 1);
+		return (NULL);
+	}
 
 	/*
-	 * Something we can maybe free or reuse.
+	 * Wake-up the bufspace daemon on transition.
 	 */
-	if (bp->b_bufsize && !(bp->b_flags & B_DELWRI))
-		bufspacewakeup();
+	if (atomic_fetchadd_int(&numfreebuffers, -1) == lofreebuffers)
+		bufspace_daemonwakeup();
 
-	if ((bp->b_flags & B_INVAL) || !(bp->b_flags & B_DELWRI))
-		bufcountadd(bp);
+	if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0)
+		panic("getnewbuf_empty: Locked buf %p on free queue.", bp);
+	
+	KASSERT(bp->b_vp == NULL,
+	    ("bp: %p still has vnode %p.", bp, bp->b_vp));
+	KASSERT((bp->b_flags & (B_DELWRI | B_NOREUSE)) == 0,
+	    ("invalid buffer %p flags %#x", bp, bp->b_flags));
+	KASSERT((bp->b_xflags & (BX_VNCLEAN|BX_VNDIRTY)) == 0,
+	    ("bp: %p still on a buffer list. xflags %X", bp, bp->b_xflags));
+	KASSERT(bp->b_npages == 0,
+	    ("bp: %p still has %d vm pages\n", bp, bp->b_npages));
+	KASSERT(bp->b_kvasize == 0, ("bp: %p still has kva\n", bp));
+	KASSERT(bp->b_bufsize == 0, ("bp: %p still has bufspace\n", bp));
+
+	bp->b_flags = 0;
+	bp->b_ioflags = 0;
+	bp->b_xflags = 0;
+	bp->b_vflags = 0;
+	bp->b_vp = NULL;
+	bp->b_blkno = bp->b_lblkno = 0;
+	bp->b_offset = NOOFFSET;
+	bp->b_iodone = 0;
+	bp->b_error = 0;
+	bp->b_resid = 0;
+	bp->b_bcount = 0;
+	bp->b_npages = 0;
+	bp->b_dirtyoff = bp->b_dirtyend = 0;
+	bp->b_bufobj = NULL;
+	bp->b_pin_count = 0;
+	bp->b_data = bp->b_kvabase = unmapped_buf;
+	bp->b_fsprivate1 = NULL;
+	bp->b_fsprivate2 = NULL;
+	bp->b_fsprivate3 = NULL;
+	LIST_INIT(&bp->b_dep);
+
+	return (bp);
+}
+
+/*
+ *	buf_qrecycle:
+ *
+ *	Free a buffer from the given bufqueue.  kva controls whether the
+ *	freed buf must own some kva resources.  This is used for
+ *	defragmenting.
+ */
+static int
+buf_qrecycle(int qindex, bool kva)
+{
+	struct buf *bp, *nbp;
+
+	if (kva)
+		atomic_add_int(&bufdefragcnt, 1);
+	nbp = NULL;
+	mtx_lock(&bqlocks[qindex]);
+	nbp = TAILQ_FIRST(&bufqueues[qindex]);
+
+	/*
+	 * Run scan, possibly freeing data and/or kva mappings on the fly
+	 * depending.
+	 */
+	while ((bp = nbp) != NULL) {
+		/*
+		 * Calculate next bp (we can only use it if we do not
+		 * release the bqlock).
+		 */
+		nbp = TAILQ_NEXT(bp, b_freelist);
+
+		/*
+		 * If we are defragging then we need a buffer with 
+		 * some kva to reclaim.
+		 */
+		if (kva && bp->b_kvasize == 0)
+			continue;
+
+		if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0)
+			continue;
+
+		/*
+		 * Skip buffers with background writes in progress.
+		 */
+		if ((bp->b_vflags & BV_BKGRDINPROG) != 0) {
+			BUF_UNLOCK(bp);
+			continue;
+		}
+
+		KASSERT(bp->b_qindex == qindex,
+		    ("getnewbuf: inconsistent queue %d bp %p", qindex, bp));
+		/*
+		 * NOTE:  nbp is now entirely invalid.  We can only restart
+		 * the scan from this point on.
+		 */
+		bremfreel(bp);
+		mtx_unlock(&bqlocks[qindex]);
+
+		/*
+		 * Requeue the background write buffer with error and
+		 * restart the scan.
+		 */
+		if ((bp->b_vflags & BV_BKGRDERR) != 0) {
+			bqrelse(bp);
+			mtx_lock(&bqlocks[qindex]);
+			nbp = TAILQ_FIRST(&bufqueues[qindex]);
+			continue;
+		}
+		bp->b_flags |= B_INVAL;
+		brelse(bp);
+		return (0);
+	}
+	mtx_unlock(&bqlocks[qindex]);
+
+	return (ENOBUFS);
+}
+
+/*
+ *	buf_recycle:
+ *
+ *	Iterate through all clean queues until we find a buf to recycle or
+ *	exhaust the search.
+ */
+static int
+buf_recycle(bool kva)
+{
+	int qindex, first_qindex;
+
+	qindex = first_qindex = bqcleanq();
+	do {
+		if (buf_qrecycle(qindex, kva) == 0)
+			return (0);
+		if (++qindex == QUEUE_CLEAN + clean_queues)
+			qindex = QUEUE_CLEAN;
+	} while (qindex != first_qindex);
+
+	return (ENOBUFS);
+}
+
+/*
+ *	buf_scan:
+ *
+ *	Scan the clean queues looking for a buffer to recycle.  needsbuffer
+ *	is set on failure so that the caller may optionally bufspace_wait()
+ *	in a race-free fashion.
+ */
+static int
+buf_scan(bool defrag)
+{
+	int error;
+
+	/*
+	 * To avoid heavy synchronization and wakeup races we set
+	 * needsbuffer and re-poll before failing.  This ensures that
+	 * no frees can be missed between an unsuccessful poll and
+	 * going to sleep in a synchronized fashion.
+	 */
+	if ((error = buf_recycle(defrag)) != 0) {
+		atomic_set_int(&needsbuffer, 1);
+		bufspace_daemonwakeup();
+		error = buf_recycle(defrag);
+	}
+	if (error == 0)
+		atomic_add_int(&getnewbufrestarts, 1);
+	return (error);
 }
 
 /*
@@ -1185,7 +1632,6 @@ bremfree(struct buf *bp)
 	BUF_ASSERT_XLOCKED(bp);
 
 	bp->b_flags |= B_REMFREE;
-	bufcountsub(bp);
 }
 
 /*
@@ -1219,7 +1665,9 @@ bremfreel(struct buf *bp)
 	    bp, bp->b_vp, bp->b_flags);
 	KASSERT(bp->b_qindex != QUEUE_NONE,
 	    ("bremfreel: buffer %p not on a queue.", bp));
-	BUF_ASSERT_XLOCKED(bp);
+	if (bp->b_qindex != QUEUE_EMPTY) {
+		BUF_ASSERT_XLOCKED(bp);
+	}
 	mtx_assert(bqlock(bp->b_qindex), MA_OWNED);
 
 	TAILQ_REMOVE(&bufqueues[bp->b_qindex], bp, b_freelist);
@@ -1229,25 +1677,17 @@ bremfreel(struct buf *bp)
 	bq_len[bp->b_qindex]--;
 #endif
 	bp->b_qindex = QUEUE_NONE;
-	/*
-	 * If this was a delayed bremfree() we only need to remove the buffer
-	 * from the queue and return the stats are already done.
-	 */
-	if (bp->b_flags & B_REMFREE) {
-		bp->b_flags &= ~B_REMFREE;
-		return;
-	}
-	bufcountsub(bp);
+	bp->b_flags &= ~B_REMFREE;
 }
 
 /*
- *	bufkvafree:
+ *	bufkva_free:
  *
  *	Free the kva allocation for a buffer.
  *
  */
 static void
-bufkvafree(struct buf *bp)
+bufkva_free(struct buf *bp)
 {
 
 #ifdef INVARIANTS
@@ -1271,12 +1711,12 @@ bufkvafree(struct buf *bp)
 }
 
 /*
- *	bufkvaalloc:
+ *	bufkva_alloc:
  *
  *	Allocate the buffer KVA and set b_kvasize and b_kvabase.
  */
 static int
-bufkvaalloc(struct buf *bp, int maxsize, int gbflags)
+bufkva_alloc(struct buf *bp, int maxsize, int gbflags)
 {
 	vm_offset_t addr;
 	int error;
@@ -1284,7 +1724,7 @@ bufkvaalloc(struct buf *bp, int maxsize, int gbflags)
 	KASSERT((gbflags & GB_UNMAPPED) == 0 || (gbflags & GB_KVAALLOC) != 0,
 	    ("Invalid gbflags 0x%x in %s", gbflags, __func__));
 
-	bufkvafree(bp);
+	bufkva_free(bp);
 
 	addr = 0;
 	error = vmem_alloc(buffer_arena, maxsize, M_BESTFIT | M_NOWAIT, &addr);
@@ -1293,7 +1733,6 @@ bufkvaalloc(struct buf *bp, int maxsize, int gbflags)
 		 * Buffer map is too fragmented.  Request the caller
 		 * to defragment the map.
 		 */
-		atomic_add_int(&bufdefragcnt, 1);
 		return (error);
 	}
 	bp->b_kvabase = (caddr_t)addr;
@@ -1309,6 +1748,24 @@ bufkvaalloc(struct buf *bp, int maxsize, int gbflags)
 	return (0);
 }
 
+/*
+ *	bufkva_reclaim:
+ *
+ *	Reclaim buffer kva by freeing buffers holding kva.  This is a vmem
+ *	callback that fires to avoid returning failure.
+ */
+static void
+bufkva_reclaim(vmem_t *vmem, int flags)
+{
+	int i;
+
+	for (i = 0; i < 5; i++)
+		if (buf_scan(true) != 0)
+			break;
+	return;
+}
+
+
 /*
  * Attempt to initiate asynchronous I/O on read-ahead blocks.  We must
  * clear BIO_ERROR and B_INVAL prior to initiating I/O . If B_CACHE is set,
@@ -1900,14 +2357,11 @@ brelse(struct buf *bp)
 
 	/* buffers with no memory */
 	if (bp->b_bufsize == 0) {
-		bp->b_xflags &= ~(BX_BKGRDWRITE | BX_ALTDATA);
-		if (bp->b_vflags & BV_BKGRDINPROG)
-			panic("losing buffer 1");
-		bufkvafree(bp);
-		qindex = QUEUE_EMPTY;
-		bp->b_flags |= B_AGE;
+		buf_free(bp);
+		return;
+	}
 	/* buffers with junk contents */
-	} else if (bp->b_flags & (B_INVAL | B_NOCACHE | B_RELBUF) ||
+	if (bp->b_flags & (B_INVAL | B_NOCACHE | B_RELBUF) ||
 	    (bp->b_ioflags & BIO_ERROR)) {
 		bp->b_xflags &= ~(BX_BKGRDWRITE | BX_ALTDATA);
 		if (bp->b_vflags & BV_BKGRDINPROG)
@@ -1927,6 +2381,8 @@ brelse(struct buf *bp)
 		panic("brelse: not dirty");
 	/* unlock */
 	BUF_UNLOCK(bp);
+	if (qindex == QUEUE_CLEAN)
+		bufspace_wakeup();
 }
 
 /*
@@ -1949,6 +2405,7 @@ bqrelse(struct buf *bp)
 	KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)),
 	    ("bqrelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
 
+	qindex = QUEUE_NONE;
 	if (BUF_LOCKRECURSED(bp)) {
 		/* do not release to free list */
 		BUF_UNLOCK(bp);
@@ -1984,6 +2441,8 @@ bqrelse(struct buf *bp)
 out:
 	/* unlock */
 	BUF_UNLOCK(bp);
+	if (qindex == QUEUE_CLEAN)
+		bufspace_wakeup();
 }
 
 /*
@@ -2383,297 +2842,26 @@ vfs_bio_awrite(struct buf *bp)
 }
 
 /*
- * Ask the bufdaemon for help, or act as bufdaemon itself, when a
- * locked vnode is supplied.
+ *	getnewbuf_kva:
+ *
+ *	Allocate KVA for an empty buf header according to gbflags.
  */
-static void
-getnewbuf_bufd_help(struct vnode *vp, int gbflags, int slpflag, int slptimeo,
-    int defrag)
-{
-	struct thread *td;
-	char *waitmsg;
-	int error, fl, flags, norunbuf;
-
-	mtx_assert(&bqclean, MA_OWNED);
-
-	if (defrag) {
-		flags = VFS_BIO_NEED_BUFSPACE;
-		waitmsg = "nbufkv";
-	} else if (bufspace >= hibufspace) {
-		waitmsg = "nbufbs";
-		flags = VFS_BIO_NEED_BUFSPACE;
-	} else {
-		waitmsg = "newbuf";
-		flags = VFS_BIO_NEED_ANY;
-	}
-	atomic_set_int(&needsbuffer, flags);
-	mtx_unlock(&bqclean);
-
-	bd_speedup();	/* heeeelp */
-	if ((gbflags & GB_NOWAIT_BD) != 0)
-		return;
-
-	td = curthread;
-	rw_wlock(&nblock);
-	while ((needsbuffer & flags) != 0) {
-		if (vp != NULL && vp->v_type != VCHR &&
-		    (td->td_pflags & TDP_BUFNEED) == 0) {
-			rw_wunlock(&nblock);
-			/*
-			 * getblk() is called with a vnode locked, and
-			 * some majority of the dirty buffers may as
-			 * well belong to the vnode.  Flushing the
-			 * buffers there would make a progress that
-			 * cannot be achieved by the buf_daemon, that
-			 * cannot lock the vnode.
-			 */
-			norunbuf = ~(TDP_BUFNEED | TDP_NORUNNINGBUF) |
-			    (td->td_pflags & TDP_NORUNNINGBUF);
-
-			/*
-			 * Play bufdaemon.  The getnewbuf() function
-			 * may be called while the thread owns lock
-			 * for another dirty buffer for the same
-			 * vnode, which makes it impossible to use
-			 * VOP_FSYNC() there, due to the buffer lock
-			 * recursion.
-			 */
-			td->td_pflags |= TDP_BUFNEED | TDP_NORUNNINGBUF;
-			fl = buf_flush(vp, flushbufqtarget);
-			td->td_pflags &= norunbuf;
-			rw_wlock(&nblock);
-			if (fl != 0)
-				continue;
-			if ((needsbuffer & flags) == 0)
-				break;
-		}
-		error = rw_sleep(__DEVOLATILE(void *, &needsbuffer), &nblock,
-		    (PRIBIO + 4) | slpflag, waitmsg, slptimeo);
-		if (error != 0)
-			break;
-	}
-	rw_wunlock(&nblock);
-}
-
-static void
-getnewbuf_reuse_bp(struct buf *bp, int qindex)
+static int
+getnewbuf_kva(struct buf *bp, int gbflags, int maxsize)
 {
 
-	CTR6(KTR_BUF, "getnewbuf(%p) vp %p flags %X kvasize %d bufsize %d "
-	    "queue %d (recycling)", bp, bp->b_vp, bp->b_flags,
-	     bp->b_kvasize, bp->b_bufsize, qindex);
-	mtx_assert(&bqclean, MA_NOTOWNED);
+	if ((gbflags & (GB_UNMAPPED | GB_KVAALLOC)) != GB_UNMAPPED) {
+		/*
+		 * In order to keep fragmentation sane we only allocate kva
+		 * in BKVASIZE chunks.  XXX with vmem we can do page size.
+		 */
+		maxsize = (maxsize + BKVAMASK) & ~BKVAMASK;
 
-	/*
-	 * Note: we no longer distinguish between VMIO and non-VMIO
-	 * buffers.
-	 */
-	KASSERT((bp->b_flags & (B_DELWRI | B_NOREUSE)) == 0,
-	    ("invalid buffer %p flags %#x found in queue %d", bp, bp->b_flags,
-	    qindex));
-
-	/*
-	 * When recycling a clean buffer we have to truncate it and
-	 * release the vnode.
-	 */
-	if (qindex == QUEUE_CLEAN) {
-		allocbuf(bp, 0);
-		if (bp->b_vp != NULL)
-			brelvp(bp);
+		if (maxsize != bp->b_kvasize &&
+		    bufkva_alloc(bp, maxsize, gbflags))
+			return (ENOSPC);
 	}
-
-	/*
-	 * Get the rest of the buffer freed up.  b_kva* is still valid
-	 * after this operation.
-	 */
-	if (bp->b_rcred != NOCRED) {
-		crfree(bp->b_rcred);
-		bp->b_rcred = NOCRED;
-	}
-	if (bp->b_wcred != NOCRED) {
-		crfree(bp->b_wcred);
-		bp->b_wcred = NOCRED;
-	}
-	if (!LIST_EMPTY(&bp->b_dep))
-		buf_deallocate(bp);
-	if (bp->b_vflags & BV_BKGRDINPROG)
-		panic("losing buffer 3");
-	KASSERT(bp->b_vp == NULL, ("bp: %p still has vnode %p.  qindex: %d",
-	    bp, bp->b_vp, qindex));
-	KASSERT((bp->b_xflags & (BX_VNCLEAN|BX_VNDIRTY)) == 0,
-	    ("bp: %p still on a buffer list. xflags %X", bp, bp->b_xflags));
-	KASSERT(bp->b_npages == 0,
-	    ("bp: %p still has %d vm pages\n", bp, bp->b_npages));
-
-	bp->b_flags = 0;
-	bp->b_ioflags = 0;
-	bp->b_xflags = 0;
-	KASSERT((bp->b_flags & B_INFREECNT) == 0,
-	    ("buf %p still counted as free?", bp));
-	bp->b_vflags = 0;
-	bp->b_vp = NULL;
-	bp->b_blkno = bp->b_lblkno = 0;
-	bp->b_offset = NOOFFSET;
-	bp->b_iodone = 0;
-	bp->b_error = 0;
-	bp->b_resid = 0;
-	bp->b_bcount = 0;
-	bp->b_npages = 0;
-	bp->b_dirtyoff = bp->b_dirtyend = 0;
-	bp->b_bufobj = NULL;
-	bp->b_pin_count = 0;
-	bp->b_data = bp->b_kvabase;
-	bp->b_fsprivate1 = NULL;
-	bp->b_fsprivate2 = NULL;
-	bp->b_fsprivate3 = NULL;
-
-	LIST_INIT(&bp->b_dep);
-}
-
-static struct buf *
-getnewbuf_scan(int maxsize, int defrag, int unmapped, int metadata)
-{
-	struct buf *bp, *nbp;
-	int nqindex, qindex, pass;
-
-	KASSERT(!unmapped || !defrag, ("both unmapped and defrag"));
-
-	pass = 0;
-restart:
-	if (pass != 0)
-		atomic_add_int(&getnewbufrestarts, 1);
-
-	nbp = NULL;
-	mtx_lock(&bqclean);
-	/*
-	 * If we're not defragging or low on bufspace attempt to make a new
-	 * buf from a header.
-	 */
-	if (defrag == 0 && bufspace + maxsize < hibufspace) {
-		nqindex = QUEUE_EMPTY;
-		nbp = TAILQ_FIRST(&bufqueues[nqindex]);
-	}
-	/*
-	 * All available buffers might be clean or we need to start recycling.
-	 */
-	if (nbp == NULL) {
-		nqindex = QUEUE_CLEAN;
-		nbp = TAILQ_FIRST(&bufqueues[QUEUE_CLEAN]);
-	}
-
-	/*
-	 * Run scan, possibly freeing data and/or kva mappings on the fly
-	 * depending.
-	 */
-	while ((bp = nbp) != NULL) {
-		qindex = nqindex;
-
-		/*
-		 * Calculate next bp (we can only use it if we do not
-		 * release the bqlock)
-		 */
-		if ((nbp = TAILQ_NEXT(bp, b_freelist)) == NULL) {
-			switch (qindex) {
-			case QUEUE_EMPTY:
-				nqindex = QUEUE_CLEAN;
-				nbp = TAILQ_FIRST(&bufqueues[nqindex]);
-				if (nbp != NULL)
-					break;
-				/* FALLTHROUGH */
-			case QUEUE_CLEAN:
-				if (metadata && pass == 0) {
-					pass = 1;
-					nqindex = QUEUE_EMPTY;
-					nbp = TAILQ_FIRST(&bufqueues[nqindex]);
-				}
-				/*
-				 * nbp is NULL. 
-				 */
-				break;
-			}
-		}
-		/*
-		 * If we are defragging then we need a buffer with 
-		 * b_kvasize != 0.  This situation occurs when we
-		 * have many unmapped bufs.
-		 */
-		if (defrag && bp->b_kvasize == 0)
-			continue;
-
-		/*
-		 * Start freeing the bp.  This is somewhat involved.  nbp
-		 * remains valid only for QUEUE_EMPTY[KVA] bp's.
-		 */
-		if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0)
-			continue;
-		/*
-		 * BKGRDINPROG can only be set with the buf and bufobj
-		 * locks both held.  We tolerate a race to clear it here.
-		 */
-		if (bp->b_vflags & BV_BKGRDINPROG) {
-			BUF_UNLOCK(bp);
-			continue;
-		}
-
-		/*
-		 * Requeue the background write buffer with error.
-		 */
-		if ((bp->b_vflags & BV_BKGRDERR) != 0) {
-			bremfreel(bp);
-			mtx_unlock(&bqclean);
-			bqrelse(bp);
-			continue;
-		}
-
-		KASSERT(bp->b_qindex == qindex,
-		    ("getnewbuf: inconsistent queue %d bp %p", qindex, bp));
-
-		bremfreel(bp);
-		mtx_unlock(&bqclean);
-
-		/*
-		 * NOTE:  nbp is now entirely invalid.  We can only restart
-		 * the scan from this point on.
-		 */
-		getnewbuf_reuse_bp(bp, qindex);
-		mtx_assert(&bqclean, MA_NOTOWNED);
-
-		/*
-		 * If we are defragging then free the buffer.
-		 */
-		if (defrag) {
-			bp->b_flags |= B_INVAL;
-			brelse(bp);
-			defrag = 0;
-			goto restart;
-		}
-
-		/*
-		 * Notify any waiters for the buffer lock about
-		 * identity change by freeing the buffer.
-		 */
-		if (qindex == QUEUE_CLEAN && BUF_LOCKWAITERS(bp)) {
-			bp->b_flags |= B_INVAL;
-			brelse(bp);
-			goto restart;
-		}
-
-		if (metadata)
-			break;
-
-		/*
-		 * If we are overcomitted then recover the buffer and its
-		 * KVM space.  This occurs in rare situations when multiple
-		 * processes are blocked in getnewbuf() or allocbuf().
-		 */
-		if (bufspace >= hibufspace && bp->b_kvasize != 0) {
-			bp->b_flags |= B_INVAL;
-			brelse(bp);
-			goto restart;
-		}
-		break;
-	}
-	return (bp);
+	return (0);
 }
 
 /*
@@ -2682,86 +2870,54 @@ restart:
  *	Find and initialize a new buffer header, freeing up existing buffers
  *	in the bufqueues as necessary.  The new buffer is returned locked.
  *
- *	Important:  B_INVAL is not set.  If the caller wishes to throw the
- *	buffer away, the caller must set B_INVAL prior to calling brelse().
- *
  *	We block if:
  *		We have insufficient buffer headers
  *		We have insufficient buffer space
  *		buffer_arena is too fragmented ( space reservation fails )
  *		If we have to flush dirty buffers ( but we try to avoid this )
+ *
+ *	The caller is responsible for releasing the reserved bufspace after
+ *	allocbuf() is called.
  */
 static struct buf *
-getnewbuf(struct vnode *vp, int slpflag, int slptimeo, int size, int maxsize,
-    int gbflags)
+getnewbuf(struct vnode *vp, int slpflag, int slptimeo, int maxsize, int gbflags)
 {
 	struct buf *bp;
-	int defrag, metadata;
+	bool metadata, reserved;
 
 	KASSERT((gbflags & (GB_UNMAPPED | GB_KVAALLOC)) != GB_KVAALLOC,
 	    ("GB_KVAALLOC only makes sense with GB_UNMAPPED"));
 	if (!unmapped_buf_allowed)
 		gbflags &= ~(GB_UNMAPPED | GB_KVAALLOC);
 
-	defrag = 0;
 	if (vp == NULL || (vp->v_vflag & (VV_MD | VV_SYSTEM)) != 0 ||
 	    vp->v_type == VCHR)
-		metadata = 1;
+		metadata = true;
 	else
-		metadata = 0;
-	/*
-	 * We can't afford to block since we might be holding a vnode lock,
-	 * which may prevent system daemons from running.  We deal with
-	 * low-memory situations by proactively returning memory and running
-	 * async I/O rather then sync I/O.
-	 */
+		metadata = false;
 	atomic_add_int(&getnewbufcalls, 1);
-restart:
-	bp = getnewbuf_scan(maxsize, defrag, (gbflags & (GB_UNMAPPED |
-	    GB_KVAALLOC)) == GB_UNMAPPED, metadata);
-	if (bp != NULL)
-		defrag = 0;
+	reserved = false;
+	do {
+		if (reserved == false &&
+		    bufspace_reserve(maxsize, metadata) != 0)
+			continue;
+		reserved = true;
+		if ((bp = buf_alloc()) == NULL)
+			continue;
+		if (getnewbuf_kva(bp, gbflags, maxsize) == 0)
+			return (bp);
+		break;
+	} while(buf_scan(false) == 0);
 
-	/*
-	 * If we exhausted our list, sleep as appropriate.  We may have to
-	 * wakeup various daemons and write out some dirty buffers.
-	 *
-	 * Generally we are sleeping due to insufficient buffer space.
-	 */
-	if (bp == NULL) {
-		mtx_assert(&bqclean, MA_OWNED);
-		getnewbuf_bufd_help(vp, gbflags, slpflag, slptimeo, defrag);
-		mtx_assert(&bqclean, MA_NOTOWNED);
-	} else if ((gbflags & (GB_UNMAPPED | GB_KVAALLOC)) == GB_UNMAPPED) {
-		mtx_assert(&bqclean, MA_NOTOWNED);
-
-		bufkvafree(bp);
-		atomic_add_int(&bufreusecnt, 1);
-	} else {
-		mtx_assert(&bqclean, MA_NOTOWNED);
-
-		/*
-		 * We finally have a valid bp.  We aren't quite out of the
-		 * woods, we still have to reserve kva space. In order to
-		 * keep fragmentation sane we only allocate kva in BKVASIZE
-		 * chunks.
-		 */
-		maxsize = (maxsize + BKVAMASK) & ~BKVAMASK;
-
-		if (maxsize != bp->b_kvasize &&
-		    bufkvaalloc(bp, maxsize, gbflags)) {
-			defrag = 1;
-			bp->b_flags |= B_INVAL;
-			brelse(bp);
-			goto restart;
-		} else if ((gbflags & (GB_UNMAPPED | GB_KVAALLOC)) ==
-		    (GB_UNMAPPED | GB_KVAALLOC)) {
-			bp->b_data = unmapped_buf;
-			BUF_CHECK_UNMAPPED(bp);
-		}
-		atomic_add_int(&bufreusecnt, 1);
+	if (reserved)
+		bufspace_release(maxsize);
+	if (bp != NULL) {
+		bp->b_flags |= B_INVAL;
+		brelse(bp);
 	}
-	return (bp);
+	bufspace_wait(vp, gbflags, slpflag, slptimeo);
+
+	return (NULL);
 }
 
 /*
@@ -2771,7 +2927,6 @@ restart:
  *	update daemon but if it cannot keep up this process starts to
  *	take the load in an attempt to prevent getnewbuf() from blocking.
  */
-
 static struct kproc_desc buf_kp = {
 	"bufdaemon",
 	buf_daemon,
@@ -2902,19 +3057,19 @@ flushbufqueues(struct vnode *lvp, int target, int flushdeps)
 	bp = NULL;
 	sentinel = malloc(sizeof(struct buf), M_TEMP, M_WAITOK | M_ZERO);
 	sentinel->b_qindex = QUEUE_SENTINEL;
-	mtx_lock(&bqdirty);
+	mtx_lock(&bqlocks[queue]);
 	TAILQ_INSERT_HEAD(&bufqueues[queue], sentinel, b_freelist);
-	mtx_unlock(&bqdirty);
+	mtx_unlock(&bqlocks[queue]);
 	while (flushed != target) {
 		maybe_yield();
-		mtx_lock(&bqdirty);
+		mtx_lock(&bqlocks[queue]);
 		bp = TAILQ_NEXT(sentinel, b_freelist);
 		if (bp != NULL) {
 			TAILQ_REMOVE(&bufqueues[queue], sentinel, b_freelist);
 			TAILQ_INSERT_AFTER(&bufqueues[queue], bp, sentinel,
 			    b_freelist);
 		} else {
-			mtx_unlock(&bqdirty);
+			mtx_unlock(&bqlocks[queue]);
 			break;
 		}
 		/*
@@ -2926,11 +3081,11 @@ flushbufqueues(struct vnode *lvp, int target, int flushdeps)
 		 */
 		if (bp->b_qindex == QUEUE_SENTINEL || (lvp != NULL &&
 		    bp->b_vp != lvp)) {
-			mtx_unlock(&bqdirty);
+			mtx_unlock(&bqlocks[queue]);
  			continue;
 		}
 		error = BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL);
-		mtx_unlock(&bqdirty);
+		mtx_unlock(&bqlocks[queue]);
 		if (error != 0)
 			continue;
 		if (bp->b_pin_count > 0) {
@@ -3013,9 +3168,9 @@ flushbufqueues(struct vnode *lvp, int target, int flushdeps)
 		vn_finished_write(mp);
 		BUF_UNLOCK(bp);
 	}
-	mtx_lock(&bqdirty);
+	mtx_lock(&bqlocks[queue]);
 	TAILQ_REMOVE(&bufqueues[queue], sentinel, b_freelist);
-	mtx_unlock(&bqdirty);
+	mtx_unlock(&bqlocks[queue]);
 	free(sentinel, M_TEMP);
 	return (flushed);
 }
@@ -3196,7 +3351,6 @@ vfs_setdirty_locked_object(struct buf *bp)
 static void
 bp_unmapped_get_kva(struct buf *bp, daddr_t blkno, int size, int gbflags)
 {
-	struct buf *scratch_bp;
 	int bsize, maxsize, need_mapping, need_kva;
 	off_t offset;
 
@@ -3229,37 +3383,16 @@ bp_unmapped_get_kva(struct buf *bp, daddr_t blkno, int size, int gbflags)
 	maxsize = size + (offset & PAGE_MASK);
 	maxsize = imax(maxsize, bsize);
 
-mapping_loop:
-	if (bufkvaalloc(bp, maxsize, gbflags)) {
-		/*
-		 * Request defragmentation. getnewbuf() returns us the
-		 * allocated space by the scratch buffer KVA.
-		 */
-		scratch_bp = getnewbuf(bp->b_vp, 0, 0, size, maxsize, gbflags |
-		    (GB_UNMAPPED | GB_KVAALLOC));
-		if (scratch_bp == NULL) {
-			if ((gbflags & GB_NOWAIT_BD) != 0) {
-				/*
-				 * XXXKIB: defragmentation cannot
-				 * succeed, not sure what else to do.
-				 */
-				panic("GB_NOWAIT_BD and GB_UNMAPPED %p", bp);
-			}
-			atomic_add_int(&mappingrestarts, 1);
-			goto mapping_loop;
+	while (bufkva_alloc(bp, maxsize, gbflags) != 0) {
+		if ((gbflags & GB_NOWAIT_BD) != 0) {
+			/*
+			 * XXXKIB: defragmentation cannot
+			 * succeed, not sure what else to do.
+			 */
+			panic("GB_NOWAIT_BD and GB_UNMAPPED %p", bp);
 		}
-		KASSERT(scratch_bp->b_kvabase != unmapped_buf,
-		    ("scratch bp has no KVA %p", scratch_bp));
-		/* Grab pointers. */
-		bp->b_kvabase = scratch_bp->b_kvabase;
-		bp->b_kvasize = scratch_bp->b_kvasize;
-		bp->b_data = scratch_bp->b_data;
-
-		/* Get rid of the scratch buffer. */
-		scratch_bp->b_kvasize = 0;
-		scratch_bp->b_flags |= B_INVAL;
-		scratch_bp->b_data = scratch_bp->b_kvabase = unmapped_buf;
-		brelse(scratch_bp);
+		atomic_add_int(&mappingrestarts, 1);
+		bufspace_wait(bp->b_vp, gbflags, 0, 0);
 	}
 has_addr:
 	if (need_mapping) {
@@ -3486,7 +3619,7 @@ loop:
 		}
 		maxsize = imax(maxsize, bsize);
 
-		bp = getnewbuf(vp, slpflag, slptimeo, size, maxsize, flags);
+		bp = getnewbuf(vp, slpflag, slptimeo, maxsize, flags);
 		if (bp == NULL) {
 			if (slpflag || slptimeo)
 				return NULL;
@@ -3510,6 +3643,7 @@ loop:
 			BO_UNLOCK(bo);
 			bp->b_flags |= B_INVAL;
 			brelse(bp);
+			bufspace_release(maxsize);
 			goto loop;
 		}
 
@@ -3543,6 +3677,7 @@ loop:
 		}
 
 		allocbuf(bp, size);
+		bufspace_release(maxsize);
 		bp->b_flags &= ~B_DONE;
 	}
 	CTR4(KTR_BUF, "getblk(%p, %ld, %d) = %p", vp, (long)blkno, size, bp);
@@ -3564,12 +3699,13 @@ geteblk(int size, int flags)
 	int maxsize;
 
 	maxsize = (size + BKVAMASK) & ~BKVAMASK;
-	while ((bp = getnewbuf(NULL, 0, 0, size, maxsize, flags)) == NULL) {
+	while ((bp = getnewbuf(NULL, 0, 0, maxsize, flags)) == NULL) {
 		if ((flags & GB_NOWAIT_BD) &&
 		    (curthread->td_pflags & TDP_BUFNEED) != 0)
 			return (NULL);
 	}
 	allocbuf(bp, size);
+	bufspace_release(maxsize);
 	bp->b_flags |= B_INVAL;	/* b_dep cleared by getnewbuf() */
 	BUF_ASSERT_HELD(bp);
 	return (bp);
@@ -3595,7 +3731,7 @@ vfs_nonvmio_truncate(struct buf *bp, int newbsize)
 		return;
 	}
 	vm_hold_free_pages(bp, newbsize);
-	bufspaceadjust(bp, newbsize);
+	bufspace_adjust(bp, newbsize);
 }
 
 /*
@@ -3646,7 +3782,7 @@ vfs_nonvmio_extend(struct buf *bp, int newbsize)
 		bcopy(origbuf, bp->b_data, origbufsize);
 		free(origbuf, M_BIOBUF);
 	}
-	bufspaceadjust(bp, newbsize);
+	bufspace_adjust(bp, newbsize);
 }
 
 /*
@@ -3708,7 +3844,7 @@ allocbuf(struct buf *bp, int size)
 		/* XXX This looks as if it should be newbsize > b_bufsize */
 		else if (size > bp->b_bcount)
 			vfs_vmio_extend(bp, desiredpages, size);
-		bufspaceadjust(bp, newbsize);
+		bufspace_adjust(bp, newbsize);
 	}
 	bp->b_bcount = size;		/* requested buffer size. */
 	return (1);
@@ -4596,7 +4732,7 @@ DB_COMMAND(countfreebufs, db_coundfreebufs)
 
 	for (i = 0; i < nbuf; i++) {
 		bp = &buf[i];
-		if ((bp->b_flags & B_INFREECNT) != 0)
+		if (bp->b_qindex == QUEUE_EMPTY)
 			nfree++;
 		else
 			used++;
diff --git a/sys/vm/vm_init.c b/sys/vm/vm_init.c
index be1038758824..e6645858ad37 100644
--- a/sys/vm/vm_init.c
+++ b/sys/vm/vm_init.c
@@ -74,6 +74,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysctl.h>
 #include <sys/systm.h>
 #include <sys/selinfo.h>
+#include <sys/smp.h>
 #include <sys/pipe.h>
 #include <sys/bio.h>
 #include <sys/buf.h>
@@ -229,12 +230,15 @@ again:
 
 	/*
 	 * Allocate the buffer arena.
+	 *
+	 * Enable the quantum cache if we have more than 4 cpus.  This
+	 * avoids lock contention at the expense of some fragmentation.
 	 */
 	size = (long)nbuf * BKVASIZE;
 	kmi->buffer_sva = firstaddr;
 	kmi->buffer_eva = kmi->buffer_sva + size;
 	vmem_init(buffer_arena, "buffer arena", kmi->buffer_sva, size,
-	    PAGE_SIZE, 0, 0);
+	    PAGE_SIZE, (mp_ncpus > 4) ? BKVASIZE * 8 : 0, 0);
 	firstaddr += size;
 
 	/*

From ab645378841d73aef1fd4345eed23d39ae3ecffe Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Wed, 14 Oct 2015 02:14:15 +0000
Subject: [PATCH 182/380] NTB: MFV 78958433: Enable Snoop on Primary Side

Enable Snoop from Primary to Secondary side on BAR23 and BAR45 on all
TLPs.  Previously, Snoop was only enabled from Secondary to Primary
side.  This can have a performance improvement on some workloads.

Also, make the code more obvious about how the link is being enabled.

Authored by:	Jon Mason
Obtained from:	Linux (Dual BSD/GPL driver)
Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/ntb_hw/ntb_hw.c   | 19 +++++++++++++------
 sys/dev/ntb/ntb_hw/ntb_regs.h | 12 +++++++-----
 2 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c
index b45e8258f3c0..7395fe0daf79 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.c
@@ -1050,12 +1050,18 @@ ntb_handle_link_event(struct ntb_softc *ntb, int link_state)
 static void
 ntb_hw_link_up(struct ntb_softc *ntb)
 {
+	uint32_t cntl;
 
-	if (ntb->conn_type == NTB_CONN_TRANSPARENT)
+	if (ntb->conn_type == NTB_CONN_TRANSPARENT) {
 		ntb_handle_link_event(ntb, NTB_LINK_UP);
-	else
-		ntb_reg_write(4, ntb->reg_ofs.lnk_cntl,
-		    NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP);
+		return;
+	}
+
+	cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl);
+	cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK);
+	cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP;
+	cntl |= NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP;
+	ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, cntl);
 }
 
 static void
@@ -1069,8 +1075,9 @@ ntb_hw_link_down(struct ntb_softc *ntb)
 	}
 
 	cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl);
-	cntl &= ~(NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP);
-	cntl |= NTB_CNTL_LINK_DISABLE;
+	cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP);
+	cntl &= ~(NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP);
+	cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK;
 	ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, cntl);
 }
 
diff --git a/sys/dev/ntb/ntb_hw/ntb_regs.h b/sys/dev/ntb/ntb_hw/ntb_regs.h
index d97d504b7db7..ad6a2a671126 100644
--- a/sys/dev/ntb/ntb_hw/ntb_regs.h
+++ b/sys/dev/ntb/ntb_hw/ntb_regs.h
@@ -116,11 +116,13 @@
 #define SOC_LTSSMSTATEJMP_FORCEDETECT	(1 << 2)
 #define SOC_IBIST_ERR_OFLOW	0x7fff7fff
 
-#define NTB_CNTL_CFG_LOCK	(1 << 0)
-#define NTB_CNTL_LINK_DISABLE	(1 << 1)
-#define NTB_CNTL_BAR23_SNOOP	(1 << 2)
-#define NTB_CNTL_BAR45_SNOOP	(1 << 6)
-#define SOC_CNTL_LINK_DOWN	(1 << 16)
+#define NTB_CNTL_CFG_LOCK		(1 << 0)
+#define NTB_CNTL_LINK_DISABLE		(1 << 1)
+#define NTB_CNTL_S2P_BAR23_SNOOP	(1 << 2)
+#define NTB_CNTL_P2S_BAR23_SNOOP	(1 << 4)
+#define NTB_CNTL_S2P_BAR45_SNOOP	(1 << 6)
+#define NTB_CNTL_P2S_BAR45_SNOOP	(1 << 8)
+#define SOC_CNTL_LINK_DOWN		(1 << 16)
 
 #define XEON_PBAR23SZ_OFFSET	0x00d0
 #define XEON_PBAR45SZ_OFFSET	0x00d1

From 466958dfd388d19f4159dc5caf4c9fe0263dff2a Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Wed, 14 Oct 2015 02:14:45 +0000
Subject: [PATCH 183/380] NTB: MFV e8aeb60c: Disable interrupts and poll under
 high load

Authored by:	Jon Mason
Obtained from:	Linux (Dual BSD/GPL driver)
Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/if_ntb/if_ntb.c | 23 +++++--------
 sys/dev/ntb/ntb_hw/ntb_hw.c | 68 ++++++++++++++++++++++++++++++-------
 sys/dev/ntb/ntb_hw/ntb_hw.h |  2 +-
 3 files changed, 65 insertions(+), 28 deletions(-)

diff --git a/sys/dev/ntb/if_ntb/if_ntb.c b/sys/dev/ntb/if_ntb/if_ntb.c
index 02bab46f8b6a..321460700011 100644
--- a/sys/dev/ntb/if_ntb/if_ntb.c
+++ b/sys/dev/ntb/if_ntb/if_ntb.c
@@ -32,6 +32,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/bus.h>
 #include <sys/ktr.h>
+#include <sys/limits.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/module.h>
@@ -249,9 +250,8 @@ static int ntb_process_tx(struct ntb_transport_qp *qp,
 static void ntb_tx_copy_task(struct ntb_transport_qp *qp,
     struct ntb_queue_entry *entry, void *offset);
 static void ntb_qp_full(void *arg);
-static void ntb_transport_rxc_db(void *data, int db_num);
+static int ntb_transport_rxc_db(void *arg, int dummy);
 static void ntb_rx_pendq_full(void *arg);
-static void ntb_transport_rx(struct ntb_transport_qp *qp);
 static int ntb_process_rxc(struct ntb_transport_qp *qp);
 static void ntb_rx_copy_task(struct ntb_transport_qp *qp,
     struct ntb_queue_entry *entry, void *offset);
@@ -839,25 +839,18 @@ ntb_qp_full(void *arg)
 }
 
 /* Transport Rx */
-static void
-ntb_transport_rxc_db(void *data, int db_num)
-{
-	struct ntb_transport_qp *qp = data;
-
-	ntb_transport_rx(qp);
-}
-
 static void
 ntb_rx_pendq_full(void *arg)
 {
 
 	CTR0(KTR_NTB, "RX: ntb_rx_pendq_full callout");
-	ntb_transport_rx(arg);
+	ntb_transport_rxc_db(arg, 0);
 }
 
-static void
-ntb_transport_rx(struct ntb_transport_qp *qp)
+static int
+ntb_transport_rxc_db(void *arg, int dummy __unused)
 {
+	struct ntb_transport_qp *qp = arg;
 	uint64_t i;
 	int rc;
 
@@ -867,7 +860,7 @@ ntb_transport_rx(struct ntb_transport_qp *qp)
 	 */
 	mtx_lock(&qp->transport->rx_lock);
 	CTR0(KTR_NTB, "RX: transport_rx");
-	for (i = 0; i < qp->rx_max_entry; i++) {
+	for (i = 0; i < MIN(qp->rx_max_entry, INT_MAX); i++) {
 		rc = ntb_process_rxc(qp);
 		if (rc != 0) {
 			CTR0(KTR_NTB, "RX: process_rxc failed");
@@ -875,6 +868,8 @@ ntb_transport_rx(struct ntb_transport_qp *qp)
 		}
 	}
 	mtx_unlock(&qp->transport->rx_lock);
+
+	return ((int)i);
 }
 
 static int
diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c
index 7395fe0daf79..599e06858394 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.c
@@ -109,6 +109,7 @@ struct ntb_db_cb {
 	unsigned int		db_num;
 	void			*data;
 	struct ntb_softc	*ntb;
+	struct callout		irq_work;
 };
 
 struct ntb_softc {
@@ -204,6 +205,9 @@ static void handle_soc_irq(void *arg);
 static void handle_xeon_irq(void *arg);
 static void handle_xeon_event_irq(void *arg);
 static void ntb_handle_legacy_interrupt(void *arg);
+static void ntb_irq_work(void *arg);
+static void mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx);
+static void unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx);
 static int ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors);
 static void ntb_free_callbacks(struct ntb_softc *ntb);
 static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id);
@@ -579,6 +583,26 @@ ntb_teardown_interrupts(struct ntb_softc *ntb)
 	pci_release_msi(ntb->device);
 }
 
+static void
+mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx)
+{
+	unsigned long mask;
+
+	mask = ntb_reg_read(2, ntb->reg_ofs.ldb_mask);
+	mask |= 1 << (idx * ntb->bits_per_vector);
+	ntb_reg_write(2, ntb->reg_ofs.ldb_mask, mask);
+}
+
+static void
+unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx)
+{
+	unsigned long mask;
+
+	mask = ntb_reg_read(2, ntb->reg_ofs.ldb_mask);
+	mask &= ~(1 << (idx * ntb->bits_per_vector));
+	ntb_reg_write(2, ntb->reg_ofs.ldb_mask, mask);
+}
+
 static void
 handle_soc_irq(void *arg)
 {
@@ -587,8 +611,10 @@ handle_soc_irq(void *arg)
 
 	ntb_reg_write(8, ntb->reg_ofs.ldb, (uint64_t) 1 << db_cb->db_num);
 
-	if (db_cb->callback != NULL)
-		db_cb->callback(db_cb->data, db_cb->db_num);
+	if (db_cb->callback != NULL) {
+		mask_ldb_interrupt(ntb, db_cb->db_num);
+		callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb);
+	}
 }
 
 static void
@@ -607,8 +633,10 @@ handle_xeon_irq(void *arg)
 	    ((1 << ntb->bits_per_vector) - 1) <<
 	    (db_cb->db_num * ntb->bits_per_vector));
 
-	if (db_cb->callback != NULL)
-		db_cb->callback(db_cb->data, db_cb->db_num);
+	if (db_cb->callback != NULL) {
+		mask_ldb_interrupt(ntb, db_cb->db_num);
+		callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb);
+	}
 }
 
 /* Since we do not have a HW doorbell in SOC, this is only used in JF/JT */
@@ -1191,6 +1219,25 @@ ntb_unregister_event_callback(struct ntb_softc *ntb)
 	ntb->event_cb = NULL;
 }
 
+static void
+ntb_irq_work(void *arg)
+{
+	struct ntb_db_cb *db_cb = arg;
+	struct ntb_softc *ntb;
+	int rc;
+
+	rc = db_cb->callback(db_cb->data, db_cb->db_num);
+	/* Poll if forward progress was made. */
+	if (rc != 0) {
+		callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb);
+		return;
+	}
+
+	/* Unmask interrupt if no progress was made. */
+	ntb = db_cb->ntb;
+	unmask_ldb_interrupt(ntb, db_cb->db_num);
+}
+
 /**
  * ntb_register_db_callback() - register a callback for doorbell interrupt
  * @ntb: pointer to ntb_softc instance
@@ -1208,7 +1255,6 @@ int
 ntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data,
     ntb_db_callback func)
 {
-	uint16_t mask;
 
 	if (idx >= ntb->allocated_interrupts || ntb->db_cb[idx].callback) {
 		device_printf(ntb->device, "Invalid Index.\n");
@@ -1217,11 +1263,9 @@ ntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data,
 
 	ntb->db_cb[idx].callback = func;
 	ntb->db_cb[idx].data = data;
+	callout_init(&ntb->db_cb[idx].irq_work, 1);
 
-	/* unmask interrupt */
-	mask = ntb_reg_read(2, ntb->reg_ofs.ldb_mask);
-	mask &= ~(1 << (idx * ntb->bits_per_vector));
-	ntb_reg_write(2, ntb->reg_ofs.ldb_mask, mask);
+	unmask_ldb_interrupt(ntb, idx);
 
 	return (0);
 }
@@ -1237,15 +1281,13 @@ ntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data,
 void
 ntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx)
 {
-	unsigned long mask;
 
 	if (idx >= ntb->allocated_interrupts || !ntb->db_cb[idx].callback)
 		return;
 
-	mask = ntb_reg_read(2, ntb->reg_ofs.ldb_mask);
-	mask |= 1 << (idx * ntb->bits_per_vector);
-	ntb_reg_write(2, ntb->reg_ofs.ldb_mask, mask);
+	mask_ldb_interrupt(ntb, idx);
 
+	callout_drain(&ntb->db_cb[idx].irq_work);
 	ntb->db_cb[idx].callback = NULL;
 }
 
diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.h b/sys/dev/ntb/ntb_hw/ntb_hw.h
index b3eee7a35a08..861d17863746 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.h
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.h
@@ -46,7 +46,7 @@ enum ntb_hw_event {
 
 SYSCTL_DECL(_hw_ntb);
 
-typedef void (*ntb_db_callback)(void *data, int db_num);
+typedef int (*ntb_db_callback)(void *data, int db_num);
 typedef void (*ntb_event_callback)(void *data, enum ntb_hw_event event);
 
 int ntb_register_event_callback(struct ntb_softc *ntb, ntb_event_callback func);

From e08f0663f752a84e4cf96c8fa65bddf7c12648a3 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Wed, 14 Oct 2015 02:37:30 +0000
Subject: [PATCH 184/380] Replace the out-of-place includes/files/config
 handling in bsd.subdir.mk with more typical ALL_SUBDIR_TARGETS entries and
 target hooks in bsd.incs.mk, bsd.files.mk and bsd.confs.mk.

This allows the targets to be NOPs if unneeded and still work with the
shortcut 'make includes' to build and then install in a parallel-safe manner.

Sort and re-indent the ALL_SUBDIR_TARGETS with the new entries.

Sponsored by:	EMC / Isilon Storage Division
---
 share/mk/bsd.confs.mk  |  3 +++
 share/mk/bsd.files.mk  |  3 +++
 share/mk/bsd.incs.mk   |  3 +++
 share/mk/bsd.subdir.mk | 26 ++++++--------------------
 4 files changed, 15 insertions(+), 20 deletions(-)

diff --git a/share/mk/bsd.confs.mk b/share/mk/bsd.confs.mk
index 0750a4e4fa57..bcc36bc18e6f 100644
--- a/share/mk/bsd.confs.mk
+++ b/share/mk/bsd.confs.mk
@@ -84,4 +84,7 @@ STAGE_TARGETS+= stage_config
 .endif
 .endif
 
+config: buildconfig installconfig
+.ORDER: buildconfig installconfig
+
 .endif # ${MK_INCLUDES} != "no"
diff --git a/share/mk/bsd.files.mk b/share/mk/bsd.files.mk
index b9379d92b59a..d1d3263c4444 100644
--- a/share/mk/bsd.files.mk
+++ b/share/mk/bsd.files.mk
@@ -94,4 +94,7 @@ buildfiles: stage_as
 .endif
 .endif
 
+files: buildfiles installfiles
+.ORDER: buildfiles installfiles
+
 .endif # !target(__<bsd.files.mk>__)
diff --git a/share/mk/bsd.incs.mk b/share/mk/bsd.incs.mk
index e1398c95dcd1..c985668c7c4b 100644
--- a/share/mk/bsd.incs.mk
+++ b/share/mk/bsd.incs.mk
@@ -99,4 +99,7 @@ STAGE_SYMLINKS.INCS= ${INCSLINKS}
 .endif
 .endif
 
+includes: buildincludes installincludes
+.ORDER: buildincludes installincludes
+
 .endif # ${MK_INCLUDES} != "no"
diff --git a/share/mk/bsd.subdir.mk b/share/mk/bsd.subdir.mk
index 8afe40a693b0..e4c3e69d1c27 100644
--- a/share/mk/bsd.subdir.mk
+++ b/share/mk/bsd.subdir.mk
@@ -32,9 +32,12 @@
 .if !target(__<bsd.subdir.mk>__)
 __<bsd.subdir.mk>__:
 
-ALL_SUBDIR_TARGETS= all all-man checkdpadd clean cleandepend cleandir \
-	cleanilinks cleanobj depend distribute lint maninstall manlint obj \
-	objlink realinstall regress tags ${SUBDIR_TARGETS}
+ALL_SUBDIR_TARGETS= all all-man buildconfig buildfiles buildincludes \
+		    checkdpadd clean cleandepend cleandir cleanilinks \
+		    cleanobj config depend distribute files includes \
+		    installconfig installfiles installincludes lint \
+		    maninstall manlint obj objlink realinstall regress tags \
+		    ${SUBDIR_TARGETS}
 
 .include <bsd.init.mk>
 
@@ -123,23 +126,6 @@ _sub.${__target}: _SUBDIR
 .endif
 .endfor
 
-# This is to support 'make includes' calling 'make buildincludes' and
-# 'make installincludes' in the proper order, and to support these
-# targets as SUBDIR_TARGETS.
-.for __target in files includes config
-.for __stage in build install
-${__stage}${__target}:
-.if make(${__stage}${__target})
-${__stage}${__target}: _sub.${__stage}${__target}
-_sub.${__stage}${__target}: _SUBDIR
-.endif
-.endfor
-.if !target(${__target})
-${__target}: .MAKE
-	${_+_}cd ${.CURDIR}; ${MAKE} build${__target}; ${MAKE} install${__target}
-.endif
-.endfor
-
 .endif
 
 .if !target(install)

From e146e19fb5fee8587e3eca2c4ba6d73628e244eb Mon Sep 17 00:00:00 2001
From: adrian <adrian@FreeBSD.org>
Date: Wed, 14 Oct 2015 02:43:04 +0000
Subject: [PATCH 185/380] rsu(4) manpage updates: add me, add 802.11n support,
 update caveats.

* Add that I indeed added 802.11n support.
* Update caveats - we support 1x1, 1x2 and 2x2 operation now, but
  there's no transmit aggregation support.
---
 share/man/man4/rsu.4 | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/share/man/man4/rsu.4 b/share/man/man4/rsu.4
index 2bd2f947c2a2..cfaf43e602d9 100644
--- a/share/man/man4/rsu.4
+++ b/share/man/man4/rsu.4
@@ -175,7 +175,10 @@ driver was written by
 .An Damien Bergamini Aq Mt damien@openbsd.org
 and ported by
 .An Rui Paulo Aq Mt rpaulo@freebsd.org .
+The 802.11n support was added by
+.An Adrian Chadd Aq Mt adrian@freebsd.org .
 .Sh CAVEATS
 The
 .Nm
-driver only supports 1T1R 802.11n operation.
+driver currently does not support 802.11n transmit aggregation,
+either A-MSDU or A-MPDU.

From 7726170474c6c89d386ba77b425d186e993a03a0 Mon Sep 17 00:00:00 2001
From: rpaulo <rpaulo@FreeBSD.org>
Date: Wed, 14 Oct 2015 04:30:17 +0000
Subject: [PATCH 186/380] Import wpa_supplicant/hostapd 2.5.

Major changes: bunch of CVEs fixed, tab completion for wpa_cli and
misc bug fixes.
---
 hostapd/ChangeLog                             |   36 +
 hostapd/Makefile                              |   55 +-
 hostapd/config_file.c                         |  164 +-
 hostapd/config_file.h                         |    2 +-
 hostapd/ctrl_iface.c                          |  814 +++++-
 hostapd/defconfig                             |   12 +
 hostapd/hlr_auc_gw.c                          |   48 +-
 hostapd/hlr_auc_gw.milenage_db                |    4 +-
 hostapd/hostapd.conf                          |  115 +-
 hostapd/hostapd_cli.c                         |   91 +-
 hostapd/main.c                                |   43 +-
 hs20/client/Makefile                          |    6 +
 hs20/client/osu_client.c                      |   77 +-
 hs20/client/spp_client.c                      |   15 +-
 patches/openssl-0.9.8zf-tls-extensions.patch  |  398 +++
 src/Makefile                                  |    1 +
 src/ap/Makefile                               |   65 +-
 src/ap/accounting.c                           |   12 +-
 src/ap/acs.c                                  |   13 +-
 src/ap/ap_config.c                            |   53 +-
 src/ap/ap_config.h                            |   35 +-
 src/ap/ap_drv_ops.c                           |  123 +-
 src/ap/ap_drv_ops.h                           |    1 +
 src/ap/ap_list.c                              |   13 +-
 src/ap/ap_list.h                              |    5 +
 src/ap/authsrv.c                              |   18 +-
 src/ap/beacon.c                               |  165 +-
 src/ap/beacon.h                               |    5 +
 src/ap/ctrl_iface_ap.c                        |   13 +-
 src/ap/dfs.c                                  |   32 +-
 src/ap/drv_callbacks.c                        |  172 +-
 src/ap/eap_user_db.c                          |   16 +-
 src/ap/hostapd.c                              |  245 +-
 src/ap/hostapd.h                              |   27 +
 src/ap/hw_features.c                          |   63 +-
 src/ap/hw_features.h                          |    5 +
 src/ap/ieee802_11.c                           |  162 +-
 src/ap/ieee802_11.h                           |    8 +-
 src/ap/ieee802_11_auth.c                      |   25 +-
 src/ap/ieee802_11_auth.h                      |    1 +
 src/ap/ieee802_11_ht.c                        |   25 +-
 src/ap/ieee802_11_vht.c                       |    3 +-
 src/ap/ieee802_1x.c                           |   86 +-
 src/ap/ieee802_1x.h                           |    1 -
 src/ap/ndisc_snoop.c                          |   51 +-
 src/ap/sta_info.c                             |   67 +-
 src/ap/sta_info.h                             |   12 +-
 src/ap/utils.c                                |   11 +
 src/ap/vlan_init.c                            |  157 +-
 src/ap/vlan_init.h                            |    2 -
 src/ap/vlan_util.c                            |   55 +-
 src/ap/wmm.c                                  |    3 +
 src/ap/wpa_auth.c                             |  118 +-
 src/ap/wpa_auth.h                             |   17 +-
 src/ap/wpa_auth_ft.c                          |   21 +-
 src/ap/wpa_auth_glue.c                        |   24 +-
 src/ap/wpa_auth_i.h                           |    2 +
 src/ap/wpa_auth_ie.c                          |   20 +-
 src/ap/wps_hostapd.c                          |   89 +-
 src/ap/x_snoop.c                              |    8 +
 src/common/Makefile                           |   26 +-
 src/common/common_module_tests.c              |   37 +-
 src/common/defs.h                             |   10 +-
 src/common/hw_features_common.c               |  103 +-
 src/common/hw_features_common.h               |    1 -
 src/common/ieee802_11_common.c                |  296 +-
 src/common/ieee802_11_common.h                |   26 +-
 src/common/ieee802_11_defs.h                  |  117 +-
 src/common/privsep_commands.h                 |   35 +-
 src/common/qca-vendor.h                       |  113 +-
 src/common/sae.c                              |  391 ++-
 src/common/sae.h                              |    3 +
 src/common/version.h                          |    2 +-
 src/common/wpa_common.c                       |  182 +-
 src/common/wpa_common.h                       |   16 +-
 src/common/wpa_ctrl.c                         |   26 +-
 src/common/wpa_ctrl.h                         |   20 +
 src/crypto/crypto.h                           |   30 +
 src/crypto/crypto_cryptoapi.c                 |  783 ------
 src/crypto/crypto_module_tests.c              |   59 +-
 src/crypto/crypto_openssl.c                   |  169 +-
 src/crypto/dh_groups.c                        |    2 +-
 src/crypto/fips_prf_openssl.c                 |   16 +-
 src/crypto/ms_funcs.c                         |   29 +-
 src/crypto/ms_funcs.h                         |    2 +
 src/crypto/random.c                           |    1 +
 src/crypto/sha1-tlsprf.c                      |    5 +
 src/crypto/sha1-tprf.c                        |    2 +
 src/crypto/sha256-kdf.c                       |    3 +
 src/crypto/sha384-prf.c                       |  100 +
 src/crypto/sha384.h                           |    5 +
 src/crypto/tls.h                              |   94 +-
 src/crypto/tls_gnutls.c                       |   56 +-
 src/crypto/tls_internal.c                     |  118 +-
 src/crypto/tls_none.c                         |   52 +-
 src/crypto/tls_openssl.c                      |  841 ++++--
 src/crypto/tls_schannel.c                     |  763 ------
 src/drivers/driver.h                          |  100 +-
 src/drivers/driver_atheros.c                  |   93 +-
 src/drivers/driver_bsd.c                      |    6 +-
 src/drivers/driver_hostap.c                   |   43 +-
 src/drivers/driver_hostap.h                   |    2 +-
 src/drivers/driver_ndis.c                     |    8 +-
 src/drivers/driver_nl80211.c                  |  457 +++-
 src/drivers/driver_nl80211.h                  |    9 +-
 src/drivers/driver_nl80211_android.c          |   34 +-
 src/drivers/driver_nl80211_capa.c             |   90 +-
 src/drivers/driver_nl80211_event.c            |   63 +
 src/drivers/driver_nl80211_scan.c             |   10 +-
 src/drivers/driver_privsep.c                  |   94 +-
 src/drivers/driver_wext.c                     |  133 +-
 src/drivers/drivers.c                         |    2 +-
 src/drivers/drivers.mak                       |    4 +-
 src/drivers/linux_ioctl.c                     |   23 +
 src/drivers/linux_ioctl.h                     |    1 +
 src/drivers/nl80211_copy.h                    |  236 +-
 src/eap_common/Makefile                       |   29 +-
 src/eap_common/eap_common.c                   |    2 +-
 src/eap_common/eap_fast_common.c              |   41 +-
 src/eap_common/eap_pwd_common.c               |   17 +-
 src/eap_common/eap_pwd_common.h               |   13 +-
 src/eap_common/eap_sake_common.c              |   86 +-
 src/eap_common/ikev2_common.c                 |    6 +-
 src/eap_peer/Makefile                         |   18 +-
 src/eap_peer/eap.c                            |   14 +-
 src/eap_peer/eap.h                            |    2 +-
 src/eap_peer/eap_aka.c                        |    2 +-
 src/eap_peer/eap_eke.c                        |  105 +-
 src/eap_peer/eap_fast.c                       |   78 +-
 src/eap_peer/eap_gpsk.c                       |   27 +-
 src/eap_peer/eap_i.h                          |    8 +-
 src/eap_peer/eap_mschapv2.c                   |    6 +
 src/eap_peer/eap_pax.c                        |    2 +-
 src/eap_peer/eap_peap.c                       |   18 +-
 src/eap_peer/eap_pwd.c                        |  116 +-
 src/eap_peer/eap_sake.c                       |   25 +-
 src/eap_peer/eap_sim.c                        |    2 +-
 src/eap_peer/eap_tls.c                        |   18 +-
 src/eap_peer/eap_tls_common.c                 |  139 +-
 src/eap_peer/eap_tls_common.h                 |    2 +-
 src/eap_peer/eap_ttls.c                       |   41 +-
 src/eap_peer/eap_wsc.c                        |    3 +
 src/eap_server/Makefile                       |   19 +-
 src/eap_server/eap.h                          |    6 +-
 src/eap_server/eap_i.h                        |    3 +-
 src/eap_server/eap_server.c                   |   47 +-
 src/eap_server/eap_server_eke.c               |   24 +
 src/eap_server/eap_server_fast.c              |    2 +-
 src/eap_server/eap_server_mschapv2.c          |   13 +
 src/eap_server/eap_server_peap.c              |  147 +-
 src/eap_server/eap_server_pwd.c               |   64 +-
 src/eap_server/eap_server_tls.c               |   58 +-
 src/eap_server/eap_server_tls_common.c        |   53 +-
 src/eap_server/eap_server_ttls.c              |  111 +-
 src/eap_server/eap_tls_common.h               |    2 +-
 src/eapol_auth/Makefile                       |   14 +-
 src/eapol_auth/eapol_auth_sm.c                |  101 +-
 src/eapol_auth/eapol_auth_sm.h                |    6 +-
 src/eapol_supp/Makefile                       |   16 +-
 src/eapol_supp/eapol_supp_sm.c                |   15 +-
 src/fst/Makefile                              |    8 +
 src/fst/fst.c                                 |  225 ++
 src/fst/fst.h                                 |  296 ++
 src/fst/fst_ctrl_aux.c                        |   69 +
 src/fst/fst_ctrl_aux.h                        |   91 +
 src/fst/fst_ctrl_defs.h                       |  109 +
 src/fst/fst_ctrl_iface.c                      |  948 +++++++
 src/fst/fst_ctrl_iface.h                      |   45 +
 src/fst/fst_defs.h                            |   87 +
 src/fst/fst_group.c                           |  462 ++++
 src/fst/fst_group.h                           |   75 +
 src/fst/fst_iface.c                           |   79 +
 src/fst/fst_iface.h                           |  135 +
 src/fst/fst_internal.h                        |   49 +
 src/fst/fst_session.c                         | 1620 +++++++++++
 src/fst/fst_session.h                         |   80 +
 src/l2_packet/Makefile                        |   14 +-
 src/p2p/Makefile                              |   27 +-
 src/p2p/p2p.c                                 |  282 +-
 src/p2p/p2p.h                                 |  136 +-
 src/p2p/p2p_build.c                           |  358 ++-
 src/p2p/p2p_dev_disc.c                        |    2 +-
 src/p2p/p2p_go_neg.c                          |  213 +-
 src/p2p/p2p_group.c                           |   40 +
 src/p2p/p2p_i.h                               |   38 +-
 src/p2p/p2p_invitation.c                      |   23 +-
 src/p2p/p2p_parse.c                           |   25 +-
 src/p2p/p2p_pd.c                              |  400 ++-
 src/p2p/p2p_utils.c                           |   58 +-
 src/radius/Makefile                           |    1 +
 src/radius/radius.c                           |   12 +-
 src/radius/radius_das.c                       |    2 +-
 src/radius/radius_server.c                    |   20 +-
 src/radius/radius_server.h                    |    2 +
 src/rsn_supp/Makefile                         |   28 +-
 src/rsn_supp/tdls.c                           |   23 +-
 src/rsn_supp/wpa.c                            |   27 +-
 src/rsn_supp/wpa_ft.c                         |    4 +-
 src/rsn_supp/wpa_ie.c                         |   18 +-
 src/rsn_supp/wpa_ie.h                         |    2 -
 src/tls/libtommath.c                          |    5 +-
 src/tls/tlsv1_client.c                        |    8 +-
 src/tls/tlsv1_client.h                        |    2 +-
 src/tls/tlsv1_server.c                        |    8 +-
 src/tls/tlsv1_server.h                        |    2 +-
 src/tls/x509v3.c                              |    2 +-
 src/utils/browser-wpadebug.c                  |    6 +-
 src/utils/common.c                            |   97 +-
 src/utils/common.h                            |   41 +-
 src/utils/eloop.c                             |   45 +-
 src/utils/http_curl.c                         |   14 +-
 src/utils/includes.h                          |    7 +-
 src/utils/os.h                                |   15 +
 src/utils/os_internal.c                       |    6 +
 src/utils/os_none.c                           |    6 +
 src/utils/os_unix.c                           |  120 +-
 src/utils/os_win32.c                          |   18 +
 src/utils/radiotap.c                          |    4 +-
 src/utils/utils_module_tests.c                |  440 ++-
 src/utils/wpa_debug.c                         |   90 +-
 src/utils/wpa_debug.h                         |   27 +-
 src/utils/wpabuf.c                            |    2 +-
 src/wps/Makefile                              |   39 +-
 src/wps/http_client.c                         |   42 +-
 src/wps/http_server.c                         |    8 +-
 src/wps/httpread.c                            |  125 +-
 src/wps/ndef.c                                |   16 +-
 src/wps/wps.c                                 |   15 +-
 src/wps/wps.h                                 |    7 +-
 src/wps/wps_attr_parse.c                      |   43 +-
 src/wps/wps_attr_parse.h                      |   41 +-
 src/wps/wps_common.c                          |    4 +-
 src/wps/wps_defs.h                            |    6 +
 src/wps/wps_enrollee.c                        |    7 +-
 src/wps/wps_er.c                              |   27 +-
 src/wps/wps_er_ssdp.c                         |    4 +-
 src/wps/wps_module_tests.c                    |    4 +-
 src/wps/wps_registrar.c                       |   14 +-
 src/wps/wps_upnp.c                            |    7 +-
 src/wps/wps_upnp_ap.c                         |    6 +-
 src/wps/wps_upnp_event.c                      |    8 +-
 src/wps/wps_upnp_ssdp.c                       |   19 +-
 src/wps/wps_upnp_web.c                        |   15 +-
 src/wps/wps_validate.c                        |    2 +
 wpa_supplicant/ChangeLog                      |   63 +
 wpa_supplicant/Makefile                       |  102 +-
 wpa_supplicant/ap.c                           |   46 +-
 wpa_supplicant/ap.h                           |    2 +
 wpa_supplicant/bss.c                          |   76 +-
 wpa_supplicant/bss.h                          |    2 +-
 wpa_supplicant/config.c                       |   69 +-
 wpa_supplicant/config.h                       |   86 +-
 wpa_supplicant/config_file.c                  |   67 +-
 wpa_supplicant/config_ssid.h                  |   17 +-
 wpa_supplicant/ctrl_iface.c                   |  761 +++++-
 wpa_supplicant/ctrl_iface_named_pipe.c        |    3 +-
 wpa_supplicant/ctrl_iface_udp.c               |    3 +-
 wpa_supplicant/ctrl_iface_unix.c              |  118 +-
 wpa_supplicant/dbus/dbus_new.c                |  378 ++-
 wpa_supplicant/dbus/dbus_new.h                |   43 +-
 wpa_supplicant/dbus/dbus_new_handlers.c       |  128 +-
 wpa_supplicant/dbus/dbus_new_handlers.h       |    6 +
 wpa_supplicant/dbus/dbus_new_handlers_p2p.c   |  317 ++-
 wpa_supplicant/dbus/dbus_new_handlers_p2p.h   |   22 +
 wpa_supplicant/dbus/dbus_new_handlers_wps.c   |   26 +-
 wpa_supplicant/dbus/dbus_new_helpers.c        |    7 +-
 wpa_supplicant/dbus/dbus_new_introspect.c     |    2 +-
 wpa_supplicant/dbus/dbus_old.c                |   10 +-
 wpa_supplicant/dbus/dbus_old_handlers.c       |   26 +-
 wpa_supplicant/defconfig                      |    9 +
 wpa_supplicant/doc/docbook/eapol_test.8       |    2 +-
 wpa_supplicant/doc/docbook/wpa_background.8   |    2 +-
 wpa_supplicant/doc/docbook/wpa_cli.8          |    2 +-
 wpa_supplicant/doc/docbook/wpa_gui.8          |   13 +-
 wpa_supplicant/doc/docbook/wpa_gui.sgml       |   17 +
 wpa_supplicant/doc/docbook/wpa_passphrase.8   |    2 +-
 wpa_supplicant/doc/docbook/wpa_priv.8         |    2 +-
 wpa_supplicant/doc/docbook/wpa_supplicant.8   |    2 +-
 .../doc/docbook/wpa_supplicant.conf.5         |    2 +-
 wpa_supplicant/driver_i.h                     |   40 +-
 wpa_supplicant/eapol_test.c                   |  131 +-
 wpa_supplicant/eapol_test.py                  |  142 +
 wpa_supplicant/events.c                       |  279 +-
 wpa_supplicant/hs20_supplicant.c              |   18 +-
 wpa_supplicant/ibss_rsn.c                     |    6 +-
 wpa_supplicant/interworking.c                 |   16 +-
 wpa_supplicant/main.c                         |   18 +-
 wpa_supplicant/mesh.c                         |   17 +-
 wpa_supplicant/mesh_mpm.c                     |    9 +-
 wpa_supplicant/mesh_rsn.c                     |    8 +-
 wpa_supplicant/notify.c                       |   58 +-
 wpa_supplicant/notify.h                       |    9 +-
 wpa_supplicant/p2p_supplicant.c               | 2424 +++++++----------
 wpa_supplicant/p2p_supplicant.h               |   33 +-
 wpa_supplicant/p2p_supplicant_sd.c            | 1273 +++++++++
 wpa_supplicant/preauth_test.c                 |    2 +-
 wpa_supplicant/scan.c                         |   76 +-
 wpa_supplicant/sme.c                          |   70 +-
 wpa_supplicant/wpa_cli.c                      |  511 +++-
 wpa_supplicant/wpa_priv.c                     |  155 +-
 wpa_supplicant/wpa_supplicant.c               |  379 ++-
 wpa_supplicant/wpa_supplicant.conf            |   47 +-
 wpa_supplicant/wpa_supplicant_i.h             |   81 +-
 wpa_supplicant/wpas_glue.c                    |   61 +-
 wpa_supplicant/wpas_glue.h                    |    3 +
 wpa_supplicant/wps_supplicant.c               |  122 +-
 wpa_supplicant/wps_supplicant.h               |    1 +
 307 files changed, 20881 insertions(+), 6491 deletions(-)
 create mode 100644 patches/openssl-0.9.8zf-tls-extensions.patch
 delete mode 100644 src/crypto/crypto_cryptoapi.c
 create mode 100644 src/crypto/sha384-prf.c
 delete mode 100644 src/crypto/tls_schannel.c
 create mode 100644 src/fst/Makefile
 create mode 100644 src/fst/fst.c
 create mode 100644 src/fst/fst.h
 create mode 100644 src/fst/fst_ctrl_aux.c
 create mode 100644 src/fst/fst_ctrl_aux.h
 create mode 100644 src/fst/fst_ctrl_defs.h
 create mode 100644 src/fst/fst_ctrl_iface.c
 create mode 100644 src/fst/fst_ctrl_iface.h
 create mode 100644 src/fst/fst_defs.h
 create mode 100644 src/fst/fst_group.c
 create mode 100644 src/fst/fst_group.h
 create mode 100644 src/fst/fst_iface.c
 create mode 100644 src/fst/fst_iface.h
 create mode 100644 src/fst/fst_internal.h
 create mode 100644 src/fst/fst_session.c
 create mode 100644 src/fst/fst_session.h
 create mode 100755 wpa_supplicant/eapol_test.py
 create mode 100644 wpa_supplicant/p2p_supplicant_sd.c

diff --git a/hostapd/ChangeLog b/hostapd/ChangeLog
index e6f8c6a03e07..af54e1e5b4e4 100644
--- a/hostapd/ChangeLog
+++ b/hostapd/ChangeLog
@@ -1,5 +1,41 @@
 ChangeLog for hostapd
 
+2015-09-27 - v2.5
+	* fixed WPS UPnP vulnerability with HTTP chunked transfer encoding
+	  [http://w1.fi/security/2015-2/] (CVE-2015-4141)
+	* fixed WMM Action frame parser
+	  [http://w1.fi/security/2015-3/] (CVE-2015-4142)
+	* fixed EAP-pwd server missing payload length validation
+	  [http://w1.fi/security/2015-4/]
+	  (CVE-2015-4143, CVE-2015-4144, CVE-2015-4145)
+	* fixed validation of WPS and P2P NFC NDEF record payload length
+	  [http://w1.fi/security/2015-5/]
+	* nl80211:
+	  - fixed vendor command handling to check OUI properly
+	* fixed hlr_auc_gw build with OpenSSL
+	* hlr_auc_gw: allow Milenage RES length to be reduced
+	* disable HT for a station that does not support WMM/QoS
+	* added support for hashed password (NtHash) in EAP-pwd server
+	* fixed and extended dynamic VLAN cases
+	* added EAP-EKE server support for deriving Session-Id
+	* set Acct-Session-Id to a random value to make it more likely to be
+	  unique even if the device does not have a proper clock
+	* added more 2.4 GHz channels for 20/40 MHz HT co-ex scan
+	* modified SAE routines to be more robust and PWE generation to be
+	  stronger against timing attacks
+	* added support for Brainpool Elliptic Curves with SAE
+	* increases maximum value accepted for cwmin/cwmax
+	* added support for CCMP-256 and GCMP-256 as group ciphers with FT
+	* added Fast Session Transfer (FST) module
+	* removed optional fields from RSNE when using FT with PMF
+	  (workaround for interoperability issues with iOS 8.4)
+	* added EAP server support for TLS session resumption
+	* fixed key derivation for Suite B 192-bit AKM (this breaks
+	  compatibility with the earlier version)
+	* added mechanism to track unconnected stations and do minimal band
+	  steering
+	* number of small fixes
+
 2015-03-15 - v2.4
 	* allow OpenSSL cipher configuration to be set for internal EAP server
 	  (openssl_ciphers parameter)
diff --git a/hostapd/Makefile b/hostapd/Makefile
index eace68cd051b..a812b9d661c3 100644
--- a/hostapd/Makefile
+++ b/hostapd/Makefile
@@ -107,7 +107,16 @@ LIBS_h += -lrt
 LIBS_n += -lrt
 endif
 
+ifdef CONFIG_ELOOP_POLL
+CFLAGS += -DCONFIG_ELOOP_POLL
+endif
+
+ifdef CONFIG_ELOOP_EPOLL
+CFLAGS += -DCONFIG_ELOOP_EPOLL
+endif
+
 OBJS += ../src/utils/common.o
+OBJS_c += ../src/utils/common.o
 OBJS += ../src/utils/wpa_debug.o
 OBJS_c += ../src/utils/wpa_debug.o
 OBJS += ../src/utils/wpabuf.o
@@ -227,6 +236,7 @@ CFLAGS += -DCONFIG_SAE
 OBJS += ../src/common/sae.o
 NEED_ECC=y
 NEED_DH_GROUPS=y
+NEED_AP_MLME=y
 endif
 
 ifdef CONFIG_WNM
@@ -531,8 +541,14 @@ HOBJS += ../src/crypto/crypto_openssl.o
 ifdef NEED_FIPS186_2_PRF
 OBJS += ../src/crypto/fips_prf_openssl.o
 endif
+NEED_SHA256=y
+NEED_TLS_PRF_SHA256=y
 LIBS += -lcrypto
 LIBS_h += -lcrypto
+ifdef CONFIG_TLS_ADD_DL
+LIBS += -ldl
+LIBS_h += -ldl
+endif
 endif
 
 ifeq ($(CONFIG_TLS), gnutls)
@@ -553,17 +569,6 @@ CONFIG_INTERNAL_RC4=y
 CONFIG_INTERNAL_DH_GROUP5=y
 endif
 
-ifeq ($(CONFIG_TLS), schannel)
-ifdef TLS_FUNCS
-OBJS += ../src/crypto/tls_schannel.o
-endif
-OBJS += ../src/crypto/crypto_cryptoapi.o
-OBJS_p += ../src/crypto/crypto_cryptoapi.o
-CONFIG_INTERNAL_SHA256=y
-CONFIG_INTERNAL_RC4=y
-CONFIG_INTERNAL_DH_GROUP5=y
-endif
-
 ifeq ($(CONFIG_TLS), internal)
 ifndef CONFIG_CRYPTO
 CONFIG_CRYPTO=internal
@@ -694,8 +699,10 @@ endif
 endif
 ifdef NEED_AES_CBC
 NEED_AES_DEC=y
+ifneq ($(CONFIG_TLS), openssl)
 AESOBJS += ../src/crypto/aes-cbc.o
 endif
+endif
 ifdef NEED_AES_DEC
 ifdef CONFIG_INTERNAL_AES
 AESOBJS += ../src/crypto/aes-internal-dec.o
@@ -754,11 +761,17 @@ OBJS += ../src/crypto/des-internal.o
 endif
 endif
 
+ifdef CONFIG_NO_RC4
+CFLAGS += -DCONFIG_NO_RC4
+endif
+
 ifdef NEED_RC4
 ifdef CONFIG_INTERNAL_RC4
+ifndef CONFIG_NO_RC4
 OBJS += ../src/crypto/rc4.o
 endif
 endif
+endif
 
 ifdef NEED_SHA256
 CFLAGS += -DCONFIG_SHA256
@@ -778,6 +791,7 @@ endif
 endif
 ifdef NEED_SHA384
 CFLAGS += -DCONFIG_SHA384
+OBJS += ../src/crypto/sha384-prf.o
 endif
 
 ifdef NEED_DH_GROUPS
@@ -803,8 +817,10 @@ OBJS += ../src/crypto/random.o
 HOBJS += ../src/crypto/random.o
 HOBJS += ../src/utils/eloop.o
 HOBJS += $(SHA1OBJS)
+ifneq ($(CONFIG_TLS), openssl)
 HOBJS += ../src/crypto/md5.o
 endif
+endif
 
 ifdef CONFIG_RADIUS_SERVER
 CFLAGS += -DRADIUS_SERVER
@@ -903,6 +919,21 @@ LIBS += -lsqlite3
 LIBS_h += -lsqlite3
 endif
 
+ifdef CONFIG_FST
+CFLAGS += -DCONFIG_FST
+OBJS += ../src/fst/fst.o
+OBJS += ../src/fst/fst_group.o
+OBJS += ../src/fst/fst_iface.o
+OBJS += ../src/fst/fst_session.o
+OBJS += ../src/fst/fst_ctrl_aux.o
+ifdef CONFIG_FST_TEST
+CFLAGS += -DCONFIG_FST_TEST
+endif
+ifndef CONFIG_NO_CTRL_IFACE
+OBJS += ../src/fst/fst_ctrl_iface.o
+endif
+endif
+
 ALL=hostapd hostapd_cli
 
 all: verify_config $(ALL)
@@ -965,9 +996,11 @@ NOBJS = nt_password_hash.o ../src/crypto/ms_funcs.o $(SHA1OBJS)
 NOBJS += ../src/utils/common.o
 ifdef NEED_RC4
 ifdef CONFIG_INTERNAL_RC4
+ifndef CONFIG_NO_RC4
 NOBJS += ../src/crypto/rc4.o
 endif
 endif
+endif
 ifdef CONFIG_INTERNAL_MD5
 NOBJS += ../src/crypto/md5-internal.o
 endif
diff --git a/hostapd/config_file.c b/hostapd/config_file.c
index 53143f76cfe8..82ac61d7729a 100644
--- a/hostapd/config_file.c
+++ b/hostapd/config_file.c
@@ -222,9 +222,15 @@ static int hostapd_config_read_eap_user(const char *fname,
 		return 0;
 
 	if (os_strncmp(fname, "sqlite:", 7) == 0) {
+#ifdef CONFIG_SQLITE
 		os_free(conf->eap_user_sqlite);
 		conf->eap_user_sqlite = os_strdup(fname + 7);
 		return 0;
+#else /* CONFIG_SQLITE */
+		wpa_printf(MSG_ERROR,
+			   "EAP user file in SQLite DB, but CONFIG_SQLITE was not enabled in the build.");
+		return -1;
+#endif /* CONFIG_SQLITE */
 	}
 
 	f = fopen(fname, "r");
@@ -775,6 +781,24 @@ static int hostapd_config_read_wep(struct hostapd_wep_keys *wep, int keyidx,
 }
 
 
+static int hostapd_parse_chanlist(struct hostapd_config *conf, char *val)
+{
+	char *pos;
+
+	/* for backwards compatibility, translate ' ' in conf str to ',' */
+	pos = val;
+	while (pos) {
+		pos = os_strchr(pos, ' ');
+		if (pos)
+			*pos++ = ',';
+	}
+	if (freq_range_list_parse(&conf->acs_ch_list, val))
+		return -1;
+
+	return 0;
+}
+
+
 static int hostapd_parse_intlist(int **int_list, char *val)
 {
 	int *list;
@@ -875,7 +899,9 @@ static int hostapd_config_read_int10(const char *value)
 static int valid_cw(int cw)
 {
 	return (cw == 1 || cw == 3 || cw == 7 || cw == 15 || cw == 31 ||
-		cw == 63 || cw == 127 || cw == 255 || cw == 511 || cw == 1023);
+		cw == 63 || cw == 127 || cw == 255 || cw == 511 || cw == 1023 ||
+		cw == 2047 || cw == 4095 || cw == 8191 || cw == 16383 ||
+		cw == 32767);
 }
 
 
@@ -886,11 +912,11 @@ enum {
 	IEEE80211_TX_QUEUE_DATA3 = 3 /* used for EDCA AC_BK data */
 };
 
-static int hostapd_config_tx_queue(struct hostapd_config *conf, char *name,
-				   char *val)
+static int hostapd_config_tx_queue(struct hostapd_config *conf,
+				   const char *name, const char *val)
 {
 	int num;
-	char *pos;
+	const char *pos;
 	struct hostapd_tx_queue_params *queue;
 
 	/* skip 'tx_queue_' prefix */
@@ -1134,13 +1160,23 @@ static int hostapd_config_vht_capab(struct hostapd_config *conf,
 	if (os_strstr(capab, "[BF-ANTENNA-2]") &&
 	    (conf->vht_capab & VHT_CAP_SU_BEAMFORMEE_CAPABLE))
 		conf->vht_capab |= (1 << VHT_CAP_BEAMFORMEE_STS_OFFSET);
+	if (os_strstr(capab, "[BF-ANTENNA-3]") &&
+	    (conf->vht_capab & VHT_CAP_SU_BEAMFORMEE_CAPABLE))
+		conf->vht_capab |= (2 << VHT_CAP_BEAMFORMEE_STS_OFFSET);
+	if (os_strstr(capab, "[BF-ANTENNA-4]") &&
+	    (conf->vht_capab & VHT_CAP_SU_BEAMFORMEE_CAPABLE))
+		conf->vht_capab |= (3 << VHT_CAP_BEAMFORMEE_STS_OFFSET);
 	if (os_strstr(capab, "[SOUNDING-DIMENSION-2]") &&
 	    (conf->vht_capab & VHT_CAP_SU_BEAMFORMER_CAPABLE))
 		conf->vht_capab |= (1 << VHT_CAP_SOUNDING_DIMENSION_OFFSET);
+	if (os_strstr(capab, "[SOUNDING-DIMENSION-3]") &&
+	    (conf->vht_capab & VHT_CAP_SU_BEAMFORMER_CAPABLE))
+		conf->vht_capab |= (2 << VHT_CAP_SOUNDING_DIMENSION_OFFSET);
+	if (os_strstr(capab, "[SOUNDING-DIMENSION-4]") &&
+	    (conf->vht_capab & VHT_CAP_SU_BEAMFORMER_CAPABLE))
+		conf->vht_capab |= (3 << VHT_CAP_SOUNDING_DIMENSION_OFFSET);
 	if (os_strstr(capab, "[MU-BEAMFORMER]"))
 		conf->vht_capab |= VHT_CAP_MU_BEAMFORMER_CAPABLE;
-	if (os_strstr(capab, "[MU-BEAMFORMEE]"))
-		conf->vht_capab |= VHT_CAP_MU_BEAMFORMEE_CAPABLE;
 	if (os_strstr(capab, "[VHT-TXOP-PS]"))
 		conf->vht_capab |= VHT_CAP_VHT_TXOP_PS;
 	if (os_strstr(capab, "[HTC-VHT]"))
@@ -1699,7 +1735,7 @@ static int hs20_parse_osu_ssid(struct hostapd_bss_config *bss,
 	char *str;
 
 	str = wpa_config_parse_string(pos, &slen);
-	if (str == NULL || slen < 1 || slen > HOSTAPD_MAX_SSID_LEN) {
+	if (str == NULL || slen < 1 || slen > SSID_MAX_LEN) {
 		wpa_printf(MSG_ERROR, "Line %d: Invalid SSID '%s'", line, pos);
 		os_free(str);
 		return -1;
@@ -1900,7 +1936,7 @@ fail:
 
 static int hostapd_config_fill(struct hostapd_config *conf,
 			       struct hostapd_bss_config *bss,
-			       char *buf, char *pos, int line)
+			       const char *buf, char *pos, int line)
 {
 	if (os_strcmp(buf, "interface") == 0) {
 		os_strlcpy(conf->bss[0]->iface, pos,
@@ -1946,7 +1982,7 @@ static int hostapd_config_fill(struct hostapd_config *conf,
 			   line);
 	} else if (os_strcmp(buf, "ssid") == 0) {
 		bss->ssid.ssid_len = os_strlen(pos);
-		if (bss->ssid.ssid_len > HOSTAPD_MAX_SSID_LEN ||
+		if (bss->ssid.ssid_len > SSID_MAX_LEN ||
 		    bss->ssid.ssid_len < 1) {
 			wpa_printf(MSG_ERROR, "Line %d: invalid SSID '%s'",
 				   line, pos);
@@ -1957,7 +1993,7 @@ static int hostapd_config_fill(struct hostapd_config *conf,
 	} else if (os_strcmp(buf, "ssid2") == 0) {
 		size_t slen;
 		char *str = wpa_config_parse_string(pos, &slen);
-		if (str == NULL || slen < 1 || slen > HOSTAPD_MAX_SSID_LEN) {
+		if (str == NULL || slen < 1 || slen > SSID_MAX_LEN) {
 			wpa_printf(MSG_ERROR, "Line %d: invalid SSID '%s'",
 				   line, pos);
 			os_free(str);
@@ -2043,6 +2079,8 @@ static int hostapd_config_fill(struct hostapd_config *conf,
 		bss->private_key_passwd = os_strdup(pos);
 	} else if (os_strcmp(buf, "check_crl") == 0) {
 		bss->check_crl = atoi(pos);
+	} else if (os_strcmp(buf, "tls_session_lifetime") == 0) {
+		bss->tls_session_lifetime = atoi(pos);
 	} else if (os_strcmp(buf, "ocsp_stapling_response") == 0) {
 		os_free(bss->ocsp_stapling_response);
 		bss->ocsp_stapling_response = os_strdup(pos);
@@ -2515,13 +2553,17 @@ static int hostapd_config_fill(struct hostapd_config *conf,
 			conf->hw_mode = HOSTAPD_MODE_IEEE80211G;
 		else if (os_strcmp(pos, "ad") == 0)
 			conf->hw_mode = HOSTAPD_MODE_IEEE80211AD;
+		else if (os_strcmp(pos, "any") == 0)
+			conf->hw_mode = HOSTAPD_MODE_IEEE80211ANY;
 		else {
 			wpa_printf(MSG_ERROR, "Line %d: unknown hw_mode '%s'",
 				   line, pos);
 			return 1;
 		}
 	} else if (os_strcmp(buf, "wps_rf_bands") == 0) {
-		if (os_strcmp(pos, "a") == 0)
+		if (os_strcmp(pos, "ad") == 0)
+			bss->wps_rf_bands = WPS_RF_60GHZ;
+		else if (os_strcmp(pos, "a") == 0)
 			bss->wps_rf_bands = WPS_RF_50GHZ;
 		else if (os_strcmp(pos, "g") == 0 ||
 			 os_strcmp(pos, "b") == 0)
@@ -2542,12 +2584,15 @@ static int hostapd_config_fill(struct hostapd_config *conf,
 				   line);
 			return 1;
 #else /* CONFIG_ACS */
+			conf->acs = 1;
 			conf->channel = 0;
 #endif /* CONFIG_ACS */
-		} else
+		} else {
 			conf->channel = atoi(pos);
+			conf->acs = conf->channel == 0;
+		}
 	} else if (os_strcmp(buf, "chanlist") == 0) {
-		if (hostapd_parse_intlist(&conf->chanlist, pos)) {
+		if (hostapd_parse_chanlist(conf, pos)) {
 			wpa_printf(MSG_ERROR, "Line %d: invalid channel list",
 				   line);
 			return 1;
@@ -2810,7 +2855,7 @@ static int hostapd_config_fill(struct hostapd_config *conf,
 		os_free(bss->wps_pin_requests);
 		bss->wps_pin_requests = os_strdup(pos);
 	} else if (os_strcmp(buf, "device_name") == 0) {
-		if (os_strlen(pos) > 32) {
+		if (os_strlen(pos) > WPS_DEV_NAME_MAX_LEN) {
 			wpa_printf(MSG_ERROR, "Line %d: Too long "
 				   "device_name", line);
 			return 1;
@@ -3111,6 +3156,8 @@ static int hostapd_config_fill(struct hostapd_config *conf,
 		bss->disable_dgaf = atoi(pos);
 	} else if (os_strcmp(buf, "proxy_arp") == 0) {
 		bss->proxy_arp = atoi(pos);
+	} else if (os_strcmp(buf, "na_mcast_to_ucast") == 0) {
+		bss->na_mcast_to_ucast = atoi(pos);
 	} else if (os_strcmp(buf, "osen") == 0) {
 		bss->osen = atoi(pos);
 	} else if (os_strcmp(buf, "anqp_domain_id") == 0) {
@@ -3223,6 +3270,24 @@ static int hostapd_config_fill(struct hostapd_config *conf,
 		bss->bss_load_test_set = 1;
 	} else if (os_strcmp(buf, "radio_measurements") == 0) {
 		bss->radio_measurements = atoi(pos);
+	} else if (os_strcmp(buf, "own_ie_override") == 0) {
+		struct wpabuf *tmp;
+		size_t len = os_strlen(pos) / 2;
+
+		tmp = wpabuf_alloc(len);
+		if (!tmp)
+			return 1;
+
+		if (hexstr2bin(pos, wpabuf_put(tmp, len), len)) {
+			wpabuf_free(tmp);
+			wpa_printf(MSG_ERROR,
+				   "Line %d: Invalid own_ie_override '%s'",
+				   line, pos);
+			return 1;
+		}
+
+		wpabuf_free(bss->own_ie_override);
+		bss->own_ie_override = tmp;
 #endif /* CONFIG_TESTING_OPTIONS */
 	} else if (os_strcmp(buf, "vendor_elements") == 0) {
 		struct wpabuf *elems;
@@ -3276,6 +3341,74 @@ static int hostapd_config_fill(struct hostapd_config *conf,
 	} else if (os_strcmp(buf, "wowlan_triggers") == 0) {
 		os_free(bss->wowlan_triggers);
 		bss->wowlan_triggers = os_strdup(pos);
+#ifdef CONFIG_FST
+	} else if (os_strcmp(buf, "fst_group_id") == 0) {
+		size_t len = os_strlen(pos);
+
+		if (!len || len >= sizeof(conf->fst_cfg.group_id)) {
+			wpa_printf(MSG_ERROR,
+				   "Line %d: Invalid fst_group_id value '%s'",
+				   line, pos);
+			return 1;
+		}
+
+		if (conf->fst_cfg.group_id[0]) {
+			wpa_printf(MSG_ERROR,
+				   "Line %d: Duplicate fst_group value '%s'",
+				   line, pos);
+			return 1;
+		}
+
+		os_strlcpy(conf->fst_cfg.group_id, pos,
+			   sizeof(conf->fst_cfg.group_id));
+	} else if (os_strcmp(buf, "fst_priority") == 0) {
+		char *endp;
+		long int val;
+
+		if (!*pos) {
+			wpa_printf(MSG_ERROR,
+				   "Line %d: fst_priority value not supplied (expected 1..%u)",
+				   line, FST_MAX_PRIO_VALUE);
+			return -1;
+		}
+
+		val = strtol(pos, &endp, 0);
+		if (*endp || val < 1 || val > FST_MAX_PRIO_VALUE) {
+			wpa_printf(MSG_ERROR,
+				   "Line %d: Invalid fst_priority %ld (%s) (expected 1..%u)",
+				   line, val, pos, FST_MAX_PRIO_VALUE);
+			return 1;
+		}
+		conf->fst_cfg.priority = (u8) val;
+	} else if (os_strcmp(buf, "fst_llt") == 0) {
+		char *endp;
+		long int val;
+
+		if (!*pos) {
+			wpa_printf(MSG_ERROR,
+				   "Line %d: fst_llt value not supplied (expected 1..%u)",
+				   line, FST_MAX_LLT_MS);
+			return -1;
+		}
+		val = strtol(pos, &endp, 0);
+		if (*endp || val < 1 || val > FST_MAX_LLT_MS) {
+			wpa_printf(MSG_ERROR,
+				   "Line %d: Invalid fst_llt %ld (%s) (expected 1..%u)",
+				   line, val, pos, FST_MAX_LLT_MS);
+			return 1;
+		}
+		conf->fst_cfg.llt = (u32) val;
+#endif /* CONFIG_FST */
+	} else if (os_strcmp(buf, "track_sta_max_num") == 0) {
+		conf->track_sta_max_num = atoi(pos);
+	} else if (os_strcmp(buf, "track_sta_max_age") == 0) {
+		conf->track_sta_max_age = atoi(pos);
+	} else if (os_strcmp(buf, "no_probe_resp_if_seen_on") == 0) {
+		os_free(bss->no_probe_resp_if_seen_on);
+		bss->no_probe_resp_if_seen_on = os_strdup(pos);
+	} else if (os_strcmp(buf, "no_auth_if_seen_on") == 0) {
+		os_free(bss->no_auth_if_seen_on);
+		bss->no_auth_if_seen_on = os_strdup(pos);
 	} else {
 		wpa_printf(MSG_ERROR,
 			   "Line %d: unknown configuration item '%s'",
@@ -3378,7 +3511,8 @@ struct hostapd_config * hostapd_config_read(const char *fname)
 
 
 int hostapd_set_iface(struct hostapd_config *conf,
-		      struct hostapd_bss_config *bss, char *field, char *value)
+		      struct hostapd_bss_config *bss, const char *field,
+		      char *value)
 {
 	int errors;
 	size_t i;
diff --git a/hostapd/config_file.h b/hostapd/config_file.h
index fba57b87ad7c..c98bdb683ba1 100644
--- a/hostapd/config_file.h
+++ b/hostapd/config_file.h
@@ -11,7 +11,7 @@
 
 struct hostapd_config * hostapd_config_read(const char *fname);
 int hostapd_set_iface(struct hostapd_config *conf,
-		      struct hostapd_bss_config *bss, char *field,
+		      struct hostapd_bss_config *bss, const char *field,
 		      char *value);
 
 #endif /* CONFIG_FILE_H */
diff --git a/hostapd/ctrl_iface.c b/hostapd/ctrl_iface.c
index 86f1aa6dfdb2..cb6fb1757708 100644
--- a/hostapd/ctrl_iface.c
+++ b/hostapd/ctrl_iface.c
@@ -25,6 +25,7 @@
 #include "common/ieee802_11_defs.h"
 #include "crypto/tls.h"
 #include "drivers/driver.h"
+#include "eapol_auth/eapol_auth_sm.h"
 #include "radius/radius_client.h"
 #include "radius/radius_server.h"
 #include "l2_packet/l2_packet.h"
@@ -43,10 +44,13 @@
 #include "ap/beacon.h"
 #include "wps/wps_defs.h"
 #include "wps/wps.h"
+#include "fst/fst_ctrl_iface.h"
 #include "config_file.h"
 #include "ctrl_iface.h"
 
 
+#define HOSTAPD_CLI_DUP_VALUE_MAX_LEN 256
+
 struct wpa_ctrl_dst {
 	struct wpa_ctrl_dst *next;
 	struct sockaddr_un addr;
@@ -57,6 +61,7 @@ struct wpa_ctrl_dst {
 
 
 static void hostapd_ctrl_iface_send(struct hostapd_data *hapd, int level,
+				    enum wpa_msg_type type,
 				    const char *buf, size_t len);
 
 
@@ -1055,6 +1060,97 @@ static int hostapd_ctrl_iface_bss_tm_req(struct hostapd_data *hapd,
 #endif /* CONFIG_WNM */
 
 
+static int hostapd_ctrl_iface_get_key_mgmt(struct hostapd_data *hapd,
+					   char *buf, size_t buflen)
+{
+	int ret = 0;
+	char *pos, *end;
+
+	pos = buf;
+	end = buf + buflen;
+
+	WPA_ASSERT(hapd->conf->wpa_key_mgmt);
+
+	if (hapd->conf->wpa_key_mgmt & WPA_KEY_MGMT_PSK) {
+		ret = os_snprintf(pos, end - pos, "WPA-PSK ");
+		if (os_snprintf_error(end - pos, ret))
+			return pos - buf;
+		pos += ret;
+	}
+	if (hapd->conf->wpa_key_mgmt & WPA_KEY_MGMT_IEEE8021X) {
+		ret = os_snprintf(pos, end - pos, "WPA-EAP ");
+		if (os_snprintf_error(end - pos, ret))
+			return pos - buf;
+		pos += ret;
+	}
+#ifdef CONFIG_IEEE80211R
+	if (hapd->conf->wpa_key_mgmt & WPA_KEY_MGMT_FT_PSK) {
+		ret = os_snprintf(pos, end - pos, "FT-PSK ");
+		if (os_snprintf_error(end - pos, ret))
+			return pos - buf;
+		pos += ret;
+	}
+	if (hapd->conf->wpa_key_mgmt & WPA_KEY_MGMT_FT_IEEE8021X) {
+		ret = os_snprintf(pos, end - pos, "FT-EAP ");
+		if (os_snprintf_error(end - pos, ret))
+			return pos - buf;
+		pos += ret;
+	}
+#ifdef CONFIG_SAE
+	if (hapd->conf->wpa_key_mgmt & WPA_KEY_MGMT_FT_SAE) {
+		ret = os_snprintf(pos, end - pos, "FT-SAE ");
+		if (os_snprintf_error(end - pos, ret))
+			return pos - buf;
+		pos += ret;
+	}
+#endif /* CONFIG_SAE */
+#endif /* CONFIG_IEEE80211R */
+#ifdef CONFIG_IEEE80211W
+	if (hapd->conf->wpa_key_mgmt & WPA_KEY_MGMT_PSK_SHA256) {
+		ret = os_snprintf(pos, end - pos, "WPA-PSK-SHA256 ");
+		if (os_snprintf_error(end - pos, ret))
+			return pos - buf;
+		pos += ret;
+	}
+	if (hapd->conf->wpa_key_mgmt & WPA_KEY_MGMT_IEEE8021X_SHA256) {
+		ret = os_snprintf(pos, end - pos, "WPA-EAP-SHA256 ");
+		if (os_snprintf_error(end - pos, ret))
+			return pos - buf;
+		pos += ret;
+	}
+#endif /* CONFIG_IEEE80211W */
+#ifdef CONFIG_SAE
+	if (hapd->conf->wpa_key_mgmt & WPA_KEY_MGMT_SAE) {
+		ret = os_snprintf(pos, end - pos, "SAE ");
+		if (os_snprintf_error(end - pos, ret))
+			return pos - buf;
+		pos += ret;
+	}
+#endif /* CONFIG_SAE */
+	if (hapd->conf->wpa_key_mgmt & WPA_KEY_MGMT_IEEE8021X_SUITE_B) {
+		ret = os_snprintf(pos, end - pos, "WPA-EAP-SUITE-B ");
+		if (os_snprintf_error(end - pos, ret))
+			return pos - buf;
+		pos += ret;
+	}
+	if (hapd->conf->wpa_key_mgmt &
+	    WPA_KEY_MGMT_IEEE8021X_SUITE_B_192) {
+		ret = os_snprintf(pos, end - pos,
+				  "WPA-EAP-SUITE-B-192 ");
+		if (os_snprintf_error(end - pos, ret))
+			return pos - buf;
+		pos += ret;
+	}
+
+	if (pos > buf && *(pos - 1) == ' ') {
+		*(pos - 1) = '\0';
+		pos--;
+	}
+
+	return pos - buf;
+}
+
+
 static int hostapd_ctrl_iface_get_config(struct hostapd_data *hapd,
 					 char *buf, size_t buflen)
 {
@@ -1104,82 +1200,20 @@ static int hostapd_ctrl_iface_get_config(struct hostapd_data *hapd,
 	}
 #endif /* CONFIG_WPS */
 
+	if (hapd->conf->wpa) {
+		ret = os_snprintf(pos, end - pos, "wpa=%d\n", hapd->conf->wpa);
+		if (os_snprintf_error(end - pos, ret))
+			return pos - buf;
+		pos += ret;
+	}
+
 	if (hapd->conf->wpa && hapd->conf->wpa_key_mgmt) {
 		ret = os_snprintf(pos, end - pos, "key_mgmt=");
 		if (os_snprintf_error(end - pos, ret))
 			return pos - buf;
 		pos += ret;
 
-		if (hapd->conf->wpa_key_mgmt & WPA_KEY_MGMT_PSK) {
-			ret = os_snprintf(pos, end - pos, "WPA-PSK ");
-			if (os_snprintf_error(end - pos, ret))
-				return pos - buf;
-			pos += ret;
-		}
-		if (hapd->conf->wpa_key_mgmt & WPA_KEY_MGMT_IEEE8021X) {
-			ret = os_snprintf(pos, end - pos, "WPA-EAP ");
-			if (os_snprintf_error(end - pos, ret))
-				return pos - buf;
-			pos += ret;
-		}
-#ifdef CONFIG_IEEE80211R
-		if (hapd->conf->wpa_key_mgmt & WPA_KEY_MGMT_FT_PSK) {
-			ret = os_snprintf(pos, end - pos, "FT-PSK ");
-			if (os_snprintf_error(end - pos, ret))
-				return pos - buf;
-			pos += ret;
-		}
-		if (hapd->conf->wpa_key_mgmt & WPA_KEY_MGMT_FT_IEEE8021X) {
-			ret = os_snprintf(pos, end - pos, "FT-EAP ");
-			if (os_snprintf_error(end - pos, ret))
-				return pos - buf;
-			pos += ret;
-		}
-#ifdef CONFIG_SAE
-		if (hapd->conf->wpa_key_mgmt & WPA_KEY_MGMT_FT_SAE) {
-			ret = os_snprintf(pos, end - pos, "FT-SAE ");
-			if (os_snprintf_error(end - pos, ret))
-				return pos - buf;
-			pos += ret;
-		}
-#endif /* CONFIG_SAE */
-#endif /* CONFIG_IEEE80211R */
-#ifdef CONFIG_IEEE80211W
-		if (hapd->conf->wpa_key_mgmt & WPA_KEY_MGMT_PSK_SHA256) {
-			ret = os_snprintf(pos, end - pos, "WPA-PSK-SHA256 ");
-			if (os_snprintf_error(end - pos, ret))
-				return pos - buf;
-			pos += ret;
-		}
-		if (hapd->conf->wpa_key_mgmt & WPA_KEY_MGMT_IEEE8021X_SHA256) {
-			ret = os_snprintf(pos, end - pos, "WPA-EAP-SHA256 ");
-			if (os_snprintf_error(end - pos, ret))
-				return pos - buf;
-			pos += ret;
-		}
-#endif /* CONFIG_IEEE80211W */
-#ifdef CONFIG_SAE
-		if (hapd->conf->wpa_key_mgmt & WPA_KEY_MGMT_SAE) {
-			ret = os_snprintf(pos, end - pos, "SAE ");
-			if (os_snprintf_error(end - pos, ret))
-				return pos - buf;
-			pos += ret;
-		}
-#endif /* CONFIG_SAE */
-		if (hapd->conf->wpa_key_mgmt & WPA_KEY_MGMT_IEEE8021X_SUITE_B) {
-			ret = os_snprintf(pos, end - pos, "WPA-EAP-SUITE-B ");
-			if (os_snprintf_error(end - pos, ret))
-				return pos - buf;
-			pos += ret;
-		}
-		if (hapd->conf->wpa_key_mgmt &
-		    WPA_KEY_MGMT_IEEE8021X_SUITE_B_192) {
-			ret = os_snprintf(pos, end - pos,
-					  "WPA-EAP-SUITE-B-192 ");
-			if (os_snprintf_error(end - pos, ret))
-				return pos - buf;
-			pos += ret;
-		}
+		pos += hostapd_ctrl_iface_get_key_mgmt(hapd, pos, end - pos);
 
 		ret = os_snprintf(pos, end - pos, "\n");
 		if (os_snprintf_error(end - pos, ret))
@@ -1528,7 +1562,7 @@ void hostapd_data_test_rx(void *ctx, const u8 *src_addr, const u8 *buf,
 {
 	struct hostapd_data *hapd = ctx;
 	const struct ether_header *eth;
-	const struct iphdr *ip;
+	struct iphdr ip;
 	const u8 *pos;
 	unsigned int i;
 
@@ -1536,14 +1570,14 @@ void hostapd_data_test_rx(void *ctx, const u8 *src_addr, const u8 *buf,
 		return;
 
 	eth = (const struct ether_header *) buf;
-	ip = (const struct iphdr *) (eth + 1);
-	pos = (const u8 *) (ip + 1);
+	os_memcpy(&ip, eth + 1, sizeof(ip));
+	pos = &buf[sizeof(*eth) + sizeof(ip)];
 
-	if (ip->ihl != 5 || ip->version != 4 ||
-	    ntohs(ip->tot_len) != HWSIM_IP_LEN)
+	if (ip.ihl != 5 || ip.version != 4 ||
+	    ntohs(ip.tot_len) != HWSIM_IP_LEN)
 		return;
 
-	for (i = 0; i < HWSIM_IP_LEN - sizeof(*ip); i++) {
+	for (i = 0; i < HWSIM_IP_LEN - sizeof(ip); i++) {
 		if (*pos != (u8) i)
 			return;
 		pos++;
@@ -1599,7 +1633,7 @@ static int hostapd_ctrl_iface_data_test_tx(struct hostapd_data *hapd, char *cmd)
 	int used;
 	long int val;
 	u8 tos;
-	u8 buf[HWSIM_PACKETLEN];
+	u8 buf[2 + HWSIM_PACKETLEN];
 	struct ether_header *eth;
 	struct iphdr *ip;
 	u8 *dpos;
@@ -1627,7 +1661,7 @@ static int hostapd_ctrl_iface_data_test_tx(struct hostapd_data *hapd, char *cmd)
 		return -1;
 	tos = val;
 
-	eth = (struct ether_header *) buf;
+	eth = (struct ether_header *) &buf[2];
 	os_memcpy(eth->ether_dhost, dst, ETH_ALEN);
 	os_memcpy(eth->ether_shost, src, ETH_ALEN);
 	eth->ether_type = htons(ETHERTYPE_IP);
@@ -1639,14 +1673,14 @@ static int hostapd_ctrl_iface_data_test_tx(struct hostapd_data *hapd, char *cmd)
 	ip->tos = tos;
 	ip->tot_len = htons(HWSIM_IP_LEN);
 	ip->protocol = 1;
-	ip->saddr = htonl(192 << 24 | 168 << 16 | 1 << 8 | 1);
-	ip->daddr = htonl(192 << 24 | 168 << 16 | 1 << 8 | 2);
+	ip->saddr = htonl(192U << 24 | 168 << 16 | 1 << 8 | 1);
+	ip->daddr = htonl(192U << 24 | 168 << 16 | 1 << 8 | 2);
 	ip->check = ipv4_hdr_checksum(ip, sizeof(*ip));
 	dpos = (u8 *) (ip + 1);
 	for (i = 0; i < HWSIM_IP_LEN - sizeof(*ip); i++)
 		*dpos++ = i;
 
-	if (l2_packet_send(hapd->l2_test, dst, ETHERTYPE_IP, buf,
+	if (l2_packet_send(hapd->l2_test, dst, ETHERTYPE_IP, &buf[2],
 			   HWSIM_PACKETLEN) < 0)
 		return -1;
 
@@ -1746,6 +1780,45 @@ static int hostapd_ctrl_get_alloc_fail(struct hostapd_data *hapd,
 #endif /* WPA_TRACE_BFD */
 }
 
+
+static int hostapd_ctrl_test_fail(struct hostapd_data *hapd, char *cmd)
+{
+#ifdef WPA_TRACE_BFD
+	extern char wpa_trace_test_fail_func[256];
+	extern unsigned int wpa_trace_test_fail_after;
+	char *pos;
+
+	wpa_trace_test_fail_after = atoi(cmd);
+	pos = os_strchr(cmd, ':');
+	if (pos) {
+		pos++;
+		os_strlcpy(wpa_trace_test_fail_func, pos,
+			   sizeof(wpa_trace_test_fail_func));
+	} else {
+		wpa_trace_test_fail_after = 0;
+	}
+
+	return 0;
+#else /* WPA_TRACE_BFD */
+	return -1;
+#endif /* WPA_TRACE_BFD */
+}
+
+
+static int hostapd_ctrl_get_fail(struct hostapd_data *hapd,
+				 char *buf, size_t buflen)
+{
+#ifdef WPA_TRACE_BFD
+	extern char wpa_trace_test_fail_func[256];
+	extern unsigned int wpa_trace_test_fail_after;
+
+	return os_snprintf(buf, buflen, "%u:%s", wpa_trace_test_fail_after,
+			   wpa_trace_test_fail_func);
+#else /* WPA_TRACE_BFD */
+	return -1;
+#endif /* WPA_TRACE_BFD */
+}
+
 #endif /* CONFIG_TESTING_OPTIONS */
 
 
@@ -1847,41 +1920,134 @@ static int hostapd_ctrl_iface_vendor(struct hostapd_data *hapd, char *cmd,
 }
 
 
-static void hostapd_ctrl_iface_receive(int sock, void *eloop_ctx,
-				       void *sock_ctx)
+static int hostapd_ctrl_iface_eapol_reauth(struct hostapd_data *hapd,
+					   const char *cmd)
 {
-	struct hostapd_data *hapd = eloop_ctx;
-	char buf[4096];
-	int res;
-	struct sockaddr_un from;
-	socklen_t fromlen = sizeof(from);
-	char *reply;
-	const int reply_size = 4096;
-	int reply_len;
-	int level = MSG_DEBUG;
+	u8 addr[ETH_ALEN];
+	struct sta_info *sta;
 
-	res = recvfrom(sock, buf, sizeof(buf) - 1, 0,
-		       (struct sockaddr *) &from, &fromlen);
-	if (res < 0) {
-		wpa_printf(MSG_ERROR, "recvfrom(ctrl_iface): %s",
-			   strerror(errno));
-		return;
+	if (hwaddr_aton(cmd, addr))
+		return -1;
+
+	sta = ap_get_sta(hapd, addr);
+	if (!sta || !sta->eapol_sm)
+		return -1;
+
+	eapol_auth_reauthenticate(sta->eapol_sm);
+	return 0;
+}
+
+
+static int hostapd_ctrl_iface_eapol_set(struct hostapd_data *hapd, char *cmd)
+{
+	u8 addr[ETH_ALEN];
+	struct sta_info *sta;
+	char *pos = cmd, *param;
+
+	if (hwaddr_aton(pos, addr) || pos[17] != ' ')
+		return -1;
+	pos += 18;
+	param = pos;
+	pos = os_strchr(pos, ' ');
+	if (!pos)
+		return -1;
+	*pos++ = '\0';
+
+	sta = ap_get_sta(hapd, addr);
+	if (!sta || !sta->eapol_sm)
+		return -1;
+
+	return eapol_auth_set_conf(sta->eapol_sm, param, pos);
+}
+
+
+static int hostapd_ctrl_iface_log_level(struct hostapd_data *hapd, char *cmd,
+					char *buf, size_t buflen)
+{
+	char *pos, *end, *stamp;
+	int ret;
+
+	/* cmd: "LOG_LEVEL [<level>]" */
+	if (*cmd == '\0') {
+		pos = buf;
+		end = buf + buflen;
+		ret = os_snprintf(pos, end - pos, "Current level: %s\n"
+				  "Timestamp: %d\n",
+				  debug_level_str(wpa_debug_level),
+				  wpa_debug_timestamp);
+		if (os_snprintf_error(end - pos, ret))
+			ret = 0;
+
+		return ret;
 	}
-	buf[res] = '\0';
-	if (os_strcmp(buf, "PING") == 0)
-		level = MSG_EXCESSIVE;
-	wpa_hexdump_ascii(level, "RX ctrl_iface", (u8 *) buf, res);
 
-	reply = os_malloc(reply_size);
-	if (reply == NULL) {
-		if (sendto(sock, "FAIL\n", 5, 0, (struct sockaddr *) &from,
-			   fromlen) < 0) {
-			wpa_printf(MSG_DEBUG, "CTRL: sendto failed: %s",
-				   strerror(errno));
+	while (*cmd == ' ')
+		cmd++;
+
+	stamp = os_strchr(cmd, ' ');
+	if (stamp) {
+		*stamp++ = '\0';
+		while (*stamp == ' ') {
+			stamp++;
 		}
-		return;
 	}
 
+	if (os_strlen(cmd)) {
+		int level = str_to_debug_level(cmd);
+		if (level < 0)
+			return -1;
+		wpa_debug_level = level;
+	}
+
+	if (stamp && os_strlen(stamp))
+		wpa_debug_timestamp = atoi(stamp);
+
+	os_memcpy(buf, "OK\n", 3);
+	return 3;
+}
+
+
+#ifdef NEED_AP_MLME
+static int hostapd_ctrl_iface_track_sta_list(struct hostapd_data *hapd,
+					     char *buf, size_t buflen)
+{
+	struct hostapd_iface *iface = hapd->iface;
+	char *pos, *end;
+	struct hostapd_sta_info *info;
+	struct os_reltime now;
+
+	sta_track_expire(iface, 0);
+
+	pos = buf;
+	end = buf + buflen;
+
+	os_get_reltime(&now);
+	dl_list_for_each_reverse(info, &iface->sta_seen,
+				 struct hostapd_sta_info, list) {
+		struct os_reltime age;
+		int ret;
+
+		os_reltime_sub(&now, &info->last_seen, &age);
+		ret = os_snprintf(pos, end - pos, MACSTR " %u\n",
+				  MAC2STR(info->addr), (unsigned int) age.sec);
+		if (os_snprintf_error(end - pos, ret))
+			break;
+		pos += ret;
+	}
+
+	return pos - buf;
+}
+#endif /* NEED_AP_MLME */
+
+
+static int hostapd_ctrl_iface_receive_process(struct hostapd_data *hapd,
+					      char *buf, char *reply,
+					      int reply_size,
+					      struct sockaddr_un *from,
+					      socklen_t fromlen)
+{
+	int reply_len, res;
+
 	os_memcpy(reply, "OK\n", 3);
 	reply_len = 3;
 
@@ -1938,13 +2104,13 @@ static void hostapd_ctrl_iface_receive(int sock, void *eloop_ctx,
 		reply_len = hostapd_ctrl_iface_sta_next(hapd, buf + 9, reply,
 							reply_size);
 	} else if (os_strcmp(buf, "ATTACH") == 0) {
-		if (hostapd_ctrl_iface_attach(hapd, &from, fromlen))
+		if (hostapd_ctrl_iface_attach(hapd, from, fromlen))
 			reply_len = -1;
 	} else if (os_strcmp(buf, "DETACH") == 0) {
-		if (hostapd_ctrl_iface_detach(hapd, &from, fromlen))
+		if (hostapd_ctrl_iface_detach(hapd, from, fromlen))
 			reply_len = -1;
 	} else if (os_strncmp(buf, "LEVEL ", 6) == 0) {
-		if (hostapd_ctrl_iface_level(hapd, &from, fromlen,
+		if (hostapd_ctrl_iface_level(hapd, from, fromlen,
 						    buf + 6))
 			reply_len = -1;
 	} else if (os_strncmp(buf, "NEW_STA ", 8) == 0) {
@@ -2079,6 +2245,11 @@ static void hostapd_ctrl_iface_receive(int sock, void *eloop_ctx,
 	} else if (os_strcmp(buf, "GET_ALLOC_FAIL") == 0) {
 		reply_len = hostapd_ctrl_get_alloc_fail(hapd, reply,
 							reply_size);
+	} else if (os_strncmp(buf, "TEST_FAIL ", 10) == 0) {
+		if (hostapd_ctrl_test_fail(hapd, buf + 10) < 0)
+			reply_len = -1;
+	} else if (os_strcmp(buf, "GET_FAIL") == 0) {
+		reply_len = hostapd_ctrl_get_fail(hapd, reply, reply_size);
 #endif /* CONFIG_TESTING_OPTIONS */
 	} else if (os_strncmp(buf, "CHAN_SWITCH ", 12) == 0) {
 		if (hostapd_ctrl_iface_chan_switch(hapd->iface, buf + 12))
@@ -2091,6 +2262,20 @@ static void hostapd_ctrl_iface_receive(int sock, void *eloop_ctx,
 #ifdef RADIUS_SERVER
 		radius_server_erp_flush(hapd->radius_srv);
 #endif /* RADIUS_SERVER */
+	} else if (os_strncmp(buf, "EAPOL_REAUTH ", 13) == 0) {
+		if (hostapd_ctrl_iface_eapol_reauth(hapd, buf + 13))
+			reply_len = -1;
+	} else if (os_strncmp(buf, "EAPOL_SET ", 10) == 0) {
+		if (hostapd_ctrl_iface_eapol_set(hapd, buf + 10))
+			reply_len = -1;
+	} else if (os_strncmp(buf, "LOG_LEVEL", 9) == 0) {
+		reply_len = hostapd_ctrl_iface_log_level(
+			hapd, buf + 9, reply, reply_size);
+#ifdef NEED_AP_MLME
+	} else if (os_strcmp(buf, "TRACK_STA_LIST") == 0) {
+		reply_len = hostapd_ctrl_iface_track_sta_list(
+			hapd, reply, reply_size);
+#endif /* NEED_AP_MLME */
 	} else {
 		os_memcpy(reply, "UNKNOWN COMMAND\n", 16);
 		reply_len = 16;
@@ -2100,6 +2285,50 @@ static void hostapd_ctrl_iface_receive(int sock, void *eloop_ctx,
 		os_memcpy(reply, "FAIL\n", 5);
 		reply_len = 5;
 	}
+
+	return reply_len;
+}
+
+
+static void hostapd_ctrl_iface_receive(int sock, void *eloop_ctx,
+				       void *sock_ctx)
+{
+	struct hostapd_data *hapd = eloop_ctx;
+	char buf[4096];
+	int res;
+	struct sockaddr_un from;
+	socklen_t fromlen = sizeof(from);
+	char *reply;
+	const int reply_size = 4096;
+	int reply_len;
+	int level = MSG_DEBUG;
+
+	res = recvfrom(sock, buf, sizeof(buf) - 1, 0,
+		       (struct sockaddr *) &from, &fromlen);
+	if (res < 0) {
+		wpa_printf(MSG_ERROR, "recvfrom(ctrl_iface): %s",
+			   strerror(errno));
+		return;
+	}
+	buf[res] = '\0';
+	if (os_strcmp(buf, "PING") == 0)
+		level = MSG_EXCESSIVE;
+	wpa_hexdump_ascii(level, "RX ctrl_iface", (u8 *) buf, res);
+
+	reply = os_malloc(reply_size);
+	if (reply == NULL) {
+		if (sendto(sock, "FAIL\n", 5, 0, (struct sockaddr *) &from,
+			   fromlen) < 0) {
+			wpa_printf(MSG_DEBUG, "CTRL: sendto failed: %s",
+				   strerror(errno));
+		}
+		return;
+	}
+
+	reply_len = hostapd_ctrl_iface_receive_process(hapd, buf,
+						       reply, reply_size,
+						       &from, fromlen);
+
 	if (sendto(sock, reply, reply_len, 0, (struct sockaddr *) &from,
 		   fromlen) < 0) {
 		wpa_printf(MSG_DEBUG, "CTRL: sendto failed: %s",
@@ -2130,13 +2359,14 @@ static char * hostapd_ctrl_iface_path(struct hostapd_data *hapd)
 }
 
 
-static void hostapd_ctrl_iface_msg_cb(void *ctx, int level, int global,
+static void hostapd_ctrl_iface_msg_cb(void *ctx, int level,
+				      enum wpa_msg_type type,
 				      const char *txt, size_t len)
 {
 	struct hostapd_data *hapd = ctx;
 	if (hapd == NULL)
 		return;
-	hostapd_ctrl_iface_send(hapd, level, txt, len);
+	hostapd_ctrl_iface_send(hapd, level, type, txt, len);
 }
 
 
@@ -2359,6 +2589,58 @@ static int hostapd_ctrl_iface_remove(struct hapd_interfaces *interfaces,
 }
 
 
+static int hostapd_global_ctrl_iface_attach(struct hapd_interfaces *interfaces,
+					    struct sockaddr_un *from,
+					    socklen_t fromlen)
+{
+	struct wpa_ctrl_dst *dst;
+
+	dst = os_zalloc(sizeof(*dst));
+	if (dst == NULL)
+		return -1;
+	os_memcpy(&dst->addr, from, sizeof(struct sockaddr_un));
+	dst->addrlen = fromlen;
+	dst->debug_level = MSG_INFO;
+	dst->next = interfaces->global_ctrl_dst;
+	interfaces->global_ctrl_dst = dst;
+	wpa_hexdump(MSG_DEBUG, "CTRL_IFACE monitor attached (global)",
+		    from->sun_path,
+		    fromlen - offsetof(struct sockaddr_un, sun_path));
+	return 0;
+}
+
+
+static int hostapd_global_ctrl_iface_detach(struct hapd_interfaces *interfaces,
+					    struct sockaddr_un *from,
+					    socklen_t fromlen)
+{
+	struct wpa_ctrl_dst *dst, *prev = NULL;
+
+	dst = interfaces->global_ctrl_dst;
+	while (dst) {
+		if (fromlen == dst->addrlen &&
+		    os_memcmp(from->sun_path, dst->addr.sun_path,
+			      fromlen - offsetof(struct sockaddr_un, sun_path))
+		    == 0) {
+			wpa_hexdump(MSG_DEBUG,
+				    "CTRL_IFACE monitor detached (global)",
+				    from->sun_path,
+				    fromlen -
+				    offsetof(struct sockaddr_un, sun_path));
+			if (prev == NULL)
+				interfaces->global_ctrl_dst = dst->next;
+			else
+				prev->next = dst->next;
+			os_free(dst);
+			return 0;
+		}
+		prev = dst;
+		dst = dst->next;
+	}
+	return -1;
+}
+
+
 static void hostapd_ctrl_iface_flush(struct hapd_interfaces *interfaces)
 {
 #ifdef CONFIG_WPS_TESTING
@@ -2369,6 +2651,214 @@ static void hostapd_ctrl_iface_flush(struct hapd_interfaces *interfaces)
 }
 
 
+#ifdef CONFIG_FST
+
+static int
+hostapd_global_ctrl_iface_fst_attach(struct hapd_interfaces *interfaces,
+				     const char *cmd)
+{
+	char ifname[IFNAMSIZ + 1];
+	struct fst_iface_cfg cfg;
+	struct hostapd_data *hapd;
+	struct fst_wpa_obj iface_obj;
+
+	if (!fst_parse_attach_command(cmd, ifname, sizeof(ifname), &cfg)) {
+		hapd = hostapd_get_iface(interfaces, ifname);
+		if (hapd) {
+			if (hapd->iface->fst) {
+				wpa_printf(MSG_INFO, "FST: Already attached");
+				return -1;
+			}
+			fst_hostapd_fill_iface_obj(hapd, &iface_obj);
+			hapd->iface->fst = fst_attach(ifname, hapd->own_addr,
+						      &iface_obj, &cfg);
+			if (hapd->iface->fst)
+				return 0;
+		}
+	}
+
+	return -EINVAL;
+}
+
+
+static int
+hostapd_global_ctrl_iface_fst_detach(struct hapd_interfaces *interfaces,
+				     const char *cmd)
+{
+	char ifname[IFNAMSIZ + 1];
+	struct hostapd_data * hapd;
+
+	if (!fst_parse_detach_command(cmd, ifname, sizeof(ifname))) {
+		hapd = hostapd_get_iface(interfaces, ifname);
+		if (hapd) {
+			if (!fst_iface_detach(ifname)) {
+				hapd->iface->fst = NULL;
+				hapd->iface->fst_ies = NULL;
+				return 0;
+			}
+		}
+	}
+
+	return -EINVAL;
+}
+
+#endif /* CONFIG_FST */
+
+
+static struct hostapd_data *
+hostapd_interfaces_get_hapd(struct hapd_interfaces *interfaces,
+			    const char *ifname)
+{
+	size_t i, j;
+
+	for (i = 0; i < interfaces->count; i++) {
+		struct hostapd_iface *iface = interfaces->iface[i];
+
+		for (j = 0; j < iface->num_bss; j++) {
+			struct hostapd_data *hapd;
+
+			hapd = iface->bss[j];
+			if (os_strcmp(ifname, hapd->conf->iface) == 0)
+				return hapd;
+		}
+	}
+
+	return NULL;
+}
+
+
+static int hostapd_ctrl_iface_dup_param(struct hostapd_data *src_hapd,
+					struct hostapd_data *dst_hapd,
+					const char *param)
+{
+	int res;
+	char *value;
+
+	value = os_zalloc(HOSTAPD_CLI_DUP_VALUE_MAX_LEN);
+	if (!value) {
+		wpa_printf(MSG_ERROR,
+			   "DUP: cannot allocate buffer to stringify %s",
+			   param);
+		goto error_return;
+	}
+
+	if (os_strcmp(param, "wpa") == 0) {
+		os_snprintf(value, HOSTAPD_CLI_DUP_VALUE_MAX_LEN, "%d",
+			    src_hapd->conf->wpa);
+	} else if (os_strcmp(param, "wpa_key_mgmt") == 0 &&
+		   src_hapd->conf->wpa_key_mgmt) {
+		res = hostapd_ctrl_iface_get_key_mgmt(
+			src_hapd, value, HOSTAPD_CLI_DUP_VALUE_MAX_LEN);
+		if (os_snprintf_error(HOSTAPD_CLI_DUP_VALUE_MAX_LEN, res))
+			goto error_stringify;
+	} else if (os_strcmp(param, "wpa_pairwise") == 0 &&
+		   src_hapd->conf->wpa_pairwise) {
+		res = wpa_write_ciphers(value,
+					value + HOSTAPD_CLI_DUP_VALUE_MAX_LEN,
+					src_hapd->conf->wpa_pairwise, " ");
+		if (res < 0)
+			goto error_stringify;
+	} else if (os_strcmp(param, "rsn_pairwise") == 0 &&
+		   src_hapd->conf->rsn_pairwise) {
+		res = wpa_write_ciphers(value,
+					value + HOSTAPD_CLI_DUP_VALUE_MAX_LEN,
+					src_hapd->conf->rsn_pairwise, " ");
+		if (res < 0)
+			goto error_stringify;
+	} else if (os_strcmp(param, "wpa_passphrase") == 0 &&
+		   src_hapd->conf->ssid.wpa_passphrase) {
+		os_snprintf(value, HOSTAPD_CLI_DUP_VALUE_MAX_LEN, "%s",
+			    src_hapd->conf->ssid.wpa_passphrase);
+	} else if (os_strcmp(param, "wpa_psk") == 0 &&
+		   src_hapd->conf->ssid.wpa_psk_set) {
+		wpa_snprintf_hex(value, HOSTAPD_CLI_DUP_VALUE_MAX_LEN,
+			src_hapd->conf->ssid.wpa_psk->psk, PMK_LEN);
+	} else {
+		wpa_printf(MSG_WARNING, "DUP: %s cannot be duplicated", param);
+		goto error_return;
+	}
+
+	res = hostapd_set_iface(dst_hapd->iconf, dst_hapd->conf, param, value);
+	os_free(value);
+	return res;
+
+error_stringify:
+	wpa_printf(MSG_ERROR, "DUP: cannot stringify %s", param);
+error_return:
+	os_free(value);
+	return -1;
+}
+
+
+static int
+hostapd_global_ctrl_iface_dup_network(struct hapd_interfaces *interfaces,
+				      char *cmd)
+{
+	char *p_start = cmd, *p_end;
+	struct hostapd_data *src_hapd, *dst_hapd;
+
+	/* cmd: "<src ifname> <dst ifname> <variable name> */
+
+	p_end = os_strchr(p_start, ' ');
+	if (!p_end) {
+		wpa_printf(MSG_ERROR, "DUP: no src ifname found in cmd: '%s'",
+			   cmd);
+		return -1;
+	}
+
+	*p_end = '\0';
+	src_hapd = hostapd_interfaces_get_hapd(interfaces, p_start);
+	if (!src_hapd) {
+		wpa_printf(MSG_ERROR, "DUP: no src ifname found: '%s'",
+			   p_start);
+		return -1;
+	}
+
+	p_start = p_end + 1;
+	p_end = os_strchr(p_start, ' ');
+	if (!p_end) {
+		wpa_printf(MSG_ERROR, "DUP: no dst ifname found in cmd: '%s'",
+			   cmd);
+		return -1;
+	}
+
+	*p_end = '\0';
+	dst_hapd = hostapd_interfaces_get_hapd(interfaces, p_start);
+	if (!dst_hapd) {
+		wpa_printf(MSG_ERROR, "DUP: no dst ifname found: '%s'",
+			   p_start);
+		return -1;
+	}
+
+	p_start = p_end + 1;
+	return hostapd_ctrl_iface_dup_param(src_hapd, dst_hapd, p_start);
+}
+
+
+static int hostapd_global_ctrl_iface_ifname(struct hapd_interfaces *interfaces,
+					    const char *ifname,
+					    char *buf, char *reply,
+					    int reply_size,
+					    struct sockaddr_un *from,
+					    socklen_t fromlen)
+{
+	struct hostapd_data *hapd;
+
+	hapd = hostapd_interfaces_get_hapd(interfaces, ifname);
+	if (hapd == NULL) {
+		int res;
+
+		res = os_snprintf(reply, reply_size, "FAIL-NO-IFNAME-MATCH\n");
+		if (os_snprintf_error(reply_size, res))
+			return -1;
+		return res;
+	}
+
+	return hostapd_ctrl_iface_receive_process(hapd, buf, reply,reply_size,
+						  from, fromlen);
+}
+
+
 static void hostapd_global_ctrl_iface_receive(int sock, void *eloop_ctx,
 					      void *sock_ctx)
 {
@@ -2377,8 +2867,9 @@ static void hostapd_global_ctrl_iface_receive(int sock, void *eloop_ctx,
 	int res;
 	struct sockaddr_un from;
 	socklen_t fromlen = sizeof(from);
-	char reply[24];
+	char *reply;
 	int reply_len;
+	const int reply_size = 4096;
 
 	res = recvfrom(sock, buf, sizeof(buf) - 1, 0,
 		       (struct sockaddr *) &from, &fromlen);
@@ -2390,9 +2881,31 @@ static void hostapd_global_ctrl_iface_receive(int sock, void *eloop_ctx,
 	buf[res] = '\0';
 	wpa_printf(MSG_DEBUG, "Global ctrl_iface command: %s", buf);
 
+	reply = os_malloc(reply_size);
+	if (reply == NULL) {
+		if (sendto(sock, "FAIL\n", 5, 0, (struct sockaddr *) &from,
+			   fromlen) < 0) {
+			wpa_printf(MSG_DEBUG, "CTRL: sendto failed: %s",
+				   strerror(errno));
+		}
+		return;
+	}
+
 	os_memcpy(reply, "OK\n", 3);
 	reply_len = 3;
 
+	if (os_strncmp(buf, "IFNAME=", 7) == 0) {
+		char *pos = os_strchr(buf + 7, ' ');
+
+		if (pos) {
+			*pos++ = '\0';
+			reply_len = hostapd_global_ctrl_iface_ifname(
+				interfaces, buf + 7, pos, reply, reply_size,
+				&from, fromlen);
+			goto send_reply;
+		}
+	}
+
 	if (os_strcmp(buf, "PING") == 0) {
 		os_memcpy(reply, "PONG\n", 5);
 		reply_len = 5;
@@ -2407,18 +2920,47 @@ static void hostapd_global_ctrl_iface_receive(int sock, void *eloop_ctx,
 	} else if (os_strncmp(buf, "REMOVE ", 7) == 0) {
 		if (hostapd_ctrl_iface_remove(interfaces, buf + 7) < 0)
 			reply_len = -1;
+	} else if (os_strcmp(buf, "ATTACH") == 0) {
+		if (hostapd_global_ctrl_iface_attach(interfaces, &from,
+						     fromlen))
+			reply_len = -1;
+	} else if (os_strcmp(buf, "DETACH") == 0) {
+		if (hostapd_global_ctrl_iface_detach(interfaces, &from,
+			fromlen))
+			reply_len = -1;
 #ifdef CONFIG_MODULE_TESTS
 	} else if (os_strcmp(buf, "MODULE_TESTS") == 0) {
 		int hapd_module_tests(void);
 		if (hapd_module_tests() < 0)
 			reply_len = -1;
 #endif /* CONFIG_MODULE_TESTS */
+#ifdef CONFIG_FST
+	} else if (os_strncmp(buf, "FST-ATTACH ", 11) == 0) {
+		if (!hostapd_global_ctrl_iface_fst_attach(interfaces, buf + 11))
+			reply_len = os_snprintf(reply, reply_size, "OK\n");
+		else
+			reply_len = -1;
+	} else if (os_strncmp(buf, "FST-DETACH ", 11) == 0) {
+		if (!hostapd_global_ctrl_iface_fst_detach(interfaces, buf + 11))
+			reply_len = os_snprintf(reply, reply_size, "OK\n");
+		else
+			reply_len = -1;
+	} else if (os_strncmp(buf, "FST-MANAGER ", 12) == 0) {
+		reply_len = fst_ctrl_iface_receive(buf + 12, reply, reply_size);
+#endif /* CONFIG_FST */
+	} else if (os_strncmp(buf, "DUP_NETWORK ", 12) == 0) {
+		if (!hostapd_global_ctrl_iface_dup_network(interfaces,
+							   buf + 12))
+			reply_len = os_snprintf(reply, reply_size, "OK\n");
+		else
+			reply_len = -1;
 	} else {
 		wpa_printf(MSG_DEBUG, "Unrecognized global ctrl_iface command "
 			   "ignored");
 		reply_len = -1;
 	}
 
+send_reply:
 	if (reply_len < 0) {
 		os_memcpy(reply, "FAIL\n", 5);
 		reply_len = 5;
@@ -2429,6 +2971,7 @@ static void hostapd_global_ctrl_iface_receive(int sock, void *eloop_ctx,
 		wpa_printf(MSG_DEBUG, "CTRL: sendto failed: %s",
 			   strerror(errno));
 	}
+	os_free(reply);
 }
 
 
@@ -2566,6 +3109,7 @@ fail:
 void hostapd_global_ctrl_iface_deinit(struct hapd_interfaces *interfaces)
 {
 	char *fname = NULL;
+	struct wpa_ctrl_dst *dst, *prev;
 
 	if (interfaces->global_ctrl_sock > -1) {
 		eloop_unregister_read_sock(interfaces->global_ctrl_sock);
@@ -2590,13 +3134,23 @@ void hostapd_global_ctrl_iface_deinit(struct hapd_interfaces *interfaces)
 					   strerror(errno));
 			}
 		}
-		os_free(interfaces->global_iface_path);
-		interfaces->global_iface_path = NULL;
+	}
+
+	os_free(interfaces->global_iface_path);
+	interfaces->global_iface_path = NULL;
+
+	dst = interfaces->global_ctrl_dst;
+	interfaces->global_ctrl_dst = NULL;
+	while (dst) {
+		prev = dst;
+		dst = dst->next;
+		os_free(prev);
 	}
 }
 
 
 static void hostapd_ctrl_iface_send(struct hostapd_data *hapd, int level,
+				    enum wpa_msg_type type,
 				    const char *buf, size_t len)
 {
 	struct wpa_ctrl_dst *dst, *next;
@@ -2604,9 +3158,17 @@ static void hostapd_ctrl_iface_send(struct hostapd_data *hapd, int level,
 	int idx;
 	struct iovec io[2];
 	char levelstr[10];
+	int s;
 
-	dst = hapd->ctrl_dst;
-	if (hapd->ctrl_sock < 0 || dst == NULL)
+	if (type != WPA_MSG_ONLY_GLOBAL) {
+		s = hapd->ctrl_sock;
+		dst = hapd->ctrl_dst;
+	} else {
+		s = hapd->iface->interfaces->global_ctrl_sock;
+		dst = hapd->iface->interfaces->global_ctrl_dst;
+	}
+
+	if (s < 0 || dst == NULL)
 		return;
 
 	os_snprintf(levelstr, sizeof(levelstr), "<%d>", level);
@@ -2627,16 +3189,22 @@ static void hostapd_ctrl_iface_send(struct hostapd_data *hapd, int level,
 				    offsetof(struct sockaddr_un, sun_path));
 			msg.msg_name = &dst->addr;
 			msg.msg_namelen = dst->addrlen;
-			if (sendmsg(hapd->ctrl_sock, &msg, 0) < 0) {
+			if (sendmsg(s, &msg, 0) < 0) {
 				int _errno = errno;
 				wpa_printf(MSG_INFO, "CTRL_IFACE monitor[%d]: "
 					   "%d - %s",
 					   idx, errno, strerror(errno));
 				dst->errors++;
 				if (dst->errors > 10 || _errno == ENOENT) {
-					hostapd_ctrl_iface_detach(
-						hapd, &dst->addr,
-						dst->addrlen);
+					if (type != WPA_MSG_ONLY_GLOBAL)
+						hostapd_ctrl_iface_detach(
+							hapd, &dst->addr,
+							dst->addrlen);
+					else
+						hostapd_global_ctrl_iface_detach(
+							hapd->iface->interfaces,
+							&dst->addr,
+							dst->addrlen);
 				}
 			} else
 				dst->errors = 0;
diff --git a/hostapd/defconfig b/hostapd/defconfig
index 4cde2b563483..430f7584b167 100644
--- a/hostapd/defconfig
+++ b/hostapd/defconfig
@@ -240,6 +240,12 @@ CONFIG_IPV6=y
 # requirements described above.
 #CONFIG_NO_RANDOM_POOL=y
 
+# Should we use poll instead of select? Select is used by default.
+#CONFIG_ELOOP_POLL=y
+
+# Should we use epoll instead of select? Select is used by default.
+#CONFIG_ELOOP_EPOLL=y
+
 # Select TLS implementation
 # openssl = OpenSSL (default)
 # gnutls = GnuTLS
@@ -283,6 +289,12 @@ CONFIG_IPV6=y
 # Enable SQLite database support in hlr_auc_gw, EAP-SIM DB, and eap_user_file
 #CONFIG_SQLITE=y
 
+# Enable Fast Session Transfer (FST)
+#CONFIG_FST=y
+
+# Enable CLI commands for FST testing
+#CONFIG_FST_TEST=y
+
 # Testing options
 # This can be used to enable some testing options (see also the example
 # configuration file) that are really useful only for testing clients that
diff --git a/hostapd/hlr_auc_gw.c b/hostapd/hlr_auc_gw.c
index 42d59dba7ede..84d0308262e6 100644
--- a/hostapd/hlr_auc_gw.c
+++ b/hostapd/hlr_auc_gw.c
@@ -87,6 +87,7 @@ struct milenage_parameters {
 	u8 amf[2];
 	u8 sqn[6];
 	int set;
+	size_t res_len;
 };
 
 static struct milenage_parameters *milenage_db = NULL;
@@ -96,6 +97,7 @@ static struct milenage_parameters *milenage_db = NULL;
 #define EAP_AKA_RAND_LEN 16
 #define EAP_AKA_AUTN_LEN 16
 #define EAP_AKA_AUTS_LEN 14
+#define EAP_AKA_RES_MIN_LEN 4
 #define EAP_AKA_RES_MAX_LEN 16
 #define EAP_AKA_IK_LEN 16
 #define EAP_AKA_CK_LEN 16
@@ -124,7 +126,8 @@ static int db_table_create_milenage(sqlite3 *db)
 		"  ki CHAR(32) NOT NULL,"
 		"  opc CHAR(32) NOT NULL,"
 		"  amf CHAR(4) NOT NULL,"
-		"  sqn CHAR(12) NOT NULL"
+		"  sqn CHAR(12) NOT NULL,"
+		"  res_len INTEGER"
 		");";
 
 	printf("Adding database table for milenage information\n");
@@ -190,6 +193,10 @@ static int get_milenage_cb(void *ctx, int argc, char *argv[], char *col[])
 			printf("Invalid sqn value in database\n");
 			return -1;
 		}
+
+		if (os_strcmp(col[i], "res_len") == 0 && argv[i]) {
+			m->res_len = atoi(argv[i]);
+		}
 	}
 
 	return 0;
@@ -206,8 +213,7 @@ static struct milenage_parameters * db_get_milenage(const char *imsi_txt)
 	os_snprintf(db_tmp_milenage.imsi, sizeof(db_tmp_milenage.imsi),
 		    "%llu", imsi);
 	os_snprintf(cmd, sizeof(cmd),
-		    "SELECT ki,opc,amf,sqn FROM milenage WHERE imsi=%llu;",
-		    imsi);
+		    "SELECT * FROM milenage WHERE imsi=%llu;", imsi);
 	if (sqlite3_exec(sqlite_db, cmd, get_milenage_cb, &db_tmp_milenage,
 			 NULL) != SQLITE_OK)
 		return NULL;
@@ -424,7 +430,7 @@ static int read_milenage(const char *fname)
 	while (fgets(buf, sizeof(buf), f)) {
 		line++;
 
-		/* Parse IMSI Ki OPc AMF SQN */
+		/* Parse IMSI Ki OPc AMF SQN [RES_len] */
 		buf[sizeof(buf) - 1] = '\0';
 		if (buf[0] == '#')
 			continue;
@@ -515,7 +521,19 @@ static int read_milenage(const char *fname)
 			ret = -1;
 			break;
 		}
-		pos = pos2 + 1;
+
+		if (pos2) {
+			pos = pos2 + 1;
+			m->res_len = atoi(pos);
+			if (m->res_len &&
+			    (m->res_len < EAP_AKA_RES_MIN_LEN ||
+			     m->res_len > EAP_AKA_RES_MAX_LEN)) {
+				printf("%s:%d - Invalid RES_len (%s)\n",
+				       fname, line, pos);
+				ret = -1;
+				break;
+			}
+		}
 
 		m->next = milenage_db;
 		milenage_db = m;
@@ -532,7 +550,7 @@ static int read_milenage(const char *fname)
 static void update_milenage_file(const char *fname)
 {
 	FILE *f, *f2;
-	char buf[500], *pos;
+	char name[500], buf[500], *pos;
 	char *end = buf + sizeof(buf);
 	struct milenage_parameters *m;
 	size_t imsi_len;
@@ -543,10 +561,10 @@ static void update_milenage_file(const char *fname)
 		return;
 	}
 
-	snprintf(buf, sizeof(buf), "%s.new", fname);
-	f2 = fopen(buf, "w");
+	snprintf(name, sizeof(name), "%s.new", fname);
+	f2 = fopen(name, "w");
 	if (f2 == NULL) {
-		printf("Could not write Milenage data file '%s'\n", buf);
+		printf("Could not write Milenage data file '%s'\n", name);
 		fclose(f);
 		return;
 	}
@@ -588,14 +606,14 @@ static void update_milenage_file(const char *fname)
 	fclose(f2);
 	fclose(f);
 
-	snprintf(buf, sizeof(buf), "%s.bak", fname);
-	if (rename(fname, buf) < 0) {
+	snprintf(name, sizeof(name), "%s.bak", fname);
+	if (rename(fname, name) < 0) {
 		perror("rename");
 		return;
 	}
 
-	snprintf(buf, sizeof(buf), "%s.new", fname);
-	if (rename(buf, fname) < 0) {
+	snprintf(name, sizeof(name), "%s.new", fname);
+	if (rename(name, fname) < 0) {
 		perror("rename");
 		return;
 	}
@@ -798,6 +816,10 @@ static int aka_req_auth(char *imsi, char *resp, size_t resp_len)
 		}
 		milenage_generate(m->opc, m->amf, m->ki, m->sqn, _rand,
 				  autn, ik, ck, res, &res_len);
+		if (m->res_len >= EAP_AKA_RES_MIN_LEN &&
+		    m->res_len <= EAP_AKA_RES_MAX_LEN &&
+		    m->res_len < res_len)
+			res_len = m->res_len;
 	} else {
 		printf("Unknown IMSI: %s\n", imsi);
 #ifdef AKA_USE_FIXED_TEST_VALUES
diff --git a/hostapd/hlr_auc_gw.milenage_db b/hostapd/hlr_auc_gw.milenage_db
index ecd06d72096d..c156a29aeda0 100644
--- a/hostapd/hlr_auc_gw.milenage_db
+++ b/hostapd/hlr_auc_gw.milenage_db
@@ -5,8 +5,10 @@
 # authentication. In case of GSM/EAP-SIM, AMF and SQN values are not used, but
 # dummy values will need to be included in this file.
 
-# IMSI Ki OPc AMF SQN
+# IMSI Ki OPc AMF SQN [RES_len]
 232010000000000 90dca4eda45b53cf0f12d7c9c3bc6a89 cb9cccc4b9258e6dca4760379fb82581 61df 000000000000
+# Example using truncated 32-bit RES instead of 64-bit default
+232010000000001 90dca4eda45b53cf0f12d7c9c3bc6a89 cb9cccc4b9258e6dca4760379fb82581 61df 000000000000 4
 
 # These values are from Test Set 19 which has the AMF separation bit set to 1
 # and as such, is suitable for EAP-AKA' test.
diff --git a/hostapd/hostapd.conf b/hostapd/hostapd.conf
index 9e81e9e98683..a0071f7d82c4 100644
--- a/hostapd/hostapd.conf
+++ b/hostapd/hostapd.conf
@@ -127,7 +127,9 @@ ssid=test
 
 # Operation mode (a = IEEE 802.11a, b = IEEE 802.11b, g = IEEE 802.11g,
 # ad = IEEE 802.11ad (60 GHz); a/g options are used with IEEE 802.11n, too, to
-# specify band)
+# specify band). When using ACS (see channel parameter), a special value "any"
+# can be used to indicate that any support band can be used. This special case
+# is currently supported only with drivers with which offloaded ACS is used.
 # Default: IEEE 802.11b
 hw_mode=g
 
@@ -170,8 +172,11 @@ channel=1
 
 # Channel list restriction. This option allows hostapd to select one of the
 # provided channels when a channel should be automatically selected.
-# Default: not set (allow any enabled channel to be selected)
+# Channel list can be provided as range using hyphen ('-') or individual
+# channels can be specified by space (' ') seperated values
+# Default: all channels allowed in selected hw_mode
 #chanlist=100 104 108 112 116
+#chanlist=1 6 11-13
 
 # Beacon interval in kus (1.024 ms) (default: 100; range 15..65535)
 beacon_int=100
@@ -275,8 +280,9 @@ ignore_broadcast_ssid=0
 #		(data0 is the highest priority queue)
 # parameters:
 #   aifs: AIFS (default 2)
-#   cwmin: cwMin (1, 3, 7, 15, 31, 63, 127, 255, 511, 1023)
-#   cwmax: cwMax (1, 3, 7, 15, 31, 63, 127, 255, 511, 1023); cwMax >= cwMin
+#   cwmin: cwMin (1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 4095, 8191,
+#	   16383, 32767)
+#   cwmax: cwMax (same values as cwMin, cwMax >= cwMin)
 #   burst: maximum length (in milliseconds with precision of up to 0.1 ms) for
 #          bursting
 #
@@ -337,8 +343,9 @@ ignore_broadcast_ssid=0
 # note - txop_limit is in units of 32microseconds
 # note - acm is admission control mandatory flag. 0 = admission control not
 # required, 1 = mandatory
-# note - here cwMin and cmMax are in exponent form. the actual cw value used
-# will be (2^n)-1 where n is the value given here
+# note - Here cwMin and cmMax are in exponent form. The actual cw value used
+# will be (2^n)-1 where n is the value given here. The allowed range for these
+# wmm_ac_??_{cwmin,cwmax} is 0..15 with cwmax >= cwmin.
 #
 wmm_enabled=1
 #
@@ -575,14 +582,16 @@ wmm_ac_vo_acm=0
 # 0 = Not supported (default)
 # 1 = Supported
 #
-# Compressed Steering Number of Beamformer Antennas Supported: [BF-ANTENNA-2]
+# Compressed Steering Number of Beamformer Antennas Supported:
+# [BF-ANTENNA-2] [BF-ANTENNA-3] [BF-ANTENNA-4]
 #   Beamformee's capability indicating the maximum number of beamformer
 #   antennas the beamformee can support when sending compressed beamforming
 #   feedback
 # If SU beamformer capable, set to maximum value minus 1
 # else reserved (default)
 #
-# Number of Sounding Dimensions: [SOUNDING-DIMENSION-2]
+# Number of Sounding Dimensions:
+# [SOUNDING-DIMENSION-2] [SOUNDING-DIMENSION-3] [SOUNDING-DIMENSION-4]
 # Beamformer's capability indicating the maximum value of the NUM_STS parameter
 # in the TXVECTOR of a VHT NDP
 # If SU beamformer capable, set to maximum value minus 1
@@ -593,11 +602,6 @@ wmm_ac_vo_acm=0
 # 0 = Not supported or sent by Non-AP STA (default)
 # 1 = Supported
 #
-# MU Beamformee Capable: [MU-BEAMFORMEE]
-# Indicates support for operation as an MU beamformee
-# 0 = Not supported or sent by AP (default)
-# 1 = Supported
-#
 # VHT TXOP PS: [VHT-TXOP-PS]
 # Indicates whether or not the AP supports VHT TXOP Power Save Mode
 #  or whether or not the STA is in VHT TXOP Power Save mode
@@ -764,6 +768,12 @@ eap_server=0
 # 2 = check all CRLs in the certificate path
 #check_crl=1
 
+# TLS Session Lifetime in seconds
+# This can be used to allow TLS sessions to be cached and resumed with an
+# abbreviated handshake when using EAP-TLS/TTLS/PEAP.
+# (default: 0 = session caching and resumption disabled)
+#tls_session_lifetime=3600
+
 # Cached OCSP stapling response (DER encoded)
 # If set, this file is sent as a certificate status response by the EAP server
 # if the EAP peer requests certificate status in the ClientHello message.
@@ -787,7 +797,7 @@ eap_server=0
 # is in DSA parameters format, it will be automatically converted into DH
 # params. This parameter is required if anonymous EAP-FAST is used.
 # You can generate DH parameters file with OpenSSL, e.g.,
-# "openssl dhparam -out /etc/hostapd.dh.pem 1024"
+# "openssl dhparam -out /etc/hostapd.dh.pem 2048"
 #dh_file=/etc/hostapd.dh.pem
 
 # OpenSSL cipher string
@@ -1247,6 +1257,11 @@ own_ip_addr=127.0.0.1
 # 1 = push PMK-R1 to all configured R1KHs whenever a new PMK-R0 is derived
 #pmk_r1_push=1
 
+# Whether to enable FT-over-DS
+# 0 = FT-over-DS disabled
+# 1 = FT-over-DS enabled (default)
+#ft_over_ds=1
+
 ##### Neighbor table ##########################################################
 # Maximum number of entries kept in AP table (either for neigbor table or for
 # detecting Overlapping Legacy BSS Condition). The oldest entry will be
@@ -1264,6 +1279,43 @@ own_ip_addr=127.0.0.1
 # default: 60
 #ap_table_expiration_time=3600
 
+# Maximum number of stations to track on the operating channel
+# This can be used to detect dualband capable stations before they have
+# associated, e.g., to provide guidance on which colocated BSS to use.
+# Default: 0 (disabled)
+#track_sta_max_num=100
+
+# Maximum age of a station tracking entry in seconds
+# Default: 180
+#track_sta_max_age=180
+
+# Do not reply to group-addressed Probe Request from a station that was seen on
+# another radio.
+# Default: Disabled
+#
+# This can be used with enabled track_sta_max_num configuration on another
+# interface controlled by the same hostapd process to restrict Probe Request
+# frame handling from replying to group-addressed Probe Request frames from a
+# station that has been detected to be capable of operating on another band,
+# e.g., to try to reduce likelihood of the station selecting a 2.4 GHz BSS when
+# the AP operates both a 2.4 GHz and 5 GHz BSS concurrently.
+#
+# Note: Enabling this can cause connectivity issues and increase latency for
+# discovering the AP.
+#no_probe_resp_if_seen_on=wlan1
+
+# Reject authentication from a station that was seen on another radio.
+# Default: Disabled
+#
+# This can be used with enabled track_sta_max_num configuration on another
+# interface controlled by the same hostapd process to reject authentication
+# attempts from a station that has been detected to be capable of operating on
+# another band, e.g., to try to reduce likelihood of the station selecting a
+# 2.4 GHz BSS when the AP operates both a 2.4 GHz and 5 GHz BSS concurrently.
+#
+# Note: Enabling this can cause connectivity issues and increase latency for
+# connecting with the AP.
+#no_auth_if_seen_on=wlan1
 
 ##### Wi-Fi Protected Setup (WPS) #############################################
 
@@ -1430,7 +1482,7 @@ own_ip_addr=127.0.0.1
 # 12-digit, all-numeric code that identifies the consumer package.
 #upc=123456789012
 
-# WPS RF Bands (a = 5G, b = 2.4G, g = 2.4G, ag = dual band)
+# WPS RF Bands (a = 5G, b = 2.4G, g = 2.4G, ag = dual band, ad = 60 GHz)
 # This value should be set according to RF band(s) supported by the AP if
 # hw_mode is not set. For dual band dual concurrent devices, this needs to be
 # set to ag to allow both RF bands to be advertized.
@@ -1490,6 +1542,13 @@ own_ip_addr=127.0.0.1
 # 1 = enabled
 #proxy_arp=1
 
+# IPv6 Neighbor Advertisement multicast-to-unicast conversion
+# This can be used with Proxy ARP to allow multicast NAs to be forwarded to
+# associated STAs using link layer unicast delivery.
+# 0 = disabled (default)
+# 1 = enabled
+#na_mcast_to_ucast=0
+
 ##### IEEE 802.11u-2011 #######################################################
 
 # Enable Interworking service
@@ -1738,6 +1797,32 @@ own_ip_addr=127.0.0.1
 #
 #osu_server_uri=...
 
+##### Fast Session Transfer (FST) support #####################################
+#
+# The options in this section are only available when the build configuration
+# option CONFIG_FST is set while compiling hostapd. They allow this interface
+# to be a part of FST setup.
+#
+# FST is the transfer of a session from a channel to another channel, in the
+# same or different frequency bands.
+#
+# For detals, see IEEE Std 802.11ad-2012.
+
+# Identifier of an FST Group the interface belongs to.
+#fst_group_id=bond0
+
+# Interface priority within the FST Group.
+# Announcing a higher priority for an interface means declaring it more
+# preferable for FST switch.
+# fst_priority is in 1..255 range with 1 being the lowest priority.
+#fst_priority=100
+
+# Default LLT value for this interface in milliseconds. The value used in case
+# no value provided during session setup. Default is 50 ms.
+# fst_llt is in 1..4294967 range (due to spec limitation, see 10.32.2.2
+# Transitioning between states).
+#fst_llt=100
+
 ##### TESTING OPTIONS #########################################################
 #
 # The options in this section are only available when the build configuration
diff --git a/hostapd/hostapd_cli.c b/hostapd/hostapd_cli.c
index 3f00cbbf0cc7..46c2f37e4601 100644
--- a/hostapd/hostapd_cli.c
+++ b/hostapd/hostapd_cli.c
@@ -10,22 +10,23 @@
 #include <dirent.h>
 
 #include "common/wpa_ctrl.h"
+#include "common/ieee802_11_defs.h"
 #include "utils/common.h"
 #include "utils/eloop.h"
 #include "utils/edit.h"
 #include "common/version.h"
 
 
-static const char *hostapd_cli_version =
+static const char *const hostapd_cli_version =
 "hostapd_cli v" VERSION_STR "\n"
 "Copyright (c) 2004-2015, Jouni Malinen <j@w1.fi> and contributors";
 
 
-static const char *hostapd_cli_license =
+static const char *const hostapd_cli_license =
 "This software may be distributed under the terms of the BSD license.\n"
 "See README for more details.\n";
 
-static const char *hostapd_cli_full_license =
+static const char *const hostapd_cli_full_license =
 "This software may be distributed under the terms of the BSD license.\n"
 "\n"
 "Redistribution and use in source and binary forms, with or without\n"
@@ -56,7 +57,7 @@ static const char *hostapd_cli_full_license =
 "OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n"
 "\n";
 
-static const char *commands_help =
+static const char *const commands_help =
 "Commands:\n"
 "   mib                  get MIB variables (dot1x, dot11, radius)\n"
 "   sta <addr>           get MIB variables for one station\n"
@@ -96,6 +97,7 @@ static int hostapd_cli_attached = 0;
 #define CONFIG_CTRL_IFACE_DIR "/var/run/hostapd"
 #endif /* CONFIG_CTRL_IFACE_DIR */
 static const char *ctrl_iface_dir = CONFIG_CTRL_IFACE_DIR;
+static const char *client_socket_dir = NULL;
 
 static char *ctrl_ifname = NULL;
 static const char *pid_file = NULL;
@@ -111,13 +113,15 @@ static void usage(void)
 		"\n"
 		"usage: hostapd_cli [-p<path>] [-i<ifname>] [-hvB] "
 		"[-a<path>] \\\n"
-		"                   [-G<ping interval>] [command..]\n"
+		"                   [-P<pid file>] [-G<ping interval>] [command..]\n"
 		"\n"
 		"Options:\n"
 		"   -h           help (show this usage text)\n"
 		"   -v           shown version information\n"
 		"   -p<path>     path to find control sockets (default: "
 		"/var/run/hostapd)\n"
+		"   -s<dir_path> dir path to open client sockets (default: "
+		CONFIG_CTRL_IFACE_DIR ")\n"
 		"   -a<file>     run in daemon mode executing the action file "
 		"based on events\n"
 		"                from hostapd\n"
@@ -144,7 +148,14 @@ static struct wpa_ctrl * hostapd_cli_open_connection(const char *ifname)
 		return NULL;
 	snprintf(cfile, flen, "%s/%s", ctrl_iface_dir, ifname);
 
-	ctrl_conn = wpa_ctrl_open(cfile);
+	if (client_socket_dir && client_socket_dir[0] &&
+	    access(client_socket_dir, F_OK) < 0) {
+		perror(client_socket_dir);
+		free(cfile);
+		return NULL;
+	}
+
+	ctrl_conn = wpa_ctrl_open2(cfile, client_socket_dir);
 	free(cfile);
 	return ctrl_conn;
 }
@@ -541,7 +552,7 @@ static int hostapd_cli_cmd_wps_config(struct wpa_ctrl *ctrl, int argc,
 				      char *argv[])
 {
 	char buf[256];
-	char ssid_hex[2 * 32 + 1];
+	char ssid_hex[2 * SSID_MAX_LEN + 1];
 	char key_hex[2 * 64 + 1];
 	int i;
 
@@ -552,7 +563,7 @@ static int hostapd_cli_cmd_wps_config(struct wpa_ctrl *ctrl, int argc,
 	}
 
 	ssid_hex[0] = '\0';
-	for (i = 0; i < 32; i++) {
+	for (i = 0; i < SSID_MAX_LEN; i++) {
 		if (argv[0][i] == '\0')
 			break;
 		os_snprintf(&ssid_hex[i * 2], 3, "%02x", argv[0][i]);
@@ -921,6 +932,35 @@ static int hostapd_cli_cmd_get(struct wpa_ctrl *ctrl, int argc, char *argv[])
 }
 
 
+#ifdef CONFIG_FST
+static int hostapd_cli_cmd_fst(struct wpa_ctrl *ctrl, int argc, char *argv[])
+{
+	char cmd[256];
+	int res;
+	int i;
+	int total;
+
+	if (argc <= 0) {
+		printf("FST command: parameters are required.\n");
+		return -1;
+	}
+
+	total = os_snprintf(cmd, sizeof(cmd), "FST-MANAGER");
+
+	for (i = 0; i < argc; i++) {
+		res = os_snprintf(cmd + total, sizeof(cmd) - total, " %s",
+				  argv[i]);
+		if (os_snprintf_error(sizeof(cmd) - total, res)) {
+			printf("Too long fst command.\n");
+			return -1;
+		}
+		total += res;
+	}
+	return wpa_ctrl_command(ctrl, cmd);
+}
+#endif /* CONFIG_FST */
+
+
 static int hostapd_cli_cmd_chan_switch(struct wpa_ctrl *ctrl,
 				       int argc, char *argv[])
 {
@@ -1009,12 +1049,31 @@ static int hostapd_cli_cmd_erp_flush(struct wpa_ctrl *ctrl, int argc,
 }
 
 
+static int hostapd_cli_cmd_log_level(struct wpa_ctrl *ctrl, int argc,
+				     char *argv[])
+{
+	char cmd[256];
+	int res;
+
+	res = os_snprintf(cmd, sizeof(cmd), "LOG_LEVEL%s%s%s%s",
+			  argc >= 1 ? " " : "",
+			  argc >= 1 ? argv[0] : "",
+			  argc == 2 ? " " : "",
+			  argc == 2 ? argv[1] : "");
+	if (os_snprintf_error(sizeof(cmd), res)) {
+		printf("Too long option\n");
+		return -1;
+	}
+	return wpa_ctrl_command(ctrl, cmd);
+}
+
+
 struct hostapd_cli_cmd {
 	const char *cmd;
 	int (*handler)(struct wpa_ctrl *ctrl, int argc, char *argv[]);
 };
 
-static struct hostapd_cli_cmd hostapd_cli_commands[] = {
+static const struct hostapd_cli_cmd hostapd_cli_commands[] = {
 	{ "ping", hostapd_cli_cmd_ping },
 	{ "mib", hostapd_cli_cmd_mib },
 	{ "relog", hostapd_cli_cmd_relog },
@@ -1048,6 +1107,9 @@ static struct hostapd_cli_cmd hostapd_cli_commands[] = {
 	{ "get_config", hostapd_cli_cmd_get_config },
 	{ "help", hostapd_cli_cmd_help },
 	{ "interface", hostapd_cli_cmd_interface },
+#ifdef CONFIG_FST
+	{ "fst", hostapd_cli_cmd_fst },
+#endif /* CONFIG_FST */
 	{ "level", hostapd_cli_cmd_level },
 	{ "license", hostapd_cli_cmd_license },
 	{ "quit", hostapd_cli_cmd_quit },
@@ -1063,13 +1125,14 @@ static struct hostapd_cli_cmd hostapd_cli_commands[] = {
 	{ "reload", hostapd_cli_cmd_reload },
 	{ "disable", hostapd_cli_cmd_disable },
 	{ "erp_flush", hostapd_cli_cmd_erp_flush },
+	{ "log_level", hostapd_cli_cmd_log_level },
 	{ NULL, NULL }
 };
 
 
 static void wpa_request(struct wpa_ctrl *ctrl, int argc, char *argv[])
 {
-	struct hostapd_cli_cmd *cmd, *match = NULL;
+	const struct hostapd_cli_cmd *cmd, *match = NULL;
 	int count;
 
 	count = 0;
@@ -1284,7 +1347,7 @@ int main(int argc, char *argv[])
 		return -1;
 
 	for (;;) {
-		c = getopt(argc, argv, "a:BhG:i:p:v");
+		c = getopt(argc, argv, "a:BhG:i:p:P:s:v");
 		if (c < 0)
 			break;
 		switch (c) {
@@ -1310,6 +1373,12 @@ int main(int argc, char *argv[])
 		case 'p':
 			ctrl_iface_dir = optarg;
 			break;
+		case 'P':
+			pid_file = optarg;
+			break;
+		case 's':
+			client_socket_dir = optarg;
+			break;
 		default:
 			usage();
 			return -1;
diff --git a/hostapd/main.c b/hostapd/main.c
index dd389a8d66a3..6c7406af447e 100644
--- a/hostapd/main.c
+++ b/hostapd/main.c
@@ -24,6 +24,7 @@
 #include "ap/hostapd.h"
 #include "ap/ap_config.h"
 #include "ap/ap_drv_ops.h"
+#include "fst/fst.h"
 #include "config_file.h"
 #include "eap_register.h"
 #include "ctrl_iface.h"
@@ -533,6 +534,28 @@ static int gen_uuid(const char *txt_addr)
 #endif /* CONFIG_WPS */
 
 
+#ifndef HOSTAPD_CLEANUP_INTERVAL
+#define HOSTAPD_CLEANUP_INTERVAL 10
+#endif /* HOSTAPD_CLEANUP_INTERVAL */
+
+static int hostapd_periodic_call(struct hostapd_iface *iface, void *ctx)
+{
+	hostapd_periodic_iface(iface);
+	return 0;
+}
+
+
+/* Periodic cleanup tasks */
+static void hostapd_periodic(void *eloop_ctx, void *timeout_ctx)
+{
+	struct hapd_interfaces *interfaces = eloop_ctx;
+
+	eloop_register_timeout(HOSTAPD_CLEANUP_INTERVAL, 0,
+			       hostapd_periodic, interfaces, NULL);
+	hostapd_for_each_interface(interfaces, hostapd_periodic_call, NULL);
+}
+
+
 int main(int argc, char *argv[])
 {
 	struct hapd_interfaces interfaces;
@@ -561,6 +584,7 @@ int main(int argc, char *argv[])
 	interfaces.global_iface_path = NULL;
 	interfaces.global_iface_name = NULL;
 	interfaces.global_ctrl_sock = -1;
+	interfaces.global_ctrl_dst = NULL;
 
 	for (;;) {
 		c = getopt(argc, argv, "b:Bde:f:hKP:Ttu:vg:G:");
@@ -661,10 +685,24 @@ int main(int argc, char *argv[])
 	}
 
 	if (hostapd_global_init(&interfaces, entropy_file)) {
-		wpa_printf(MSG_ERROR, "Failed to initilize global context");
+		wpa_printf(MSG_ERROR, "Failed to initialize global context");
 		return -1;
 	}
 
+	eloop_register_timeout(HOSTAPD_CLEANUP_INTERVAL, 0,
+			       hostapd_periodic, &interfaces, NULL);
+
+	if (fst_global_init()) {
+		wpa_printf(MSG_ERROR,
+			   "Failed to initialize global FST context");
+		goto out;
+	}
+
+#if defined(CONFIG_FST) && defined(CONFIG_CTRL_IFACE)
+	if (!fst_global_add_ctrl(fst_ctrl_cli))
+		wpa_printf(MSG_WARNING, "Failed to add CLI FST ctrl");
+#endif /* CONFIG_FST && CONFIG_CTRL_IFACE */
+
 	/* Allocate and parse configuration for full interface files */
 	for (i = 0; i < interfaces.count; i++) {
 		interfaces.iface[i] = hostapd_interface_init(&interfaces,
@@ -749,6 +787,7 @@ int main(int argc, char *argv[])
 	}
 	os_free(interfaces.iface);
 
+	eloop_cancel_timeout(hostapd_periodic, &interfaces, NULL);
 	hostapd_global_deinit(pid_file);
 	os_free(pid_file);
 
@@ -758,6 +797,8 @@ int main(int argc, char *argv[])
 
 	os_free(bss_config);
 
+	fst_global_deinit();
+
 	os_program_deinit();
 
 	return ret;
diff --git a/hs20/client/Makefile b/hs20/client/Makefile
index ca67b54da2ee..94cd5f14df14 100644
--- a/hs20/client/Makefile
+++ b/hs20/client/Makefile
@@ -67,7 +67,13 @@ OBJS += ../../src/crypto/sha256-internal.o
 
 CFLAGS += $(shell xml2-config --cflags)
 LIBS += $(shell xml2-config --libs)
+
+# Allow static/custom linking of libcurl.
+ifdef CUST_CURL_LINKAGE
+LIBS += ${CUST_CURL_LINKAGE}
+else
 LIBS += -lcurl
+endif
 
 CFLAGS += -DEAP_TLS_OPENSSL
 LIBS += -lssl -lcrypto
diff --git a/hs20/client/osu_client.c b/hs20/client/osu_client.c
index de7f351da244..0315f7b75ad4 100644
--- a/hs20/client/osu_client.c
+++ b/hs20/client/osu_client.c
@@ -25,6 +25,8 @@
 #include "crypto/sha256.h"
 #include "osu_client.h"
 
+const char *spp_xsd_fname = "spp.xsd";
+
 
 void write_result(struct hs20_osu_client *ctx, const char *fmt, ...)
 {
@@ -540,6 +542,7 @@ int hs20_add_pps_mo(struct hs20_osu_client *ctx, const char *uri,
 				   uri);
 			write_result(ctx, "Unsupported location for addMO to "
 				     "add PPS MO (extra directory): '%s'", uri);
+			free(fqdn);
 			return -1;
 		}
 		*pos = '\0'; /* remove trailing slash and PPS node name */
@@ -547,8 +550,9 @@ int hs20_add_pps_mo(struct hs20_osu_client *ctx, const char *uri,
 	wpa_printf(MSG_INFO, "SP FQDN: %s", fqdn);
 
 	if (!server_dnsname_suffix_match(ctx, fqdn)) {
-		wpa_printf(MSG_INFO, "FQDN '%s' for new PPS MO did not have suffix match with server's dNSName values",
-			   fqdn);
+		wpa_printf(MSG_INFO,
+			   "FQDN '%s' for new PPS MO did not have suffix match with server's dNSName values, count: %d",
+			   fqdn, (int) ctx->server_dnsname_count);
 		write_result(ctx, "FQDN '%s' for new PPS MO did not have suffix match with server's dNSName values",
 			     fqdn);
 		free(fqdn);
@@ -2094,10 +2098,14 @@ static int osu_connect(struct hs20_osu_client *ctx, const char *bssid,
 	}
 
 	ctx->no_reconnect = 1;
-	if (methods & 0x02)
+	if (methods & 0x02) {
+		wpa_printf(MSG_DEBUG, "Calling cmd_prov from osu_connect");
 		res = cmd_prov(ctx, url);
-	else if (methods & 0x01)
+	} else if (methods & 0x01) {
+		wpa_printf(MSG_DEBUG,
+			   "Calling cmd_oma_dm_prov from osu_connect");
 		res = cmd_oma_dm_prov(ctx, url);
+	}
 
 	wpa_printf(MSG_INFO, "Remove OSU network connection");
 	write_summary(ctx, "Remove OSU network connection");
@@ -2139,7 +2147,7 @@ static int cmd_osu_select(struct hs20_osu_client *ctx, const char *dir,
 	snprintf(fname, sizeof(fname), "%s/osu-providers.txt", dir);
 	osu = parse_osu_providers(fname, &osu_count);
 	if (osu == NULL) {
-		wpa_printf(MSG_INFO, "Could not any OSU providers from %s",
+		wpa_printf(MSG_INFO, "Could not find any OSU providers from %s",
 			   fname);
 		write_result(ctx, "No OSU providers available");
 		return -1;
@@ -2290,12 +2298,19 @@ selected:
 		}
 
 		if (connect == 2) {
-			if (last->methods & 0x02)
+			if (last->methods & 0x02) {
+				wpa_printf(MSG_DEBUG,
+					   "Calling cmd_prov from cmd_osu_select");
 				ret = cmd_prov(ctx, last->url);
-			else if (last->methods & 0x01)
+			} else if (last->methods & 0x01) {
+				wpa_printf(MSG_DEBUG,
+					   "Calling cmd_oma_dm_prov from cmd_osu_select");
 				ret = cmd_oma_dm_prov(ctx, last->url);
-			else
+			} else {
+				wpa_printf(MSG_DEBUG,
+					   "No supported OSU provisioning method");
 				ret = -1;
+			}
 		} else if (connect)
 			ret = osu_connect(ctx, last->bssid, last->osu_ssid,
 					  last->url, last->methods,
@@ -2690,7 +2705,7 @@ static char * get_hostname(const char *url)
 
 	end = os_strchr(pos, '/');
 	end2 = os_strchr(pos, ':');
-	if (end && end2 && end2 < end)
+	if ((end && end2 && end2 < end) || (!end && end2))
 		end = end2;
 	if (end)
 		end--;
@@ -2720,8 +2735,8 @@ static int osu_cert_cb(void *_ctx, struct http_cert *cert)
 	int found;
 	char *host = NULL;
 
-	wpa_printf(MSG_INFO, "osu_cert_cb(osu_cert_validation=%d)",
-		   !ctx->no_osu_cert_validation);
+	wpa_printf(MSG_INFO, "osu_cert_cb(osu_cert_validation=%d, url=%s)",
+		   !ctx->no_osu_cert_validation, ctx->server_url);
 
 	host = get_hostname(ctx->server_url);
 
@@ -2810,17 +2825,21 @@ static int osu_cert_cb(void *_ctx, struct http_cert *cert)
 		char *name = ctx->icon_filename[j];
 		size_t name_len = os_strlen(name);
 
-		wpa_printf(MSG_INFO, "Looking for icon file name '%s' match",
-			   name);
+		wpa_printf(MSG_INFO,
+			   "[%i] Looking for icon file name '%s' match",
+			   j, name);
 		for (i = 0; i < cert->num_logo; i++) {
 			struct http_logo *logo = &cert->logo[i];
 			size_t uri_len = os_strlen(logo->uri);
 			char *pos;
 
-			wpa_printf(MSG_INFO, "Comparing to '%s' uri_len=%d name_len=%d",
-				   logo->uri, (int) uri_len, (int) name_len);
-			if (uri_len < 1 + name_len)
+			wpa_printf(MSG_INFO,
+				   "[%i] Comparing to '%s' uri_len=%d name_len=%d",
+				   i, logo->uri, (int) uri_len, (int) name_len);
+			if (uri_len < 1 + name_len) {
+				wpa_printf(MSG_INFO, "URI Length is too short");
 				continue;
+			}
 			pos = &logo->uri[uri_len - name_len - 1];
 			if (*pos != '/')
 				continue;
@@ -2847,17 +2866,30 @@ static int osu_cert_cb(void *_ctx, struct http_cert *cert)
 		for (i = 0; i < cert->num_logo; i++) {
 			struct http_logo *logo = &cert->logo[i];
 
-			if (logo->hash_len != 32)
+			if (logo->hash_len != 32) {
+				wpa_printf(MSG_INFO,
+					   "[%i][%i] Icon hash length invalid (should be 32): %d",
+					   j, i, (int) logo->hash_len);
 				continue;
+			}
 			if (os_memcmp(logo->hash, ctx->icon_hash[j], 32) == 0) {
 				found = 1;
 				break;
 			}
+
+			wpa_printf(MSG_DEBUG,
+				   "[%u][%u] Icon hash did not match", j, i);
+			wpa_hexdump_ascii(MSG_DEBUG, "logo->hash",
+					  logo->hash, 32);
+			wpa_hexdump_ascii(MSG_DEBUG, "ctx->icon_hash[j]",
+					  ctx->icon_hash[j], 32);
 		}
 
 		if (!found) {
-			wpa_printf(MSG_INFO, "No icon hash match found");
-			write_result(ctx, "No icon hash match found");
+			wpa_printf(MSG_INFO,
+				   "No icon hash match (by hash) found");
+			write_result(ctx,
+				     "No icon hash match (by hash) found");
 			return -1;
 		}
 	}
@@ -2955,6 +2987,7 @@ static void usage(void)
 	       "    [-w<wpa_supplicant ctrl_iface dir>] "
 	       "[-r<result file>] [-f<debug file>] \\\n"
 	       "    [-s<summary file>] \\\n"
+	       "    [-x<spp.xsd file name>] \\\n"
 	       "    <command> [arguments..]\n"
 	       "commands:\n"
 	       "- to_tnds <XML MO> <XML MO in TNDS format> [URN]\n"
@@ -2996,7 +3029,7 @@ int main(int argc, char *argv[])
 		return -1;
 
 	for (;;) {
-		c = getopt(argc, argv, "df:hi:KNO:qr:s:S:tw:");
+		c = getopt(argc, argv, "df:hKNO:qr:s:S:tw:x:");
 		if (c < 0)
 			break;
 		switch (c) {
@@ -3034,6 +3067,9 @@ int main(int argc, char *argv[])
 		case 'w':
 			wpas_ctrl_path = optarg;
 			break;
+		case 'x':
+			spp_xsd_fname = optarg;
+			break;
 		case 'h':
 		default:
 			usage();
@@ -3108,6 +3144,7 @@ int main(int argc, char *argv[])
 			exit(0);
 		}
 		ctx.ca_fname = argv[optind + 2];
+		wpa_printf(MSG_DEBUG, "Calling cmd_prov from main");
 		cmd_prov(&ctx, argv[optind + 1]);
 	} else if (strcmp(argv[optind], "sim_prov") == 0) {
 		if (argc - optind < 2) {
diff --git a/hs20/client/spp_client.c b/hs20/client/spp_client.c
index 302a05040df6..c619541ae286 100644
--- a/hs20/client/spp_client.c
+++ b/hs20/client/spp_client.c
@@ -21,6 +21,8 @@
 #include "osu_client.h"
 
 
+extern const char *spp_xsd_fname;
+
 static int hs20_spp_update_response(struct hs20_osu_client *ctx,
 				    const char *session_id,
 				    const char *spp_status,
@@ -59,7 +61,7 @@ static int hs20_spp_validate(struct hs20_osu_client *ctx, xml_node_t *node,
 		return -1;
 	}
 
-	ret = xml_validate(xctx, node, "spp.xsd", &err);
+	ret = xml_validate(xctx, node, spp_xsd_fname, &err);
 	if (ret < 0) {
 		wpa_printf(MSG_INFO, "XML schema validation error(s)\n%s", err);
 		write_summary(ctx, "SPP XML schema validation failed");
@@ -77,9 +79,14 @@ static void add_mo_container(struct xml_node_ctx *ctx, xml_namespace_t *ns,
 	xml_node_t *fnode, *tnds;
 	char *str;
 
+	errno = 0;
 	fnode = node_from_file(ctx, fname);
-	if (!fnode)
+	if (!fnode) {
+		wpa_printf(MSG_ERROR,
+			   "Failed to create XML node from file: %s, possible error: %s",
+			   fname, strerror(errno));
 		return;
+	}
 	tnds = mo_to_tnds(ctx, fnode, 0, urn, "syncml:dmddf1.2");
 	xml_node_free(ctx, fnode);
 	if (!tnds)
@@ -952,7 +959,9 @@ int cmd_prov(struct hs20_osu_client *ctx, const char *url)
 		return -1;
 	}
 
-	wpa_printf(MSG_INFO, "Credential provisioning requested");
+	wpa_printf(MSG_INFO,
+		   "Credential provisioning requested - URL: %s ca_fname: %s",
+		   url, ctx->ca_fname ? ctx->ca_fname : "N/A");
 
 	os_free(ctx->server_url);
 	ctx->server_url = os_strdup(url);
diff --git a/patches/openssl-0.9.8zf-tls-extensions.patch b/patches/openssl-0.9.8zf-tls-extensions.patch
new file mode 100644
index 000000000000..3a8f90e40ce2
--- /dev/null
+++ b/patches/openssl-0.9.8zf-tls-extensions.patch
@@ -0,0 +1,398 @@
+This patch adds support for TLS SessionTicket extension (RFC 5077) for
+the parts used by EAP-FAST (RFC 4851).
+
+This is based on the patch from Alexey Kobozev <akobozev@cisco.com>
+(sent to openssl-dev mailing list on Tue, 07 Jun 2005 15:40:58 +0300).
+
+OpenSSL 0.9.8zf does not enable TLS extension support by default, so it
+will need to be enabled by adding enable-tlsext to config script
+command line.
+
+
+diff -upr openssl-0.9.8zf.orig/ssl/s3_clnt.c openssl-0.9.8zf/ssl/s3_clnt.c
+--- openssl-0.9.8zf.orig/ssl/s3_clnt.c	2015-03-19 15:46:46.000000000 +0200
++++ openssl-0.9.8zf/ssl/s3_clnt.c	2015-03-24 16:19:14.043911769 +0200
+@@ -760,6 +760,23 @@ int ssl3_get_server_hello(SSL *s)
+         goto f_err;
+     }
+ 
++#ifndef OPENSSL_NO_TLSEXT
++    /* check if we want to resume the session based on external pre-shared secret */
++    if (s->version >= TLS1_VERSION && s->tls_session_secret_cb) {
++        SSL_CIPHER *pref_cipher = NULL;
++
++        s->session->master_key_length = sizeof(s->session->master_key);
++        if (s->tls_session_secret_cb(s, s->session->master_key,
++				     &s->session->master_key_length,
++				     NULL, &pref_cipher,
++				     s->tls_session_secret_cb_arg)) {
++            s->session->cipher = pref_cipher ?
++		    pref_cipher : ssl_get_cipher_by_char(s, p + j);
++	    s->s3->flags |= SSL3_FLAGS_CCS_OK;
++	}
++    }
++#endif /* OPENSSL_NO_TLSEXT */
++
+     if (j != 0 && j == s->session->session_id_length
+         && memcmp(p, s->session->session_id, j) == 0) {
+         if (s->sid_ctx_length != s->session->sid_ctx_length
+@@ -2684,12 +2701,8 @@ int ssl3_check_finished(SSL *s)
+ {
+     int ok;
+     long n;
+-    /*
+-     * If we have no ticket or session ID is non-zero length (a match of a
+-     * non-zero session length would never reach here) it cannot be a resumed
+-     * session.
+-     */
+-    if (!s->session->tlsext_tick || s->session->session_id_length)
++    /* If we have no ticket it cannot be a resumed session. */
++    if (!s->session->tlsext_tick)
+         return 1;
+     /*
+      * this function is called when we really expect a Certificate message,
+diff -upr openssl-0.9.8zf.orig/ssl/s3_srvr.c openssl-0.9.8zf/ssl/s3_srvr.c
+--- openssl-0.9.8zf.orig/ssl/s3_srvr.c	2015-03-19 15:46:46.000000000 +0200
++++ openssl-0.9.8zf/ssl/s3_srvr.c	2015-03-24 16:23:34.567909681 +0200
+@@ -999,6 +999,59 @@ int ssl3_get_client_hello(SSL *s)
+         SSLerr(SSL_F_SSL3_GET_CLIENT_HELLO, SSL_R_CLIENTHELLO_TLSEXT);
+         goto err;
+     }
++
++    /* Check if we want to use external pre-shared secret for this
++     * handshake for not reused session only. We need to generate
++     * server_random before calling tls_session_secret_cb in order to allow
++     * SessionTicket processing to use it in key derivation. */
++    {
++        unsigned long Time;
++	unsigned char *pos;
++	Time = (unsigned long)time(NULL);			/* Time */
++	pos = s->s3->server_random;
++	l2n(Time, pos);
++	if (RAND_pseudo_bytes(pos, SSL3_RANDOM_SIZE - 4) <= 0) {
++		al = SSL_AD_INTERNAL_ERROR;
++		goto f_err;
++	}
++    }
++
++    if (!s->hit && s->version >= TLS1_VERSION && s->tls_session_secret_cb) {
++	SSL_CIPHER *pref_cipher = NULL;
++
++	s->session->master_key_length = sizeof(s->session->master_key);
++	if (s->tls_session_secret_cb(s, s->session->master_key,
++				     &s->session->master_key_length, 
++				     ciphers, &pref_cipher,
++				     s->tls_session_secret_cb_arg)) {
++	    s->hit = 1;
++	    s->session->ciphers = ciphers;
++	    s->session->verify_result = X509_V_OK;
++
++	    ciphers = NULL;
++
++	    /* check if some cipher was preferred by call back */
++	    pref_cipher = pref_cipher ? pref_cipher :
++		    ssl3_choose_cipher(s, s->session->ciphers,
++				       SSL_get_ciphers(s));
++	    if (pref_cipher == NULL) {
++		al = SSL_AD_HANDSHAKE_FAILURE;
++		SSLerr(SSL_F_SSL3_GET_CLIENT_HELLO, SSL_R_NO_SHARED_CIPHER);
++		goto f_err;
++	    }
++
++	    s->session->cipher = pref_cipher;
++
++	    if (s->cipher_list)
++		sk_SSL_CIPHER_free(s->cipher_list);
++
++	    if (s->cipher_list_by_id)
++		sk_SSL_CIPHER_free(s->cipher_list_by_id);
++
++	    s->cipher_list = sk_SSL_CIPHER_dup(s->session->ciphers);
++	    s->cipher_list_by_id = sk_SSL_CIPHER_dup(s->session->ciphers);
++	}
++    }
+ #endif
+     /*
+      * Worst case, we will use the NULL compression, but if we have other
+@@ -1143,15 +1196,21 @@ int ssl3_send_server_hello(SSL *s)
+     unsigned char *buf;
+     unsigned char *p, *d;
+     int i, sl;
+-    unsigned long l, Time;
++    unsigned long l;
++#ifdef OPENSSL_NO_TLSEXT
++    unsigned long Time;
++#endif
+ 
+     if (s->state == SSL3_ST_SW_SRVR_HELLO_A) {
+         buf = (unsigned char *)s->init_buf->data;
++#ifdef OPENSSL_NO_TLSEXT
+         p = s->s3->server_random;
++        /* Generate server_random if it was not needed previously */
+         Time = (unsigned long)time(NULL); /* Time */
+         l2n(Time, p);
+         if (RAND_pseudo_bytes(p, SSL3_RANDOM_SIZE - 4) <= 0)
+             return -1;
++#endif
+         /* Do the message type and length last */
+         d = p = &(buf[4]);
+ 
+diff -upr openssl-0.9.8zf.orig/ssl/ssl_err.c openssl-0.9.8zf/ssl/ssl_err.c
+--- openssl-0.9.8zf.orig/ssl/ssl_err.c	2015-03-19 15:46:46.000000000 +0200
++++ openssl-0.9.8zf/ssl/ssl_err.c	2015-03-24 16:35:58.627903717 +0200
+@@ -316,6 +316,7 @@ static ERR_STRING_DATA SSL_str_functs[]
+     {ERR_FUNC(SSL_F_TLS1_ENC), "TLS1_ENC"},
+     {ERR_FUNC(SSL_F_TLS1_SETUP_KEY_BLOCK), "TLS1_SETUP_KEY_BLOCK"},
+     {ERR_FUNC(SSL_F_WRITE_PENDING), "WRITE_PENDING"},
++    {ERR_FUNC(SSL_F_SSL_SET_SESSION_TICKET_EXT), "SSL_set_session_ticket_ext"},
+     {0, NULL}
+ };
+ 
+diff -upr openssl-0.9.8zf.orig/ssl/ssl.h openssl-0.9.8zf/ssl/ssl.h
+--- openssl-0.9.8zf.orig/ssl/ssl.h	2015-03-19 15:46:46.000000000 +0200
++++ openssl-0.9.8zf/ssl/ssl.h	2015-03-24 16:25:44.339908641 +0200
+@@ -349,6 +349,7 @@ extern "C" {
+  * function parameters used to prototype callbacks in SSL_CTX.
+  */
+ typedef struct ssl_st *ssl_crock_st;
++typedef struct tls_session_ticket_ext_st TLS_SESSION_TICKET_EXT;
+ 
+ /* used to hold info on the particular ciphers used */
+ typedef struct ssl_cipher_st {
+@@ -366,6 +367,12 @@ typedef struct ssl_cipher_st {
+ 
+ DECLARE_STACK_OF(SSL_CIPHER)
+ 
++typedef int (*tls_session_ticket_ext_cb_fn)(SSL *s, const unsigned char *data,
++					    int len, void *arg);
++typedef int (*tls_session_secret_cb_fn)(SSL *s, void *secret, int *secret_len,
++					STACK_OF(SSL_CIPHER) *peer_ciphers,
++					SSL_CIPHER **cipher, void *arg);
++
+ /* Used to hold functions for SSLv2 or SSLv3/TLSv1 functions */
+ typedef struct ssl_method_st {
+     int version;
+@@ -1116,6 +1123,18 @@ struct ssl_st {
+     int tlsext_ocsp_resplen;
+     /* RFC4507 session ticket expected to be received or sent */
+     int tlsext_ticket_expected;
++
++    /* TLS Session Ticket extension override */
++    TLS_SESSION_TICKET_EXT *tlsext_session_ticket;
++
++    /* TLS Session Ticket extension callback */
++    tls_session_ticket_ext_cb_fn tls_session_ticket_ext_cb;
++    void *tls_session_ticket_ext_cb_arg;
++
++    /* TLS pre-shared secret session resumption */
++    tls_session_secret_cb_fn tls_session_secret_cb;
++    void *tls_session_secret_cb_arg;
++
+     SSL_CTX *initial_ctx;       /* initial ctx, used to store sessions */
+ #  define session_ctx initial_ctx
+ # else
+@@ -1772,6 +1791,17 @@ void *SSL_COMP_get_compression_methods(v
+ int SSL_COMP_add_compression_method(int id, void *cm);
+ # endif
+ 
++/* TLS extensions functions */
++int SSL_set_session_ticket_ext(SSL *s, void *ext_data, int ext_len);
++
++int SSL_set_session_ticket_ext_cb(SSL *s, tls_session_ticket_ext_cb_fn cb,
++				  void *arg);
++
++/* Pre-shared secret session resumption functions */
++int SSL_set_session_secret_cb(SSL *s,
++			      tls_session_secret_cb_fn tls_session_secret_cb,
++			      void *arg);
++
+ /* BEGIN ERROR CODES */
+ /*
+  * The following lines are auto generated by the script mkerr.pl. Any changes
+@@ -1977,6 +2007,7 @@ void ERR_load_SSL_strings(void);
+ # define SSL_F_TLS1_ENC                                   210
+ # define SSL_F_TLS1_SETUP_KEY_BLOCK                       211
+ # define SSL_F_WRITE_PENDING                              212
++#define SSL_F_SSL_SET_SESSION_TICKET_EXT                  213
+ 
+ /* Reason codes. */
+ # define SSL_R_APP_DATA_IN_HANDSHAKE                      100
+diff -upr openssl-0.9.8zf.orig/ssl/ssl_sess.c openssl-0.9.8zf/ssl/ssl_sess.c
+--- openssl-0.9.8zf.orig/ssl/ssl_sess.c	2015-03-19 15:46:46.000000000 +0200
++++ openssl-0.9.8zf/ssl/ssl_sess.c	2015-03-24 16:28:04.819907515 +0200
+@@ -716,6 +716,61 @@ long SSL_CTX_get_timeout(const SSL_CTX *
+     return (s->session_timeout);
+ }
+ 
++#ifndef OPENSSL_NO_TLSEXT
++int SSL_set_session_secret_cb(
++	SSL *s,
++	int (*tls_session_secret_cb)(SSL *s, void *secret, int *secret_len,
++				     STACK_OF(SSL_CIPHER) *peer_ciphers,
++				     SSL_CIPHER **cipher, void *arg), void *arg)
++{
++    if (s == NULL)
++	    return 0;
++    s->tls_session_secret_cb = tls_session_secret_cb;
++    s->tls_session_secret_cb_arg = arg;
++    return 1;
++}
++
++int SSL_set_session_ticket_ext_cb(SSL *s, tls_session_ticket_ext_cb_fn cb,
++				  void *arg)
++{
++    if (s == NULL)
++	    return 0;
++    s->tls_session_ticket_ext_cb = cb;
++    s->tls_session_ticket_ext_cb_arg = arg;
++    return 1;
++}
++
++int SSL_set_session_ticket_ext(SSL *s, void *ext_data, int ext_len)
++{
++    if (s->version >= TLS1_VERSION) {
++	if (s->tlsext_session_ticket) {
++	    OPENSSL_free(s->tlsext_session_ticket);
++	    s->tlsext_session_ticket = NULL;
++	}
++
++	s->tlsext_session_ticket = OPENSSL_malloc(
++		sizeof(TLS_SESSION_TICKET_EXT) + ext_len);
++	if (!s->tlsext_session_ticket) {
++	    SSLerr(SSL_F_SSL_SET_SESSION_TICKET_EXT, ERR_R_MALLOC_FAILURE);
++	    return 0;
++	}
++
++	if (ext_data) {
++	    s->tlsext_session_ticket->length = ext_len;
++	    s->tlsext_session_ticket->data = s->tlsext_session_ticket + 1;
++	    memcpy(s->tlsext_session_ticket->data, ext_data, ext_len);
++	} else {
++		s->tlsext_session_ticket->length = 0;
++		s->tlsext_session_ticket->data = NULL;
++	}
++
++	return 1;
++    }
++
++    return 0;
++}
++#endif /* OPENSSL_NO_TLSEXT */
++
+ typedef struct timeout_param_st {
+     SSL_CTX *ctx;
+     long time;
+diff -upr openssl-0.9.8zf.orig/ssl/t1_lib.c openssl-0.9.8zf/ssl/t1_lib.c
+--- openssl-0.9.8zf.orig/ssl/t1_lib.c	2015-03-19 15:46:46.000000000 +0200
++++ openssl-0.9.8zf/ssl/t1_lib.c	2015-03-24 16:32:46.923905254 +0200
+@@ -108,6 +108,11 @@ int tls1_new(SSL *s)
+ 
+ void tls1_free(SSL *s)
+ {
++#ifndef OPENSSL_NO_TLSEXT
++    if (s->tlsext_session_ticket) {
++	OPENSSL_free(s->tlsext_session_ticket);
++    }
++#endif
+     ssl3_free(s);
+ }
+ 
+@@ -206,8 +211,20 @@ unsigned char *ssl_add_clienthello_tlsex
+         int ticklen;
+         if (!s->new_session && s->session && s->session->tlsext_tick)
+             ticklen = s->session->tlsext_ticklen;
+-        else
++	else if (s->session && s->tlsext_session_ticket &&
++		 s->tlsext_session_ticket->data) {
++	    ticklen = s->tlsext_session_ticket->length;
++	    s->session->tlsext_tick = OPENSSL_malloc(ticklen);
++	    if (!s->session->tlsext_tick)
++		return NULL;
++	    memcpy(s->session->tlsext_tick, s->tlsext_session_ticket->data,
++		   ticklen);
++	    s->session->tlsext_ticklen = ticklen;
++	} else
+             ticklen = 0;
++	if (ticklen == 0 && s->tlsext_session_ticket &&
++	    s->tlsext_session_ticket->data == NULL)
++	    goto skip_ext;
+         /*
+          * Check for enough room 2 for extension type, 2 for len rest for
+          * ticket
+@@ -221,6 +238,7 @@ unsigned char *ssl_add_clienthello_tlsex
+             ret += ticklen;
+         }
+     }
++skip_ext:
+ 
+     if (s->tlsext_status_type == TLSEXT_STATUSTYPE_ocsp &&
+         s->version != DTLS1_VERSION) {
+@@ -560,6 +578,14 @@ int ssl_parse_clienthello_tlsext(SSL *s,
+             if (!ssl_parse_clienthello_renegotiate_ext(s, data, size, al))
+                 return 0;
+             renegotiate_seen = 1;
++	} else if (type == TLSEXT_TYPE_session_ticket) {
++	    if (s->tls_session_ticket_ext_cb &&
++		!s->tls_session_ticket_ext_cb(s, data, size,
++					      s->tls_session_ticket_ext_cb_arg))
++	    {
++		*al = TLS1_AD_INTERNAL_ERROR;
++		return 0;
++	    }
+         } else if (type == TLSEXT_TYPE_status_request &&
+                    s->version != DTLS1_VERSION && s->ctx->tlsext_status_cb) {
+ 
+@@ -710,6 +736,13 @@ int ssl_parse_serverhello_tlsext(SSL *s,
+             }
+             tlsext_servername = 1;
+         } else if (type == TLSEXT_TYPE_session_ticket) {
++	    if (s->tls_session_ticket_ext_cb &&
++		!s->tls_session_ticket_ext_cb(
++			s, data, size,
++			s->tls_session_ticket_ext_cb_arg)) {
++		*al = TLS1_AD_INTERNAL_ERROR;
++		return 0;
++	    }
+             if ((SSL_get_options(s) & SSL_OP_NO_TICKET)
+                 || (size > 0)) {
+                 *al = TLS1_AD_UNSUPPORTED_EXTENSION;
+@@ -993,6 +1026,14 @@ int tls1_process_ticket(SSL *s, unsigned
+                 s->tlsext_ticket_expected = 1;
+                 return 0;       /* Cache miss */
+             }
++	    if (s->tls_session_secret_cb) {
++		/* Indicate cache miss here and instead of
++		 * generating the session from ticket now,
++		 * trigger abbreviated handshake based on
++		 * external mechanism to calculate the master
++		 * secret later. */
++		return 0;
++	    }
+             return tls_decrypt_ticket(s, p, size, session_id, len, ret);
+         }
+         p += size;
+diff -upr openssl-0.9.8zf.orig/ssl/tls1.h openssl-0.9.8zf/ssl/tls1.h
+--- openssl-0.9.8zf.orig/ssl/tls1.h	2015-03-19 15:46:46.000000000 +0200
++++ openssl-0.9.8zf/ssl/tls1.h	2015-03-24 16:33:31.855904894 +0200
+@@ -460,6 +460,12 @@ SSL_CTX_callback_ctrl(ssl,SSL_CTRL_SET_T
+ #  define TLS_MD_MASTER_SECRET_CONST    "\x6d\x61\x73\x74\x65\x72\x20\x73\x65\x63\x72\x65\x74"
+ # endif
+ 
++/* TLS extension struct */
++struct tls_session_ticket_ext_st {
++    unsigned short length;
++    void *data;
++};
++
+ #ifdef  __cplusplus
+ }
+ #endif
+diff -upr openssl-0.9.8zf.orig/util/ssleay.num openssl-0.9.8zf/util/ssleay.num
+--- openssl-0.9.8zf.orig/util/ssleay.num	2015-03-19 15:47:15.000000000 +0200
++++ openssl-0.9.8zf/util/ssleay.num	2015-03-24 16:33:51.127904739 +0200
+@@ -242,3 +242,5 @@ SSL_set_SSL_CTX
+ SSL_get_servername                      291	EXIST::FUNCTION:TLSEXT
+ SSL_get_servername_type                 292	EXIST::FUNCTION:TLSEXT
+ SSL_CTX_set_client_cert_engine          293	EXIST::FUNCTION:ENGINE
++SSL_set_session_ticket_ext		306	EXIST::FUNCTION:TLSEXT
++SSL_set_session_secret_cb		307	EXIST::FUNCTION:TLSEXT
diff --git a/src/Makefile b/src/Makefile
index 10e0171b0c6d..c9e84c11de7a 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -1,4 +1,5 @@
 SUBDIRS=ap common crypto drivers eapol_auth eapol_supp eap_common eap_peer eap_server l2_packet p2p pae radius rsn_supp tls utils wps
+SUBDIRS += fst
 
 all:
 	for d in $(SUBDIRS); do [ -d $$d ] && $(MAKE) -C $$d; done
diff --git a/src/ap/Makefile b/src/ap/Makefile
index adfd3dfd5b9b..98788fef797e 100644
--- a/src/ap/Makefile
+++ b/src/ap/Makefile
@@ -1,8 +1,67 @@
-all:
-	@echo Nothing to be made.
+all: libap.a
 
 clean:
-	rm -f *~ *.o *.d *.gcno *.gcda *.gcov
+	rm -f *~ *.o *.d *.gcno *.gcda *.gcov libap.a
 
 install:
 	@echo Nothing to be made.
+
+include ../lib.rules
+
+CFLAGS += -DHOSTAPD
+CFLAGS += -DNEED_AP_MLME
+CFLAGS += -DCONFIG_HS20
+CFLAGS += -DCONFIG_INTERWORKING
+CFLAGS += -DCONFIG_IEEE80211R
+CFLAGS += -DCONFIG_IEEE80211W
+CFLAGS += -DCONFIG_WPS
+CFLAGS += -DCONFIG_PROXYARP
+CFLAGS += -DCONFIG_IAPP
+
+LIB_OBJS= \
+	accounting.o \
+	ap_config.o \
+	ap_drv_ops.o \
+	ap_list.o \
+	ap_mlme.o \
+	authsrv.o \
+	beacon.o \
+	bss_load.o \
+	ctrl_iface_ap.o \
+	dfs.o \
+	dhcp_snoop.o \
+	drv_callbacks.o \
+	eap_user_db.o \
+	gas_serv.o \
+	hostapd.o \
+	hs20.o \
+	hw_features.o \
+	iapp.o \
+	ieee802_11_auth.o \
+	ieee802_11.o \
+	ieee802_11_ht.o \
+	ieee802_11_shared.o \
+	ieee802_11_vht.o \
+	ieee802_1x.o \
+	ndisc_snoop.o \
+	p2p_hostapd.o \
+	peerkey_auth.o \
+	pmksa_cache_auth.o \
+	preauth_auth.o \
+	sta_info.o \
+	tkip_countermeasures.o \
+	utils.o \
+	vlan_init.o \
+	wmm.o \
+	wnm_ap.o \
+	wpa_auth.o \
+	wpa_auth_ft.o \
+	wpa_auth_glue.o \
+	wpa_auth_ie.o \
+	wps_hostapd.o \
+	x_snoop.o
+
+libap.a: $(LIB_OBJS)
+	$(AR) crT $@ $?
+
+-include $(OBJS:%.o=%.d)
diff --git a/src/ap/accounting.c b/src/ap/accounting.c
index 7c55146b2c5c..a096de4d3e51 100644
--- a/src/ap/accounting.c
+++ b/src/ap/accounting.c
@@ -459,10 +459,14 @@ int accounting_init(struct hostapd_data *hapd)
 {
 	struct os_time now;
 
-	/* Acct-Session-Id should be unique over reboots. If reliable clock is
-	 * not available, this could be replaced with reboot counter, etc. */
+	/* Acct-Session-Id should be unique over reboots. Using a random number
+	 * is preferred. If that is not available, take the current time. Mix
+	 * in microseconds to make this more likely to be unique. */
 	os_get_time(&now);
-	hapd->acct_session_id_hi = now.sec;
+	if (os_get_random((u8 *) &hapd->acct_session_id_hi,
+			  sizeof(hapd->acct_session_id_hi)) < 0)
+		hapd->acct_session_id_hi = now.sec;
+	hapd->acct_session_id_hi ^= now.usec;
 
 	if (radius_client_register(hapd->radius, RADIUS_ACCT,
 				   accounting_receive, hapd))
@@ -475,7 +479,7 @@ int accounting_init(struct hostapd_data *hapd)
 
 
 /**
- * accounting_deinit: Deinitilize accounting
+ * accounting_deinit: Deinitialize accounting
  * @hapd: hostapd BSS data
  */
 void accounting_deinit(struct hostapd_data *hapd)
diff --git a/src/ap/acs.c b/src/ap/acs.c
index ae7f6c309289..03d797fe8836 100644
--- a/src/ap/acs.c
+++ b/src/ap/acs.c
@@ -479,16 +479,10 @@ static int acs_usable_chan(struct hostapd_channel_data *chan)
 static int is_in_chanlist(struct hostapd_iface *iface,
 			  struct hostapd_channel_data *chan)
 {
-	int *entry;
-
-	if (!iface->conf->chanlist)
+	if (!iface->conf->acs_ch_list.num)
 		return 1;
 
-	for (entry = iface->conf->chanlist; *entry != -1; entry++) {
-		if (*entry == chan->chan)
-			return 1;
-	}
-	return 0;
+	return freq_range_list_includes(&iface->conf->acs_ch_list, chan->chan);
 }
 
 
@@ -900,6 +894,9 @@ static int acs_request_scan(struct hostapd_iface *iface)
 		if (chan->flag & HOSTAPD_CHAN_DISABLED)
 			continue;
 
+		if (!is_in_chanlist(iface, chan))
+			continue;
+
 		*freq++ = chan->freq;
 	}
 	*freq = 0;
diff --git a/src/ap/ap_config.c b/src/ap/ap_config.c
index 76011dc07fd4..9a96e50b7385 100644
--- a/src/ap/ap_config.c
+++ b/src/ap/ap_config.c
@@ -172,6 +172,7 @@ struct hostapd_config * hostapd_config_defaults(void)
 
 	conf->ap_table_max_size = 255;
 	conf->ap_table_expiration_time = 60;
+	conf->track_sta_max_age = 180;
 
 #ifdef CONFIG_TESTING_OPTIONS
 	conf->ignore_probe_probability = 0.0;
@@ -181,6 +182,8 @@ struct hostapd_config * hostapd_config_defaults(void)
 	conf->corrupt_gtk_rekey_mic_probability = 0.0;
 #endif /* CONFIG_TESTING_OPTIONS */
 
+	conf->acs = 0;
+	conf->acs_ch_list.num = 0;
 #ifdef CONFIG_ACS
 	conf->acs_num_scans = 5;
 #endif /* CONFIG_ACS */
@@ -559,6 +562,13 @@ void hostapd_config_free_bss(struct hostapd_bss_config *conf)
 
 	os_free(conf->server_id);
 
+#ifdef CONFIG_TESTING_OPTIONS
+	wpabuf_free(conf->own_ie_override);
+#endif /* CONFIG_TESTING_OPTIONS */
+
+	os_free(conf->no_probe_resp_if_seen_on);
+	os_free(conf->no_auth_if_seen_on);
+
 	os_free(conf);
 }
 
@@ -579,7 +589,7 @@ void hostapd_config_free(struct hostapd_config *conf)
 	os_free(conf->bss);
 	os_free(conf->supported_rates);
 	os_free(conf->basic_rates);
-	os_free(conf->chanlist);
+	os_free(conf->acs_ch_list.range);
 	os_free(conf->driver_params);
 #ifdef CONFIG_ACS
 	os_free(conf->acs_chan_bias);
@@ -817,9 +827,9 @@ static int hostapd_config_check_bss(struct hostapd_bss_config *bss,
 
 	if (full_config && bss->wps_state && bss->wpa &&
 	    (!(bss->wpa & 2) ||
-	     !(bss->rsn_pairwise & WPA_CIPHER_CCMP))) {
+	     !(bss->rsn_pairwise & (WPA_CIPHER_CCMP | WPA_CIPHER_GCMP)))) {
 		wpa_printf(MSG_INFO, "WPS: WPA/TKIP configuration without "
-			   "WPA2/CCMP forced WPS to be disabled");
+			   "WPA2/CCMP/GCMP forced WPS to be disabled");
 		bss->wps_state = 0;
 	}
 #endif /* CONFIG_WPS */
@@ -841,6 +851,29 @@ static int hostapd_config_check_bss(struct hostapd_bss_config *bss,
 }
 
 
+static int hostapd_config_check_cw(struct hostapd_config *conf, int queue)
+{
+	int tx_cwmin = conf->tx_queue[queue].cwmin;
+	int tx_cwmax = conf->tx_queue[queue].cwmax;
+	int ac_cwmin = conf->wmm_ac_params[queue].cwmin;
+	int ac_cwmax = conf->wmm_ac_params[queue].cwmax;
+
+	if (tx_cwmin > tx_cwmax) {
+		wpa_printf(MSG_ERROR,
+			   "Invalid TX queue cwMin/cwMax values. cwMin(%d) greater than cwMax(%d)",
+			   tx_cwmin, tx_cwmax);
+		return -1;
+	}
+	if (ac_cwmin > ac_cwmax) {
+		wpa_printf(MSG_ERROR,
+			   "Invalid WMM AC cwMin/cwMax values. cwMin(%d) greater than cwMax(%d)",
+			   ac_cwmin, ac_cwmax);
+		return -1;
+	}
+	return 0;
+}
+
+
 int hostapd_config_check(struct hostapd_config *conf, int full_config)
 {
 	size_t i;
@@ -870,6 +903,11 @@ int hostapd_config_check(struct hostapd_config *conf, int full_config)
 		return -1;
 	}
 
+	for (i = 0; i < NUM_TX_QUEUES; i++) {
+		if (hostapd_config_check_cw(conf, i))
+			return -1;
+	}
+
 	for (i = 0; i < conf->num_bss; i++) {
 		if (hostapd_config_check_bss(conf->bss[i], conf, full_config))
 			return -1;
@@ -937,10 +975,11 @@ void hostapd_set_security_params(struct hostapd_bss_config *bss,
 		bss->rsn_pairwise = WPA_CIPHER_CCMP;
 	} else {
 		bss->ssid.security_policy = SECURITY_PLAINTEXT;
-		bss->wpa_group = WPA_CIPHER_NONE;
-		bss->wpa_pairwise = WPA_CIPHER_NONE;
-		bss->rsn_pairwise = WPA_CIPHER_NONE;
-		if (full_config)
+		if (full_config) {
+			bss->wpa_group = WPA_CIPHER_NONE;
+			bss->wpa_pairwise = WPA_CIPHER_NONE;
+			bss->rsn_pairwise = WPA_CIPHER_NONE;
 			bss->wpa_key_mgmt = WPA_KEY_MGMT_NONE;
+		}
 	}
 }
diff --git a/src/ap/ap_config.h b/src/ap/ap_config.h
index 961d2dd389f8..de470a969b50 100644
--- a/src/ap/ap_config.h
+++ b/src/ap/ap_config.h
@@ -1,6 +1,6 @@
 /*
  * hostapd / Configuration definitions and helpers functions
- * Copyright (c) 2003-2012, Jouni Malinen <j@w1.fi>
+ * Copyright (c) 2003-2015, Jouni Malinen <j@w1.fi>
  *
  * This software may be distributed under the terms of the BSD license.
  * See README for more details.
@@ -12,8 +12,10 @@
 #include "common/defs.h"
 #include "ip_addr.h"
 #include "common/wpa_common.h"
+#include "common/ieee802_11_defs.h"
 #include "common/ieee802_11_common.h"
 #include "wps/wps.h"
+#include "fst/fst.h"
 
 /**
  * mesh_conf - local MBSS state and settings
@@ -31,8 +33,8 @@ struct mesh_conf {
 	u8 mesh_sp_id;
 	/* Authentication Protocol Identifier */
 	u8 mesh_auth_id;
-	u8 *ies;
-	int ie_len;
+	u8 *rsn_ie;
+	int rsn_ie_len;
 #define MESH_CONF_SEC_NONE BIT(0)
 #define MESH_CONF_SEC_AUTH BIT(1)
 #define MESH_CONF_SEC_AMPE BIT(2)
@@ -57,8 +59,6 @@ struct hostapd_radius_servers;
 struct ft_remote_r0kh;
 struct ft_remote_r1kh;
 
-#define HOSTAPD_MAX_SSID_LEN 32
-
 #define NUM_WEP_KEYS 4
 struct hostapd_wep_keys {
 	u8 idx;
@@ -78,7 +78,7 @@ typedef enum hostap_security_policy {
 } secpolicy;
 
 struct hostapd_ssid {
-	u8 ssid[HOSTAPD_MAX_SSID_LEN];
+	u8 ssid[SSID_MAX_LEN];
 	size_t ssid_len;
 	unsigned int ssid_set:1;
 	unsigned int utf8_ssid:1;
@@ -114,12 +114,10 @@ struct hostapd_vlan {
 	struct hostapd_vlan *next;
 	int vlan_id; /* VLAN ID or -1 (VLAN_ID_WILDCARD) for wildcard entry */
 	char ifname[IFNAMSIZ + 1];
+	int configured;
 	int dynamic_vlan;
 #ifdef CONFIG_FULL_DYNAMIC_VLAN
 
-#define DVLAN_CLEAN_BR 	0x1
-#define DVLAN_CLEAN_VLAN	0x2
-#define DVLAN_CLEAN_VLAN_PORT	0x4
 #define DVLAN_CLEAN_WLAN_PORT	0x8
 	int clean;
 #endif /* CONFIG_FULL_DYNAMIC_VLAN */
@@ -332,6 +330,7 @@ struct hostapd_bss_config {
 	char *private_key;
 	char *private_key_passwd;
 	int check_crl;
+	unsigned int tls_session_lifetime;
 	char *ocsp_stapling_response;
 	char *dh_file;
 	char *openssl_ciphers;
@@ -490,6 +489,7 @@ struct hostapd_bss_config {
 
 	int osen;
 	int proxy_arp;
+	int na_mcast_to_ucast;
 #ifdef CONFIG_HS20
 	int hs20;
 	int disable_dgaf;
@@ -510,7 +510,7 @@ struct hostapd_bss_config {
 		char file[256];
 	} *hs20_icons;
 	size_t hs20_icons_count;
-	u8 osu_ssid[HOSTAPD_MAX_SSID_LEN];
+	u8 osu_ssid[SSID_MAX_LEN];
 	size_t osu_ssid_len;
 	struct hs20_osu_provider {
 		unsigned int friendly_name_count;
@@ -545,6 +545,7 @@ struct hostapd_bss_config {
 #ifdef CONFIG_TESTING_OPTIONS
 	u8 bss_load_test[5];
 	u8 bss_load_test_set;
+	struct wpabuf *own_ie_override;
 #endif /* CONFIG_TESTING_OPTIONS */
 
 #define MESH_ENABLED BIT(0)
@@ -553,6 +554,9 @@ struct hostapd_bss_config {
 	int radio_measurements;
 
 	int vendor_vht;
+
+	char *no_probe_resp_if_seen_on;
+	char *no_auth_if_seen_on;
 };
 
 
@@ -568,7 +572,8 @@ struct hostapd_config {
 	int fragm_threshold;
 	u8 send_probe_response;
 	u8 channel;
-	int *chanlist;
+	u8 acs;
+	struct wpa_freq_range_list acs_ch_list;
 	enum hostapd_hw_mode hw_mode; /* HOSTAPD_MODE_IEEE80211A, .. */
 	enum {
 		LONG_PREAMBLE = 0,
@@ -584,6 +589,9 @@ struct hostapd_config {
 	int ap_table_max_size;
 	int ap_table_expiration_time;
 
+	unsigned int track_sta_max_num;
+	unsigned int track_sta_max_age;
+
 	char country[3]; /* first two octets: country code as described in
 			  * ISO/IEC 3166-1. Third octet:
 			  * ' ' (ascii 32): all environments
@@ -620,6 +628,7 @@ struct hostapd_config {
 	u16 ht_capab;
 	int ieee80211n;
 	int secondary_channel;
+	int no_pri_sec_switch;
 	int require_ht;
 	int obss_interval;
 	u32 vht_capab;
@@ -629,6 +638,10 @@ struct hostapd_config {
 	u8 vht_oper_centr_freq_seg0_idx;
 	u8 vht_oper_centr_freq_seg1_idx;
 
+#ifdef CONFIG_FST
+	struct fst_iface_cfg fst_cfg;
+#endif /* CONFIG_FST */
+
 #ifdef CONFIG_P2P
 	u8 p2p_go_ctwindow;
 #endif /* CONFIG_P2P */
diff --git a/src/ap/ap_drv_ops.c b/src/ap/ap_drv_ops.c
index e16306c4e106..6cafcb749351 100644
--- a/src/ap/ap_drv_ops.c
+++ b/src/ap/ap_drv_ops.c
@@ -81,6 +81,22 @@ int hostapd_build_ap_extra_ies(struct hostapd_data *hapd,
 		wpabuf_put_data(proberesp, buf, pos - buf);
 	}
 
+#ifdef CONFIG_FST
+	if (hapd->iface->fst_ies) {
+		size_t add = wpabuf_len(hapd->iface->fst_ies);
+
+		if (wpabuf_resize(&beacon, add) < 0)
+			goto fail;
+		wpabuf_put_buf(beacon, hapd->iface->fst_ies);
+		if (wpabuf_resize(&proberesp, add) < 0)
+			goto fail;
+		wpabuf_put_buf(proberesp, hapd->iface->fst_ies);
+		if (wpabuf_resize(&assocresp, add) < 0)
+			goto fail;
+		wpabuf_put_buf(assocresp, hapd->iface->fst_ies);
+	}
+#endif /* CONFIG_FST */
+
 	if (hapd->wps_beacon_ie) {
 		if (wpabuf_resize(&beacon, wpabuf_len(hapd->wps_beacon_ie)) <
 		    0)
@@ -217,6 +233,15 @@ void hostapd_free_ap_extra_ies(struct hostapd_data *hapd,
 }
 
 
+int hostapd_reset_ap_wps_ie(struct hostapd_data *hapd)
+{
+	if (hapd->driver == NULL || hapd->driver->set_ap_wps_ie == NULL)
+		return 0;
+
+	return hapd->driver->set_ap_wps_ie(hapd->drv_priv, NULL, NULL, NULL);
+}
+
+
 int hostapd_set_ap_wps_ie(struct hostapd_data *hapd)
 {
 	struct wpabuf *beacon, *proberesp, *assocresp;
@@ -281,8 +306,14 @@ int hostapd_set_drv_ieee8021x(struct hostapd_data *hapd, const char *ifname,
 		params.wpa = hapd->conf->wpa;
 		params.ieee802_1x = hapd->conf->ieee802_1x;
 		params.wpa_group = hapd->conf->wpa_group;
-		params.wpa_pairwise = hapd->conf->wpa_pairwise |
-			hapd->conf->rsn_pairwise;
+		if ((hapd->conf->wpa & (WPA_PROTO_WPA | WPA_PROTO_RSN)) ==
+		    (WPA_PROTO_WPA | WPA_PROTO_RSN))
+			params.wpa_pairwise = hapd->conf->wpa_pairwise |
+				hapd->conf->rsn_pairwise;
+		else if (hapd->conf->wpa & WPA_PROTO_RSN)
+			params.wpa_pairwise = hapd->conf->rsn_pairwise;
+		else if (hapd->conf->wpa & WPA_PROTO_WPA)
+			params.wpa_pairwise = hapd->conf->wpa_pairwise;
 		params.wpa_key_mgmt = hapd->conf->wpa_key_mgmt;
 		params.rsn_preauth = hapd->conf->rsn_preauth;
 #ifdef CONFIG_IEEE80211W
@@ -618,7 +649,7 @@ int hostapd_drv_send_mlme(struct hostapd_data *hapd,
 {
 	if (hapd->driver == NULL || hapd->driver->send_mlme == NULL)
 		return 0;
-	return hapd->driver->send_mlme(hapd->drv_priv, msg, len, noack);
+	return hapd->driver->send_mlme(hapd->drv_priv, msg, len, noack, 0);
 }
 
 
@@ -712,16 +743,100 @@ int hostapd_drv_set_qos_map(struct hostapd_data *hapd,
 }
 
 
+static void hostapd_get_hw_mode_any_channels(struct hostapd_data *hapd,
+					     struct hostapd_hw_modes *mode,
+					     int acs_ch_list_all,
+					     int **freq_list)
+{
+	int i;
+
+	for (i = 0; i < mode->num_channels; i++) {
+		struct hostapd_channel_data *chan = &mode->channels[i];
+
+		if ((acs_ch_list_all ||
+		     freq_range_list_includes(&hapd->iface->conf->acs_ch_list,
+					      chan->chan)) &&
+		    !(chan->flag & HOSTAPD_CHAN_DISABLED))
+			int_array_add_unique(freq_list, chan->freq);
+	}
+}
+
+
 int hostapd_drv_do_acs(struct hostapd_data *hapd)
 {
 	struct drv_acs_params params;
+	int ret, i, acs_ch_list_all = 0;
+	u8 *channels = NULL;
+	unsigned int num_channels = 0;
+	struct hostapd_hw_modes *mode;
+	int *freq_list = NULL;
 
 	if (hapd->driver == NULL || hapd->driver->do_acs == NULL)
 		return 0;
+
 	os_memset(&params, 0, sizeof(params));
 	params.hw_mode = hapd->iface->conf->hw_mode;
+
+	/*
+	 * If no chanlist config parameter is provided, include all enabled
+	 * channels of the selected hw_mode.
+	 */
+	if (!hapd->iface->conf->acs_ch_list.num)
+		acs_ch_list_all = 1;
+
+	mode = hapd->iface->current_mode;
+	if (mode) {
+		channels = os_malloc(mode->num_channels);
+		if (channels == NULL)
+			return -1;
+
+		for (i = 0; i < mode->num_channels; i++) {
+			struct hostapd_channel_data *chan = &mode->channels[i];
+			if (!acs_ch_list_all &&
+			    !freq_range_list_includes(
+				    &hapd->iface->conf->acs_ch_list,
+				    chan->chan))
+				continue;
+			if (!(chan->flag & HOSTAPD_CHAN_DISABLED)) {
+				channels[num_channels++] = chan->chan;
+				int_array_add_unique(&freq_list, chan->freq);
+			}
+		}
+	} else {
+		for (i = 0; i < hapd->iface->num_hw_features; i++) {
+			mode = &hapd->iface->hw_features[i];
+			hostapd_get_hw_mode_any_channels(hapd, mode,
+							 acs_ch_list_all,
+							 &freq_list);
+		}
+	}
+
+	params.ch_list = channels;
+	params.ch_list_len = num_channels;
+	params.freq_list = freq_list;
+
 	params.ht_enabled = !!(hapd->iface->conf->ieee80211n);
 	params.ht40_enabled = !!(hapd->iface->conf->ht_capab &
 				 HT_CAP_INFO_SUPP_CHANNEL_WIDTH_SET);
-	return hapd->driver->do_acs(hapd->drv_priv, &params);
+	params.vht_enabled = !!(hapd->iface->conf->ieee80211ac);
+	params.ch_width = 20;
+	if (hapd->iface->conf->ieee80211n && params.ht40_enabled)
+		params.ch_width = 40;
+
+	/* Note: VHT20 is defined by combination of ht_capab & vht_oper_chwidth
+	 */
+	if (hapd->iface->conf->ieee80211ac && params.ht40_enabled) {
+		if (hapd->iface->conf->vht_oper_chwidth == VHT_CHANWIDTH_80MHZ)
+			params.ch_width = 80;
+		else if (hapd->iface->conf->vht_oper_chwidth ==
+			 VHT_CHANWIDTH_160MHZ ||
+			 hapd->iface->conf->vht_oper_chwidth ==
+			 VHT_CHANWIDTH_80P80MHZ)
+			params.ch_width = 160;
+	}
+
+	ret = hapd->driver->do_acs(hapd->drv_priv, &params);
+	os_free(channels);
+
+	return ret;
 }
diff --git a/src/ap/ap_drv_ops.h b/src/ap/ap_drv_ops.h
index 5d07e71f1bf1..82eaf3f08bb5 100644
--- a/src/ap/ap_drv_ops.h
+++ b/src/ap/ap_drv_ops.h
@@ -24,6 +24,7 @@ int hostapd_build_ap_extra_ies(struct hostapd_data *hapd,
 void hostapd_free_ap_extra_ies(struct hostapd_data *hapd, struct wpabuf *beacon,
 			       struct wpabuf *proberesp,
 			       struct wpabuf *assocresp);
+int hostapd_reset_ap_wps_ie(struct hostapd_data *hapd);
 int hostapd_set_ap_wps_ie(struct hostapd_data *hapd);
 int hostapd_set_authorized(struct hostapd_data *hapd,
 			   struct sta_info *sta, int authorized);
diff --git a/src/ap/ap_list.c b/src/ap/ap_list.c
index 04a56a95efd9..8bf6ddec8d37 100644
--- a/src/ap/ap_list.c
+++ b/src/ap/ap_list.c
@@ -193,14 +193,14 @@ void ap_list_process_beacon(struct hostapd_iface *iface,
 			  elems->supp_rates, elems->supp_rates_len,
 			  elems->ext_supp_rates, elems->ext_supp_rates_len);
 
-	if (elems->erp_info && elems->erp_info_len == 1)
+	if (elems->erp_info)
 		ap->erp = elems->erp_info[0];
 	else
 		ap->erp = -1;
 
-	if (elems->ds_params && elems->ds_params_len == 1)
+	if (elems->ds_params)
 		ap->channel = elems->ds_params[0];
-	else if (elems->ht_operation && elems->ht_operation_len >= 1)
+	else if (elems->ht_operation)
 		ap->channel = elems->ht_operation[0];
 	else if (fi)
 		ap->channel = fi->channel;
@@ -248,15 +248,12 @@ void ap_list_process_beacon(struct hostapd_iface *iface,
 }
 
 
-static void ap_list_timer(void *eloop_ctx, void *timeout_ctx)
+void ap_list_timer(struct hostapd_iface *iface)
 {
-	struct hostapd_iface *iface = eloop_ctx;
 	struct os_reltime now;
 	struct ap_info *ap;
 	int set_beacon = 0;
 
-	eloop_register_timeout(10, 0, ap_list_timer, iface, NULL);
-
 	if (!iface->ap_list)
 		return;
 
@@ -305,13 +302,11 @@ static void ap_list_timer(void *eloop_ctx, void *timeout_ctx)
 
 int ap_list_init(struct hostapd_iface *iface)
 {
-	eloop_register_timeout(10, 0, ap_list_timer, iface, NULL);
 	return 0;
 }
 
 
 void ap_list_deinit(struct hostapd_iface *iface)
 {
-	eloop_cancel_timeout(ap_list_timer, iface, NULL);
 	hostapd_free_aps(iface);
 }
diff --git a/src/ap/ap_list.h b/src/ap/ap_list.h
index 93dc0eda88d3..9e0353cfec95 100644
--- a/src/ap/ap_list.h
+++ b/src/ap/ap_list.h
@@ -39,6 +39,7 @@ void ap_list_process_beacon(struct hostapd_iface *iface,
 #ifdef NEED_AP_MLME
 int ap_list_init(struct hostapd_iface *iface);
 void ap_list_deinit(struct hostapd_iface *iface);
+void ap_list_timer(struct hostapd_iface *iface);
 #else /* NEED_AP_MLME */
 static inline int ap_list_init(struct hostapd_iface *iface)
 {
@@ -48,6 +49,10 @@ static inline int ap_list_init(struct hostapd_iface *iface)
 static inline void ap_list_deinit(struct hostapd_iface *iface)
 {
 }
+
+static inline void ap_list_timer(struct hostapd_iface *iface)
+{
+}
 #endif /* NEED_AP_MLME */
 
 #endif /* AP_LIST_H */
diff --git a/src/ap/authsrv.c b/src/ap/authsrv.c
index bd1778e41865..934dcfc8d631 100644
--- a/src/ap/authsrv.c
+++ b/src/ap/authsrv.c
@@ -55,10 +55,11 @@ static int hostapd_radius_get_eap_user(void *ctx, const u8 *identity,
 {
 	const struct hostapd_eap_user *eap_user;
 	int i;
+	int rv = -1;
 
 	eap_user = hostapd_get_eap_user(ctx, identity, identity_len, phase2);
 	if (eap_user == NULL)
-		return -1;
+		goto out;
 
 	if (user == NULL)
 		return 0;
@@ -72,7 +73,7 @@ static int hostapd_radius_get_eap_user(void *ctx, const u8 *identity,
 	if (eap_user->password) {
 		user->password = os_malloc(eap_user->password_len);
 		if (user->password == NULL)
-			return -1;
+			goto out;
 		os_memcpy(user->password, eap_user->password,
 			  eap_user->password_len);
 		user->password_len = eap_user->password_len;
@@ -83,8 +84,13 @@ static int hostapd_radius_get_eap_user(void *ctx, const u8 *identity,
 	user->ttls_auth = eap_user->ttls_auth;
 	user->remediation = eap_user->remediation;
 	user->accept_attr = eap_user->accept_attr;
+	rv = 0;
 
-	return 0;
+out:
+	if (rv)
+		wpa_printf(MSG_DEBUG, "%s: Failed to find user", __func__);
+
+	return rv;
 }
 
 
@@ -126,6 +132,7 @@ static int hostapd_setup_radius_srv(struct hostapd_data *hapd)
 #endif /* CONFIG_HS20 */
 	srv.erp = conf->eap_server_erp;
 	srv.erp_domain = conf->erp_domain;
+	srv.tls_session_lifetime = conf->tls_session_lifetime;
 
 	hapd->radius_srv = radius_server_init(&srv);
 	if (hapd->radius_srv == NULL) {
@@ -145,9 +152,12 @@ int authsrv_init(struct hostapd_data *hapd)
 	if (hapd->conf->eap_server &&
 	    (hapd->conf->ca_cert || hapd->conf->server_cert ||
 	     hapd->conf->private_key || hapd->conf->dh_file)) {
+		struct tls_config conf;
 		struct tls_connection_params params;
 
-		hapd->ssl_ctx = tls_init(NULL);
+		os_memset(&conf, 0, sizeof(conf));
+		conf.tls_session_lifetime = hapd->conf->tls_session_lifetime;
+		hapd->ssl_ctx = tls_init(&conf);
 		if (hapd->ssl_ctx == NULL) {
 			wpa_printf(MSG_ERROR, "Failed to initialize TLS");
 			authsrv_deinit(hapd);
diff --git a/src/ap/beacon.c b/src/ap/beacon.c
index e575b65cbf3a..5fe8fd5660b4 100644
--- a/src/ap/beacon.c
+++ b/src/ap/beacon.c
@@ -360,7 +360,6 @@ static u8 * hostapd_add_csa_elems(struct hostapd_data *hapd, u8 *pos,
 
 
 static u8 * hostapd_gen_probe_resp(struct hostapd_data *hapd,
-				   struct sta_info *sta,
 				   const struct ieee80211_mgmt *req,
 				   int is_p2p, size_t *resp_len)
 {
@@ -378,6 +377,10 @@ static u8 * hostapd_gen_probe_resp(struct hostapd_data *hapd,
 	if (hapd->p2p_probe_resp_ie)
 		buflen += wpabuf_len(hapd->p2p_probe_resp_ie);
 #endif /* CONFIG_P2P */
+#ifdef CONFIG_FST
+	if (hapd->iface->fst_ies)
+		buflen += wpabuf_len(hapd->iface->fst_ies);
+#endif /* CONFIG_FST */
 	if (hapd->conf->vendor_elements)
 		buflen += wpabuf_len(hapd->conf->vendor_elements);
 	if (hapd->conf->vendor_vht) {
@@ -402,7 +405,7 @@ static u8 * hostapd_gen_probe_resp(struct hostapd_data *hapd,
 
 	/* hardware or low-level driver will setup seq_ctrl and timestamp */
 	resp->u.probe_resp.capab_info =
-		host_to_le16(hostapd_own_capab_info(hapd, sta, 1));
+		host_to_le16(hostapd_own_capab_info(hapd));
 
 	pos = resp->u.probe_resp.variable;
 	*pos++ = WLAN_EID_SSID;
@@ -450,6 +453,15 @@ static u8 * hostapd_gen_probe_resp(struct hostapd_data *hapd,
 
 	pos = hostapd_add_csa_elems(hapd, pos, (u8 *)resp,
 				    &hapd->cs_c_off_proberesp);
+
+#ifdef CONFIG_FST
+	if (hapd->iface->fst_ies) {
+		os_memcpy(pos, wpabuf_head(hapd->iface->fst_ies),
+			  wpabuf_len(hapd->iface->fst_ies));
+		pos += wpabuf_len(hapd->iface->fst_ies);
+	}
+#endif /* CONFIG_FST */
+
 #ifdef CONFIG_IEEE80211AC
 	if (hapd->iconf->ieee80211ac && !hapd->conf->disable_11ac) {
 		pos = hostapd_eid_vht_capabilities(hapd, pos);
@@ -540,6 +552,102 @@ static enum ssid_match_result ssid_match(struct hostapd_data *hapd,
 }
 
 
+void sta_track_expire(struct hostapd_iface *iface, int force)
+{
+	struct os_reltime now;
+	struct hostapd_sta_info *info;
+
+	if (!iface->num_sta_seen)
+		return;
+
+	os_get_reltime(&now);
+	while ((info = dl_list_first(&iface->sta_seen, struct hostapd_sta_info,
+				     list))) {
+		if (!force &&
+		    !os_reltime_expired(&now, &info->last_seen,
+					iface->conf->track_sta_max_age))
+			break;
+		force = 0;
+
+		wpa_printf(MSG_MSGDUMP, "%s: Expire STA tracking entry for "
+			   MACSTR, iface->bss[0]->conf->iface,
+			   MAC2STR(info->addr));
+		dl_list_del(&info->list);
+		iface->num_sta_seen--;
+		os_free(info);
+	}
+}
+
+
+static struct hostapd_sta_info * sta_track_get(struct hostapd_iface *iface,
+					       const u8 *addr)
+{
+	struct hostapd_sta_info *info;
+
+	dl_list_for_each(info, &iface->sta_seen, struct hostapd_sta_info, list)
+		if (os_memcmp(addr, info->addr, ETH_ALEN) == 0)
+			return info;
+
+	return NULL;
+}
+
+
+void sta_track_add(struct hostapd_iface *iface, const u8 *addr)
+{
+	struct hostapd_sta_info *info;
+
+	info = sta_track_get(iface, addr);
+	if (info) {
+		/* Move the most recent entry to the end of the list */
+		dl_list_del(&info->list);
+		dl_list_add_tail(&iface->sta_seen, &info->list);
+		os_get_reltime(&info->last_seen);
+		return;
+	}
+
+	/* Add a new entry */
+	info = os_zalloc(sizeof(*info));
+	os_memcpy(info->addr, addr, ETH_ALEN);
+	os_get_reltime(&info->last_seen);
+
+	if (iface->num_sta_seen >= iface->conf->track_sta_max_num) {
+		/* Expire oldest entry to make room for a new one */
+		sta_track_expire(iface, 1);
+	}
+
+	wpa_printf(MSG_MSGDUMP, "%s: Add STA tracking entry for "
+		   MACSTR, iface->bss[0]->conf->iface, MAC2STR(addr));
+	dl_list_add_tail(&iface->sta_seen, &info->list);
+	iface->num_sta_seen++;
+}
+
+
+struct hostapd_data *
+sta_track_seen_on(struct hostapd_iface *iface, const u8 *addr,
+		  const char *ifname)
+{
+	struct hapd_interfaces *interfaces = iface->interfaces;
+	size_t i, j;
+
+	for (i = 0; i < interfaces->count; i++) {
+		struct hostapd_data *hapd = NULL;
+
+		iface = interfaces->iface[i];
+		for (j = 0; j < iface->num_bss; j++) {
+			hapd = iface->bss[j];
+			if (os_strcmp(ifname, hapd->conf->iface) == 0)
+				break;
+			hapd = NULL;
+		}
+
+		if (hapd && sta_track_get(iface, addr))
+			return hapd;
+	}
+
+	return NULL;
+}
+
+
 void handle_probe_req(struct hostapd_data *hapd,
 		      const struct ieee80211_mgmt *mgmt, size_t len,
 		      int ssi_signal)
@@ -548,7 +656,6 @@ void handle_probe_req(struct hostapd_data *hapd,
 	struct ieee802_11_elems elems;
 	const u8 *ie;
 	size_t ie_len;
-	struct sta_info *sta = NULL;
 	size_t i, resp_len;
 	int noack;
 	enum ssid_match_result res;
@@ -556,6 +663,8 @@ void handle_probe_req(struct hostapd_data *hapd,
 	ie = mgmt->u.probe_req.variable;
 	if (len < IEEE80211_HDRLEN + sizeof(mgmt->u.probe_req))
 		return;
+	if (hapd->iconf->track_sta_max_num)
+		sta_track_add(hapd->iface, mgmt->sa);
 	ie_len = len - (IEEE80211_HDRLEN + sizeof(mgmt->u.probe_req));
 
 	for (i = 0; hapd->probereq_cb && i < hapd->num_probereq_cb; i++)
@@ -590,7 +699,7 @@ void handle_probe_req(struct hostapd_data *hapd,
 	 * is less likely to see them (Probe Request frame sent on a
 	 * neighboring, but partially overlapping, channel).
 	 */
-	if (elems.ds_params && elems.ds_params_len == 1 &&
+	if (elems.ds_params &&
 	    hapd->iface->current_mode &&
 	    (hapd->iface->current_mode->mode == HOSTAPD_MODE_IEEE80211G ||
 	     hapd->iface->current_mode->mode == HOSTAPD_MODE_IEEE80211B) &&
@@ -635,8 +744,6 @@ void handle_probe_req(struct hostapd_data *hapd,
 		return;
 	}
 
-	sta = ap_get_sta(hapd, mgmt->sa);
-
 #ifdef CONFIG_P2P
 	if ((hapd->conf->p2p & P2P_GROUP_OWNER) &&
 	    elems.ssid_len == P2P_WILDCARD_SSID_LEN &&
@@ -649,10 +756,7 @@ void handle_probe_req(struct hostapd_data *hapd,
 
 	res = ssid_match(hapd, elems.ssid, elems.ssid_len,
 			 elems.ssid_list, elems.ssid_list_len);
-	if (res != NO_SSID_MATCH) {
-		if (sta)
-			sta->ssid_probe = &hapd->conf->ssid;
-	} else {
+	if (res == NO_SSID_MATCH) {
 		if (!(mgmt->da[0] & 0x01)) {
 			wpa_printf(MSG_MSGDUMP, "Probe Request from " MACSTR
 				   " for foreign SSID '%s' (DA " MACSTR ")%s",
@@ -709,6 +813,18 @@ void handle_probe_req(struct hostapd_data *hapd,
 	/* TODO: verify that supp_rates contains at least one matching rate
 	 * with AP configuration */
 
+	if (hapd->conf->no_probe_resp_if_seen_on &&
+	    is_multicast_ether_addr(mgmt->da) &&
+	    is_multicast_ether_addr(mgmt->bssid) &&
+	    sta_track_seen_on(hapd->iface, mgmt->sa,
+			      hapd->conf->no_probe_resp_if_seen_on)) {
+		wpa_printf(MSG_MSGDUMP, "%s: Ignore Probe Request from " MACSTR
+			   " since STA has been seen on %s",
+			   hapd->conf->iface, MAC2STR(mgmt->sa),
+			   hapd->conf->no_probe_resp_if_seen_on);
+		return;
+	}
+
 #ifdef CONFIG_TESTING_OPTIONS
 	if (hapd->iconf->ignore_probe_probability > 0.0 &&
 	    drand48() < hapd->iconf->ignore_probe_probability) {
@@ -719,7 +835,7 @@ void handle_probe_req(struct hostapd_data *hapd,
 	}
 #endif /* CONFIG_TESTING_OPTIONS */
 
-	resp = hostapd_gen_probe_resp(hapd, sta, mgmt, elems.p2p != NULL,
+	resp = hostapd_gen_probe_resp(hapd, mgmt, elems.p2p != NULL,
 				      &resp_len);
 	if (resp == NULL)
 		return;
@@ -774,7 +890,7 @@ static u8 * hostapd_probe_resp_offloads(struct hostapd_data *hapd,
 			   "this");
 
 	/* Generate a Probe Response template for the non-P2P case */
-	return hostapd_gen_probe_resp(hapd, NULL, NULL, 0, resp_len);
+	return hostapd_gen_probe_resp(hapd, NULL, 0, resp_len);
 }
 
 #endif /* NEED_AP_MLME */
@@ -804,6 +920,10 @@ int ieee802_11_build_ap_params(struct hostapd_data *hapd,
 	if (hapd->p2p_beacon_ie)
 		tail_len += wpabuf_len(hapd->p2p_beacon_ie);
 #endif /* CONFIG_P2P */
+#ifdef CONFIG_FST
+	if (hapd->iface->fst_ies)
+		tail_len += wpabuf_len(hapd->iface->fst_ies);
+#endif /* CONFIG_FST */
 	if (hapd->conf->vendor_elements)
 		tail_len += wpabuf_len(hapd->conf->vendor_elements);
 
@@ -833,7 +953,7 @@ int ieee802_11_build_ap_params(struct hostapd_data *hapd,
 		host_to_le16(hapd->iconf->beacon_int);
 
 	/* hardware or low-level driver will setup seq_ctrl and timestamp */
-	capab_info = hostapd_own_capab_info(hapd, NULL, 0);
+	capab_info = hostapd_own_capab_info(hapd);
 	head->u.beacon.capab_info = host_to_le16(capab_info);
 	pos = &head->u.beacon.variable[0];
 
@@ -902,6 +1022,15 @@ int ieee802_11_build_ap_params(struct hostapd_data *hapd,
 	tailpos = hostapd_eid_roaming_consortium(hapd, tailpos);
 	tailpos = hostapd_add_csa_elems(hapd, tailpos, tail,
 					&hapd->cs_c_off_beacon);
+
+#ifdef CONFIG_FST
+	if (hapd->iface->fst_ies) {
+		os_memcpy(tailpos, wpabuf_head(hapd->iface->fst_ies),
+			  wpabuf_len(hapd->iface->fst_ies));
+		tailpos += wpabuf_len(hapd->iface->fst_ies);
+	}
+#endif /* CONFIG_FST */
+
 #ifdef CONFIG_IEEE80211AC
 	if (hapd->iconf->ieee80211ac && !hapd->conf->disable_11ac) {
 		tailpos = hostapd_eid_vht_capabilities(hapd, tailpos);
@@ -963,8 +1092,14 @@ int ieee802_11_build_ap_params(struct hostapd_data *hapd,
 	params->basic_rates = hapd->iface->basic_rates;
 	params->ssid = hapd->conf->ssid.ssid;
 	params->ssid_len = hapd->conf->ssid.ssid_len;
-	params->pairwise_ciphers = hapd->conf->wpa_pairwise |
-		hapd->conf->rsn_pairwise;
+	if ((hapd->conf->wpa & (WPA_PROTO_WPA | WPA_PROTO_RSN)) ==
+	    (WPA_PROTO_WPA | WPA_PROTO_RSN))
+		params->pairwise_ciphers = hapd->conf->wpa_pairwise |
+			hapd->conf->rsn_pairwise;
+	else if (hapd->conf->wpa & WPA_PROTO_RSN)
+		params->pairwise_ciphers = hapd->conf->rsn_pairwise;
+	else if (hapd->conf->wpa & WPA_PROTO_WPA)
+		params->pairwise_ciphers = hapd->conf->wpa_pairwise;
 	params->group_cipher = hapd->conf->wpa_group;
 	params->key_mgmt_suites = hapd->conf->wpa_key_mgmt;
 	params->auth_algs = hapd->conf->auth_algs;
diff --git a/src/ap/beacon.h b/src/ap/beacon.h
index 722159a7500c..d98f42e8157a 100644
--- a/src/ap/beacon.h
+++ b/src/ap/beacon.h
@@ -21,5 +21,10 @@ int ieee802_11_update_beacons(struct hostapd_iface *iface);
 int ieee802_11_build_ap_params(struct hostapd_data *hapd,
 			       struct wpa_driver_ap_params *params);
 void ieee802_11_free_ap_params(struct wpa_driver_ap_params *params);
+void sta_track_add(struct hostapd_iface *iface, const u8 *addr);
+void sta_track_expire(struct hostapd_iface *iface, int force);
+struct hostapd_data *
+sta_track_seen_on(struct hostapd_iface *iface, const u8 *addr,
+		  const char *ifname);
 
 #endif /* BEACON_H */
diff --git a/src/ap/ctrl_iface_ap.c b/src/ap/ctrl_iface_ap.c
index 41ab988277bb..c98978f33d05 100644
--- a/src/ap/ctrl_iface_ap.c
+++ b/src/ap/ctrl_iface_ap.c
@@ -12,6 +12,7 @@
 #include "common/ieee802_11_defs.h"
 #include "common/sae.h"
 #include "eapol_auth/eapol_auth_sm.h"
+#include "fst/fst_ctrl_iface.h"
 #include "hostapd.h"
 #include "ieee802_1x.h"
 #include "wpa_auth.h"
@@ -153,6 +154,13 @@ static int hostapd_ctrl_iface_sta_mib(struct hostapd_data *hapd,
 	}
 #endif /* CONFIG_SAE */
 
+	if (sta->vlan_id > 0) {
+		res = os_snprintf(buf + len, buflen - len, "vlan_id=%d\n",
+				  sta->vlan_id);
+		if (!os_snprintf_error(buflen - len, res))
+			len += res;
+	}
+
 	return len;
 }
 
@@ -199,7 +207,10 @@ int hostapd_ctrl_iface_sta(struct hostapd_data *hapd, const char *txtaddr,
 		return -1;
 	}
 
-	return hostapd_ctrl_iface_sta_mib(hapd, sta, buf, buflen);
+	ret = hostapd_ctrl_iface_sta_mib(hapd, sta, buf, buflen);
+	ret += fst_ctrl_iface_mb_info(addr, buf + ret, buflen - ret);
+
+	return ret;
 }
 
 
diff --git a/src/ap/dfs.c b/src/ap/dfs.c
index da6fd4646710..715f19b6ac7b 100644
--- a/src/ap/dfs.c
+++ b/src/ap/dfs.c
@@ -122,6 +122,20 @@ static int dfs_is_chan_allowed(struct hostapd_channel_data *chan, int n_chans)
 }
 
 
+static struct hostapd_channel_data *
+dfs_get_chan_data(struct hostapd_hw_modes *mode, int freq, int first_chan_idx)
+{
+	int i;
+
+	for (i = first_chan_idx; i < mode->num_channels; i++) {
+		if (mode->channels[i].freq == freq)
+			return &mode->channels[i];
+	}
+
+	return NULL;
+}
+
+
 static int dfs_chan_range_available(struct hostapd_hw_modes *mode,
 				    int first_chan_idx, int num_chans,
 				    int skip_radar)
@@ -129,15 +143,15 @@ static int dfs_chan_range_available(struct hostapd_hw_modes *mode,
 	struct hostapd_channel_data *first_chan, *chan;
 	int i;
 
-	if (first_chan_idx + num_chans >= mode->num_channels)
+	if (first_chan_idx + num_chans > mode->num_channels)
 		return 0;
 
 	first_chan = &mode->channels[first_chan_idx];
 
 	for (i = 0; i < num_chans; i++) {
-		chan = &mode->channels[first_chan_idx + i];
-
-		if (first_chan->freq + i * 20 != chan->freq)
+		chan = dfs_get_chan_data(mode, first_chan->freq + i * 20,
+					 first_chan_idx);
+		if (!chan)
 			return 0;
 
 		if (!dfs_channel_available(chan, skip_radar))
@@ -151,16 +165,10 @@ static int dfs_chan_range_available(struct hostapd_hw_modes *mode,
 static int is_in_chanlist(struct hostapd_iface *iface,
 			  struct hostapd_channel_data *chan)
 {
-	int *entry;
-
-	if (!iface->conf->chanlist)
+	if (!iface->conf->acs_ch_list.num)
 		return 1;
 
-	for (entry = iface->conf->chanlist; *entry != -1; entry++) {
-		if (*entry == chan->chan)
-			return 1;
-	}
-	return 0;
+	return freq_range_list_includes(&iface->conf->acs_ch_list, chan->chan);
 }
 
 
diff --git a/src/ap/drv_callbacks.c b/src/ap/drv_callbacks.c
index a0adc67db430..ca8b75c83906 100644
--- a/src/ap/drv_callbacks.c
+++ b/src/ap/drv_callbacks.c
@@ -18,6 +18,7 @@
 #include "crypto/random.h"
 #include "p2p/p2p.h"
 #include "wps/wps.h"
+#include "fst/fst.h"
 #include "wnm_ap.h"
 #include "hostapd.h"
 #include "ieee802_11.h"
@@ -42,10 +43,10 @@ int hostapd_notif_assoc(struct hostapd_data *hapd, const u8 *addr,
 	struct ieee802_11_elems elems;
 	const u8 *ie;
 	size_t ielen;
-#ifdef CONFIG_IEEE80211R
+#if defined(CONFIG_IEEE80211R) || defined(CONFIG_IEEE80211W)
 	u8 buf[sizeof(struct ieee80211_mgmt) + 1024];
 	u8 *p = buf;
-#endif /* CONFIG_IEEE80211R */
+#endif /* CONFIG_IEEE80211R || CONFIG_IEEE80211W */
 	u16 reason = WLAN_REASON_UNSPECIFIED;
 	u16 status = WLAN_STATUS_SUCCESS;
 	const u8 *p2p_dev_addr = NULL;
@@ -58,8 +59,8 @@ int hostapd_notif_assoc(struct hostapd_data *hapd, const u8 *addr,
 		 * running, so better make sure we stop processing such an
 		 * event here.
 		 */
-		wpa_printf(MSG_DEBUG, "hostapd_notif_assoc: Skip event with "
-			   "no address");
+		wpa_printf(MSG_DEBUG,
+			   "hostapd_notif_assoc: Skip event with no address");
 		return -1;
 	}
 	random_add_randomness(addr, ETH_ALEN);
@@ -89,8 +90,8 @@ int hostapd_notif_assoc(struct hostapd_data *hapd, const u8 *addr,
 	} else {
 		ie = NULL;
 		ielen = 0;
-		wpa_printf(MSG_DEBUG, "STA did not include WPS/RSN/WPA IE in "
-			   "(Re)AssocReq");
+		wpa_printf(MSG_DEBUG,
+			   "STA did not include WPS/RSN/WPA IE in (Re)AssocReq");
 	}
 
 	sta = ap_get_sta(hapd, addr);
@@ -126,8 +127,6 @@ int hostapd_notif_assoc(struct hostapd_data *hapd, const u8 *addr,
 #ifdef CONFIG_IEEE80211N
 #ifdef NEED_AP_MLME
 	if (elems.ht_capabilities &&
-	    elems.ht_capabilities_len >=
-	    sizeof(struct ieee80211_ht_capabilities) &&
 	    (hapd->iface->conf->ht_capab &
 	     HT_CAP_INFO_SUPP_CHANNEL_WIDTH_SET)) {
 		struct ieee80211_ht_capabilities *ht_cap =
@@ -157,13 +156,20 @@ int hostapd_notif_assoc(struct hostapd_data *hapd, const u8 *addr,
 		sta->hs20_ie = NULL;
 #endif /* CONFIG_HS20 */
 
+#ifdef CONFIG_FST
+	wpabuf_free(sta->mb_ies);
+	if (hapd->iface->fst)
+		sta->mb_ies = mb_ies_by_info(&elems.mb_ies);
+	else
+		sta->mb_ies = NULL;
+#endif /* CONFIG_FST */
+
 	if (hapd->conf->wpa) {
 		if (ie == NULL || ielen == 0) {
 #ifdef CONFIG_WPS
 			if (hapd->conf->wps_state) {
-				wpa_printf(MSG_DEBUG, "STA did not include "
-					   "WPA/RSN IE in (Re)Association "
-					   "Request - possible WPS use");
+				wpa_printf(MSG_DEBUG,
+					   "STA did not include WPA/RSN IE in (Re)Association Request - possible WPS use");
 				sta->flags |= WLAN_STA_MAYBE_WPS;
 				goto skip_wpa_check;
 			}
@@ -176,13 +182,14 @@ int hostapd_notif_assoc(struct hostapd_data *hapd, const u8 *addr,
 		if (hapd->conf->wps_state && ie[0] == 0xdd && ie[1] >= 4 &&
 		    os_memcmp(ie + 2, "\x00\x50\xf2\x04", 4) == 0) {
 			struct wpabuf *wps;
+
 			sta->flags |= WLAN_STA_WPS;
 			wps = ieee802_11_vendor_ie_concat(ie, ielen,
 							  WPS_IE_VENDOR_TYPE);
 			if (wps) {
 				if (wps_is_20(wps)) {
-					wpa_printf(MSG_DEBUG, "WPS: STA "
-						   "supports WPS 2.0");
+					wpa_printf(MSG_DEBUG,
+						   "WPS: STA supports WPS 2.0");
 					sta->flags |= WLAN_STA_WPS2;
 				}
 				wpabuf_free(wps);
@@ -196,16 +203,17 @@ int hostapd_notif_assoc(struct hostapd_data *hapd, const u8 *addr,
 							sta->addr,
 							p2p_dev_addr);
 		if (sta->wpa_sm == NULL) {
-			wpa_printf(MSG_ERROR, "Failed to initialize WPA state "
-				   "machine");
+			wpa_printf(MSG_ERROR,
+				   "Failed to initialize WPA state machine");
 			return -1;
 		}
 		res = wpa_validate_wpa_ie(hapd->wpa_auth, sta->wpa_sm,
 					  ie, ielen,
 					  elems.mdie, elems.mdie_len);
 		if (res != WPA_IE_OK) {
-			wpa_printf(MSG_DEBUG, "WPA/RSN information element "
-				   "rejected? (res %u)", res);
+			wpa_printf(MSG_DEBUG,
+				   "WPA/RSN information element rejected? (res %u)",
+				   res);
 			wpa_hexdump(MSG_DEBUG, "IE", ie, ielen);
 			if (res == WPA_INVALID_GROUP) {
 				reason = WLAN_REASON_GROUP_CIPHER_NOT_VALID;
@@ -248,14 +256,12 @@ int hostapd_notif_assoc(struct hostapd_data *hapd, const u8 *addr,
 			if (sta->sa_query_count == 0)
 				ap_sta_start_sa_query(hapd, sta);
 
-#ifdef CONFIG_IEEE80211R
 			status = WLAN_STATUS_ASSOC_REJECTED_TEMPORARILY;
 
 			p = hostapd_eid_assoc_comeback_time(hapd, sta, p);
 
 			hostapd_sta_assoc(hapd, addr, reassoc, status, buf,
 					  p - buf);
-#endif /* CONFIG_IEEE80211R */
 			return 0;
 		}
 
@@ -283,6 +289,7 @@ int hostapd_notif_assoc(struct hostapd_data *hapd, const u8 *addr,
 	} else if (hapd->conf->wps_state) {
 #ifdef CONFIG_WPS
 		struct wpabuf *wps;
+
 		if (req_ies)
 			wps = ieee802_11_vendor_ie_concat(req_ies, req_ies_len,
 							  WPS_IE_VENDOR_TYPE);
@@ -299,8 +306,8 @@ int hostapd_notif_assoc(struct hostapd_data *hapd, const u8 *addr,
 		if (wps) {
 			sta->flags |= WLAN_STA_WPS;
 			if (wps_is_20(wps)) {
-				wpa_printf(MSG_DEBUG, "WPS: STA supports "
-					   "WPS 2.0");
+				wpa_printf(MSG_DEBUG,
+					   "WPS: STA supports WPS 2.0");
 				sta->flags |= WLAN_STA_WPS2;
 			}
 		} else
@@ -322,8 +329,8 @@ int hostapd_notif_assoc(struct hostapd_data *hapd, const u8 *addr,
 			sta->wpa_sm = wpa_auth_sta_init(hapd->wpa_auth,
 							sta->addr, NULL);
 		if (sta->wpa_sm == NULL) {
-			wpa_printf(MSG_WARNING, "Failed to initialize WPA "
-				   "state machine");
+			wpa_printf(MSG_WARNING,
+				   "Failed to initialize WPA state machine");
 			return WLAN_STATUS_UNSPECIFIED_FAILURE;
 		}
 		if (wpa_validate_osen(hapd->wpa_auth, sta->wpa_sm,
@@ -395,8 +402,8 @@ void hostapd_notif_disassoc(struct hostapd_data *hapd, const u8 *addr)
 		 * was running, so better make sure we stop processing such an
 		 * event here.
 		 */
-		wpa_printf(MSG_DEBUG, "hostapd_notif_disassoc: Skip event "
-			   "with no address");
+		wpa_printf(MSG_DEBUG,
+			   "hostapd_notif_disassoc: Skip event with no address");
 		return;
 	}
 
@@ -405,8 +412,9 @@ void hostapd_notif_disassoc(struct hostapd_data *hapd, const u8 *addr)
 
 	sta = ap_get_sta(hapd, addr);
 	if (sta == NULL) {
-		wpa_printf(MSG_DEBUG, "Disassociation notification for "
-			   "unknown STA " MACSTR, MAC2STR(addr));
+		wpa_printf(MSG_DEBUG,
+			   "Disassociation notification for unknown STA "
+			   MACSTR, MAC2STR(addr));
 		return;
 	}
 
@@ -427,8 +435,8 @@ void hostapd_event_sta_low_ack(struct hostapd_data *hapd, const u8 *addr)
 		return;
 
 	hostapd_logger(hapd, addr, HOSTAPD_MODULE_IEEE80211,
-		       HOSTAPD_LEVEL_INFO, "disconnected due to excessive "
-		       "missing ACKs");
+		       HOSTAPD_LEVEL_INFO,
+		       "disconnected due to excessive missing ACKs");
 	hostapd_drv_sta_disassoc(hapd, addr, WLAN_REASON_DISASSOC_LOW_ACK);
 	if (sta)
 		ap_sta_disassociate(hapd, sta, WLAN_REASON_DISASSOC_LOW_ACK);
@@ -452,8 +460,8 @@ void hostapd_event_ch_switch(struct hostapd_data *hapd, int freq, int ht,
 	channel = hostapd_hw_get_channel(hapd, freq);
 	if (!channel) {
 		hostapd_logger(hapd, NULL, HOSTAPD_MODULE_IEEE80211,
-			       HOSTAPD_LEVEL_WARNING, "driver switched to "
-			       "bad channel!");
+			       HOSTAPD_LEVEL_WARNING,
+			       "driver switched to bad channel!");
 		return;
 	}
 
@@ -532,10 +540,9 @@ void hostapd_event_connect_failed_reason(struct hostapd_data *hapd,
 
 #ifdef CONFIG_ACS
 static void hostapd_acs_channel_selected(struct hostapd_data *hapd,
-					 u8 pri_channel, u8 sec_channel)
+					 struct acs_selected_channels *acs_res)
 {
-	int channel;
-	int ret;
+	int ret, i;
 
 	if (hapd->iconf->channel) {
 		wpa_printf(MSG_INFO, "ACS: Channel was already set to %d",
@@ -543,29 +550,73 @@ static void hostapd_acs_channel_selected(struct hostapd_data *hapd,
 		return;
 	}
 
-	hapd->iface->freq = hostapd_hw_get_freq(hapd, pri_channel);
+	if (!hapd->iface->current_mode) {
+		for (i = 0; i < hapd->iface->num_hw_features; i++) {
+			struct hostapd_hw_modes *mode =
+				&hapd->iface->hw_features[i];
 
-	channel = pri_channel;
-	if (!channel) {
+			if (mode->mode == acs_res->hw_mode) {
+				hapd->iface->current_mode = mode;
+				break;
+			}
+		}
+		if (!hapd->iface->current_mode) {
+			hostapd_logger(hapd, NULL, HOSTAPD_MODULE_IEEE80211,
+				       HOSTAPD_LEVEL_WARNING,
+				       "driver selected to bad hw_mode");
+			return;
+		}
+	}
+
+	hapd->iface->freq = hostapd_hw_get_freq(hapd, acs_res->pri_channel);
+
+	if (!acs_res->pri_channel) {
 		hostapd_logger(hapd, NULL, HOSTAPD_MODULE_IEEE80211,
 			       HOSTAPD_LEVEL_WARNING,
 			       "driver switched to bad channel");
 		return;
 	}
 
-	hapd->iconf->channel = channel;
+	hapd->iconf->channel = acs_res->pri_channel;
+	hapd->iconf->acs = 1;
 
-	if (sec_channel == 0)
+	if (acs_res->sec_channel == 0)
 		hapd->iconf->secondary_channel = 0;
-	else if (sec_channel < pri_channel)
+	else if (acs_res->sec_channel < acs_res->pri_channel)
 		hapd->iconf->secondary_channel = -1;
-	else if (sec_channel > pri_channel)
+	else if (acs_res->sec_channel > acs_res->pri_channel)
 		hapd->iconf->secondary_channel = 1;
 	else {
 		wpa_printf(MSG_ERROR, "Invalid secondary channel!");
 		return;
 	}
 
+	if (hapd->iface->conf->ieee80211ac) {
+		/* set defaults for backwards compatibility */
+		hapd->iconf->vht_oper_centr_freq_seg1_idx = 0;
+		hapd->iconf->vht_oper_centr_freq_seg0_idx = 0;
+		hapd->iconf->vht_oper_chwidth = VHT_CHANWIDTH_USE_HT;
+		if (acs_res->ch_width == 80) {
+			hapd->iconf->vht_oper_centr_freq_seg0_idx =
+				acs_res->vht_seg0_center_ch;
+			hapd->iconf->vht_oper_chwidth = VHT_CHANWIDTH_80MHZ;
+		} else if (acs_res->ch_width == 160) {
+			if (acs_res->vht_seg1_center_ch == 0) {
+				hapd->iconf->vht_oper_centr_freq_seg0_idx =
+					acs_res->vht_seg0_center_ch;
+				hapd->iconf->vht_oper_chwidth =
+					VHT_CHANWIDTH_160MHZ;
+			} else {
+				hapd->iconf->vht_oper_centr_freq_seg0_idx =
+					acs_res->vht_seg0_center_ch;
+				hapd->iconf->vht_oper_centr_freq_seg1_idx =
+					acs_res->vht_seg1_center_ch;
+				hapd->iconf->vht_oper_chwidth =
+					VHT_CHANWIDTH_80P80MHZ;
+			}
+		}
+	}
+
 	ret = hostapd_acs_completed(hapd->iface, 0);
 	if (ret) {
 		wpa_printf(MSG_ERROR,
@@ -647,8 +698,8 @@ static void hostapd_notif_auth(struct hostapd_data *hapd,
 			sta->wpa_sm = wpa_auth_sta_init(hapd->wpa_auth,
 							sta->addr, NULL);
 		if (sta->wpa_sm == NULL) {
-			wpa_printf(MSG_DEBUG, "FT: Failed to initialize WPA "
-				   "state machine");
+			wpa_printf(MSG_DEBUG,
+				   "FT: Failed to initialize WPA state machine");
 			status = WLAN_STATUS_UNSPECIFIED_FAILURE;
 			goto fail;
 		}
@@ -683,7 +734,7 @@ static void hostapd_action_rx(struct hostapd_data *hapd,
 	if (WLAN_FC_GET_STYPE(fc) != WLAN_FC_STYPE_ACTION)
 		return; /* handled by the driver */
 
-        wpa_printf(MSG_DEBUG, "RX_ACTION cat %d action plen %d",
+	wpa_printf(MSG_DEBUG, "RX_ACTION cat %d action plen %d",
 		   mgmt->u.action.category, (int) plen);
 
 	sta = ap_get_sta(hapd, mgmt->sa);
@@ -694,6 +745,7 @@ static void hostapd_action_rx(struct hostapd_data *hapd,
 #ifdef CONFIG_IEEE80211R
 	if (mgmt->u.action.category == WLAN_ACTION_FT) {
 		const u8 *payload = drv_mgmt->frame + 24 + 1;
+
 		wpa_ft_action_rx(sta->wpa_sm, payload, plen);
 	}
 #endif /* CONFIG_IEEE80211R */
@@ -710,6 +762,13 @@ static void hostapd_action_rx(struct hostapd_data *hapd,
 		ieee802_11_rx_wnm_action_ap(hapd, mgmt, drv_mgmt->frame_len);
 	}
 #endif /* CONFIG_WNM */
+#ifdef CONFIG_FST
+	if (mgmt->u.action.category == WLAN_ACTION_FST && hapd->iface->fst) {
+		fst_rx_action(hapd->iface->fst, mgmt, drv_mgmt->frame_len);
+		return;
+	}
+#endif /* CONFIG_FST */
+
 }
 
 
@@ -761,6 +820,7 @@ static int hostapd_mgmt_rx(struct hostapd_data *hapd, struct rx_mgmt *rx_mgmt)
 	if (hapd->ext_mgmt_frame_handling) {
 		size_t hex_len = 2 * rx_mgmt->frame_len + 1;
 		char *hex = os_malloc(hex_len);
+
 		if (hex) {
 			wpa_snprintf_hex(hex, hex_len, rx_mgmt->frame,
 					 rx_mgmt->frame_len);
@@ -778,8 +838,7 @@ static int hostapd_mgmt_rx(struct hostapd_data *hapd, struct rx_mgmt *rx_mgmt)
 
 	hapd = get_hapd_bssid(iface, bssid);
 	if (hapd == NULL) {
-		u16 fc;
-		fc = le_to_host16(hdr->frame_control);
+		u16 fc = le_to_host16(hdr->frame_control);
 
 		/*
 		 * Drop frames to unknown BSSIDs except for Beacon frames which
@@ -798,6 +857,7 @@ static int hostapd_mgmt_rx(struct hostapd_data *hapd, struct rx_mgmt *rx_mgmt)
 
 	if (hapd == HAPD_BROADCAST) {
 		size_t i;
+
 		ret = 0;
 		for (i = 0; i < iface->num_bss; i++) {
 			/* if bss is set, driver will call this function for
@@ -824,6 +884,7 @@ static void hostapd_mgmt_tx_cb(struct hostapd_data *hapd, const u8 *buf,
 			       size_t len, u16 stype, int ok)
 {
 	struct ieee80211_hdr *hdr;
+
 	hdr = (struct ieee80211_hdr *) buf;
 	hapd = get_hapd_bssid(hapd->iface, get_hdr_bssid(hdr, len));
 	if (hapd == NULL || hapd == HAPD_BROADCAST)
@@ -837,6 +898,7 @@ static void hostapd_mgmt_tx_cb(struct hostapd_data *hapd, const u8 *buf,
 static int hostapd_event_new_sta(struct hostapd_data *hapd, const u8 *addr)
 {
 	struct sta_info *sta = ap_get_sta(hapd, addr);
+
 	if (sta)
 		return 0;
 
@@ -863,11 +925,10 @@ static void hostapd_event_eapol_rx(struct hostapd_data *hapd, const u8 *src,
 	size_t j;
 
 	for (j = 0; j < iface->num_bss; j++) {
-		if ((sta = ap_get_sta(iface->bss[j], src))) {
-			if (sta->flags & WLAN_STA_ASSOC) {
-				hapd = iface->bss[j];
-				break;
-			}
+		sta = ap_get_sta(iface->bss[j], src);
+		if (sta && sta->flags & WLAN_STA_ASSOC) {
+			hapd = iface->bss[j];
+			break;
 		}
 	}
 
@@ -927,7 +988,8 @@ static void hostapd_single_channel_get_survey(struct hostapd_iface *iface,
 	if (!chan || chan->flag & HOSTAPD_CHAN_DISABLED)
 		return;
 
-	wpa_printf(MSG_DEBUG, "Single Channel Survey: (freq=%d channel_time=%ld channel_time_busy=%ld)",
+	wpa_printf(MSG_DEBUG,
+		   "Single Channel Survey: (freq=%d channel_time=%ld channel_time_busy=%ld)",
 		   survey->freq,
 		   (unsigned long int) survey->channel_time,
 		   (unsigned long int) survey->channel_time_busy);
@@ -1061,6 +1123,7 @@ void wpa_supplicant_event(void *ctx, enum wpa_event_type event,
 	    data->rx_mgmt.frame_len >= 24) {
 		const struct ieee80211_hdr *hdr;
 		u16 fc;
+
 		hdr = (const struct ieee80211_hdr *) data->rx_mgmt.frame;
 		fc = le_to_host16(hdr->frame_control);
 		if (WLAN_FC_GET_TYPE(fc) == WLAN_FC_TYPE_MGMT &&
@@ -1248,9 +1311,8 @@ void wpa_supplicant_event(void *ctx, enum wpa_event_type event,
 		break;
 #ifdef CONFIG_ACS
 	case EVENT_ACS_CHANNEL_SELECTED:
-		hostapd_acs_channel_selected(
-			hapd, data->acs_selected_channels.pri_channel,
-			data->acs_selected_channels.sec_channel);
+		hostapd_acs_channel_selected(hapd,
+					     &data->acs_selected_channels);
 		break;
 #endif /* CONFIG_ACS */
 	default:
diff --git a/src/ap/eap_user_db.c b/src/ap/eap_user_db.c
index 559d77f9ef2b..082d0f53175e 100644
--- a/src/ap/eap_user_db.c
+++ b/src/ap/eap_user_db.c
@@ -138,8 +138,12 @@ eap_user_sqlite_get(struct hostapd_data *hapd, const u8 *identity,
 	char id_str[256], cmd[300];
 	size_t i;
 
-	if (identity_len >= sizeof(id_str))
+	if (identity_len >= sizeof(id_str)) {
+		wpa_printf(MSG_DEBUG, "%s: identity len too big: %d >= %d",
+			   __func__, (int) identity_len,
+			   (int) (sizeof(id_str)));
 		return NULL;
+	}
 	os_memcpy(id_str, identity, identity_len);
 	id_str[identity_len] = '\0';
 	for (i = 0; i < identity_len; i++) {
@@ -182,7 +186,9 @@ eap_user_sqlite_get(struct hostapd_data *hapd, const u8 *identity,
 	wpa_printf(MSG_DEBUG, "DB: %s", cmd);
 	if (sqlite3_exec(db, cmd, get_user_cb, &hapd->tmp_eap_user, NULL) !=
 	    SQLITE_OK) {
-		wpa_printf(MSG_DEBUG, "DB: Failed to complete SQL operation");
+		wpa_printf(MSG_DEBUG,
+			   "DB: Failed to complete SQL operation: %s  db: %s",
+			   sqlite3_errmsg(db), hapd->conf->eap_user_sqlite);
 	} else if (hapd->tmp_eap_user.next)
 		user = &hapd->tmp_eap_user;
 
@@ -192,8 +198,10 @@ eap_user_sqlite_get(struct hostapd_data *hapd, const u8 *identity,
 		wpa_printf(MSG_DEBUG, "DB: %s", cmd);
 		if (sqlite3_exec(db, cmd, get_wildcard_cb, &hapd->tmp_eap_user,
 				 NULL) != SQLITE_OK) {
-			wpa_printf(MSG_DEBUG, "DB: Failed to complete SQL "
-				   "operation");
+			wpa_printf(MSG_DEBUG,
+				   "DB: Failed to complete SQL operation: %s  db: %s",
+				   sqlite3_errmsg(db),
+				   hapd->conf->eap_user_sqlite);
 		} else if (hapd->tmp_eap_user.next) {
 			user = &hapd->tmp_eap_user;
 			os_free(user->identity);
diff --git a/src/ap/hostapd.c b/src/ap/hostapd.c
index 3e4e16b4f396..c09c17a44696 100644
--- a/src/ap/hostapd.c
+++ b/src/ap/hostapd.c
@@ -17,6 +17,7 @@
 #include "eap_server/tncs.h"
 #include "eapol_auth/eapol_auth_sm.h"
 #include "eapol_auth/eapol_auth_sm_i.h"
+#include "fst/fst.h"
 #include "hostapd.h"
 #include "authsrv.h"
 #include "sta_info.h"
@@ -179,6 +180,7 @@ int hostapd_reload_config(struct hostapd_iface *iface)
 		hapd = iface->bss[j];
 		hapd->iconf = newconf;
 		hapd->iconf->channel = oldconf->channel;
+		hapd->iconf->acs = oldconf->acs;
 		hapd->iconf->secondary_channel = oldconf->secondary_channel;
 		hapd->iconf->ieee80211n = oldconf->ieee80211n;
 		hapd->iconf->ieee80211ac = oldconf->ieee80211ac;
@@ -259,6 +261,7 @@ static void hostapd_free_hapd_data(struct hostapd_data *hapd)
 {
 	os_free(hapd->probereq_cb);
 	hapd->probereq_cb = NULL;
+	hapd->num_probereq_cb = 0;
 
 #ifdef CONFIG_P2P
 	wpabuf_free(hapd->p2p_beacon_ie);
@@ -353,6 +356,22 @@ static void hostapd_cleanup(struct hostapd_data *hapd)
 }
 
 
+static void sta_track_deinit(struct hostapd_iface *iface)
+{
+	struct hostapd_sta_info *info;
+
+	if (!iface->num_sta_seen)
+		return;
+
+	while ((info = dl_list_first(&iface->sta_seen, struct hostapd_sta_info,
+				     list))) {
+		dl_list_del(&info->list);
+		iface->num_sta_seen--;
+		os_free(info);
+	}
+}
+
+
 static void hostapd_cleanup_iface_partial(struct hostapd_iface *iface)
 {
 	wpa_printf(MSG_DEBUG, "%s(%p)", __func__, iface);
@@ -368,6 +387,7 @@ static void hostapd_cleanup_iface_partial(struct hostapd_iface *iface)
 	os_free(iface->basic_rates);
 	iface->basic_rates = NULL;
 	ap_list_deinit(iface);
+	sta_track_deinit(iface);
 }
 
 
@@ -861,7 +881,7 @@ hostapd_das_disconnect(void *ctx, struct radius_das_attrs *attr)
 static int hostapd_setup_bss(struct hostapd_data *hapd, int first)
 {
 	struct hostapd_bss_config *conf = hapd->conf;
-	u8 ssid[HOSTAPD_MAX_SSID_LEN + 1];
+	u8 ssid[SSID_MAX_LEN + 1];
 	int ssid_len, set_ssid;
 	char force_ifname[IFNAMSIZ];
 	u8 if_addr[ETH_ALEN];
@@ -1363,6 +1383,132 @@ fail:
 }
 
 
+#ifdef CONFIG_FST
+
+static const u8 * fst_hostapd_get_bssid_cb(void *ctx)
+{
+	struct hostapd_data *hapd = ctx;
+
+	return hapd->own_addr;
+}
+
+
+static void fst_hostapd_get_channel_info_cb(void *ctx,
+					    enum hostapd_hw_mode *hw_mode,
+					    u8 *channel)
+{
+	struct hostapd_data *hapd = ctx;
+
+	*hw_mode = ieee80211_freq_to_chan(hapd->iface->freq, channel);
+}
+
+
+static void fst_hostapd_set_ies_cb(void *ctx, const struct wpabuf *fst_ies)
+{
+	struct hostapd_data *hapd = ctx;
+
+	if (hapd->iface->fst_ies != fst_ies) {
+		hapd->iface->fst_ies = fst_ies;
+		if (ieee802_11_set_beacon(hapd))
+			wpa_printf(MSG_WARNING, "FST: Cannot set beacon");
+	}
+}
+
+
+static int fst_hostapd_send_action_cb(void *ctx, const u8 *da,
+				      struct wpabuf *buf)
+{
+	struct hostapd_data *hapd = ctx;
+
+	return hostapd_drv_send_action(hapd, hapd->iface->freq, 0, da,
+				       wpabuf_head(buf), wpabuf_len(buf));
+}
+
+
+static const struct wpabuf * fst_hostapd_get_mb_ie_cb(void *ctx, const u8 *addr)
+{
+	struct hostapd_data *hapd = ctx;
+	struct sta_info *sta = ap_get_sta(hapd, addr);
+
+	return sta ? sta->mb_ies : NULL;
+}
+
+
+static void fst_hostapd_update_mb_ie_cb(void *ctx, const u8 *addr,
+					const u8 *buf, size_t size)
+{
+	struct hostapd_data *hapd = ctx;
+	struct sta_info *sta = ap_get_sta(hapd, addr);
+
+	if (sta) {
+		struct mb_ies_info info;
+
+		if (!mb_ies_info_by_ies(&info, buf, size)) {
+			wpabuf_free(sta->mb_ies);
+			sta->mb_ies = mb_ies_by_info(&info);
+		}
+	}
+}
+
+
+static const u8 * fst_hostapd_get_sta(struct fst_get_peer_ctx **get_ctx,
+				      Boolean mb_only)
+{
+	struct sta_info *s = (struct sta_info *) *get_ctx;
+
+	if (mb_only) {
+		for (; s && !s->mb_ies; s = s->next)
+			;
+	}
+
+	if (s) {
+		*get_ctx = (struct fst_get_peer_ctx *) s->next;
+
+		return s->addr;
+	}
+
+	*get_ctx = NULL;
+	return NULL;
+}
+
+
+static const u8 * fst_hostapd_get_peer_first(void *ctx,
+					     struct fst_get_peer_ctx **get_ctx,
+					     Boolean mb_only)
+{
+	struct hostapd_data *hapd = ctx;
+
+	*get_ctx = (struct fst_get_peer_ctx *) hapd->sta_list;
+
+	return fst_hostapd_get_sta(get_ctx, mb_only);
+}
+
+
+static const u8 * fst_hostapd_get_peer_next(void *ctx,
+					    struct fst_get_peer_ctx **get_ctx,
+					    Boolean mb_only)
+{
+	return fst_hostapd_get_sta(get_ctx, mb_only);
+}
+
+
+void fst_hostapd_fill_iface_obj(struct hostapd_data *hapd,
+				struct fst_wpa_obj *iface_obj)
+{
+	iface_obj->ctx = hapd;
+	iface_obj->get_bssid = fst_hostapd_get_bssid_cb;
+	iface_obj->get_channel_info = fst_hostapd_get_channel_info_cb;
+	iface_obj->set_ies = fst_hostapd_set_ies_cb;
+	iface_obj->send_action = fst_hostapd_send_action_cb;
+	iface_obj->get_mb_ie = fst_hostapd_get_mb_ie_cb;
+	iface_obj->update_mb_ie = fst_hostapd_update_mb_ie_cb;
+	iface_obj->get_peer_first = fst_hostapd_get_peer_first;
+	iface_obj->get_peer_next = fst_hostapd_get_peer_next;
+}
+
+#endif /* CONFIG_FST */
+
+
 /**
  * hostapd_setup_interface_complete - Complete interface setup
  *
@@ -1495,6 +1641,7 @@ int hostapd_setup_interface_complete(struct hostapd_iface *iface, int err)
 	hostapd_tx_queue_params(iface);
 
 	ap_list_init(iface);
+	dl_list_init(&iface->sta_seen);
 
 	hostapd_set_acl(hapd);
 
@@ -1528,6 +1675,22 @@ int hostapd_setup_interface_complete(struct hostapd_iface *iface, int err)
 #ifdef NEED_AP_MLME
 dfs_offload:
 #endif /* NEED_AP_MLME */
+
+#ifdef CONFIG_FST
+	if (hapd->iconf->fst_cfg.group_id[0]) {
+		struct fst_wpa_obj iface_obj;
+
+		fst_hostapd_fill_iface_obj(hapd, &iface_obj);
+		iface->fst = fst_attach(hapd->conf->iface, hapd->own_addr,
+					&iface_obj, &hapd->iconf->fst_cfg);
+		if (!iface->fst) {
+			wpa_printf(MSG_ERROR, "Could not attach to FST %s",
+				   hapd->iconf->fst_cfg.group_id);
+			goto fail;
+		}
+	}
+#endif /* CONFIG_FST */
+
 	hostapd_set_state(iface, HAPD_IFACE_ENABLED);
 	wpa_msg(iface->bss[0]->msg_ctx, MSG_INFO, AP_EVENT_ENABLED);
 	if (hapd->setup_complete_cb)
@@ -1544,6 +1707,12 @@ fail:
 	wpa_printf(MSG_ERROR, "Interface initialization failed");
 	hostapd_set_state(iface, HAPD_IFACE_DISABLED);
 	wpa_msg(hapd->msg_ctx, MSG_INFO, AP_EVENT_DISABLED);
+#ifdef CONFIG_FST
+	if (iface->fst) {
+		fst_detach(iface->fst);
+		iface->fst = NULL;
+	}
+#endif /* CONFIG_FST */
 	if (iface->interfaces && iface->interfaces->terminate_on_error)
 		eloop_terminate();
 	return -1;
@@ -1643,6 +1812,13 @@ void hostapd_interface_deinit(struct hostapd_iface *iface)
 	eloop_cancel_timeout(channel_list_update_timeout, iface, NULL);
 	iface->wait_channel_update = 0;
 
+#ifdef CONFIG_FST
+	if (iface->fst) {
+		fst_detach(iface->fst);
+		iface->fst = NULL;
+	}
+#endif /* CONFIG_FST */
+
 	for (j = iface->num_bss - 1; j >= 0; j--)
 		hostapd_bss_deinit(iface->bss[j]);
 }
@@ -2029,7 +2205,7 @@ hostapd_iface_alloc(struct hapd_interfaces *interfaces)
 
 static struct hostapd_config *
 hostapd_config_alloc(struct hapd_interfaces *interfaces, const char *ifname,
-		     const char *ctrl_iface)
+		     const char *ctrl_iface, const char *driver)
 {
 	struct hostapd_bss_config *bss;
 	struct hostapd_config *conf;
@@ -2042,6 +2218,21 @@ hostapd_config_alloc(struct hapd_interfaces *interfaces, const char *ifname,
 		return NULL;
 	}
 
+	if (driver) {
+		int j;
+
+		for (j = 0; wpa_drivers[j]; j++) {
+			if (os_strcmp(driver, wpa_drivers[j]->name) == 0) {
+				conf->driver = wpa_drivers[j];
+				goto skip;
+			}
+		}
+
+		wpa_printf(MSG_ERROR,
+			   "Invalid/unknown driver '%s' - registering the default driver",
+			   driver);
+	}
+
 	conf->driver = wpa_drivers[0];
 	if (conf->driver == NULL) {
 		wpa_printf(MSG_ERROR, "No driver wrappers registered!");
@@ -2049,6 +2240,7 @@ hostapd_config_alloc(struct hapd_interfaces *interfaces, const char *ifname,
 		return NULL;
 	}
 
+skip:
 	bss = conf->last_bss = conf->bss[0];
 
 	os_strlcpy(bss->iface, ifname, sizeof(bss->iface));
@@ -2209,8 +2401,14 @@ int hostapd_add_iface(struct hapd_interfaces *interfaces, char *buf)
 		if (conf && conf->bss)
 			os_strlcpy(conf->bss[0]->iface, buf,
 				   sizeof(conf->bss[0]->iface));
-	} else
-		conf = hostapd_config_alloc(interfaces, buf, ptr);
+	} else {
+		char *driver = os_strchr(ptr, ' ');
+
+		if (driver)
+			*driver++ = '\0';
+		conf = hostapd_config_alloc(interfaces, buf, ptr, driver);
+	}
+
 	if (conf == NULL || conf->bss == NULL) {
 		wpa_printf(MSG_ERROR, "%s: Failed to allocate memory "
 			   "for configuration", __func__);
@@ -2722,4 +2920,43 @@ hostapd_switch_channel_fallback(struct hostapd_iface *iface,
 	hostapd_enable_iface(iface);
 }
 
+
+struct hostapd_data * hostapd_get_iface(struct hapd_interfaces *interfaces,
+					const char *ifname)
+{
+	size_t i, j;
+
+	for (i = 0; i < interfaces->count; i++) {
+		struct hostapd_iface *iface = interfaces->iface[i];
+
+		for (j = 0; j < iface->num_bss; j++) {
+			struct hostapd_data *hapd = iface->bss[j];
+
+			if (os_strcmp(ifname, hapd->conf->iface) == 0)
+				return hapd;
+		}
+	}
+
+	return NULL;
+}
+
 #endif /* NEED_AP_MLME */
+
+
+void hostapd_periodic_iface(struct hostapd_iface *iface)
+{
+	size_t i;
+
+	ap_list_timer(iface);
+
+	for (i = 0; i < iface->num_bss; i++) {
+		struct hostapd_data *hapd = iface->bss[i];
+
+		if (!hapd->started)
+			continue;
+
+#ifndef CONFIG_NO_RADIUS
+		hostapd_acl_expire(hapd);
+#endif /* CONFIG_NO_RADIUS */
+	}
+}
diff --git a/src/ap/hostapd.h b/src/ap/hostapd.h
index 75cc24edb665..dcf51f00f78d 100644
--- a/src/ap/hostapd.h
+++ b/src/ap/hostapd.h
@@ -41,6 +41,7 @@ struct hapd_interfaces {
 
 	size_t count;
 	int global_ctrl_sock;
+	struct wpa_ctrl_dst *global_ctrl_dst;
 	char *global_iface_path;
 	char *global_iface_name;
 #ifndef CONFIG_NATIVE_WINDOWS
@@ -49,6 +50,9 @@ struct hapd_interfaces {
 	struct hostapd_iface **iface;
 
 	size_t terminate_on_error;
+#ifndef CONFIG_NO_VLAN
+	struct dynamic_iface *vlan_priv;
+#endif /* CONFIG_NO_VLAN */
 };
 
 enum hostapd_chan_status {
@@ -265,6 +269,7 @@ struct hostapd_data {
 	/** Key used for generating SAE anti-clogging tokens */
 	u8 sae_token_key[8];
 	struct os_reltime last_sae_token_key_update;
+	int dot11RSNASAERetransPeriod; /* msec */
 #endif /* CONFIG_SAE */
 
 #ifdef CONFIG_TESTING_OPTIONS
@@ -276,6 +281,12 @@ struct hostapd_data {
 };
 
 
+struct hostapd_sta_info {
+	struct dl_list list;
+	u8 addr[ETH_ALEN];
+	struct os_reltime last_seen;
+};
+
 /**
  * struct hostapd_iface - hostapd per-interface data structure
  */
@@ -305,6 +316,10 @@ struct hostapd_iface {
 
 	unsigned int wait_channel_update:1;
 	unsigned int cac_started:1;
+#ifdef CONFIG_FST
+	struct fst_iface *fst;
+	const struct wpabuf *fst_ies;
+#endif /* CONFIG_FST */
 
 	/*
 	 * When set, indicates that the driver will handle the AP
@@ -400,6 +415,9 @@ struct hostapd_iface {
 
 	void (*scan_cb)(struct hostapd_iface *iface);
 	int num_ht40_scan_tries;
+
+	struct dl_list sta_seen; /* struct hostapd_sta_info */
+	unsigned int num_sta_seen;
 };
 
 /* hostapd.c */
@@ -437,6 +455,7 @@ void
 hostapd_switch_channel_fallback(struct hostapd_iface *iface,
 				const struct hostapd_freq_params *freq_params);
 void hostapd_cleanup_cs_params(struct hostapd_data *hapd);
+void hostapd_periodic_iface(struct hostapd_iface *iface);
 
 /* utils.c */
 int hostapd_register_probereq_cb(struct hostapd_data *hapd,
@@ -464,4 +483,12 @@ const struct hostapd_eap_user *
 hostapd_get_eap_user(struct hostapd_data *hapd, const u8 *identity,
 		     size_t identity_len, int phase2);
 
+struct hostapd_data * hostapd_get_iface(struct hapd_interfaces *interfaces,
+					const char *ifname);
+
+#ifdef CONFIG_FST
+void fst_hostapd_fill_iface_obj(struct hostapd_data *hapd,
+				struct fst_wpa_obj *iface_obj);
+#endif /* CONFIG_FST */
+
 #endif /* HOSTAPD_H */
diff --git a/src/ap/hw_features.c b/src/ap/hw_features.c
index 05431d32c205..fc8786dc311c 100644
--- a/src/ap/hw_features.c
+++ b/src/ap/hw_features.c
@@ -260,8 +260,14 @@ static int ieee80211n_check_40mhz_5g(struct hostapd_iface *iface,
 
 	res = check_40mhz_5g(iface->current_mode, scan_res, pri_chan, sec_chan);
 
-	if (res == 2)
-		ieee80211n_switch_pri_sec(iface);
+	if (res == 2) {
+		if (iface->conf->no_pri_sec_switch) {
+			wpa_printf(MSG_DEBUG,
+				   "Cannot switch PRI/SEC channels due to local constraint");
+		} else {
+			ieee80211n_switch_pri_sec(iface);
+		}
+	}
 
 	return !!res;
 }
@@ -347,8 +353,13 @@ static void ieee80211n_scan_channels_2g4(struct hostapd_iface *iface,
 		sec_freq = pri_freq + 20;
 	else
 		sec_freq = pri_freq - 20;
-	affected_start = (pri_freq + sec_freq) / 2 - 25;
-	affected_end = (pri_freq + sec_freq) / 2 + 25;
+	/*
+	 * Note: Need to find the PRI channel also in cases where the affected
+	 * channel is the SEC channel of a 40 MHz BSS, so need to include the
+	 * scanning coverage here to be 40 MHz from the center frequency.
+	 */
+	affected_start = (pri_freq + sec_freq) / 2 - 40;
+	affected_end = (pri_freq + sec_freq) / 2 + 40;
 	wpa_printf(MSG_DEBUG, "40 MHz affected channel range: [%d,%d] MHz",
 		   affected_start, affected_end);
 
@@ -510,7 +521,11 @@ static int ieee80211n_supported_ht_capab(struct hostapd_iface *iface)
 		return 0;
 	}
 
-	if ((conf & HT_CAP_INFO_SUPP_CHANNEL_WIDTH_SET) &&
+	/*
+	 * Driver ACS chosen channel may not be HT40 due to internal driver
+	 * restrictions.
+	 */
+	if (!iface->conf->acs && (conf & HT_CAP_INFO_SUPP_CHANNEL_WIDTH_SET) &&
 	    !(hw & HT_CAP_INFO_SUPP_CHANNEL_WIDTH_SET)) {
 		wpa_printf(MSG_ERROR, "Driver does not support configured "
 			   "HT capability [HT40*]");
@@ -717,6 +732,15 @@ int hostapd_check_ht_capab(struct hostapd_iface *iface)
 	int ret;
 	if (!iface->conf->ieee80211n)
 		return 0;
+
+	if (iface->current_mode->mode != HOSTAPD_MODE_IEEE80211B &&
+	    iface->current_mode->mode != HOSTAPD_MODE_IEEE80211G &&
+	    (iface->conf->ht_capab & HT_CAP_INFO_DSSS_CCK40MHZ)) {
+		wpa_printf(MSG_DEBUG,
+			   "Disable HT capability [DSSS_CCK-40] on 5 GHz band");
+		iface->conf->ht_capab &= ~HT_CAP_INFO_DSSS_CCK40MHZ;
+	}
+
 	if (!ieee80211n_supported_ht_capab(iface))
 		return -1;
 #ifdef CONFIG_IEEE80211AC
@@ -740,6 +764,9 @@ static int hostapd_is_usable_chan(struct hostapd_iface *iface,
 	int i;
 	struct hostapd_channel_data *chan;
 
+	if (!iface->current_mode)
+		return 0;
+
 	for (i = 0; i < iface->current_mode->num_channels; i++) {
 		chan = &iface->current_mode->channels[i];
 		if (chan->chan != channel)
@@ -801,6 +828,12 @@ hostapd_check_chans(struct hostapd_iface *iface)
 
 static void hostapd_notify_bad_chans(struct hostapd_iface *iface)
 {
+	if (!iface->current_mode) {
+		hostapd_logger(iface->bss[0], NULL, HOSTAPD_MODULE_IEEE80211,
+			       HOSTAPD_LEVEL_WARNING,
+			       "Hardware does not support configured mode");
+		return;
+	}
 	hostapd_logger(iface->bss[0], NULL,
 		       HOSTAPD_MODULE_IEEE80211,
 		       HOSTAPD_LEVEL_WARNING,
@@ -891,14 +924,18 @@ int hostapd_select_hw_mode(struct hostapd_iface *iface)
 	}
 
 	if (iface->current_mode == NULL) {
-		wpa_printf(MSG_ERROR, "Hardware does not support configured "
-			   "mode");
-		hostapd_logger(iface->bss[0], NULL, HOSTAPD_MODULE_IEEE80211,
-			       HOSTAPD_LEVEL_WARNING,
-			       "Hardware does not support configured mode "
-			       "(%d) (hw_mode in hostapd.conf)",
-			       (int) iface->conf->hw_mode);
-		return -2;
+		if (!(iface->drv_flags & WPA_DRIVER_FLAGS_ACS_OFFLOAD) ||
+		    !(iface->drv_flags & WPA_DRIVER_FLAGS_SUPPORT_HW_MODE_ANY))
+		{
+			wpa_printf(MSG_ERROR,
+				   "Hardware does not support configured mode");
+			hostapd_logger(iface->bss[0], NULL,
+				       HOSTAPD_MODULE_IEEE80211,
+				       HOSTAPD_LEVEL_WARNING,
+				       "Hardware does not support configured mode (%d) (hw_mode in hostapd.conf)",
+				       (int) iface->conf->hw_mode);
+			return -2;
+		}
 	}
 
 	switch (hostapd_check_chans(iface)) {
diff --git a/src/ap/hw_features.h b/src/ap/hw_features.h
index 0f67ab8e5f37..ca7f22ba205b 100644
--- a/src/ap/hw_features.h
+++ b/src/ap/hw_features.h
@@ -36,6 +36,11 @@ static inline int hostapd_get_hw_features(struct hostapd_iface *iface)
 	return -1;
 }
 
+static inline int hostapd_acs_completed(struct hostapd_iface *iface, int err)
+{
+	return -1;
+}
+
 static inline int hostapd_select_hw_mode(struct hostapd_iface *iface)
 {
 	return -100;
diff --git a/src/ap/ieee802_11.c b/src/ap/ieee802_11.c
index 89911b1fdd42..7bb18c01d1a1 100644
--- a/src/ap/ieee802_11.c
+++ b/src/ap/ieee802_11.c
@@ -23,6 +23,7 @@
 #include "radius/radius_client.h"
 #include "p2p/p2p.h"
 #include "wps/wps.h"
+#include "fst/fst.h"
 #include "hostapd.h"
 #include "beacon.h"
 #include "ieee802_11_auth.h"
@@ -38,6 +39,7 @@
 #include "p2p_hostapd.h"
 #include "ap_drv_ops.h"
 #include "wnm_ap.h"
+#include "hw_features.h"
 #include "ieee802_11.h"
 #include "dfs.h"
 
@@ -132,8 +134,7 @@ u8 * hostapd_eid_ext_supp_rates(struct hostapd_data *hapd, u8 *eid)
 }
 
 
-u16 hostapd_own_capab_info(struct hostapd_data *hapd, struct sta_info *sta,
-			   int probe)
+u16 hostapd_own_capab_info(struct hostapd_data *hapd)
 {
 	int capab = WLAN_CAPABILITY_ESS;
 	int privacy;
@@ -166,20 +167,6 @@ u16 hostapd_own_capab_info(struct hostapd_data *hapd, struct sta_info *sta,
 		privacy = 1;
 #endif /* CONFIG_HS20 */
 
-	if (sta) {
-		int policy, def_klen;
-		if (probe && sta->ssid_probe) {
-			policy = sta->ssid_probe->security_policy;
-			def_klen = sta->ssid_probe->wep.default_len;
-		} else {
-			policy = sta->ssid->security_policy;
-			def_klen = sta->ssid->wep.default_len;
-		}
-		privacy = policy != SECURITY_PLAINTEXT;
-		if (policy == SECURITY_IEEE_802_1X && def_klen == 0)
-			privacy = 0;
-	}
-
 	if (privacy)
 		capab |= WLAN_CAPABILITY_PRIVACY;
 
@@ -206,6 +193,7 @@ u16 hostapd_own_capab_info(struct hostapd_data *hapd, struct sta_info *sta,
 }
 
 
+#ifndef CONFIG_NO_RC4
 static u16 auth_shared_key(struct hostapd_data *hapd, struct sta_info *sta,
 			   u16 auth_transaction, const u8 *challenge,
 			   int iswep)
@@ -259,6 +247,7 @@ static u16 auth_shared_key(struct hostapd_data *hapd, struct sta_info *sta,
 
 	return 0;
 }
+#endif /* CONFIG_NO_RC4 */
 
 
 static void send_auth_reply(struct hostapd_data *hapd,
@@ -328,7 +317,6 @@ static void handle_auth_ft_finish(void *ctx, const u8 *dst, const u8 *bssid,
 
 #ifdef CONFIG_SAE
 
-#define dot11RSNASAERetransPeriod 40	/* msec */
 #define dot11RSNASAESync 5		/* attempts */
 
 
@@ -511,12 +499,14 @@ static void auth_sae_retransmit_timer(void *eloop_ctx, void *eloop_data)
 	switch (sta->sae->state) {
 	case SAE_COMMITTED:
 		ret = auth_sae_send_commit(hapd, sta, hapd->own_addr, 0);
-		eloop_register_timeout(0, dot11RSNASAERetransPeriod * 1000,
+		eloop_register_timeout(0,
+				       hapd->dot11RSNASAERetransPeriod * 1000,
 				       auth_sae_retransmit_timer, hapd, sta);
 		break;
 	case SAE_CONFIRMED:
 		ret = auth_sae_send_confirm(hapd, sta, hapd->own_addr);
-		eloop_register_timeout(0, dot11RSNASAERetransPeriod * 1000,
+		eloop_register_timeout(0,
+				       hapd->dot11RSNASAERetransPeriod * 1000,
 				       auth_sae_retransmit_timer, hapd, sta);
 		break;
 	default:
@@ -542,7 +532,7 @@ static void sae_set_retransmit_timer(struct hostapd_data *hapd,
 		return;
 
 	eloop_cancel_timeout(auth_sae_retransmit_timer, hapd, sta);
-	eloop_register_timeout(0, dot11RSNASAERetransPeriod * 1000,
+	eloop_register_timeout(0, hapd->dot11RSNASAERetransPeriod * 1000,
 			       auth_sae_retransmit_timer, hapd, sta);
 }
 
@@ -624,7 +614,7 @@ static int sae_sm_step(struct hostapd_data *hapd, struct sta_info *sta,
 				return WLAN_STATUS_SUCCESS;
 			sta->sae->sync++;
 
-			ret = auth_sae_send_commit(hapd, sta, bssid, 1);
+			ret = auth_sae_send_commit(hapd, sta, bssid, 0);
 			if (ret)
 				return ret;
 
@@ -784,6 +774,12 @@ static void handle_auth_sae(struct hostapd_data *hapd, struct sta_info *sta,
 					((const u8 *) mgmt) + len -
 					mgmt->u.auth.variable, &token,
 					&token_len, hapd->conf->sae_groups);
+		if (resp == SAE_SILENTLY_DISCARD) {
+			wpa_printf(MSG_DEBUG,
+				   "SAE: Drop commit message from " MACSTR " due to reflection attack",
+				   MAC2STR(sta->addr));
+			return;
+		}
 		if (token && check_sae_token(hapd, sta->addr, token, token_len)
 		    < 0) {
 			wpa_printf(MSG_DEBUG, "SAE: Drop commit message with "
@@ -934,6 +930,16 @@ static void handle_auth(struct hostapd_data *hapd,
 		   challenge ? " challenge" : "",
 		   seq_ctrl, (fc & WLAN_FC_RETRY) ? " retry" : "");
 
+#ifdef CONFIG_NO_RC4
+	if (auth_alg == WLAN_AUTH_SHARED_KEY) {
+		wpa_printf(MSG_INFO,
+			   "Unsupported authentication algorithm (%d)",
+			   auth_alg);
+		resp = WLAN_STATUS_NOT_SUPPORTED_AUTH_ALG;
+		goto fail;
+	}
+#endif /* CONFIG_NO_RC4 */
+
 	if (hapd->tkip_countermeasures) {
 		resp = WLAN_REASON_MICHAEL_MIC_FAILURE;
 		goto fail;
@@ -972,6 +978,61 @@ static void handle_auth(struct hostapd_data *hapd,
 		goto fail;
 	}
 
+	if (hapd->conf->no_auth_if_seen_on) {
+		struct hostapd_data *other;
+
+		other = sta_track_seen_on(hapd->iface, mgmt->sa,
+					  hapd->conf->no_auth_if_seen_on);
+		if (other) {
+			u8 *pos;
+			u32 info;
+			u8 op_class, channel, phytype;
+
+			wpa_printf(MSG_DEBUG, "%s: Reject authentication from "
+				   MACSTR " since STA has been seen on %s",
+				   hapd->conf->iface, MAC2STR(mgmt->sa),
+				   hapd->conf->no_auth_if_seen_on);
+
+			resp = WLAN_STATUS_REJECTED_WITH_SUGGESTED_BSS_TRANSITION;
+			pos = &resp_ies[0];
+			*pos++ = WLAN_EID_NEIGHBOR_REPORT;
+			*pos++ = 13;
+			os_memcpy(pos, other->own_addr, ETH_ALEN);
+			pos += ETH_ALEN;
+			info = 0; /* TODO: BSSID Information */
+			WPA_PUT_LE32(pos, info);
+			pos += 4;
+			if (other->iconf->hw_mode == HOSTAPD_MODE_IEEE80211AD)
+				phytype = 8; /* dmg */
+			else if (other->iconf->ieee80211ac)
+				phytype = 9; /* vht */
+			else if (other->iconf->ieee80211n)
+				phytype = 7; /* ht */
+			else if (other->iconf->hw_mode ==
+				 HOSTAPD_MODE_IEEE80211A)
+				phytype = 4; /* ofdm */
+			else if (other->iconf->hw_mode ==
+				 HOSTAPD_MODE_IEEE80211G)
+				phytype = 6; /* erp */
+			else
+				phytype = 5; /* hrdsss */
+			if (ieee80211_freq_to_channel_ext(
+				    hostapd_hw_get_freq(other,
+							other->iconf->channel),
+				    other->iconf->secondary_channel,
+				    other->iconf->ieee80211ac,
+				    &op_class, &channel) == NUM_HOSTAPD_MODES) {
+				op_class = 0;
+				channel = other->iconf->channel;
+			}
+			*pos++ = op_class;
+			*pos++ = channel;
+			*pos++ = phytype;
+			resp_ies_len = pos - &resp_ies[0];
+			goto fail;
+		}
+	}
+
 	res = hostapd_allowed_address(hapd, mgmt->sa, (u8 *) mgmt, len,
 				      &session_timeout,
 				      &acct_interim_interval, &vlan_id,
@@ -1081,6 +1142,7 @@ static void handle_auth(struct hostapd_data *hapd,
 		sta->auth_alg = WLAN_AUTH_OPEN;
 		mlme_authenticate_indication(hapd, sta);
 		break;
+#ifndef CONFIG_NO_RC4
 	case WLAN_AUTH_SHARED_KEY:
 		resp = auth_shared_key(hapd, sta, auth_transaction, challenge,
 				       fc & WLAN_FC_ISWEP);
@@ -1094,6 +1156,7 @@ static void handle_auth(struct hostapd_data *hapd,
 			resp_ies_len = 2 + WLAN_AUTH_CHALLENGE_LEN;
 		}
 		break;
+#endif /* CONFIG_NO_RC4 */
 #ifdef CONFIG_IEEE80211R
 	case WLAN_AUTH_FT:
 		sta->auth_alg = WLAN_AUTH_FT;
@@ -1297,8 +1360,7 @@ static u16 check_assoc_ies(struct hostapd_data *hapd, struct sta_info *sta,
 	if (resp != WLAN_STATUS_SUCCESS)
 		return resp;
 #ifdef CONFIG_IEEE80211N
-	resp = copy_sta_ht_capab(hapd, sta, elems.ht_capabilities,
-				 elems.ht_capabilities_len);
+	resp = copy_sta_ht_capab(hapd, sta, elems.ht_capabilities);
 	if (resp != WLAN_STATUS_SUCCESS)
 		return resp;
 	if (hapd->iconf->ieee80211n && hapd->iconf->require_ht &&
@@ -1311,14 +1373,15 @@ static u16 check_assoc_ies(struct hostapd_data *hapd, struct sta_info *sta,
 #endif /* CONFIG_IEEE80211N */
 
 #ifdef CONFIG_IEEE80211AC
-	resp = copy_sta_vht_capab(hapd, sta, elems.vht_capabilities,
-				  elems.vht_capabilities_len);
-	if (resp != WLAN_STATUS_SUCCESS)
-		return resp;
+	if (hapd->iconf->ieee80211ac) {
+		resp = copy_sta_vht_capab(hapd, sta, elems.vht_capabilities);
+		if (resp != WLAN_STATUS_SUCCESS)
+			return resp;
 
-	resp = set_sta_vht_opmode(hapd, sta, elems.vht_opmode_notif);
-	if (resp != WLAN_STATUS_SUCCESS)
-		return resp;
+		resp = set_sta_vht_opmode(hapd, sta, elems.vht_opmode_notif);
+		if (resp != WLAN_STATUS_SUCCESS)
+			return resp;
+	}
 
 	if (hapd->iconf->ieee80211ac && hapd->iconf->require_vht &&
 	    !(sta->flags & WLAN_STA_VHT)) {
@@ -1546,6 +1609,14 @@ static u16 check_assoc_ies(struct hostapd_data *hapd, struct sta_info *sta,
 		sta->hs20_ie = NULL;
 #endif /* CONFIG_HS20 */
 
+#ifdef CONFIG_FST
+	wpabuf_free(sta->mb_ies);
+	if (hapd->iface->fst)
+		sta->mb_ies = mb_ies_by_info(&elems.mb_ies);
+	else
+		sta->mb_ies = NULL;
+#endif /* CONFIG_FST */
+
 	return WLAN_STATUS_SUCCESS;
 }
 
@@ -1594,7 +1665,7 @@ static void send_assoc_resp(struct hostapd_data *hapd, struct sta_info *sta,
 	send_len = IEEE80211_HDRLEN;
 	send_len += sizeof(reply->u.assoc_resp);
 	reply->u.assoc_resp.capab_info =
-		host_to_le16(hostapd_own_capab_info(hapd, sta, 0));
+		host_to_le16(hostapd_own_capab_info(hapd));
 	reply->u.assoc_resp.status_code = host_to_le16(status_code);
 	reply->u.assoc_resp.aid = host_to_le16(sta->aid | BIT(14) | BIT(15));
 	/* Supported rates */
@@ -1634,6 +1705,14 @@ static void send_assoc_resp(struct hostapd_data *hapd, struct sta_info *sta,
 	if (sta->qos_map_enabled)
 		p = hostapd_eid_qos_map_set(hapd, p);
 
+#ifdef CONFIG_FST
+	if (hapd->iface->fst_ies) {
+		os_memcpy(p, wpabuf_head(hapd->iface->fst_ies),
+			  wpabuf_len(hapd->iface->fst_ies));
+		p += wpabuf_len(hapd->iface->fst_ies);
+	}
+#endif /* CONFIG_FST */
+
 #ifdef CONFIG_IEEE80211AC
 	if (hapd->conf->vendor_vht && (sta->flags & WLAN_STA_VENDOR_VHT))
 		p = hostapd_eid_vendor_vht(hapd, p);
@@ -2112,10 +2191,20 @@ static int handle_action(struct hostapd_data *hapd,
 		ieee802_11_rx_wnm_action_ap(hapd, mgmt, len);
 		return 1;
 #endif /* CONFIG_WNM */
+#ifdef CONFIG_FST
+	case WLAN_ACTION_FST:
+		if (hapd->iface->fst)
+			fst_rx_action(hapd->iface->fst, mgmt, len);
+		else
+			wpa_printf(MSG_DEBUG,
+				   "FST: Ignore FST Action frame - no FST attached");
+		return 1;
+#endif /* CONFIG_FST */
 	case WLAN_ACTION_PUBLIC:
 	case WLAN_ACTION_PROTECTED_DUAL:
 #ifdef CONFIG_IEEE80211N
-		if (mgmt->u.action.u.public_action.action ==
+		if (len >= IEEE80211_HDRLEN + 2 &&
+		    mgmt->u.action.u.public_action.action ==
 		    WLAN_PA_20_40_BSS_COEX) {
 			wpa_printf(MSG_DEBUG,
 				   "HT20/40 coex mgmt frame received from STA "
@@ -2248,6 +2337,9 @@ int ieee802_11_mgmt(struct hostapd_data *hapd, const u8 *buf, size_t len,
 		return 0;
 	}
 
+	if (hapd->iconf->track_sta_max_num)
+		sta_track_add(hapd->iface, mgmt->sa);
+
 	switch (stype) {
 	case WLAN_FC_STYPE_AUTH:
 		wpa_printf(MSG_DEBUG, "mgmt::auth");
@@ -2335,7 +2427,7 @@ static void hostapd_set_wds_encryption(struct hostapd_data *hapd,
 				       char *ifname_wds)
 {
 	int i;
-	struct hostapd_ssid *ssid = sta->ssid;
+	struct hostapd_ssid *ssid = &hapd->conf->ssid;
 
 	if (hapd->conf->ieee802_1x || hapd->conf->wpa)
 		return;
@@ -2473,11 +2565,11 @@ static void handle_assoc_cb(struct hostapd_data *hapd,
 		 * so bind it to the selected VLAN interface now, since the
 		 * interface selection is not going to change anymore.
 		 */
-		if (ap_sta_bind_vlan(hapd, sta, 0) < 0)
+		if (ap_sta_bind_vlan(hapd, sta) < 0)
 			return;
 	} else if (sta->vlan_id) {
 		/* VLAN ID already set (e.g., by PMKSA caching), so bind STA */
-		if (ap_sta_bind_vlan(hapd, sta, 0) < 0)
+		if (ap_sta_bind_vlan(hapd, sta) < 0)
 			return;
 	}
 
diff --git a/src/ap/ieee802_11.h b/src/ap/ieee802_11.h
index 41c27d906e72..44c1bff364ac 100644
--- a/src/ap/ieee802_11.h
+++ b/src/ap/ieee802_11.h
@@ -14,6 +14,7 @@ struct hostapd_data;
 struct sta_info;
 struct hostapd_frame_info;
 struct ieee80211_ht_capabilities;
+struct ieee80211_vht_capabilities;
 struct ieee80211_mgmt;
 
 int ieee802_11_mgmt(struct hostapd_data *hapd, const u8 *buf, size_t len,
@@ -40,8 +41,7 @@ static inline int ieee802_11_get_mib_sta(struct hostapd_data *hapd,
 	return 0;
 }
 #endif /* NEED_AP_MLME */
-u16 hostapd_own_capab_info(struct hostapd_data *hapd, struct sta_info *sta,
-			   int probe);
+u16 hostapd_own_capab_info(struct hostapd_data *hapd);
 void ap_ht2040_timeout(void *eloop_data, void *user_data);
 u8 * hostapd_eid_ext_capab(struct hostapd_data *hapd, u8 *eid);
 u8 * hostapd_eid_qos_map_set(struct hostapd_data *hapd, u8 *eid);
@@ -62,7 +62,7 @@ void hostapd_get_vht_capab(struct hostapd_data *hapd,
 			   struct ieee80211_vht_capabilities *vht_cap,
 			   struct ieee80211_vht_capabilities *neg_vht_cap);
 u16 copy_sta_ht_capab(struct hostapd_data *hapd, struct sta_info *sta,
-		      const u8 *ht_capab, size_t ht_capab_len);
+		      const u8 *ht_capab);
 u16 copy_sta_vendor_vht(struct hostapd_data *hapd, struct sta_info *sta,
 			const u8 *ie, size_t len);
 
@@ -70,7 +70,7 @@ void update_ht_state(struct hostapd_data *hapd, struct sta_info *sta);
 void ht40_intolerant_add(struct hostapd_iface *iface, struct sta_info *sta);
 void ht40_intolerant_remove(struct hostapd_iface *iface, struct sta_info *sta);
 u16 copy_sta_vht_capab(struct hostapd_data *hapd, struct sta_info *sta,
-		       const u8 *vht_capab, size_t vht_capab_len);
+		       const u8 *vht_capab);
 u16 set_sta_vht_opmode(struct hostapd_data *hapd, struct sta_info *sta,
 		       const u8 *vht_opmode);
 void hostapd_tx_status(struct hostapd_data *hapd, const u8 *addr,
diff --git a/src/ap/ieee802_11_auth.c b/src/ap/ieee802_11_auth.c
index 56c3ce0313d4..531a67da412c 100644
--- a/src/ap/ieee802_11_auth.c
+++ b/src/ap/ieee802_11_auth.c
@@ -399,19 +399,15 @@ static void hostapd_acl_expire_queries(struct hostapd_data *hapd,
 
 /**
  * hostapd_acl_expire - ACL cache expiration callback
- * @eloop_ctx: struct hostapd_data *
- * @timeout_ctx: Not used
+ * @hapd: struct hostapd_data *
  */
-static void hostapd_acl_expire(void *eloop_ctx, void *timeout_ctx)
+void hostapd_acl_expire(struct hostapd_data *hapd)
 {
-	struct hostapd_data *hapd = eloop_ctx;
 	struct os_reltime now;
 
 	os_get_reltime(&now);
 	hostapd_acl_expire_cache(hapd, &now);
 	hostapd_acl_expire_queries(hapd, &now);
-
-	eloop_register_timeout(10, 0, hostapd_acl_expire, hapd, NULL);
 }
 
 
@@ -561,6 +557,19 @@ hostapd_acl_recv_radius(struct radius_msg *msg, struct radius_msg *req,
 		if (hapd->conf->wpa_psk_radius == PSK_RADIUS_REQUIRED &&
 		    !cache->psk)
 			cache->accepted = HOSTAPD_ACL_REJECT;
+
+		if (cache->vlan_id &&
+		    !hostapd_vlan_id_valid(hapd->conf->vlan, cache->vlan_id)) {
+			hostapd_logger(hapd, query->addr,
+				       HOSTAPD_MODULE_RADIUS,
+				       HOSTAPD_LEVEL_INFO,
+				       "Invalid VLAN ID %d received from RADIUS server",
+				       cache->vlan_id);
+			cache->vlan_id = 0;
+		}
+		if (hapd->conf->ssid.dynamic_vlan == DYNAMIC_VLAN_REQUIRED &&
+		    !cache->vlan_id)
+			cache->accepted = HOSTAPD_ACL_REJECT;
 	} else
 		cache->accepted = HOSTAPD_ACL_REJECT;
 	cache->next = hapd->acl_cache;
@@ -602,8 +611,6 @@ int hostapd_acl_init(struct hostapd_data *hapd)
 	if (radius_client_register(hapd->radius, RADIUS_AUTH,
 				   hostapd_acl_recv_radius, hapd))
 		return -1;
-
-	eloop_register_timeout(10, 0, hostapd_acl_expire, hapd, NULL);
 #endif /* CONFIG_NO_RADIUS */
 
 	return 0;
@@ -619,8 +626,6 @@ void hostapd_acl_deinit(struct hostapd_data *hapd)
 	struct hostapd_acl_query_data *query, *prev;
 
 #ifndef CONFIG_NO_RADIUS
-	eloop_cancel_timeout(hostapd_acl_expire, hapd, NULL);
-
 	hostapd_acl_cache_free(hapd->acl_cache);
 #endif /* CONFIG_NO_RADIUS */
 
diff --git a/src/ap/ieee802_11_auth.h b/src/ap/ieee802_11_auth.h
index 2bc1065a226c..b66f244b3ebc 100644
--- a/src/ap/ieee802_11_auth.h
+++ b/src/ap/ieee802_11_auth.h
@@ -24,5 +24,6 @@ int hostapd_allowed_address(struct hostapd_data *hapd, const u8 *addr,
 int hostapd_acl_init(struct hostapd_data *hapd);
 void hostapd_acl_deinit(struct hostapd_data *hapd);
 void hostapd_free_psk_list(struct hostapd_sta_wpa_psk_short *psk);
+void hostapd_acl_expire(struct hostapd_data *hapd);
 
 #endif /* IEEE802_11_AUTH_H */
diff --git a/src/ap/ieee802_11_ht.c b/src/ap/ieee802_11_ht.c
index 4b0653de95db..11fde2a26394 100644
--- a/src/ap/ieee802_11_ht.c
+++ b/src/ap/ieee802_11_ht.c
@@ -209,7 +209,7 @@ void hostapd_2040_coex_action(struct hostapd_data *hapd,
 	struct hostapd_iface *iface = hapd->iface;
 	struct ieee80211_2040_bss_coex_ie *bc_ie;
 	struct ieee80211_2040_intol_chan_report *ic_report;
-	int is_ht_allowed = 1;
+	int is_ht40_allowed = 1;
 	int i;
 	const u8 *start = (const u8 *) mgmt;
 	const u8 *data = start + IEEE80211_HDRLEN + 2;
@@ -242,7 +242,7 @@ void hostapd_2040_coex_action(struct hostapd_data *hapd,
 			       HOSTAPD_MODULE_IEEE80211,
 			       HOSTAPD_LEVEL_DEBUG,
 			       "20 MHz BSS width request bit is set in BSS coexistence information field");
-		is_ht_allowed = 0;
+		is_ht40_allowed = 0;
 	}
 
 	if (bc_ie->coex_param & WLAN_20_40_BSS_COEX_40MHZ_INTOL) {
@@ -250,7 +250,7 @@ void hostapd_2040_coex_action(struct hostapd_data *hapd,
 			       HOSTAPD_MODULE_IEEE80211,
 			       HOSTAPD_LEVEL_DEBUG,
 			       "40 MHz intolerant bit is set in BSS coexistence information field");
-		is_ht_allowed = 0;
+		is_ht40_allowed = 0;
 	}
 
 	if (start + len - data >= 3 &&
@@ -276,13 +276,13 @@ void hostapd_2040_coex_action(struct hostapd_data *hapd,
 				       HOSTAPD_LEVEL_DEBUG,
 				       "20_40_INTOLERANT channel %d reported",
 				       chan);
-			is_ht_allowed = 0;
+			is_ht40_allowed = 0;
 		}
 	}
-	wpa_printf(MSG_DEBUG, "is_ht_allowed=%d num_sta_ht40_intolerant=%d",
-		   is_ht_allowed, iface->num_sta_ht40_intolerant);
+	wpa_printf(MSG_DEBUG, "is_ht40_allowed=%d num_sta_ht40_intolerant=%d",
+		   is_ht40_allowed, iface->num_sta_ht40_intolerant);
 
-	if (!is_ht_allowed &&
+	if (!is_ht40_allowed &&
 	    (iface->drv_flags & WPA_DRIVER_FLAGS_HT_2040_COEX)) {
 		if (iface->conf->secondary_channel) {
 			hostapd_logger(hapd, mgmt->sa,
@@ -310,12 +310,15 @@ void hostapd_2040_coex_action(struct hostapd_data *hapd,
 
 
 u16 copy_sta_ht_capab(struct hostapd_data *hapd, struct sta_info *sta,
-		      const u8 *ht_capab, size_t ht_capab_len)
+		      const u8 *ht_capab)
 {
-	/* Disable HT caps for STAs associated to no-HT BSSes. */
+	/*
+	 * Disable HT caps for STAs associated to no-HT BSSes, or for stations
+	 * that did not specify a valid WMM IE in the (Re)Association Request
+	 * frame.
+	 */
 	if (!ht_capab ||
-	    ht_capab_len < sizeof(struct ieee80211_ht_capabilities) ||
-	    hapd->conf->disable_11n) {
+	    !(sta->flags & WLAN_STA_WMM) || hapd->conf->disable_11n) {
 		sta->flags &= ~WLAN_STA_HT;
 		os_free(sta->ht_capabilities);
 		sta->ht_capabilities = NULL;
diff --git a/src/ap/ieee802_11_vht.c b/src/ap/ieee802_11_vht.c
index 171538ad74a7..5bf1b5d72002 100644
--- a/src/ap/ieee802_11_vht.c
+++ b/src/ap/ieee802_11_vht.c
@@ -132,11 +132,10 @@ static int check_valid_vht_mcs(struct hostapd_hw_modes *mode,
 
 
 u16 copy_sta_vht_capab(struct hostapd_data *hapd, struct sta_info *sta,
-		       const u8 *vht_capab, size_t vht_capab_len)
+		       const u8 *vht_capab)
 {
 	/* Disable VHT caps for STAs associated to no-VHT BSSes. */
 	if (!vht_capab ||
-	    vht_capab_len < sizeof(struct ieee80211_vht_capabilities) ||
 	    hapd->conf->disable_11ac ||
 	    !check_valid_vht_mcs(hapd->iface->current_mode, vht_capab)) {
 		sta->flags &= ~WLAN_STA_VHT;
diff --git a/src/ap/ieee802_1x.c b/src/ap/ieee802_1x.c
index 79dc0f957488..0f2d428cf752 100644
--- a/src/ap/ieee802_1x.c
+++ b/src/ap/ieee802_1x.c
@@ -125,6 +125,9 @@ void ieee802_1x_set_sta_authorized(struct hostapd_data *hapd,
 }
 
 
+#ifndef CONFIG_FIPS
+#ifndef CONFIG_NO_RC4
+
 static void ieee802_1x_tx_key_one(struct hostapd_data *hapd,
 				  struct sta_info *sta,
 				  int idx, int broadcast,
@@ -204,7 +207,7 @@ static void ieee802_1x_tx_key_one(struct hostapd_data *hapd,
 }
 
 
-void ieee802_1x_tx_key(struct hostapd_data *hapd, struct sta_info *sta)
+static void ieee802_1x_tx_key(struct hostapd_data *hapd, struct sta_info *sta)
 {
 	struct eapol_authenticator *eapol = hapd->eapol_auth;
 	struct eapol_state_machine *sm = sta->eapol_sm;
@@ -259,6 +262,9 @@ void ieee802_1x_tx_key(struct hostapd_data *hapd, struct sta_info *sta)
 	}
 }
 
+#endif /* CONFIG_NO_RC4 */
+#endif /* CONFIG_FIPS */
+
 
 const char *radius_mode_txt(struct hostapd_data *hapd)
 {
@@ -346,7 +352,8 @@ static int add_common_radius_sta_attr_rsn(struct hostapd_data *hapd,
 		return -1;
 	}
 
-	suite = wpa_cipher_to_suite((hapd->conf->wpa & 0x2) ?
+	suite = wpa_cipher_to_suite(((hapd->conf->wpa & 0x2) ||
+				     hapd->conf->osen) ?
 				    WPA_PROTO_RSN : WPA_PROTO_WPA,
 				    hapd->conf->wpa_group);
 	if (!hostapd_config_get_radius_attr(req_attr,
@@ -453,7 +460,7 @@ static int add_common_radius_sta_attr(struct hostapd_data *hapd,
 	}
 #endif /* CONFIG_IEEE80211R */
 
-	if (hapd->conf->wpa && sta->wpa_sm &&
+	if ((hapd->conf->wpa || hapd->conf->osen) && sta->wpa_sm &&
 	    add_common_radius_sta_attr_rsn(hapd, req_attr, sta, msg) < 0)
 		return -1;
 
@@ -599,7 +606,7 @@ static void ieee802_1x_encapsulate_radius(struct hostapd_data *hapd,
 		goto fail;
 	}
 
-	if (eap && !radius_msg_add_eap(msg, eap, len)) {
+	if (!radius_msg_add_eap(msg, eap, len)) {
 		wpa_printf(MSG_INFO, "Could not add EAP-Message");
 		goto fail;
 	}
@@ -1108,8 +1115,6 @@ void ieee802_1x_new_station(struct hostapd_data *hapd, struct sta_info *sta)
 
 	pmksa = wpa_auth_sta_get_pmksa(sta->wpa_sm);
 	if (pmksa) {
-		int old_vlanid;
-
 		hostapd_logger(hapd, sta->addr, HOSTAPD_MODULE_IEEE8021X,
 			       HOSTAPD_LEVEL_DEBUG,
 			       "PMK from PMKSA cache - skip IEEE 802.1X/EAP");
@@ -1123,11 +1128,8 @@ void ieee802_1x_new_station(struct hostapd_data *hapd, struct sta_info *sta)
 		sta->eapol_sm->authFail = FALSE;
 		if (sta->eapol_sm->eap)
 			eap_sm_notify_cached(sta->eapol_sm->eap);
-		old_vlanid = sta->vlan_id;
 		pmksa_cache_to_eapol_data(pmksa, sta->eapol_sm);
-		if (sta->ssid->dynamic_vlan == DYNAMIC_VLAN_DISABLED)
-			sta->vlan_id = 0;
-		ap_sta_bind_vlan(hapd, sta, old_vlanid);
+		ap_sta_bind_vlan(hapd, sta);
 	} else {
 		if (reassoc) {
 			/*
@@ -1290,7 +1292,7 @@ static void ieee802_1x_store_radius_class(struct hostapd_data *hapd,
 					  struct sta_info *sta,
 					  struct radius_msg *msg)
 {
-	u8 *class;
+	u8 *attr_class;
 	size_t class_len;
 	struct eapol_state_machine *sm = sta->eapol_sm;
 	int count, i;
@@ -1312,12 +1314,12 @@ static void ieee802_1x_store_radius_class(struct hostapd_data *hapd,
 
 	nclass_count = 0;
 
-	class = NULL;
+	attr_class = NULL;
 	for (i = 0; i < count; i++) {
 		do {
 			if (radius_msg_get_attr_ptr(msg, RADIUS_ATTR_CLASS,
-						    &class, &class_len,
-						    class) < 0) {
+						    &attr_class, &class_len,
+						    attr_class) < 0) {
 				i = count;
 				break;
 			}
@@ -1327,7 +1329,7 @@ static void ieee802_1x_store_radius_class(struct hostapd_data *hapd,
 		if (nclass[nclass_count].data == NULL)
 			break;
 
-		os_memcpy(nclass[nclass_count].data, class, class_len);
+		os_memcpy(nclass[nclass_count].data, attr_class, class_len);
 		nclass[nclass_count].len = class_len;
 		nclass_count++;
 	}
@@ -1590,7 +1592,7 @@ ieee802_1x_receive_auth(struct radius_msg *msg, struct radius_msg *req,
 	struct hostapd_data *hapd = data;
 	struct sta_info *sta;
 	u32 session_timeout = 0, termination_action, acct_interim_interval;
-	int session_timeout_set, old_vlanid = 0;
+	int session_timeout_set, vlan_id = 0;
 	struct eapol_state_machine *sm;
 	int override_eapReq = 0;
 	struct radius_hdr *hdr = radius_msg_get_hdr(msg);
@@ -1657,20 +1659,27 @@ ieee802_1x_receive_auth(struct radius_msg *msg, struct radius_msg *req,
 
 	switch (hdr->code) {
 	case RADIUS_CODE_ACCESS_ACCEPT:
-		if (sta->ssid->dynamic_vlan == DYNAMIC_VLAN_DISABLED)
-			sta->vlan_id = 0;
+		if (hapd->conf->ssid.dynamic_vlan == DYNAMIC_VLAN_DISABLED)
+			vlan_id = 0;
 #ifndef CONFIG_NO_VLAN
-		else {
-			old_vlanid = sta->vlan_id;
-			sta->vlan_id = radius_msg_get_vlanid(msg);
-		}
-		if (sta->vlan_id > 0 &&
-		    hostapd_vlan_id_valid(hapd->conf->vlan, sta->vlan_id)) {
+		else
+			vlan_id = radius_msg_get_vlanid(msg);
+		if (vlan_id > 0 &&
+		    hostapd_vlan_id_valid(hapd->conf->vlan, vlan_id)) {
 			hostapd_logger(hapd, sta->addr,
 				       HOSTAPD_MODULE_RADIUS,
 				       HOSTAPD_LEVEL_INFO,
-				       "VLAN ID %d", sta->vlan_id);
-		} else if (sta->ssid->dynamic_vlan == DYNAMIC_VLAN_REQUIRED) {
+				       "VLAN ID %d", vlan_id);
+		} else if (vlan_id > 0) {
+			sta->eapol_sm->authFail = TRUE;
+			hostapd_logger(hapd, sta->addr,
+				       HOSTAPD_MODULE_RADIUS,
+				       HOSTAPD_LEVEL_INFO,
+				       "Invalid VLAN ID %d received from RADIUS server",
+				       vlan_id);
+			break;
+		} else if (hapd->conf->ssid.dynamic_vlan ==
+			   DYNAMIC_VLAN_REQUIRED) {
 			sta->eapol_sm->authFail = TRUE;
 			hostapd_logger(hapd, sta->addr,
 				       HOSTAPD_MODULE_IEEE8021X,
@@ -1681,7 +1690,9 @@ ieee802_1x_receive_auth(struct radius_msg *msg, struct radius_msg *req,
 		}
 #endif /* CONFIG_NO_VLAN */
 
-		if (ap_sta_bind_vlan(hapd, sta, old_vlanid) < 0)
+		sta->vlan_id = vlan_id;
+		if ((sta->flags & WLAN_STA_ASSOC) &&
+		    ap_sta_bind_vlan(hapd, sta) < 0)
 			break;
 
 		sta->session_timeout_set = !!session_timeout_set;
@@ -1926,10 +1937,11 @@ static int ieee802_1x_get_eap_user(void *ctx, const u8 *identity,
 	struct hostapd_data *hapd = ctx;
 	const struct hostapd_eap_user *eap_user;
 	int i;
+	int rv = -1;
 
 	eap_user = hostapd_get_eap_user(hapd, identity, identity_len, phase2);
 	if (eap_user == NULL)
-		return -1;
+		goto out;
 
 	os_memset(user, 0, sizeof(*user));
 	user->phase2 = phase2;
@@ -1941,7 +1953,7 @@ static int ieee802_1x_get_eap_user(void *ctx, const u8 *identity,
 	if (eap_user->password) {
 		user->password = os_malloc(eap_user->password_len);
 		if (user->password == NULL)
-			return -1;
+			goto out;
 		os_memcpy(user->password, eap_user->password,
 			  eap_user->password_len);
 		user->password_len = eap_user->password_len;
@@ -1951,8 +1963,13 @@ static int ieee802_1x_get_eap_user(void *ctx, const u8 *identity,
 	user->macacl = eap_user->macacl;
 	user->ttls_auth = eap_user->ttls_auth;
 	user->remediation = eap_user->remediation;
+	rv = 0;
 
-	return 0;
+out:
+	if (rv)
+		wpa_printf(MSG_DEBUG, "%s: Failed to find user", __func__);
+
+	return rv;
 }
 
 
@@ -2012,9 +2029,13 @@ static void _ieee802_1x_abort_auth(void *ctx, void *sta_ctx)
 
 static void _ieee802_1x_tx_key(void *ctx, void *sta_ctx)
 {
+#ifndef CONFIG_FIPS
+#ifndef CONFIG_NO_RC4
 	struct hostapd_data *hapd = ctx;
 	struct sta_info *sta = sta_ctx;
 	ieee802_1x_tx_key(hapd, sta);
+#endif /* CONFIG_NO_RC4 */
+#endif /* CONFIG_FIPS */
 }
 
 
@@ -2085,6 +2106,7 @@ int ieee802_1x_init(struct hostapd_data *hapd)
 	conf.erp_send_reauth_start = hapd->conf->erp_send_reauth_start;
 	conf.erp_domain = hapd->conf->erp_domain;
 	conf.erp = hapd->conf->eap_server_erp;
+	conf.tls_session_lifetime = hapd->conf->tls_session_lifetime;
 	conf.pac_opaque_encr_key = hapd->conf->pac_opaque_encr_key;
 	conf.eap_fast_a_id = hapd->conf->eap_fast_a_id;
 	conf.eap_fast_a_id_len = hapd->conf->eap_fast_a_id_len;
@@ -2332,9 +2354,9 @@ void ieee802_1x_notify_pre_auth(struct eapol_state_machine *sm, int pre_auth)
 }
 
 
-static const char * bool_txt(Boolean bool)
+static const char * bool_txt(Boolean val)
 {
-	return bool ? "TRUE" : "FALSE";
+	return val ? "TRUE" : "FALSE";
 }
 
 
diff --git a/src/ap/ieee802_1x.h b/src/ap/ieee802_1x.h
index de6e0e75fac3..14d69556993c 100644
--- a/src/ap/ieee802_1x.h
+++ b/src/ap/ieee802_1x.h
@@ -23,7 +23,6 @@ void ieee802_1x_receive(struct hostapd_data *hapd, const u8 *sa, const u8 *buf,
 void ieee802_1x_new_station(struct hostapd_data *hapd, struct sta_info *sta);
 void ieee802_1x_free_station(struct sta_info *sta);
 
-void ieee802_1x_tx_key(struct hostapd_data *hapd, struct sta_info *sta);
 void ieee802_1x_abort_auth(struct hostapd_data *hapd, struct sta_info *sta);
 void ieee802_1x_set_sta_authorized(struct hostapd_data *hapd,
 				   struct sta_info *sta, int authorized);
diff --git a/src/ap/ndisc_snoop.c b/src/ap/ndisc_snoop.c
index b0d42dcd82c4..4a87721e2ecf 100644
--- a/src/ap/ndisc_snoop.c
+++ b/src/ap/ndisc_snoop.c
@@ -81,12 +81,24 @@ static int sta_has_ip6addr(struct sta_info *sta, struct in6_addr *addr)
 }
 
 
+static void ucast_to_stas(struct hostapd_data *hapd, const u8 *buf, size_t len)
+{
+	struct sta_info *sta;
+
+	for (sta = hapd->sta_list; sta; sta = sta->next) {
+		if (!(sta->flags & WLAN_STA_AUTHORIZED))
+			continue;
+		x_snoop_mcast_to_ucast_convert_send(hapd, sta, (u8 *) buf, len);
+	}
+}
+
+
 static void handle_ndisc(void *ctx, const u8 *src_addr, const u8 *buf,
 			 size_t len)
 {
 	struct hostapd_data *hapd = ctx;
 	struct icmpv6_ndmsg *msg;
-	struct in6_addr *saddr;
+	struct in6_addr saddr;
 	struct sta_info *sta;
 	int res;
 	char addrtxt[INET6_ADDRSTRLEN + 1];
@@ -101,25 +113,30 @@ static void handle_ndisc(void *ctx, const u8 *src_addr, const u8 *buf,
 		if (msg->opt_type != SOURCE_LL_ADDR)
 			return;
 
-		saddr = &msg->ipv6h.ip6_src;
-		if (!(saddr->s6_addr32[0] == 0 && saddr->s6_addr32[1] == 0 &&
-		      saddr->s6_addr32[2] == 0 && saddr->s6_addr32[3] == 0)) {
+		/*
+		 * IPv6 header may not be 32-bit aligned in the buffer, so use
+		 * a local copy to avoid unaligned reads.
+		 */
+		os_memcpy(&saddr, &msg->ipv6h.ip6_src, sizeof(saddr));
+		if (!(saddr.s6_addr32[0] == 0 && saddr.s6_addr32[1] == 0 &&
+		      saddr.s6_addr32[2] == 0 && saddr.s6_addr32[3] == 0)) {
 			if (len < ETH_HLEN + sizeof(*msg) + ETH_ALEN)
 				return;
 			sta = ap_get_sta(hapd, msg->opt_lladdr);
 			if (!sta)
 				return;
 
-			if (sta_has_ip6addr(sta, saddr))
+			if (sta_has_ip6addr(sta, &saddr))
 				return;
 
-			if (inet_ntop(AF_INET6, saddr, addrtxt, sizeof(addrtxt))
-			    == NULL)
+			if (inet_ntop(AF_INET6, &saddr, addrtxt,
+				      sizeof(addrtxt)) == NULL)
 				addrtxt[0] = '\0';
 			wpa_printf(MSG_DEBUG, "ndisc_snoop: Learned new IPv6 address %s for "
 				   MACSTR, addrtxt, MAC2STR(sta->addr));
-			hostapd_drv_br_delete_ip_neigh(hapd, 6, (u8 *) saddr);
-			res = hostapd_drv_br_add_ip_neigh(hapd, 6, (u8 *) saddr,
+			hostapd_drv_br_delete_ip_neigh(hapd, 6, (u8 *) &saddr);
+			res = hostapd_drv_br_add_ip_neigh(hapd, 6,
+							  (u8 *) &saddr,
 							  128, sta->addr);
 			if (res) {
 				wpa_printf(MSG_ERROR,
@@ -128,21 +145,17 @@ static void handle_ndisc(void *ctx, const u8 *src_addr, const u8 *buf,
 				return;
 			}
 
-			if (sta_ip6addr_add(sta, saddr))
+			if (sta_ip6addr_add(sta, &saddr))
 				return;
 		}
 		break;
 	case ROUTER_ADVERTISEMENT:
-		if (!hapd->conf->disable_dgaf)
-			return;
-		/* fall through */
+		if (hapd->conf->disable_dgaf)
+			ucast_to_stas(hapd, buf, len);
+		break;
 	case NEIGHBOR_ADVERTISEMENT:
-		for (sta = hapd->sta_list; sta; sta = sta->next) {
-			if (!(sta->flags & WLAN_STA_AUTHORIZED))
-				continue;
-			x_snoop_mcast_to_ucast_convert_send(hapd, sta,
-							    (u8 *) buf, len);
-		}
+		if (hapd->conf->na_mcast_to_ucast)
+			ucast_to_stas(hapd, buf, len);
 		break;
 	default:
 		break;
diff --git a/src/ap/sta_info.c b/src/ap/sta_info.c
index 7e75e1a61e1a..d64307ccfd08 100644
--- a/src/ap/sta_info.c
+++ b/src/ap/sta_info.c
@@ -16,6 +16,7 @@
 #include "radius/radius.h"
 #include "radius/radius_client.h"
 #include "p2p/p2p.h"
+#include "fst/fst.h"
 #include "hostapd.h"
 #include "accounting.h"
 #include "ieee802_1x.h"
@@ -171,6 +172,19 @@ void ap_free_sta(struct hostapd_data *hapd, struct sta_info *sta)
 	    !(sta->flags & WLAN_STA_PREAUTH))
 		hostapd_drv_sta_remove(hapd, sta->addr);
 
+#ifndef CONFIG_NO_VLAN
+	if (sta->vlan_id_bound) {
+		/*
+		 * Need to remove the STA entry before potentially removing the
+		 * VLAN.
+		 */
+		if (hapd->iface->driver_ap_teardown &&
+		    !(sta->flags & WLAN_STA_PREAUTH))
+			hostapd_drv_sta_remove(hapd, sta->addr);
+		vlan_remove_dynamic(hapd, sta->vlan_id_bound);
+	}
+#endif /* CONFIG_NO_VLAN */
+
 	ap_sta_hash_del(hapd, sta);
 	ap_sta_list_del(hapd, sta);
 
@@ -283,6 +297,9 @@ void ap_free_sta(struct hostapd_data *hapd, struct sta_info *sta)
 	wpabuf_free(sta->wps_ie);
 	wpabuf_free(sta->p2p_ie);
 	wpabuf_free(sta->hs20_ie);
+#ifdef CONFIG_FST
+	wpabuf_free(sta->mb_ies);
+#endif /* CONFIG_FST */
 
 	os_free(sta->ht_capabilities);
 	os_free(sta->vht_capabilities);
@@ -619,7 +636,6 @@ struct sta_info * ap_sta_add(struct hostapd_data *hapd, const u8 *addr)
 	hapd->sta_list = sta;
 	hapd->num_sta++;
 	ap_sta_hash_add(hapd, sta);
-	sta->ssid = &hapd->conf->ssid;
 	ap_sta_remove_in_other_bss(hapd, sta);
 	sta->last_seq_ctrl = WLAN_INVALID_MGMT_SEQ;
 	dl_list_init(&sta->ip6addr);
@@ -768,26 +784,19 @@ int ap_sta_wps_cancel(struct hostapd_data *hapd,
 #endif /* CONFIG_WPS */
 
 
-int ap_sta_bind_vlan(struct hostapd_data *hapd, struct sta_info *sta,
-		     int old_vlanid)
+int ap_sta_bind_vlan(struct hostapd_data *hapd, struct sta_info *sta)
 {
 #ifndef CONFIG_NO_VLAN
 	const char *iface;
 	struct hostapd_vlan *vlan = NULL;
 	int ret;
-
-	/*
-	 * Do not proceed furthur if the vlan id remains same. We do not want
-	 * duplicate dynamic vlan entries.
-	 */
-	if (sta->vlan_id == old_vlanid)
-		return 0;
+	int old_vlanid = sta->vlan_id_bound;
 
 	iface = hapd->conf->iface;
-	if (sta->ssid->vlan[0])
-		iface = sta->ssid->vlan;
+	if (hapd->conf->ssid.vlan[0])
+		iface = hapd->conf->ssid.vlan;
 
-	if (sta->ssid->dynamic_vlan == DYNAMIC_VLAN_DISABLED)
+	if (hapd->conf->ssid.dynamic_vlan == DYNAMIC_VLAN_DISABLED)
 		sta->vlan_id = 0;
 	else if (sta->vlan_id > 0) {
 		struct hostapd_vlan *wildcard_vlan = NULL;
@@ -805,6 +814,14 @@ int ap_sta_bind_vlan(struct hostapd_data *hapd, struct sta_info *sta,
 			iface = vlan->ifname;
 	}
 
+	/*
+	 * Do not increment ref counters if the VLAN ID remains same, but do
+	 * not skip hostapd_drv_set_sta_vlan() as hostapd_drv_sta_remove() might
+	 * have been called before.
+	 */
+	if (sta->vlan_id == old_vlanid)
+		goto skip_counting;
+
 	if (sta->vlan_id > 0 && vlan == NULL) {
 		hostapd_logger(hapd, sta->addr, HOSTAPD_MODULE_IEEE80211,
 			       HOSTAPD_LEVEL_DEBUG, "could not find VLAN for "
@@ -825,7 +842,7 @@ int ap_sta_bind_vlan(struct hostapd_data *hapd, struct sta_info *sta,
 		}
 
 		iface = vlan->ifname;
-		if (vlan_setup_encryption_dyn(hapd, sta->ssid, iface) != 0) {
+		if (vlan_setup_encryption_dyn(hapd, iface) != 0) {
 			hostapd_logger(hapd, sta->addr,
 				       HOSTAPD_MODULE_IEEE80211,
 				       HOSTAPD_LEVEL_DEBUG, "could not "
@@ -838,7 +855,7 @@ int ap_sta_bind_vlan(struct hostapd_data *hapd, struct sta_info *sta,
 			       HOSTAPD_LEVEL_DEBUG, "added new dynamic VLAN "
 			       "interface '%s'", iface);
 	} else if (vlan && vlan->vlan_id == sta->vlan_id) {
-		if (sta->vlan_id > 0) {
+		if (vlan->dynamic_vlan > 0) {
 			vlan->dynamic_vlan++;
 			hostapd_logger(hapd, sta->addr,
 				       HOSTAPD_MODULE_IEEE80211,
@@ -852,7 +869,7 @@ int ap_sta_bind_vlan(struct hostapd_data *hapd, struct sta_info *sta,
 		 * configuration for the case where hostapd did not yet know
 		 * which keys are to be used when the interface was added.
 		 */
-		if (vlan_setup_encryption_dyn(hapd, sta->ssid, iface) != 0) {
+		if (vlan_setup_encryption_dyn(hapd, iface) != 0) {
 			hostapd_logger(hapd, sta->addr,
 				       HOSTAPD_MODULE_IEEE80211,
 				       HOSTAPD_LEVEL_DEBUG, "could not "
@@ -862,6 +879,10 @@ int ap_sta_bind_vlan(struct hostapd_data *hapd, struct sta_info *sta,
 		}
 	}
 
+	/* ref counters have been increased, so mark the station */
+	sta->vlan_id_bound = sta->vlan_id;
+
+skip_counting:
 	hostapd_logger(hapd, sta->addr, HOSTAPD_MODULE_IEEE80211,
 		       HOSTAPD_LEVEL_DEBUG, "binding station to interface "
 		       "'%s'", iface);
@@ -876,10 +897,10 @@ int ap_sta_bind_vlan(struct hostapd_data *hapd, struct sta_info *sta,
 			       "entry to vlan_id=%d", sta->vlan_id);
 	}
 
-done:
 	/* During 1x reauth, if the vlan id changes, then remove the old id. */
-	if (old_vlanid > 0)
+	if (old_vlanid > 0 && old_vlanid != sta->vlan_id)
 		vlan_remove_dynamic(hapd, old_vlanid);
+done:
 
 	return ret;
 #else /* CONFIG_NO_VLAN */
@@ -1043,6 +1064,16 @@ void ap_sta_set_authorized(struct hostapd_data *hapd, struct sta_info *sta,
 			wpa_msg_no_global(hapd->msg_ctx_parent, MSG_INFO,
 					  AP_STA_DISCONNECTED "%s", buf);
 	}
+
+#ifdef CONFIG_FST
+	if (hapd->iface->fst) {
+		if (authorized)
+			fst_notify_peer_connected(hapd->iface->fst, sta->addr);
+		else
+			fst_notify_peer_disconnected(hapd->iface->fst,
+						     sta->addr);
+	}
+#endif /* CONFIG_FST */
 }
 
 
diff --git a/src/ap/sta_info.h b/src/ap/sta_info.h
index 57551ab17d5d..420d64e5793b 100644
--- a/src/ap/sta_info.h
+++ b/src/ap/sta_info.h
@@ -117,10 +117,8 @@ struct sta_info {
 	struct wpa_state_machine *wpa_sm;
 	struct rsn_preauth_interface *preauth_iface;
 
-	struct hostapd_ssid *ssid; /* SSID selection based on (Re)AssocReq */
-	struct hostapd_ssid *ssid_probe; /* SSID selection based on ProbeReq */
-
-	int vlan_id;
+	int vlan_id; /* 0: none, >0: VID */
+	int vlan_id_bound; /* updated by ap_sta_bind_vlan() */
 	 /* PSKs from RADIUS authentication server */
 	struct hostapd_sta_wpa_psk_short *psk;
 
@@ -155,6 +153,9 @@ struct sta_info {
 	struct wpabuf *hs20_deauth_req;
 	char *hs20_session_info_url;
 	int hs20_disassoc_timer;
+#ifdef CONFIG_FST
+	struct wpabuf *mb_ies; /* MB IEs from (Re)Association Request */
+#endif /* CONFIG_FST */
 
 	struct os_reltime connected_time;
 
@@ -218,8 +219,7 @@ void ap_sta_deauthenticate(struct hostapd_data *hapd, struct sta_info *sta,
 int ap_sta_wps_cancel(struct hostapd_data *hapd,
 		      struct sta_info *sta, void *ctx);
 #endif /* CONFIG_WPS */
-int ap_sta_bind_vlan(struct hostapd_data *hapd, struct sta_info *sta,
-		     int old_vlanid);
+int ap_sta_bind_vlan(struct hostapd_data *hapd, struct sta_info *sta);
 void ap_sta_start_sa_query(struct hostapd_data *hapd, struct sta_info *sta);
 void ap_sta_stop_sa_query(struct hostapd_data *hapd, struct sta_info *sta);
 int ap_check_sa_query_timeout(struct hostapd_data *hapd, struct sta_info *sta);
diff --git a/src/ap/utils.c b/src/ap/utils.c
index 931968c84b0e..fcb371bec283 100644
--- a/src/ap/utils.c
+++ b/src/ap/utils.c
@@ -10,6 +10,7 @@
 
 #include "common.h"
 #include "common/ieee802_11_defs.h"
+#include "fst/fst.h"
 #include "sta_info.h"
 #include "hostapd.h"
 
@@ -55,10 +56,20 @@ static int prune_associations(struct hostapd_iface *iface, void *ctx)
 		ohapd = iface->bss[j];
 		if (ohapd == data->hapd)
 			continue;
+#ifdef CONFIG_FST
+		/* Don't prune STAs belong to same FST */
+		if (ohapd->iface->fst &&
+		    data->hapd->iface->fst &&
+		    fst_are_ifaces_aggregated(ohapd->iface->fst,
+					      data->hapd->iface->fst))
+			continue;
+#endif /* CONFIG_FST */
 		osta = ap_get_sta(ohapd, data->addr);
 		if (!osta)
 			continue;
 
+		wpa_printf(MSG_INFO, "%s: Prune association for " MACSTR,
+			   ohapd->conf->iface, MAC2STR(osta->addr));
 		ap_sta_disassociate(ohapd, osta, WLAN_REASON_UNSPECIFIED);
 	}
 
diff --git a/src/ap/vlan_init.c b/src/ap/vlan_init.c
index dc6501997db0..fd1c8ddacee6 100644
--- a/src/ap/vlan_init.c
+++ b/src/ap/vlan_init.c
@@ -9,6 +9,13 @@
  */
 
 #include "utils/includes.h"
+#ifdef CONFIG_FULL_DYNAMIC_VLAN
+#include <net/if.h>
+#include <sys/ioctl.h>
+#include <linux/sockios.h>
+#include <linux/if_vlan.h>
+#include <linux/if_bridge.h>
+#endif /* CONFIG_FULL_DYNAMIC_VLAN */
 
 #include "utils/common.h"
 #include "hostapd.h"
@@ -20,12 +27,6 @@
 
 #ifdef CONFIG_FULL_DYNAMIC_VLAN
 
-#include <net/if.h>
-#include <sys/ioctl.h>
-#include <linux/sockios.h>
-#include <linux/if_vlan.h>
-#include <linux/if_bridge.h>
-
 #include "drivers/priv_netlink.h"
 #include "utils/eloop.h"
 
@@ -34,6 +35,90 @@ struct full_dynamic_vlan {
 	int s; /* socket on which to listen for new/removed interfaces. */
 };
 
+#define DVLAN_CLEAN_BR         0x1
+#define DVLAN_CLEAN_VLAN       0x2
+#define DVLAN_CLEAN_VLAN_PORT  0x4
+
+struct dynamic_iface {
+	char ifname[IFNAMSIZ + 1];
+	int usage;
+	int clean;
+	struct dynamic_iface *next;
+};
+
+
+/* Increment ref counter for ifname and add clean flag.
+ * If not in list, add it only if some flags are given.
+ */
+static void dyn_iface_get(struct hostapd_data *hapd, const char *ifname,
+			  int clean)
+{
+	struct dynamic_iface *next, **dynamic_ifaces;
+	struct hapd_interfaces *interfaces;
+
+	interfaces = hapd->iface->interfaces;
+	dynamic_ifaces = &interfaces->vlan_priv;
+
+	for (next = *dynamic_ifaces; next; next = next->next) {
+		if (os_strcmp(ifname, next->ifname) == 0)
+			break;
+	}
+
+	if (next) {
+		next->usage++;
+		next->clean |= clean;
+		return;
+	}
+
+	if (!clean)
+		return;
+
+	next = os_zalloc(sizeof(*next));
+	if (!next)
+		return;
+	os_strlcpy(next->ifname, ifname, sizeof(next->ifname));
+	next->usage = 1;
+	next->clean = clean;
+	next->next = *dynamic_ifaces;
+	*dynamic_ifaces = next;
+}
+
+
+/* Decrement reference counter for given ifname.
+ * Return clean flag iff reference counter was decreased to zero, else zero
+ */
+static int dyn_iface_put(struct hostapd_data *hapd, const char *ifname)
+{
+	struct dynamic_iface *next, *prev = NULL, **dynamic_ifaces;
+	struct hapd_interfaces *interfaces;
+	int clean;
+
+	interfaces = hapd->iface->interfaces;
+	dynamic_ifaces = &interfaces->vlan_priv;
+
+	for (next = *dynamic_ifaces; next; next = next->next) {
+		if (os_strcmp(ifname, next->ifname) == 0)
+			break;
+		prev = next;
+	}
+
+	if (!next)
+		return 0;
+
+	next->usage--;
+	if (next->usage)
+		return 0;
+
+	if (prev)
+		prev->next = next->next;
+	else
+		*dynamic_ifaces = next->next;
+	clean = next->clean;
+	os_free(next);
+
+	return clean;
+}
+
 
 static int ifconfig_helper(const char *if_name, int up)
 {
@@ -481,11 +566,13 @@ static void vlan_newlink(char *ifname, struct hostapd_data *hapd)
 	struct hostapd_vlan *vlan = hapd->conf->vlan;
 	char *tagged_interface = hapd->conf->ssid.vlan_tagged_interface;
 	int vlan_naming = hapd->conf->ssid.vlan_naming;
+	int clean;
 
 	wpa_printf(MSG_DEBUG, "VLAN: vlan_newlink(%s)", ifname);
 
 	while (vlan) {
-		if (os_strcmp(ifname, vlan->ifname) == 0) {
+		if (os_strcmp(ifname, vlan->ifname) == 0 && !vlan->configured) {
+			vlan->configured = 1;
 
 			if (hapd->conf->vlan_bridge[0]) {
 				os_snprintf(br_name, sizeof(br_name), "%s%d",
@@ -500,8 +587,8 @@ static void vlan_newlink(char *ifname, struct hostapd_data *hapd)
 				            "brvlan%d", vlan->vlan_id);
 			}
 
-			if (!br_addbr(br_name))
-				vlan->clean |= DVLAN_CLEAN_BR;
+			dyn_iface_get(hapd, br_name,
+				      br_addbr(br_name) ? 0 : DVLAN_CLEAN_BR);
 
 			ifconfig_up(br_name);
 
@@ -517,13 +604,16 @@ static void vlan_newlink(char *ifname, struct hostapd_data *hapd)
 						    sizeof(vlan_ifname),
 						    "vlan%d", vlan->vlan_id);
 
+				clean = 0;
 				ifconfig_up(tagged_interface);
 				if (!vlan_add(tagged_interface, vlan->vlan_id,
 					      vlan_ifname))
-					vlan->clean |= DVLAN_CLEAN_VLAN;
+					clean |= DVLAN_CLEAN_VLAN;
 
 				if (!br_addif(br_name, vlan_ifname))
-					vlan->clean |= DVLAN_CLEAN_VLAN_PORT;
+					clean |= DVLAN_CLEAN_VLAN_PORT;
+
+				dyn_iface_get(hapd, vlan_ifname, clean);
 
 				ifconfig_up(vlan_ifname);
 			}
@@ -547,13 +637,15 @@ static void vlan_dellink(char *ifname, struct hostapd_data *hapd)
 	struct hostapd_vlan *first, *prev, *vlan = hapd->conf->vlan;
 	char *tagged_interface = hapd->conf->ssid.vlan_tagged_interface;
 	int vlan_naming = hapd->conf->ssid.vlan_naming;
+	int clean;
 
 	wpa_printf(MSG_DEBUG, "VLAN: vlan_dellink(%s)", ifname);
 
 	first = prev = vlan;
 
 	while (vlan) {
-		if (os_strcmp(ifname, vlan->ifname) == 0) {
+		if (os_strcmp(ifname, vlan->ifname) == 0 &&
+		    vlan->configured) {
 			if (hapd->conf->vlan_bridge[0]) {
 				os_snprintf(br_name, sizeof(br_name), "%s%d",
 					    hapd->conf->vlan_bridge,
@@ -581,20 +673,27 @@ static void vlan_dellink(char *ifname, struct hostapd_data *hapd)
 					os_snprintf(vlan_ifname,
 						    sizeof(vlan_ifname),
 						    "vlan%d", vlan->vlan_id);
-				if (vlan->clean & DVLAN_CLEAN_VLAN_PORT)
-					br_delif(br_name, vlan_ifname);
-				ifconfig_down(vlan_ifname);
 
-				if (vlan->clean & DVLAN_CLEAN_VLAN)
+				clean = dyn_iface_put(hapd, vlan_ifname);
+
+				if (clean & DVLAN_CLEAN_VLAN_PORT)
+					br_delif(br_name, vlan_ifname);
+
+				if (clean & DVLAN_CLEAN_VLAN) {
+					ifconfig_down(vlan_ifname);
 					vlan_rem(vlan_ifname);
+				}
 			}
 
-			if ((vlan->clean & DVLAN_CLEAN_BR) &&
+			clean = dyn_iface_put(hapd, br_name);
+			if ((clean & DVLAN_CLEAN_BR) &&
 			    br_getnumports(br_name) == 0) {
 				ifconfig_down(br_name);
 				br_delbr(br_name);
 			}
+		}
 
+		if (os_strcmp(ifname, vlan->ifname) == 0) {
 			if (vlan == first) {
 				hapd->conf->vlan = vlan->next;
 			} else {
@@ -651,6 +750,11 @@ vlan_read_ifnames(struct nlmsghdr *h, size_t len, int del,
 
 	if (!ifname[0])
 		return;
+	if (del && if_nametoindex(ifname)) {
+		 /* interface still exists, race condition ->
+		  * iface has just been recreated */
+		return;
+	}
 
 	wpa_printf(MSG_DEBUG,
 		   "VLAN: RTM_%sLINK: ifi_index=%d ifname=%s ifi_family=%d ifi_flags=0x%x (%s%s%s%s)",
@@ -778,8 +882,7 @@ static void full_dynamic_vlan_deinit(struct full_dynamic_vlan *priv)
 #endif /* CONFIG_FULL_DYNAMIC_VLAN */
 
 
-int vlan_setup_encryption_dyn(struct hostapd_data *hapd,
-			      struct hostapd_ssid *mssid, const char *dyn_vlan)
+int vlan_setup_encryption_dyn(struct hostapd_data *hapd, const char *dyn_vlan)
 {
         int i;
 
@@ -789,10 +892,11 @@ int vlan_setup_encryption_dyn(struct hostapd_data *hapd,
 	/* Static WEP keys are set here; IEEE 802.1X and WPA uses their own
 	 * functions for setting up dynamic broadcast keys. */
 	for (i = 0; i < 4; i++) {
-		if (mssid->wep.key[i] &&
+		if (hapd->conf->ssid.wep.key[i] &&
 		    hostapd_drv_set_key(dyn_vlan, hapd, WPA_ALG_WEP, NULL, i,
-					i == mssid->wep.idx, NULL, 0,
-					mssid->wep.key[i], mssid->wep.len[i]))
+					i == hapd->conf->ssid.wep.idx, NULL, 0,
+					hapd->conf->ssid.wep.key[i],
+					hapd->conf->ssid.wep.len[i]))
 		{
 			wpa_printf(MSG_ERROR, "VLAN: Could not set WEP "
 				   "encryption for dynamic VLAN");
@@ -953,7 +1057,8 @@ int vlan_remove_dynamic(struct hostapd_data *hapd, int vlan_id)
 	if (vlan_id <= 0 || vlan_id > MAX_VLAN_ID)
 		return 1;
 
-	wpa_printf(MSG_DEBUG, "VLAN: %s(vlan_id=%d)", __func__, vlan_id);
+	wpa_printf(MSG_DEBUG, "VLAN: %s(ifname=%s vlan_id=%d)",
+		   __func__, hapd->conf->iface, vlan_id);
 
 	vlan = hapd->conf->vlan;
 	while (vlan) {
@@ -967,8 +1072,12 @@ int vlan_remove_dynamic(struct hostapd_data *hapd, int vlan_id)
 	if (vlan == NULL)
 		return 1;
 
-	if (vlan->dynamic_vlan == 0)
+	if (vlan->dynamic_vlan == 0) {
 		hostapd_vlan_if_remove(hapd, vlan->ifname);
+#ifdef CONFIG_FULL_DYNAMIC_VLAN
+		vlan_dellink(vlan->ifname, hapd);
+#endif /* CONFIG_FULL_DYNAMIC_VLAN */
+	}
 
 	return 0;
 }
diff --git a/src/ap/vlan_init.h b/src/ap/vlan_init.h
index 781eaac441be..fc39443e5d34 100644
--- a/src/ap/vlan_init.h
+++ b/src/ap/vlan_init.h
@@ -18,7 +18,6 @@ struct hostapd_vlan * vlan_add_dynamic(struct hostapd_data *hapd,
 				       int vlan_id);
 int vlan_remove_dynamic(struct hostapd_data *hapd, int vlan_id);
 int vlan_setup_encryption_dyn(struct hostapd_data *hapd,
-			      struct hostapd_ssid *mssid,
 			      const char *dyn_vlan);
 #else /* CONFIG_NO_VLAN */
 static inline int vlan_init(struct hostapd_data *hapd)
@@ -43,7 +42,6 @@ static inline int vlan_remove_dynamic(struct hostapd_data *hapd, int vlan_id)
 }
 
 static inline int vlan_setup_encryption_dyn(struct hostapd_data *hapd,
-					    struct hostapd_ssid *mssid,
 					    const char *dyn_vlan)
 {
 	return -1;
diff --git a/src/ap/vlan_util.c b/src/ap/vlan_util.c
index cc54051b1eca..d4e0efb9b024 100644
--- a/src/ap/vlan_util.c
+++ b/src/ap/vlan_util.c
@@ -31,7 +31,7 @@
 */
 int vlan_add(const char *if_name, int vid, const char *vlan_if_name)
 {
-	int ret = -1;
+	int err, ret = -1;
 	struct nl_sock *handle = NULL;
 	struct nl_cache *cache = NULL;
 	struct rtnl_link *rlink = NULL;
@@ -58,14 +58,18 @@ int vlan_add(const char *if_name, int vid, const char *vlan_if_name)
 		goto vlan_add_error;
 	}
 
-	if (nl_connect(handle, NETLINK_ROUTE) < 0) {
-		wpa_printf(MSG_ERROR, "VLAN: failed to connect to netlink");
+	err = nl_connect(handle, NETLINK_ROUTE);
+	if (err < 0) {
+		wpa_printf(MSG_ERROR, "VLAN: failed to connect to netlink: %s",
+			   nl_geterror(err));
 		goto vlan_add_error;
 	}
 
-	if (rtnl_link_alloc_cache(handle, AF_UNSPEC, &cache) < 0) {
+	err = rtnl_link_alloc_cache(handle, AF_UNSPEC, &cache);
+	if (err < 0) {
 		cache = NULL;
-		wpa_printf(MSG_ERROR, "VLAN: failed to alloc cache");
+		wpa_printf(MSG_ERROR, "VLAN: failed to alloc cache: %s",
+			   nl_geterror(err));
 		goto vlan_add_error;
 	}
 
@@ -92,23 +96,29 @@ int vlan_add(const char *if_name, int vid, const char *vlan_if_name)
 		goto vlan_add_error;
 	}
 
-	if (rtnl_link_set_type(rlink, "vlan") < 0) {
-		wpa_printf(MSG_ERROR, "VLAN: failed to set link type");
+	err = rtnl_link_set_type(rlink, "vlan");
+	if (err < 0) {
+		wpa_printf(MSG_ERROR, "VLAN: failed to set link type: %s",
+			   nl_geterror(err));
 		goto vlan_add_error;
 	}
 
 	rtnl_link_set_link(rlink, if_idx);
 	rtnl_link_set_name(rlink, vlan_if_name);
 
-	if (rtnl_link_vlan_set_id(rlink, vid) < 0) {
-		wpa_printf(MSG_ERROR, "VLAN: failed to set link vlan id");
+	err = rtnl_link_vlan_set_id(rlink, vid);
+	if (err < 0) {
+		wpa_printf(MSG_ERROR, "VLAN: failed to set link vlan id: %s",
+			   nl_geterror(err));
 		goto vlan_add_error;
 	}
 
-	if (rtnl_link_add(handle, rlink, NLM_F_CREATE) < 0) {
+	err = rtnl_link_add(handle, rlink, NLM_F_CREATE);
+	if (err < 0) {
 		wpa_printf(MSG_ERROR, "VLAN: failed to create link %s for "
-			   "vlan %d on %s (%d)",
-			   vlan_if_name, vid, if_name, if_idx);
+			   "vlan %d on %s (%d): %s",
+			   vlan_if_name, vid, if_name, if_idx,
+			   nl_geterror(err));
 		goto vlan_add_error;
 	}
 
@@ -127,7 +137,7 @@ vlan_add_error:
 
 int vlan_rem(const char *if_name)
 {
-	int ret = -1;
+	int err, ret = -1;
 	struct nl_sock *handle = NULL;
 	struct nl_cache *cache = NULL;
 	struct rtnl_link *rlink = NULL;
@@ -140,14 +150,18 @@ int vlan_rem(const char *if_name)
 		goto vlan_rem_error;
 	}
 
-	if (nl_connect(handle, NETLINK_ROUTE) < 0) {
-		wpa_printf(MSG_ERROR, "VLAN: failed to connect to netlink");
+	err = nl_connect(handle, NETLINK_ROUTE);
+	if (err < 0) {
+		wpa_printf(MSG_ERROR, "VLAN: failed to connect to netlink: %s",
+			   nl_geterror(err));
 		goto vlan_rem_error;
 	}
 
-	if (rtnl_link_alloc_cache(handle, AF_UNSPEC, &cache) < 0) {
+	err = rtnl_link_alloc_cache(handle, AF_UNSPEC, &cache);
+	if (err < 0) {
 		cache = NULL;
-		wpa_printf(MSG_ERROR, "VLAN: failed to alloc cache");
+		wpa_printf(MSG_ERROR, "VLAN: failed to alloc cache: %s",
+			   nl_geterror(err));
 		goto vlan_rem_error;
 	}
 
@@ -158,9 +172,10 @@ int vlan_rem(const char *if_name)
 		goto vlan_rem_error;
 	}
 
-	if (rtnl_link_delete(handle, rlink) < 0) {
-		wpa_printf(MSG_ERROR, "VLAN: failed to remove link %s",
-			   if_name);
+	err = rtnl_link_delete(handle, rlink);
+	if (err < 0) {
+		wpa_printf(MSG_ERROR, "VLAN: failed to remove link %s: %s",
+			   if_name, nl_geterror(err));
 		goto vlan_rem_error;
 	}
 
diff --git a/src/ap/wmm.c b/src/ap/wmm.c
index 6d4177c2a847..314e244bc956 100644
--- a/src/ap/wmm.c
+++ b/src/ap/wmm.c
@@ -274,6 +274,9 @@ void hostapd_wmm_action(struct hostapd_data *hapd,
 		return;
 	}
 
+	if (left < 0)
+		return; /* not a valid WMM Action frame */
+
 	/* extract the tspec info element */
 	if (ieee802_11_parse_elems(pos, left, &elems, 1) == ParseFailed) {
 		hostapd_logger(hapd, mgmt->sa, HOSTAPD_MODULE_IEEE80211,
diff --git a/src/ap/wpa_auth.c b/src/ap/wpa_auth.c
index 9c5f6094acbb..2760a3f3a00e 100644
--- a/src/ap/wpa_auth.c
+++ b/src/ap/wpa_auth.c
@@ -45,6 +45,12 @@ static int wpa_group_config_group_keys(struct wpa_authenticator *wpa_auth,
 				       struct wpa_group *group);
 static int wpa_derive_ptk(struct wpa_state_machine *sm, const u8 *snonce,
 			  const u8 *pmk, struct wpa_ptk *ptk);
+static void wpa_group_free(struct wpa_authenticator *wpa_auth,
+			   struct wpa_group *group);
+static void wpa_group_get(struct wpa_authenticator *wpa_auth,
+			  struct wpa_group *group);
+static void wpa_group_put(struct wpa_authenticator *wpa_auth,
+			  struct wpa_group *group);
 
 static const u32 dot11RSNAConfigGroupUpdateCount = 4;
 static const u32 dot11RSNAConfigPairwiseUpdateCount = 4;
@@ -67,6 +73,14 @@ static inline int wpa_auth_mic_failure_report(
 }
 
 
+static inline void wpa_auth_psk_failure_report(
+	struct wpa_authenticator *wpa_auth, const u8 *addr)
+{
+	if (wpa_auth->cb.psk_failure_report)
+		wpa_auth->cb.psk_failure_report(wpa_auth->cb.ctx, addr);
+}
+
+
 static inline void wpa_auth_set_eapol(struct wpa_authenticator *wpa_auth,
 				      const u8 *addr, wpa_eapol_variable var,
 				      int value)
@@ -254,15 +268,22 @@ static void wpa_rekey_gmk(void *eloop_ctx, void *timeout_ctx)
 static void wpa_rekey_gtk(void *eloop_ctx, void *timeout_ctx)
 {
 	struct wpa_authenticator *wpa_auth = eloop_ctx;
-	struct wpa_group *group;
+	struct wpa_group *group, *next;
 
 	wpa_auth_logger(wpa_auth, NULL, LOGGER_DEBUG, "rekeying GTK");
-	for (group = wpa_auth->group; group; group = group->next) {
+	group = wpa_auth->group;
+	while (group) {
+		wpa_group_get(wpa_auth, group);
+
 		group->GTKReKey = TRUE;
 		do {
 			group->changed = FALSE;
 			wpa_group_sm_step(wpa_auth, group);
 		} while (group->changed);
+
+		next = group->next;
+		wpa_group_put(wpa_auth, group);
+		group = next;
 	}
 
 	if (wpa_auth->conf.wpa_group_rekey) {
@@ -565,6 +586,7 @@ wpa_auth_sta_init(struct wpa_authenticator *wpa_auth, const u8 *addr,
 
 	sm->wpa_auth = wpa_auth;
 	sm->group = wpa_auth->group;
+	wpa_group_get(sm->wpa_auth, sm->group);
 
 	return sm;
 }
@@ -643,6 +665,7 @@ static void wpa_free_sta_sm(struct wpa_state_machine *sm)
 #endif /* CONFIG_IEEE80211R */
 	os_free(sm->last_rx_eapol_key);
 	os_free(sm->wpa_ie);
+	wpa_group_put(sm->wpa_auth, sm->group);
 	os_free(sm);
 }
 
@@ -1517,6 +1540,7 @@ void __wpa_send_eapol(struct wpa_authenticator *wpa_auth,
 			else
 				WPA_PUT_BE16(key->key_data_length,
 					     key_data_len);
+#ifndef CONFIG_NO_RC4
 		} else if (sm->PTK.kek_len == 16) {
 			u8 ek[32];
 			os_memcpy(key->key_iv,
@@ -1532,6 +1556,7 @@ void __wpa_send_eapol(struct wpa_authenticator *wpa_auth,
 			else
 				WPA_PUT_BE16(key->key_data_length,
 					     key_data_len);
+#endif /* CONFIG_NO_RC4 */
 		} else {
 			os_free(hdr);
 			os_free(buf);
@@ -1646,7 +1671,7 @@ void wpa_remove_ptk(struct wpa_state_machine *sm)
 }
 
 
-int wpa_auth_sm_event(struct wpa_state_machine *sm, wpa_event event)
+int wpa_auth_sm_event(struct wpa_state_machine *sm, enum wpa_event event)
 {
 	int remove_ptk = 1;
 
@@ -1734,6 +1759,14 @@ int wpa_auth_sm_event(struct wpa_state_machine *sm, wpa_event event)
 			wpa_remove_ptk(sm);
 	}
 
+	if (sm->in_step_loop) {
+		/*
+		 * wpa_sm_step() is already running - avoid recursive call to
+		 * it by making the existing loop process the new update.
+		 */
+		sm->changed = TRUE;
+		return 0;
+	}
 	return wpa_sm_step(sm);
 }
 
@@ -1818,9 +1851,13 @@ static void wpa_group_ensure_init(struct wpa_authenticator *wpa_auth,
 		group->reject_4way_hs_for_entropy = FALSE;
 	}
 
-	wpa_group_init_gmk_and_counter(wpa_auth, group);
-	wpa_gtk_update(wpa_auth, group);
-	wpa_group_config_group_keys(wpa_auth, group);
+	if (wpa_group_init_gmk_and_counter(wpa_auth, group) < 0 ||
+	    wpa_gtk_update(wpa_auth, group) < 0 ||
+	    wpa_group_config_group_keys(wpa_auth, group) < 0) {
+		wpa_printf(MSG_INFO, "WPA: GMK/GTK setup failed");
+		group->first_sta_seen = FALSE;
+		group->reject_4way_hs_for_entropy = TRUE;
+	}
 }
 
 
@@ -1985,7 +2022,7 @@ static int wpa_derive_ptk(struct wpa_state_machine *sm, const u8 *snonce,
 SM_STATE(WPA_PTK, PTKCALCNEGOTIATING)
 {
 	struct wpa_ptk PTK;
-	int ok = 0;
+	int ok = 0, psk_found = 0;
 	const u8 *pmk = NULL;
 
 	SM_ENTRY_MA(WPA_PTK, PTKCALCNEGOTIATING, wpa_ptk);
@@ -2001,6 +2038,7 @@ SM_STATE(WPA_PTK, PTKCALCNEGOTIATING)
 					       sm->p2p_dev_addr, pmk);
 			if (pmk == NULL)
 				break;
+			psk_found = 1;
 		} else
 			pmk = sm->PMK;
 
@@ -2020,6 +2058,8 @@ SM_STATE(WPA_PTK, PTKCALCNEGOTIATING)
 	if (!ok) {
 		wpa_auth_logger(sm->wpa_auth, sm->addr, LOGGER_DEBUG,
 				"invalid MIC in msg 2/4 of 4-Way Handshake");
+		if (psk_found)
+			wpa_auth_psk_failure_report(sm->wpa_auth, sm->addr);
 		return;
 	}
 
@@ -2983,9 +3023,9 @@ void wpa_gtk_rekey(struct wpa_authenticator *wpa_auth)
 }
 
 
-static const char * wpa_bool_txt(int bool)
+static const char * wpa_bool_txt(int val)
 {
-	return bool ? "TRUE" : "FALSE";
+	return val ? "TRUE" : "FALSE";
 }
 
 
@@ -3270,6 +3310,63 @@ void wpa_auth_pmksa_remove(struct wpa_authenticator *wpa_auth,
 }
 
 
+/*
+ * Remove and free the group from wpa_authenticator. This is triggered by a
+ * callback to make sure nobody is currently iterating the group list while it
+ * gets modified.
+ */
+static void wpa_group_free(struct wpa_authenticator *wpa_auth,
+			   struct wpa_group *group)
+{
+	struct wpa_group *prev = wpa_auth->group;
+
+	wpa_printf(MSG_DEBUG, "WPA: Remove group state machine for VLAN-ID %d",
+		   group->vlan_id);
+
+	while (prev) {
+		if (prev->next == group) {
+			/* This never frees the special first group as needed */
+			prev->next = group->next;
+			os_free(group);
+			break;
+		}
+		prev = prev->next;
+	}
+
+}
+
+
+/* Increase the reference counter for group */
+static void wpa_group_get(struct wpa_authenticator *wpa_auth,
+			  struct wpa_group *group)
+{
+	/* Skip the special first group */
+	if (wpa_auth->group == group)
+		return;
+
+	group->references++;
+}
+
+
+/* Decrease the reference counter and maybe free the group */
+static void wpa_group_put(struct wpa_authenticator *wpa_auth,
+			  struct wpa_group *group)
+{
+	/* Skip the special first group */
+	if (wpa_auth->group == group)
+		return;
+
+	group->references--;
+	if (group->references)
+		return;
+	wpa_group_free(wpa_auth, group);
+}
+
+
+/*
+ * Add a group that has its references counter set to zero. Caller needs to
+ * call wpa_group_get() on the return value to mark the entry in use.
+ */
 static struct wpa_group *
 wpa_auth_add_group(struct wpa_authenticator *wpa_auth, int vlan_id)
 {
@@ -3320,7 +3417,10 @@ int wpa_auth_sta_set_vlan(struct wpa_state_machine *sm, int vlan_id)
 	wpa_printf(MSG_DEBUG, "WPA: Moving STA " MACSTR " to use group state "
 		   "machine for VLAN ID %d", MAC2STR(sm->addr), vlan_id);
 
+	wpa_group_get(sm->wpa_auth, group);
+	wpa_group_put(sm->wpa_auth, sm->group);
 	sm->group = group;
+
 	return 0;
 }
 
diff --git a/src/ap/wpa_auth.h b/src/ap/wpa_auth.h
index 2788e657435d..fd04f169433a 100644
--- a/src/ap/wpa_auth.h
+++ b/src/ap/wpa_auth.h
@@ -1,6 +1,6 @@
 /*
  * hostapd - IEEE 802.11i-2004 / WPA Authenticator
- * Copyright (c) 2004-2007, Jouni Malinen <j@w1.fi>
+ * Copyright (c) 2004-2015, Jouni Malinen <j@w1.fi>
  *
  * This software may be distributed under the terms of the BSD license.
  * See README for more details.
@@ -12,6 +12,9 @@
 #include "common/defs.h"
 #include "common/eapol_common.h"
 #include "common/wpa_common.h"
+#include "common/ieee802_11_defs.h"
+
+#define MAX_OWN_IE_OVERRIDE 256
 
 #ifdef _MSC_VER
 #pragma pack(push, 1)
@@ -146,8 +149,7 @@ struct wpa_auth_config {
 	int group_mgmt_cipher;
 #endif /* CONFIG_IEEE80211W */
 #ifdef CONFIG_IEEE80211R
-#define SSID_LEN 32
-	u8 ssid[SSID_LEN];
+	u8 ssid[SSID_MAX_LEN];
 	size_t ssid_len;
 	u8 mobility_domain[MOBILITY_DOMAIN_ID_LEN];
 	u8 r0_key_holder[FT_R0KH_ID_MAX_LEN];
@@ -164,6 +166,8 @@ struct wpa_auth_config {
 	int ap_mlme;
 #ifdef CONFIG_TESTING_OPTIONS
 	double corrupt_gtk_rekey_mic_probability;
+	u8 own_ie_override[MAX_OWN_IE_OVERRIDE];
+	size_t own_ie_override_len;
 #endif /* CONFIG_TESTING_OPTIONS */
 #ifdef CONFIG_P2P
 	u8 ip_addr_go[4];
@@ -189,6 +193,7 @@ struct wpa_auth_callbacks {
 		       const char *txt);
 	void (*disconnect)(void *ctx, const u8 *addr, u16 reason);
 	int (*mic_failure_report)(void *ctx, const u8 *addr);
+	void (*psk_failure_report)(void *ctx, const u8 *addr);
 	void (*set_eapol)(void *ctx, const u8 *addr, wpa_eapol_variable var,
 			  int value);
 	int (*get_eapol)(void *ctx, const u8 *addr, wpa_eapol_variable var);
@@ -251,12 +256,12 @@ void wpa_auth_sta_deinit(struct wpa_state_machine *sm);
 void wpa_receive(struct wpa_authenticator *wpa_auth,
 		 struct wpa_state_machine *sm,
 		 u8 *data, size_t data_len);
-typedef enum {
+enum wpa_event {
 	WPA_AUTH, WPA_ASSOC, WPA_DISASSOC, WPA_DEAUTH, WPA_REAUTH,
 	WPA_REAUTH_EAPOL, WPA_ASSOC_FT
-} wpa_event;
+};
 void wpa_remove_ptk(struct wpa_state_machine *sm);
-int wpa_auth_sm_event(struct wpa_state_machine *sm, wpa_event event);
+int wpa_auth_sm_event(struct wpa_state_machine *sm, enum wpa_event event);
 void wpa_auth_sm_notify(struct wpa_state_machine *sm);
 void wpa_gtk_rekey(struct wpa_authenticator *wpa_auth);
 int wpa_get_mib(struct wpa_authenticator *wpa_auth, char *buf, size_t buflen);
diff --git a/src/ap/wpa_auth_ft.c b/src/ap/wpa_auth_ft.c
index ef3249a3eb9b..eeaffbf63516 100644
--- a/src/ap/wpa_auth_ft.c
+++ b/src/ap/wpa_auth_ft.c
@@ -534,10 +534,8 @@ static u8 * wpa_ft_process_rdie(struct wpa_state_machine *sm,
 		return pos;
 	}
 
-#ifdef NEED_AP_MLME
-	if (parse.wmm_tspec && sm->wpa_auth->conf.ap_mlme) {
+	if (parse.wmm_tspec) {
 		struct wmm_tspec_element *tspec;
-		int res;
 
 		if (parse.wmm_tspec_len + 2 < (int) sizeof(*tspec)) {
 			wpa_printf(MSG_DEBUG, "FT: Too short WMM TSPEC IE "
@@ -555,7 +553,13 @@ static u8 * wpa_ft_process_rdie(struct wpa_state_machine *sm,
 		}
 		tspec = (struct wmm_tspec_element *) pos;
 		os_memcpy(tspec, parse.wmm_tspec - 2, sizeof(*tspec));
-		res = wmm_process_tspec(tspec);
+	}
+
+#ifdef NEED_AP_MLME
+	if (parse.wmm_tspec && sm->wpa_auth->conf.ap_mlme) {
+		int res;
+
+		res = wmm_process_tspec((struct wmm_tspec_element *) pos);
 		wpa_printf(MSG_DEBUG, "FT: ADDTS processing result: %d", res);
 		if (res == WMM_ADDTS_STATUS_INVALID_PARAMETERS)
 			rdie->status_code =
@@ -566,20 +570,17 @@ static u8 * wpa_ft_process_rdie(struct wpa_state_machine *sm,
 		else {
 			/* TSPEC accepted; include updated TSPEC in response */
 			rdie->descr_count = 1;
-			pos += sizeof(*tspec);
+			pos += sizeof(struct wmm_tspec_element);
 		}
 		return pos;
 	}
 #endif /* NEED_AP_MLME */
 
 	if (parse.wmm_tspec && !sm->wpa_auth->conf.ap_mlme) {
-		struct wmm_tspec_element *tspec;
 		int res;
 
-		tspec = (struct wmm_tspec_element *) pos;
-		os_memcpy(tspec, parse.wmm_tspec - 2, sizeof(*tspec));
 		res = wpa_ft_add_tspec(sm->wpa_auth, sm->addr, pos,
-				       sizeof(*tspec));
+				       sizeof(struct wmm_tspec_element));
 		if (res >= 0) {
 			if (res)
 				rdie->status_code = host_to_le16(res);
@@ -587,7 +588,7 @@ static u8 * wpa_ft_process_rdie(struct wpa_state_machine *sm,
 				/* TSPEC accepted; include updated TSPEC in
 				 * response */
 				rdie->descr_count = 1;
-				pos += sizeof(*tspec);
+				pos += sizeof(struct wmm_tspec_element);
 			}
 			return pos;
 		}
diff --git a/src/ap/wpa_auth_glue.c b/src/ap/wpa_auth_glue.c
index 7f8320708c39..f98cc50599e3 100644
--- a/src/ap/wpa_auth_glue.c
+++ b/src/ap/wpa_auth_glue.c
@@ -11,6 +11,7 @@
 #include "utils/common.h"
 #include "common/ieee802_11_defs.h"
 #include "common/sae.h"
+#include "common/wpa_ctrl.h"
 #include "eapol_auth/eapol_auth_sm.h"
 #include "eapol_auth/eapol_auth_sm_i.h"
 #include "eap_server/eap.h"
@@ -53,8 +54,8 @@ static void hostapd_wpa_auth_conf(struct hostapd_bss_config *conf,
 #endif /* CONFIG_IEEE80211W */
 #ifdef CONFIG_IEEE80211R
 	wconf->ssid_len = conf->ssid.ssid_len;
-	if (wconf->ssid_len > SSID_LEN)
-		wconf->ssid_len = SSID_LEN;
+	if (wconf->ssid_len > SSID_MAX_LEN)
+		wconf->ssid_len = SSID_MAX_LEN;
 	os_memcpy(wconf->ssid, conf->ssid.ssid, wconf->ssid_len);
 	os_memcpy(wconf->mobility_domain, conf->mobility_domain,
 		  MOBILITY_DOMAIN_ID_LEN);
@@ -91,6 +92,13 @@ static void hostapd_wpa_auth_conf(struct hostapd_bss_config *conf,
 #ifdef CONFIG_TESTING_OPTIONS
 	wconf->corrupt_gtk_rekey_mic_probability =
 		iconf->corrupt_gtk_rekey_mic_probability;
+	if (conf->own_ie_override &&
+	    wpabuf_len(conf->own_ie_override) <= MAX_OWN_IE_OVERRIDE) {
+		wconf->own_ie_override_len = wpabuf_len(conf->own_ie_override);
+		os_memcpy(wconf->own_ie_override,
+			  wpabuf_head(conf->own_ie_override),
+			  wconf->own_ie_override_len);
+	}
 #endif /* CONFIG_TESTING_OPTIONS */
 #ifdef CONFIG_P2P
 	os_memcpy(wconf->ip_addr_go, conf->ip_addr_go, 4);
@@ -144,6 +152,14 @@ static int hostapd_wpa_auth_mic_failure_report(void *ctx, const u8 *addr)
 }
 
 
+static void hostapd_wpa_auth_psk_failure_report(void *ctx, const u8 *addr)
+{
+	struct hostapd_data *hapd = ctx;
+	wpa_msg(hapd->msg_ctx, MSG_INFO, AP_STA_POSSIBLE_PSK_MISMATCH MACSTR,
+		MAC2STR(addr));
+}
+
+
 static void hostapd_wpa_auth_set_eapol(void *ctx, const u8 *addr,
 				       wpa_eapol_variable var, int value)
 {
@@ -579,6 +595,7 @@ int hostapd_setup_wpa(struct hostapd_data *hapd)
 	cb.logger = hostapd_wpa_auth_logger;
 	cb.disconnect = hostapd_wpa_auth_disconnect;
 	cb.mic_failure_report = hostapd_wpa_auth_mic_failure_report;
+	cb.psk_failure_report = hostapd_wpa_auth_psk_failure_report;
 	cb.set_eapol = hostapd_wpa_auth_set_eapol;
 	cb.get_eapol = hostapd_wpa_auth_get_eapol;
 	cb.get_psk = hostapd_wpa_auth_get_psk;
@@ -620,7 +637,8 @@ int hostapd_setup_wpa(struct hostapd_data *hapd)
 	}
 
 #ifdef CONFIG_IEEE80211R
-	if (!hostapd_drv_none(hapd)) {
+	if (!hostapd_drv_none(hapd) && hapd->conf->ft_over_ds &&
+	    wpa_key_mgmt_ft(hapd->conf->wpa_key_mgmt)) {
 		hapd->l2 = l2_packet_init(hapd->conf->bridge[0] ?
 					  hapd->conf->bridge :
 					  hapd->conf->iface, NULL, ETH_P_RRB,
diff --git a/src/ap/wpa_auth_i.h b/src/ap/wpa_auth_i.h
index 7b2cd3ea8ed4..57b098f2ed72 100644
--- a/src/ap/wpa_auth_i.h
+++ b/src/ap/wpa_auth_i.h
@@ -169,6 +169,8 @@ struct wpa_group {
 	u8 IGTK[2][WPA_IGTK_MAX_LEN];
 	int GN_igtk, GM_igtk;
 #endif /* CONFIG_IEEE80211W */
+	/* Number of references except those in struct wpa_group->next */
+	unsigned int references;
 };
 
 
diff --git a/src/ap/wpa_auth_ie.c b/src/ap/wpa_auth_ie.c
index f2872970affe..eafb828b8d60 100644
--- a/src/ap/wpa_auth_ie.c
+++ b/src/ap/wpa_auth_ie.c
@@ -261,7 +261,8 @@ int wpa_write_rsn_ie(struct wpa_auth_config *conf, u8 *buf, size_t len,
 	}
 
 #ifdef CONFIG_IEEE80211W
-	if (conf->ieee80211w != NO_MGMT_FRAME_PROTECTION) {
+	if (conf->ieee80211w != NO_MGMT_FRAME_PROTECTION &&
+	    conf->group_mgmt_cipher != WPA_CIPHER_AES_128_CMAC) {
 		if (pos + 2 + 4 > buf + len)
 			return -1;
 		if (pmkid == NULL) {
@@ -377,6 +378,23 @@ int wpa_auth_gen_wpa_ie(struct wpa_authenticator *wpa_auth)
 	u8 *pos, buf[128];
 	int res;
 
+#ifdef CONFIG_TESTING_OPTIONS
+	if (wpa_auth->conf.own_ie_override_len) {
+		wpa_hexdump(MSG_DEBUG, "WPA: Forced own IE(s) for testing",
+			    wpa_auth->conf.own_ie_override,
+			    wpa_auth->conf.own_ie_override_len);
+		os_free(wpa_auth->wpa_ie);
+		wpa_auth->wpa_ie =
+			os_malloc(wpa_auth->conf.own_ie_override_len);
+		if (wpa_auth->wpa_ie == NULL)
+			return -1;
+		os_memcpy(wpa_auth->wpa_ie, wpa_auth->conf.own_ie_override,
+			  wpa_auth->conf.own_ie_override_len);
+		wpa_auth->wpa_ie_len = wpa_auth->conf.own_ie_override_len;
+		return 0;
+	}
+#endif /* CONFIG_TESTING_OPTIONS */
+
 	pos = buf;
 
 	if (wpa_auth->conf.wpa == WPA_PROTO_OSEN) {
diff --git a/src/ap/wps_hostapd.c b/src/ap/wps_hostapd.c
index b0e8b0bfcac3..cde31e60e03b 100644
--- a/src/ap/wps_hostapd.c
+++ b/src/ap/wps_hostapd.c
@@ -324,7 +324,7 @@ static int hapd_wps_reconfig_in_memory(struct hostapd_data *hapd,
 	wpa_printf(MSG_DEBUG, "WPS: Updating in-memory configuration");
 
 	bss->wps_state = 2;
-	if (cred->ssid_len <= HOSTAPD_MAX_SSID_LEN) {
+	if (cred->ssid_len <= SSID_MAX_LEN) {
 		os_memcpy(bss->ssid.ssid, cred->ssid, cred->ssid_len);
 		bss->ssid.ssid_len = cred->ssid_len;
 		bss->ssid.ssid_set = 1;
@@ -347,8 +347,12 @@ static int hapd_wps_reconfig_in_memory(struct hostapd_data *hapd,
 			bss->wpa_key_mgmt = WPA_KEY_MGMT_PSK;
 
 		bss->wpa_pairwise = 0;
-		if (cred->encr_type & WPS_ENCR_AES)
-			bss->wpa_pairwise |= WPA_CIPHER_CCMP;
+		if (cred->encr_type & WPS_ENCR_AES) {
+			if (hapd->iconf->hw_mode == HOSTAPD_MODE_IEEE80211AD)
+				bss->wpa_pairwise |= WPA_CIPHER_GCMP;
+			else
+				bss->wpa_pairwise |= WPA_CIPHER_CCMP;
+		}
 		if (cred->encr_type & WPS_ENCR_TKIP)
 			bss->wpa_pairwise |= WPA_CIPHER_TKIP;
 		bss->rsn_pairwise = bss->wpa_pairwise;
@@ -448,6 +452,11 @@ static int hapd_wps_cred_cb(struct hostapd_data *hapd, void *ctx)
 		os_free(hapd->wps->network_key);
 		hapd->wps->network_key = NULL;
 		hapd->wps->network_key_len = 0;
+	} else if ((cred->auth_type & (WPS_AUTH_WPA2PSK | WPS_AUTH_WPAPSK)) &&
+		   (cred->key_len < 8 || cred->key_len > 2 * PMK_LEN)) {
+		wpa_printf(MSG_INFO, "WPS: Invalid key length %lu for WPA/WPA2",
+			   (unsigned long) cred->key_len);
+		return -1;
 	} else {
 		if (hapd->wps->network_key == NULL ||
 		    hapd->wps->network_key_len < cred->key_len) {
@@ -530,7 +539,11 @@ static int hapd_wps_cred_cb(struct hostapd_data *hapd, void *ctx)
 		fprintf(nconf, "wpa_pairwise=");
 		prefix = "";
 		if (cred->encr_type & WPS_ENCR_AES) {
-			fprintf(nconf, "CCMP");
+			if (hapd->iconf->hw_mode == HOSTAPD_MODE_IEEE80211AD)
+				fprintf(nconf, "GCMP");
+			else
+				fprintf(nconf, "CCMP");
+
 			prefix = " ";
 		}
 		if (cred->encr_type & WPS_ENCR_TKIP) {
@@ -844,7 +857,9 @@ static int hostapd_wps_rf_band_cb(void *ctx)
 	struct hostapd_data *hapd = ctx;
 
 	return hapd->iconf->hw_mode == HOSTAPD_MODE_IEEE80211A ?
-		WPS_RF_50GHZ : WPS_RF_24GHZ; /* FIX: dualband AP */
+		WPS_RF_50GHZ :
+		hapd->iconf->hw_mode == HOSTAPD_MODE_IEEE80211AD ?
+		WPS_RF_60GHZ : WPS_RF_24GHZ; /* FIX: dualband AP */
 }
 
 
@@ -856,8 +871,10 @@ static void hostapd_wps_clear_ies(struct hostapd_data *hapd, int deinit_only)
 	wpabuf_free(hapd->wps_probe_resp_ie);
 	hapd->wps_probe_resp_ie = NULL;
 
-	if (deinit_only)
+	if (deinit_only) {
+		hostapd_reset_ap_wps_ie(hapd);
 		return;
+	}
 
 	hostapd_set_ap_wps_ie(hapd);
 }
@@ -1039,7 +1056,9 @@ int hostapd_init_wps(struct hostapd_data *hapd,
 	} else {
 		wps->dev.rf_bands =
 			hapd->iconf->hw_mode == HOSTAPD_MODE_IEEE80211A ?
-			WPS_RF_50GHZ : WPS_RF_24GHZ; /* FIX: dualband AP */
+			WPS_RF_50GHZ :
+			hapd->iconf->hw_mode == HOSTAPD_MODE_IEEE80211AD ?
+			WPS_RF_60GHZ : WPS_RF_24GHZ; /* FIX: dualband AP */
 	}
 
 	if (conf->wpa & WPA_PROTO_RSN) {
@@ -1285,30 +1304,53 @@ int hostapd_wps_add_pin(struct hostapd_data *hapd, const u8 *addr,
 }
 
 
+struct wps_button_pushed_ctx {
+	const u8 *p2p_dev_addr;
+	unsigned int count;
+};
+
 static int wps_button_pushed(struct hostapd_data *hapd, void *ctx)
 {
-	const u8 *p2p_dev_addr = ctx;
-	if (hapd->wps == NULL)
-		return -1;
-	return wps_registrar_button_pushed(hapd->wps->registrar, p2p_dev_addr);
+	struct wps_button_pushed_ctx *data = ctx;
+
+	if (hapd->wps) {
+		data->count++;
+		return wps_registrar_button_pushed(hapd->wps->registrar,
+						   data->p2p_dev_addr);
+	}
+
+	return 0;
 }
 
 
 int hostapd_wps_button_pushed(struct hostapd_data *hapd,
 			      const u8 *p2p_dev_addr)
 {
-	return hostapd_wps_for_each(hapd, wps_button_pushed,
-				    (void *) p2p_dev_addr);
+	struct wps_button_pushed_ctx ctx;
+	int ret;
+
+	os_memset(&ctx, 0, sizeof(ctx));
+	ctx.p2p_dev_addr = p2p_dev_addr;
+	ret = hostapd_wps_for_each(hapd, wps_button_pushed, &ctx);
+	if (ret == 0 && !ctx.count)
+		ret = -1;
+	return ret;
 }
 
 
+struct wps_cancel_ctx {
+	unsigned int count;
+};
+
 static int wps_cancel(struct hostapd_data *hapd, void *ctx)
 {
-	if (hapd->wps == NULL)
-		return -1;
+	struct wps_cancel_ctx *data = ctx;
 
-	wps_registrar_wps_cancel(hapd->wps->registrar);
-	ap_for_each_sta(hapd, ap_sta_wps_cancel, NULL);
+	if (hapd->wps) {
+		data->count++;
+		wps_registrar_wps_cancel(hapd->wps->registrar);
+		ap_for_each_sta(hapd, ap_sta_wps_cancel, NULL);
+	}
 
 	return 0;
 }
@@ -1316,7 +1358,14 @@ static int wps_cancel(struct hostapd_data *hapd, void *ctx)
 
 int hostapd_wps_cancel(struct hostapd_data *hapd)
 {
-	return hostapd_wps_for_each(hapd, wps_cancel, NULL);
+	struct wps_cancel_ctx ctx;
+	int ret;
+
+	os_memset(&ctx, 0, sizeof(ctx));
+	ret = hostapd_wps_for_each(hapd, wps_cancel, &ctx);
+	if (ret == 0 && !ctx.count)
+		ret = -1;
+	return ret;
 }
 
 
@@ -1546,6 +1595,10 @@ struct wps_ap_pin_data {
 static int wps_ap_pin_set(struct hostapd_data *hapd, void *ctx)
 {
 	struct wps_ap_pin_data *data = ctx;
+
+	if (!hapd->wps)
+		return 0;
+
 	os_free(hapd->conf->ap_pin);
 	hapd->conf->ap_pin = os_strdup(data->pin_txt);
 #ifdef CONFIG_WPS_UPNP
diff --git a/src/ap/x_snoop.c b/src/ap/x_snoop.c
index 8f77015ef57f..aef9a53c46cf 100644
--- a/src/ap/x_snoop.c
+++ b/src/ap/x_snoop.c
@@ -51,6 +51,14 @@ int x_snoop_init(struct hostapd_data *hapd)
 		return -1;
 	}
 
+#ifdef CONFIG_IPV6
+	if (hostapd_drv_br_set_net_param(hapd, DRV_BR_MULTICAST_SNOOPING, 1)) {
+		wpa_printf(MSG_DEBUG,
+			   "x_snoop: Failed to enable multicast snooping on the bridge");
+		return -1;
+	}
+#endif /* CONFIG_IPV6 */
+
 	return 0;
 }
 
diff --git a/src/common/Makefile b/src/common/Makefile
index adfd3dfd5b9b..e703630835e1 100644
--- a/src/common/Makefile
+++ b/src/common/Makefile
@@ -1,8 +1,28 @@
-all:
-	@echo Nothing to be made.
+all: libcommon.a
 
 clean:
-	rm -f *~ *.o *.d *.gcno *.gcda *.gcov
+	rm -f *~ *.o *.d *.gcno *.gcda *.gcov libcommon.a
 
 install:
 	@echo Nothing to be made.
+
+include ../lib.rules
+
+CFLAGS += -DCONFIG_IEEE80211R
+CFLAGS += -DCONFIG_IEEE80211W
+CFLAGS += -DCONFIG_HS20
+CFLAGS += -DCONFIG_SAE
+CFLAGS += -DCONFIG_SUITE
+CFLAGS += -DCONFIG_SUITEB
+
+LIB_OBJS= \
+	gas.o \
+	hw_features_common.o \
+	ieee802_11_common.o \
+	sae.o \
+	wpa_common.o
+
+libcommon.a: $(LIB_OBJS)
+	$(AR) crT $@ $?
+
+-include $(OBJS:%.o=%.d)
diff --git a/src/common/common_module_tests.c b/src/common/common_module_tests.c
index 56b11220c9ed..d69448bd3800 100644
--- a/src/common/common_module_tests.c
+++ b/src/common/common_module_tests.c
@@ -1,6 +1,6 @@
 /*
  * common module tests
- * Copyright (c) 2014, Jouni Malinen <j@w1.fi>
+ * Copyright (c) 2014-2015, Jouni Malinen <j@w1.fi>
  *
  * This software may be distributed under the terms of the BSD license.
  * See README for more details.
@@ -10,6 +10,8 @@
 
 #include "utils/common.h"
 #include "ieee802_11_common.h"
+#include "ieee802_11_defs.h"
+#include "gas.h"
 #include "wpa_common.h"
 
 
@@ -46,6 +48,10 @@ static const struct ieee802_11_parse_test_data parse_tests[] = {
 	{ (u8 *) "\x6e\x00", 2, ParseOK, 1 },
 	{ (u8 *) "\xc7\x00", 2, ParseOK, 1 },
 	{ (u8 *) "\xc7\x01\x00", 3, ParseOK, 1 },
+	{ (u8 *) "\x03\x00\x2a\x00\x36\x00\x37\x00\x38\x00\x2d\x00\x3d\x00\xbf\x00\xc0\x00",
+	  18, ParseOK, 9 },
+	{ (u8 *) "\x8b\x00", 2, ParseOK, 1 },
+	{ (u8 *) "\xdd\x04\x00\x90\x4c\x04", 6, ParseUnknown, 1 },
 	{ NULL, 0, ParseOK, 0 }
 };
 
@@ -158,6 +164,34 @@ static int rsn_ie_parse_tests(void)
 }
 
 
+static int gas_tests(void)
+{
+	struct wpabuf *buf;
+
+	wpa_printf(MSG_INFO, "gas tests");
+	gas_anqp_set_len(NULL);
+
+	buf = wpabuf_alloc(1);
+	if (buf == NULL)
+		return -1;
+	gas_anqp_set_len(buf);
+	wpabuf_free(buf);
+
+	buf = wpabuf_alloc(20);
+	if (buf == NULL)
+		return -1;
+	wpabuf_put_u8(buf, WLAN_ACTION_PUBLIC);
+	wpabuf_put_u8(buf, WLAN_PA_GAS_INITIAL_REQ);
+	wpabuf_put_u8(buf, 0);
+	wpabuf_put_be32(buf, 0);
+	wpabuf_put_u8(buf, 0);
+	gas_anqp_set_len(buf);
+	wpabuf_free(buf);
+
+	return 0;
+}
+
+
 int common_module_tests(void)
 {
 	int ret = 0;
@@ -165,6 +199,7 @@ int common_module_tests(void)
 	wpa_printf(MSG_INFO, "common module tests");
 
 	if (ieee802_11_parse_tests() < 0 ||
+	    gas_tests() < 0 ||
 	    rsn_ie_parse_tests() < 0)
 		ret = -1;
 
diff --git a/src/common/defs.h b/src/common/defs.h
index b5f4f801eda2..6aea3751a2bc 100644
--- a/src/common/defs.h
+++ b/src/common/defs.h
@@ -174,7 +174,7 @@ enum wpa_states {
 	/**
 	 * WPA_INTERFACE_DISABLED - Interface disabled
 	 *
-	 * This stat eis entered if the network interface is disabled, e.g.,
+	 * This state is entered if the network interface is disabled, e.g.,
 	 * due to rfkill. wpa_supplicant refuses any new operations that would
 	 * use the radio until the interface has been enabled.
 	 */
@@ -295,6 +295,7 @@ enum hostapd_hw_mode {
 	HOSTAPD_MODE_IEEE80211G,
 	HOSTAPD_MODE_IEEE80211A,
 	HOSTAPD_MODE_IEEE80211AD,
+	HOSTAPD_MODE_IEEE80211ANY,
 	NUM_HOSTAPD_MODES
 };
 
@@ -310,6 +311,7 @@ enum wpa_ctrl_req_type {
 	WPA_CTRL_REQ_EAP_OTP,
 	WPA_CTRL_REQ_EAP_PASSPHRASE,
 	WPA_CTRL_REQ_SIM,
+	WPA_CTRL_REQ_PSK_PASSPHRASE,
 	NUM_WPA_CTRL_REQS
 };
 
@@ -326,4 +328,10 @@ enum mesh_plink_state {
 	PLINK_BLOCKED,
 };
 
+enum set_band {
+	WPA_SETBAND_AUTO,
+	WPA_SETBAND_5G,
+	WPA_SETBAND_2G
+};
+
 #endif /* DEFS_H */
diff --git a/src/common/hw_features_common.c b/src/common/hw_features_common.c
index e8babb52a9c5..9c37ea63ca87 100644
--- a/src/common/hw_features_common.c
+++ b/src/common/hw_features_common.c
@@ -88,8 +88,8 @@ int allowed_ht40_channel_pair(struct hostapd_hw_modes *mode, int pri_chan,
 			      int sec_chan)
 {
 	int ok, j, first;
-	int allowed[] = { 36, 44, 52, 60, 100, 108, 116, 124, 132, 149, 157,
-			  184, 192 };
+	int allowed[] = { 36, 44, 52, 60, 100, 108, 116, 124, 132, 140,
+			  149, 157, 184, 192 };
 	size_t k;
 
 	if (pri_chan == sec_chan || !sec_chan)
@@ -152,8 +152,7 @@ void get_pri_sec_chan(struct wpa_scan_res *bss, int *pri_chan, int *sec_chan)
 	*pri_chan = *sec_chan = 0;
 
 	ieee802_11_parse_elems((u8 *) (bss + 1), bss->ie_len, &elems, 0);
-	if (elems.ht_operation &&
-	    elems.ht_operation_len >= sizeof(*oper)) {
+	if (elems.ht_operation) {
 		oper = (struct ieee80211_ht_operation *) elems.ht_operation;
 		*pri_chan = oper->primary_chan;
 		if (oper->ht_param & HT_INFO_HT_PARAM_STA_CHNL_WIDTH) {
@@ -177,10 +176,8 @@ int check_40mhz_5g(struct hostapd_hw_modes *mode,
 	size_t i;
 	int match;
 
-	if (!mode || !scan_res || !pri_chan || !sec_chan)
-		return 0;
-
-	if (pri_chan == sec_chan)
+	if (!mode || !scan_res || !pri_chan || !sec_chan ||
+	    pri_chan == sec_chan)
 		return 0;
 
 	pri_freq = hw_get_freq(mode, pri_chan);
@@ -238,7 +235,8 @@ int check_40mhz_5g(struct hostapd_hw_modes *mode,
 }
 
 
-int check_20mhz_bss(struct wpa_scan_res *bss, int pri_freq, int start, int end)
+static int check_20mhz_bss(struct wpa_scan_res *bss, int pri_freq, int start,
+			   int end)
 {
 	struct ieee802_11_elems elems;
 	struct ieee80211_ht_operation *oper;
@@ -253,8 +251,7 @@ int check_20mhz_bss(struct wpa_scan_res *bss, int pri_freq, int start, int end)
 		return 1;
 	}
 
-	if (elems.ht_operation &&
-	    elems.ht_operation_len >= sizeof(*oper)) {
+	if (elems.ht_operation) {
 		oper = (struct ieee80211_ht_operation *) elems.ht_operation;
 		if (oper->ht_param & HT_INFO_HT_PARAM_SECONDARY_CHNL_OFF_MASK)
 			return 0;
@@ -275,10 +272,8 @@ int check_40mhz_2g4(struct hostapd_hw_modes *mode,
 	int affected_start, affected_end;
 	size_t i;
 
-	if (!mode || !scan_res || !pri_chan || !sec_chan)
-		return 0;
-
-	if (pri_chan == sec_chan)
+	if (!mode || !scan_res || !pri_chan || !sec_chan ||
+	    pri_chan == sec_chan)
 		return 0;
 
 	pri_freq = hw_get_freq(mode, pri_chan);
@@ -335,9 +330,7 @@ int check_40mhz_2g4(struct hostapd_hw_modes *mode,
 
 		ieee802_11_parse_elems((u8 *) (bss + 1), bss->ie_len, &elems,
 				       0);
-		if (elems.ht_capabilities &&
-		    elems.ht_capabilities_len >=
-		    sizeof(struct ieee80211_ht_capabilities)) {
+		if (elems.ht_capabilities) {
 			struct ieee80211_ht_capabilities *ht_cap =
 				(struct ieee80211_ht_capabilities *)
 				elems.ht_capabilities;
@@ -363,8 +356,6 @@ int hostapd_set_freq_params(struct hostapd_freq_params *data,
 			    int vht_oper_chwidth, int center_segment0,
 			    int center_segment1, u32 vht_caps)
 {
-	int tmp;
-
 	os_memset(data, 0, sizeof(*data));
 	data->mode = mode;
 	data->freq = freq;
@@ -378,11 +369,10 @@ int hostapd_set_freq_params(struct hostapd_freq_params *data,
 
 	if (data->vht_enabled) switch (vht_oper_chwidth) {
 	case VHT_CHANWIDTH_USE_HT:
-		if (center_segment1)
-			return -1;
-		if (center_segment0 != 0 &&
-		    5000 + center_segment0 * 5 != data->center_freq1 &&
-		    2407 + center_segment0 * 5 != data->center_freq1)
+		if (center_segment1 ||
+		    (center_segment0 != 0 &&
+		     5000 + center_segment0 * 5 != data->center_freq1 &&
+		     2407 + center_segment0 * 5 != data->center_freq1))
 			return -1;
 		break;
 	case VHT_CHANWIDTH_80P80MHZ:
@@ -398,19 +388,38 @@ int hostapd_set_freq_params(struct hostapd_freq_params *data,
 		/* fall through */
 	case VHT_CHANWIDTH_80MHZ:
 		data->bandwidth = 80;
-		if (vht_oper_chwidth == 1 && center_segment1)
+		if ((vht_oper_chwidth == 1 && center_segment1) ||
+		    (vht_oper_chwidth == 3 && !center_segment1) ||
+		    !sec_channel_offset)
 			return -1;
-		if (vht_oper_chwidth == 3 && !center_segment1)
-			return -1;
-		if (!sec_channel_offset)
-			return -1;
-		/* primary 40 part must match the HT configuration */
-		tmp = (30 + freq - 5000 - center_segment0 * 5) / 20;
-		tmp /= 2;
-		if (data->center_freq1 != 5000 +
-		    center_segment0 * 5 - 20 + 40 * tmp)
-			return -1;
-		data->center_freq1 = 5000 + center_segment0 * 5;
+		if (!center_segment0) {
+			if (channel <= 48)
+				center_segment0 = 42;
+			else if (channel <= 64)
+				center_segment0 = 58;
+			else if (channel <= 112)
+				center_segment0 = 106;
+			else if (channel <= 128)
+				center_segment0 = 122;
+			else if (channel <= 144)
+				center_segment0 = 138;
+			else if (channel <= 161)
+				center_segment0 = 155;
+			data->center_freq1 = 5000 + center_segment0 * 5;
+		} else {
+			/*
+			 * Note: HT/VHT config and params are coupled. Check if
+			 * HT40 channel band is in VHT80 Pri channel band
+			 * configuration.
+			 */
+			if (center_segment0 == channel + 6 ||
+			    center_segment0 == channel + 2 ||
+			    center_segment0 == channel - 2 ||
+			    center_segment0 == channel - 6)
+				data->center_freq1 = 5000 + center_segment0 * 5;
+			else
+				return -1;
+		}
 		break;
 	case VHT_CHANWIDTH_160MHZ:
 		data->bandwidth = 160;
@@ -424,13 +433,21 @@ int hostapd_set_freq_params(struct hostapd_freq_params *data,
 			return -1;
 		if (!sec_channel_offset)
 			return -1;
-		/* primary 40 part must match the HT configuration */
-		tmp = (70 + freq - 5000 - center_segment0 * 5) / 20;
-		tmp /= 2;
-		if (data->center_freq1 != 5000 +
-		    center_segment0 * 5 - 60 + 40 * tmp)
+		/*
+		 * Note: HT/VHT config and params are coupled. Check if
+		 * HT40 channel band is in VHT160 channel band configuration.
+		 */
+		if (center_segment0 == channel + 14 ||
+		    center_segment0 == channel + 10 ||
+		    center_segment0 == channel + 6 ||
+		    center_segment0 == channel + 2 ||
+		    center_segment0 == channel - 2 ||
+		    center_segment0 == channel - 6 ||
+		    center_segment0 == channel - 10 ||
+		    center_segment0 == channel - 14)
+			data->center_freq1 = 5000 + center_segment0 * 5;
+		else
 			return -1;
-		data->center_freq1 = 5000 + center_segment0 * 5;
 		break;
 	}
 
diff --git a/src/common/hw_features_common.h b/src/common/hw_features_common.h
index 7f43d00c5b27..7360b4e3efed 100644
--- a/src/common/hw_features_common.h
+++ b/src/common/hw_features_common.h
@@ -26,7 +26,6 @@ void get_pri_sec_chan(struct wpa_scan_res *bss, int *pri_chan, int *sec_chan);
 int check_40mhz_5g(struct hostapd_hw_modes *mode,
 		   struct wpa_scan_results *scan_res, int pri_chan,
 		   int sec_chan);
-int check_20mhz_bss(struct wpa_scan_res *bss, int pri_freq, int start, int end);
 int check_40mhz_2g4(struct hostapd_hw_modes *mode,
 		    struct wpa_scan_results *scan_res, int pri_chan,
 		    int sec_chan);
diff --git a/src/common/ieee802_11_common.c b/src/common/ieee802_11_common.c
index aca0b73223bb..d07a316a7929 100644
--- a/src/common/ieee802_11_common.c
+++ b/src/common/ieee802_11_common.c
@@ -1,6 +1,6 @@
 /*
  * IEEE 802.11 Common routines
- * Copyright (c) 2002-2013, Jouni Malinen <j@w1.fi>
+ * Copyright (c) 2002-2015, Jouni Malinen <j@w1.fi>
  *
  * This software may be distributed under the terms of the BSD license.
  * See README for more details.
@@ -10,6 +10,8 @@
 
 #include "common.h"
 #include "defs.h"
+#include "wpa_common.h"
+#include "qca-vendor.h"
 #include "ieee802_11_defs.h"
 #include "ieee802_11_common.h"
 
@@ -146,6 +148,20 @@ static int ieee802_11_parse_vendor_specific(const u8 *pos, size_t elen,
 		}
 		break;
 
+	case OUI_QCA:
+		switch (pos[3]) {
+		case QCA_VENDOR_ELEM_P2P_PREF_CHAN_LIST:
+			elems->pref_freq_list = pos;
+			elems->pref_freq_list_len = elen;
+			break;
+		default:
+			wpa_printf(MSG_EXCESSIVE,
+				   "Unknown QCA information element ignored (type=%d len=%lu)",
+				   pos[3], (unsigned long) elen);
+			return -1;
+		}
+		break;
+
 	default:
 		wpa_printf(MSG_EXCESSIVE, "unknown vendor specific "
 			   "information element ignored (vendor OUI "
@@ -196,6 +212,12 @@ ParseRes ieee802_11_parse_elems(const u8 *start, size_t len,
 
 		switch (id) {
 		case WLAN_EID_SSID:
+			if (elen > SSID_MAX_LEN) {
+				wpa_printf(MSG_DEBUG,
+					   "Ignored too long SSID element (elen=%u)",
+					   elen);
+				break;
+			}
 			elems->ssid = pos;
 			elems->ssid_len = elen;
 			break;
@@ -204,8 +226,9 @@ ParseRes ieee802_11_parse_elems(const u8 *start, size_t len,
 			elems->supp_rates_len = elen;
 			break;
 		case WLAN_EID_DS_PARAMS:
+			if (elen < 1)
+				break;
 			elems->ds_params = pos;
-			elems->ds_params_len = elen;
 			break;
 		case WLAN_EID_CF_PARAMS:
 		case WLAN_EID_TIM:
@@ -215,8 +238,9 @@ ParseRes ieee802_11_parse_elems(const u8 *start, size_t len,
 			elems->challenge_len = elen;
 			break;
 		case WLAN_EID_ERP_INFO:
+			if (elen < 1)
+				break;
 			elems->erp_info = pos;
-			elems->erp_info_len = elen;
 			break;
 		case WLAN_EID_EXT_SUPP_RATES:
 			elems->ext_supp_rates = pos;
@@ -239,24 +263,31 @@ ParseRes ieee802_11_parse_elems(const u8 *start, size_t len,
 			elems->supp_channels_len = elen;
 			break;
 		case WLAN_EID_MOBILITY_DOMAIN:
+			if (elen < sizeof(struct rsn_mdie))
+				break;
 			elems->mdie = pos;
 			elems->mdie_len = elen;
 			break;
 		case WLAN_EID_FAST_BSS_TRANSITION:
+			if (elen < sizeof(struct rsn_ftie))
+				break;
 			elems->ftie = pos;
 			elems->ftie_len = elen;
 			break;
 		case WLAN_EID_TIMEOUT_INTERVAL:
+			if (elen != 5)
+				break;
 			elems->timeout_int = pos;
-			elems->timeout_int_len = elen;
 			break;
 		case WLAN_EID_HT_CAP:
+			if (elen < sizeof(struct ieee80211_ht_capabilities))
+				break;
 			elems->ht_capabilities = pos;
-			elems->ht_capabilities_len = elen;
 			break;
 		case WLAN_EID_HT_OPERATION:
+			if (elen < sizeof(struct ieee80211_ht_operation))
+				break;
 			elems->ht_operation = pos;
-			elems->ht_operation_len = elen;
 			break;
 		case WLAN_EID_MESH_CONFIG:
 			elems->mesh_config = pos;
@@ -271,12 +302,14 @@ ParseRes ieee802_11_parse_elems(const u8 *start, size_t len,
 			elems->peer_mgmt_len = elen;
 			break;
 		case WLAN_EID_VHT_CAP:
+			if (elen < sizeof(struct ieee80211_vht_capabilities))
+				break;
 			elems->vht_capabilities = pos;
-			elems->vht_capabilities_len = elen;
 			break;
 		case WLAN_EID_VHT_OPERATION:
+			if (elen < sizeof(struct ieee80211_vht_operation))
+				break;
 			elems->vht_operation = pos;
-			elems->vht_operation_len = elen;
 			break;
 		case WLAN_EID_VHT_OPERATING_MODE_NOTIFICATION:
 			if (elen != 1)
@@ -321,6 +354,18 @@ ParseRes ieee802_11_parse_elems(const u8 *start, size_t len,
 			/* after mic everything is encrypted, so stop. */
 			left = elen;
 			break;
+		case WLAN_EID_MULTI_BAND:
+			if (elems->mb_ies.nof_ies >= MAX_NOF_MB_IES_SUPPORTED) {
+				wpa_printf(MSG_MSGDUMP,
+					   "IEEE 802.11 element parse ignored MB IE (id=%d elen=%d)",
+					   id, elen);
+				break;
+			}
+
+			elems->mb_ies.ies[elems->mb_ies.nof_ies].ie = pos;
+			elems->mb_ies.ies[elems->mb_ies.nof_ies].ie_len = elen;
+			elems->mb_ies.nof_ies++;
+			break;
 		default:
 			unknown++;
 			if (!show_errors)
@@ -486,14 +531,14 @@ int hostapd_config_wmm_ac(struct hostapd_wmm_ac_params wmm_ac_params[],
 		ac->aifs = v;
 	} else if (os_strcmp(pos, "cwmin") == 0) {
 		v = atoi(val);
-		if (v < 0 || v > 12) {
+		if (v < 0 || v > 15) {
 			wpa_printf(MSG_ERROR, "Invalid cwMin value %d", v);
 			return -1;
 		}
 		ac->cwmin = v;
 	} else if (os_strcmp(pos, "cwmax") == 0) {
 		v = atoi(val);
-		if (v < 0 || v > 12) {
+		if (v < 0 || v > 15) {
 			wpa_printf(MSG_ERROR, "Invalid cwMax value %d", v);
 			return -1;
 		}
@@ -523,50 +568,163 @@ int hostapd_config_wmm_ac(struct hostapd_wmm_ac_params wmm_ac_params[],
 
 enum hostapd_hw_mode ieee80211_freq_to_chan(int freq, u8 *channel)
 {
-	enum hostapd_hw_mode mode = NUM_HOSTAPD_MODES;
+	u8 op_class;
 
-	if (freq >= 2412 && freq <= 2472) {
-		mode = HOSTAPD_MODE_IEEE80211G;
-		*channel = (freq - 2407) / 5;
-	} else if (freq == 2484) {
-		mode = HOSTAPD_MODE_IEEE80211B;
-		*channel = 14;
-	} else if (freq >= 4900 && freq < 5000) {
-		mode = HOSTAPD_MODE_IEEE80211A;
-		*channel = (freq - 4000) / 5;
-	} else if (freq >= 5000 && freq < 5900) {
-		mode = HOSTAPD_MODE_IEEE80211A;
-		*channel = (freq - 5000) / 5;
-	} else if (freq >= 56160 + 2160 * 1 && freq <= 56160 + 2160 * 4) {
-		mode = HOSTAPD_MODE_IEEE80211AD;
-		*channel = (freq - 56160) / 2160;
-	}
-
-	return mode;
+	return ieee80211_freq_to_channel_ext(freq, 0, 0, &op_class, channel);
 }
 
 
-static const char *us_op_class_cc[] = {
+/**
+ * ieee80211_freq_to_channel_ext - Convert frequency into channel info
+ * for HT40 and VHT. DFS channels are not covered.
+ * @freq: Frequency (MHz) to convert
+ * @sec_channel: 0 = non-HT40, 1 = sec. channel above, -1 = sec. channel below
+ * @vht: 0 - non-VHT, 1 - 80 MHz
+ * @op_class: Buffer for returning operating class
+ * @channel: Buffer for returning channel number
+ * Returns: hw_mode on success, NUM_HOSTAPD_MODES on failure
+ */
+enum hostapd_hw_mode ieee80211_freq_to_channel_ext(unsigned int freq,
+						   int sec_channel, int vht,
+						   u8 *op_class, u8 *channel)
+{
+	/* TODO: more operating classes */
+
+	if (sec_channel > 1 || sec_channel < -1)
+		return NUM_HOSTAPD_MODES;
+
+	if (freq >= 2412 && freq <= 2472) {
+		if ((freq - 2407) % 5)
+			return NUM_HOSTAPD_MODES;
+
+		if (vht)
+			return NUM_HOSTAPD_MODES;
+
+		/* 2.407 GHz, channels 1..13 */
+		if (sec_channel == 1)
+			*op_class = 83;
+		else if (sec_channel == -1)
+			*op_class = 84;
+		else
+			*op_class = 81;
+
+		*channel = (freq - 2407) / 5;
+
+		return HOSTAPD_MODE_IEEE80211G;
+	}
+
+	if (freq == 2484) {
+		if (sec_channel || vht)
+			return NUM_HOSTAPD_MODES;
+
+		*op_class = 82; /* channel 14 */
+		*channel = 14;
+
+		return HOSTAPD_MODE_IEEE80211B;
+	}
+
+	if (freq >= 4900 && freq < 5000) {
+		if ((freq - 4000) % 5)
+			return NUM_HOSTAPD_MODES;
+		*channel = (freq - 4000) / 5;
+		*op_class = 0; /* TODO */
+		return HOSTAPD_MODE_IEEE80211A;
+	}
+
+	/* 5 GHz, channels 36..48 */
+	if (freq >= 5180 && freq <= 5240) {
+		if ((freq - 5000) % 5)
+			return NUM_HOSTAPD_MODES;
+
+		if (sec_channel == 1)
+			*op_class = 116;
+		else if (sec_channel == -1)
+			*op_class = 117;
+		else if (vht)
+			*op_class = 128;
+		else
+			*op_class = 115;
+
+		*channel = (freq - 5000) / 5;
+
+		return HOSTAPD_MODE_IEEE80211A;
+	}
+
+	/* 5 GHz, channels 149..161 */
+	if (freq >= 5745 && freq <= 5805) {
+		if ((freq - 5000) % 5)
+			return NUM_HOSTAPD_MODES;
+
+		if (sec_channel == 1)
+			*op_class = 126;
+		else if (sec_channel == -1)
+			*op_class = 127;
+		else if (vht)
+			*op_class = 128;
+		else
+			*op_class = 124;
+
+		*channel = (freq - 5000) / 5;
+
+		return HOSTAPD_MODE_IEEE80211A;
+	}
+
+	/* 5 GHz, channels 149..169 */
+	if (freq >= 5745 && freq <= 5845) {
+		if ((freq - 5000) % 5)
+			return NUM_HOSTAPD_MODES;
+
+		*op_class = 125;
+
+		*channel = (freq - 5000) / 5;
+
+		return HOSTAPD_MODE_IEEE80211A;
+	}
+
+	if (freq >= 5000 && freq < 5900) {
+		if ((freq - 5000) % 5)
+			return NUM_HOSTAPD_MODES;
+		*channel = (freq - 5000) / 5;
+		*op_class = 0; /* TODO */
+		return HOSTAPD_MODE_IEEE80211A;
+	}
+
+	/* 56.16 GHz, channel 1..4 */
+	if (freq >= 56160 + 2160 * 1 && freq <= 56160 + 2160 * 4) {
+		if (sec_channel || vht)
+			return NUM_HOSTAPD_MODES;
+
+		*channel = (freq - 56160) / 2160;
+		*op_class = 180;
+
+		return HOSTAPD_MODE_IEEE80211AD;
+	}
+
+	return NUM_HOSTAPD_MODES;
+}
+
+
+static const char *const us_op_class_cc[] = {
 	"US", "CA", NULL
 };
 
-static const char *eu_op_class_cc[] = {
+static const char *const eu_op_class_cc[] = {
 	"AL", "AM", "AT", "AZ", "BA", "BE", "BG", "BY", "CH", "CY", "CZ", "DE",
 	"DK", "EE", "EL", "ES", "FI", "FR", "GE", "HR", "HU", "IE", "IS", "IT",
 	"LI", "LT", "LU", "LV", "MD", "ME", "MK", "MT", "NL", "NO", "PL", "PT",
 	"RO", "RS", "RU", "SE", "SI", "SK", "TR", "UA", "UK", NULL
 };
 
-static const char *jp_op_class_cc[] = {
+static const char *const jp_op_class_cc[] = {
 	"JP", NULL
 };
 
-static const char *cn_op_class_cc[] = {
-	"CN", "CA", NULL
+static const char *const cn_op_class_cc[] = {
+	"CN", NULL
 };
 
 
-static int country_match(const char *cc[], const char *country)
+static int country_match(const char *const cc[], const char *const country)
 {
 	int i;
 
@@ -612,6 +770,10 @@ static int ieee80211_chan_to_freq_us(u8 op_class, u8 chan)
 		if (chan < 149 || chan > 161)
 			return -1;
 		return 5000 + 5 * chan;
+	case 5: /* channels 149,153,157,161,165 */
+		if (chan < 149 || chan > 165)
+			return -1;
+		return 5000 + 5 * chan;
 	case 34: /* 60 GHz band, channels 1..3 */
 		if (chan < 1 || chan > 3)
 			return -1;
@@ -764,12 +926,15 @@ static int ieee80211_chan_to_freq_global(u8 op_class, u8 chan)
 			return -1;
 		return 5000 + 5 * chan;
 	case 124: /* channels 149,153,157,161 */
-	case 125: /* channels 149,153,157,161,165,169 */
 	case 126: /* channels 149,157; 40 MHz */
 	case 127: /* channels 153,161; 40 MHz */
 		if (chan < 149 || chan > 161)
 			return -1;
 		return 5000 + 5 * chan;
+	case 125: /* channels 149,153,157,161,165,169 */
+		if (chan < 149 || chan > 169)
+			return -1;
+		return 5000 + 5 * chan;
 	case 128: /* center freqs 42, 58, 106, 122, 138, 155; 80 MHz */
 	case 130: /* center freqs 42, 58, 106, 122, 138, 155; 80 MHz */
 		if (chan < 36 || chan > 161)
@@ -921,3 +1086,62 @@ const char * fc2str(u16 fc)
 	return "WLAN_FC_TYPE_UNKNOWN";
 #undef C2S
 }
+
+
+int mb_ies_info_by_ies(struct mb_ies_info *info, const u8 *ies_buf,
+		       size_t ies_len)
+{
+	os_memset(info, 0, sizeof(*info));
+
+	while (ies_buf && ies_len >= 2 &&
+	       info->nof_ies < MAX_NOF_MB_IES_SUPPORTED) {
+		size_t len = 2 + ies_buf[1];
+
+		if (len > ies_len) {
+			wpa_hexdump(MSG_DEBUG, "Truncated IEs",
+				    ies_buf, ies_len);
+			return -1;
+		}
+
+		if (ies_buf[0] == WLAN_EID_MULTI_BAND) {
+			wpa_printf(MSG_DEBUG, "MB IE of %zu bytes found", len);
+			info->ies[info->nof_ies].ie = ies_buf + 2;
+			info->ies[info->nof_ies].ie_len = ies_buf[1];
+			info->nof_ies++;
+		}
+
+		ies_len -= len;
+		ies_buf += len;
+	}
+
+	return 0;
+}
+
+
+struct wpabuf * mb_ies_by_info(struct mb_ies_info *info)
+{
+	struct wpabuf *mb_ies = NULL;
+
+	WPA_ASSERT(info != NULL);
+
+	if (info->nof_ies) {
+		u8 i;
+		size_t mb_ies_size = 0;
+
+		for (i = 0; i < info->nof_ies; i++)
+			mb_ies_size += 2 + info->ies[i].ie_len;
+
+		mb_ies = wpabuf_alloc(mb_ies_size);
+		if (mb_ies) {
+			for (i = 0; i < info->nof_ies; i++) {
+				wpabuf_put_u8(mb_ies, WLAN_EID_MULTI_BAND);
+				wpabuf_put_u8(mb_ies, info->ies[i].ie_len);
+				wpabuf_put_data(mb_ies,
+						info->ies[i].ie,
+						info->ies[i].ie_len);
+			}
+		}
+	}
+
+	return mb_ies;
+}
diff --git a/src/common/ieee802_11_common.h b/src/common/ieee802_11_common.h
index 7f0b296d2b00..55ce0223d923 100644
--- a/src/common/ieee802_11_common.h
+++ b/src/common/ieee802_11_common.h
@@ -9,6 +9,16 @@
 #ifndef IEEE802_11_COMMON_H
 #define IEEE802_11_COMMON_H
 
+#define MAX_NOF_MB_IES_SUPPORTED 5
+
+struct mb_ies_info {
+	struct {
+		const u8 *ie;
+		u8 ie_len;
+	} ies[MAX_NOF_MB_IES_SUPPORTED];
+	u8 nof_ies;
+};
+
 /* Parsed Information Elements */
 struct ieee802_11_elems {
 	const u8 *ssid;
@@ -48,12 +58,11 @@ struct ieee802_11_elems {
 	const u8 *osen;
 	const u8 *ampe;
 	const u8 *mic;
+	const u8 *pref_freq_list;
 
 	u8 ssid_len;
 	u8 supp_rates_len;
-	u8 ds_params_len;
 	u8 challenge_len;
-	u8 erp_info_len;
 	u8 ext_supp_rates_len;
 	u8 wpa_ie_len;
 	u8 rsn_ie_len;
@@ -63,14 +72,9 @@ struct ieee802_11_elems {
 	u8 supp_channels_len;
 	u8 mdie_len;
 	u8 ftie_len;
-	u8 timeout_int_len;
-	u8 ht_capabilities_len;
-	u8 ht_operation_len;
 	u8 mesh_config_len;
 	u8 mesh_id_len;
 	u8 peer_mgmt_len;
-	u8 vht_capabilities_len;
-	u8 vht_operation_len;
 	u8 vendor_ht_cap_len;
 	u8 vendor_vht_len;
 	u8 p2p_len;
@@ -83,6 +87,8 @@ struct ieee802_11_elems {
 	u8 osen_len;
 	u8 ampe_len;
 	u8 mic_len;
+	u8 pref_freq_list_len;
+	struct mb_ies_info mb_ies;
 };
 
 typedef enum { ParseOK = 0, ParseUnknown = 1, ParseFailed = -1 } ParseRes;
@@ -108,9 +114,15 @@ int hostapd_config_wmm_ac(struct hostapd_wmm_ac_params wmm_ac_params[],
 			  const char *name, const char *val);
 enum hostapd_hw_mode ieee80211_freq_to_chan(int freq, u8 *channel);
 int ieee80211_chan_to_freq(const char *country, u8 op_class, u8 chan);
+enum hostapd_hw_mode ieee80211_freq_to_channel_ext(unsigned int freq,
+						   int sec_channel, int vht,
+						   u8 *op_class, u8 *channel);
 int ieee80211_is_dfs(int freq);
 
 int supp_rates_11b_only(struct ieee802_11_elems *elems);
+int mb_ies_info_by_ies(struct mb_ies_info *info, const u8 *ies_buf,
+		       size_t ies_len);
+struct wpabuf * mb_ies_by_info(struct mb_ies_info *info);
 
 const char * fc2str(u16 fc);
 #endif /* IEEE802_11_COMMON_H */
diff --git a/src/common/ieee802_11_defs.h b/src/common/ieee802_11_defs.h
index 2e51935b8e3b..44530ce3cee6 100644
--- a/src/common/ieee802_11_defs.h
+++ b/src/common/ieee802_11_defs.h
@@ -10,6 +10,8 @@
 #ifndef IEEE802_11_DEFS_H
 #define IEEE802_11_DEFS_H
 
+#include <utils/common.h>
+
 /* IEEE 802.11 defines */
 
 #define WLAN_FC_PVER		0x0003
@@ -163,7 +165,10 @@
 #define WLAN_STATUS_ANTI_CLOGGING_TOKEN_REQ 76
 #define WLAN_STATUS_FINITE_CYCLIC_GROUP_NOT_SUPPORTED 77
 #define WLAN_STATUS_TRANSMISSION_FAILURE 79
+#define WLAN_STATUS_REJECTED_WITH_SUGGESTED_BSS_TRANSITION 82
+#define WLAN_STATUS_PENDING_ADMITTING_FST_SESSION 86
 #define WLAN_STATUS_QUERY_RESP_OUTSTANDING 95
+#define WLAN_STATUS_DENIED_WITH_SUGGESTED_BAND_AND_CHANNEL 99
 #define WLAN_STATUS_ASSOC_DENIED_NO_VHT 104
 
 /* Reason codes (IEEE 802.11-2007, 7.3.1.7, Table 7-22) */
@@ -269,6 +274,8 @@
 #define WLAN_EID_AMPE 139
 #define WLAN_EID_MIC 140
 #define WLAN_EID_CCKM 156
+#define WLAN_EID_MULTI_BAND 158
+#define WLAN_EID_SESSION_TRANSITION 164
 #define WLAN_EID_VHT_CAP 191
 #define WLAN_EID_VHT_OPERATION 192
 #define WLAN_EID_VHT_EXTENDED_BSS_LOAD 193
@@ -297,6 +304,7 @@
 #define WLAN_ACTION_TDLS 12
 #define WLAN_ACTION_SELF_PROTECTED 15
 #define WLAN_ACTION_WMM 17 /* WMM Specification 1.1 */
+#define WLAN_ACTION_FST 18
 #define WLAN_ACTION_VENDOR_SPECIFIC 127
 
 /* Public action codes */
@@ -470,35 +478,35 @@ struct ieee80211_mgmt {
 			le16 auth_transaction;
 			le16 status_code;
 			/* possibly followed by Challenge text */
-			u8 variable[0];
+			u8 variable[];
 		} STRUCT_PACKED auth;
 		struct {
 			le16 reason_code;
-			u8 variable[0];
+			u8 variable[];
 		} STRUCT_PACKED deauth;
 		struct {
 			le16 capab_info;
 			le16 listen_interval;
 			/* followed by SSID and Supported rates */
-			u8 variable[0];
+			u8 variable[];
 		} STRUCT_PACKED assoc_req;
 		struct {
 			le16 capab_info;
 			le16 status_code;
 			le16 aid;
 			/* followed by Supported rates */
-			u8 variable[0];
+			u8 variable[];
 		} STRUCT_PACKED assoc_resp, reassoc_resp;
 		struct {
 			le16 capab_info;
 			le16 listen_interval;
 			u8 current_ap[6];
 			/* followed by SSID and Supported rates */
-			u8 variable[0];
+			u8 variable[];
 		} STRUCT_PACKED reassoc_req;
 		struct {
 			le16 reason_code;
-			u8 variable[0];
+			u8 variable[];
 		} STRUCT_PACKED disassoc;
 		struct {
 			u8 timestamp[8];
@@ -506,7 +514,7 @@ struct ieee80211_mgmt {
 			le16 capab_info;
 			/* followed by some of SSID, Supported rates,
 			 * FH Params, DS Params, CF Params, IBSS Params, TIM */
-			u8 variable[0];
+			u8 variable[];
 		} STRUCT_PACKED beacon;
 		struct {
 			/* only variable items: SSID, Supported rates */
@@ -518,7 +526,7 @@ struct ieee80211_mgmt {
 			le16 capab_info;
 			/* followed by some of SSID, Supported rates,
 			 * FH Params, DS Params, CF Params, IBSS Params */
-			u8 variable[0];
+			u8 variable[];
 		} STRUCT_PACKED probe_resp;
 		struct {
 			u8 category;
@@ -527,7 +535,7 @@ struct ieee80211_mgmt {
 					u8 action_code;
 					u8 dialog_token;
 					u8 status_code;
-					u8 variable[0];
+					u8 variable[];
 				} STRUCT_PACKED wmm_action;
 				struct{
 					u8 action_code;
@@ -541,14 +549,14 @@ struct ieee80211_mgmt {
 					u8 action;
 					u8 sta_addr[ETH_ALEN];
 					u8 target_ap_addr[ETH_ALEN];
-					u8 variable[0]; /* FT Request */
+					u8 variable[]; /* FT Request */
 				} STRUCT_PACKED ft_action_req;
 				struct {
 					u8 action;
 					u8 sta_addr[ETH_ALEN];
 					u8 target_ap_addr[ETH_ALEN];
 					le16 status_code;
-					u8 variable[0]; /* FT Request */
+					u8 variable[]; /* FT Request */
 				} STRUCT_PACKED ft_action_resp;
 				struct {
 					u8 action;
@@ -561,23 +569,23 @@ struct ieee80211_mgmt {
 				struct {
 					u8 action;
 					u8 dialogtoken;
-					u8 variable[0];
+					u8 variable[];
 				} STRUCT_PACKED wnm_sleep_req;
 				struct {
 					u8 action;
 					u8 dialogtoken;
 					le16 keydata_len;
-					u8 variable[0];
+					u8 variable[];
 				} STRUCT_PACKED wnm_sleep_resp;
 				struct {
 					u8 action;
-					u8 variable[0];
+					u8 variable[];
 				} STRUCT_PACKED public_action;
 				struct {
 					u8 action; /* 9 */
 					u8 oui[3];
 					/* Vendor-specific content */
-					u8 variable[0];
+					u8 variable[];
 				} STRUCT_PACKED vs_public_action;
 				struct {
 					u8 action; /* 7 */
@@ -589,7 +597,7 @@ struct ieee80211_mgmt {
 					 * Session Information URL (optional),
 					 * BSS Transition Candidate List
 					 * Entries */
-					u8 variable[0];
+					u8 variable[];
 				} STRUCT_PACKED bss_tm_req;
 				struct {
 					u8 action; /* 8 */
@@ -599,7 +607,7 @@ struct ieee80211_mgmt {
 					/* Target BSSID (optional),
 					 * BSS Transition Candidate List
 					 * Entries (optional) */
-					u8 variable[0];
+					u8 variable[];
 				} STRUCT_PACKED bss_tm_resp;
 				struct {
 					u8 action; /* 6 */
@@ -607,12 +615,16 @@ struct ieee80211_mgmt {
 					u8 query_reason;
 					/* BSS Transition Candidate List
 					 * Entries (optional) */
-					u8 variable[0];
+					u8 variable[];
 				} STRUCT_PACKED bss_tm_query;
 				struct {
 					u8 action; /* 15 */
-					u8 variable[0];
+					u8 variable[];
 				} STRUCT_PACKED slf_prot_action;
+				struct {
+					u8 action;
+					u8 variable[];
+				} STRUCT_PACKED fst_action;
 			} u;
 		} STRUCT_PACKED action;
 	} u;
@@ -1065,6 +1077,15 @@ enum p2p_attr_id {
 #define P2P_GROUP_CAPAB_GROUP_FORMATION BIT(6)
 #define P2P_GROUP_CAPAB_IP_ADDR_ALLOCATION BIT(7)
 
+/* P2PS Coordination Protocol Transport Bitmap */
+#define P2PS_FEATURE_CAPAB_UDP_TRANSPORT BIT(0)
+#define P2PS_FEATURE_CAPAB_MAC_TRANSPORT BIT(1)
+
+struct p2ps_feature_capab {
+	u8 cpt;
+	u8 reserved;
+} STRUCT_PACKED;
+
 /* Invitation Flags */
 #define P2P_INVITATION_FLAGS_TYPE BIT(0)
 
@@ -1354,4 +1375,62 @@ struct rrm_link_measurement_report {
 	u8 variable[0];
 } STRUCT_PACKED;
 
+#define SSID_MAX_LEN 32
+
+/* IEEE Std 802.11ad-2012 - Multi-band element */
+struct multi_band_ie {
+	u8 eid; /* WLAN_EID_MULTI_BAND */
+	u8 len;
+	u8 mb_ctrl;
+	u8 band_id;
+	u8 op_class;
+	u8 chan;
+	u8 bssid[ETH_ALEN];
+	le16 beacon_int;
+	u8 tsf_offs[8];
+	u8 mb_connection_capability;
+	u8 fst_session_tmout;
+	/* Optional:
+	 *   STA MAC Address
+	 *   Pairwise Cipher Suite Count
+	 *   Pairwise Cipher Suite List
+	 */
+	u8 variable[0];
+} STRUCT_PACKED;
+
+enum mb_ctrl_sta_role {
+	MB_STA_ROLE_AP = 0,
+	MB_STA_ROLE_TDLS_STA = 1,
+	MB_STA_ROLE_IBSS_STA = 2,
+	MB_STA_ROLE_PCP = 3,
+	MB_STA_ROLE_NON_PCP_NON_AP = 4
+};
+
+#define MB_CTRL_ROLE_MASK (BIT(0) | BIT(1) | BIT(2))
+#define MB_CTRL_ROLE(ctrl) ((u8) ((ctrl) & MB_CTRL_ROLE_MASK))
+#define MB_CTRL_STA_MAC_PRESENT ((u8) (BIT(3)))
+#define MB_CTRL_PAIRWISE_CIPHER_SUITE_PRESENT ((u8) (BIT(4)))
+
+enum mb_band_id {
+	MB_BAND_ID_WIFI_2_4GHZ = 2, /* 2.4 GHz */
+	MB_BAND_ID_WIFI_5GHZ = 4, /* 4.9 and 5 GHz */
+	MB_BAND_ID_WIFI_60GHZ = 5, /* 60 GHz */
+};
+
+#define MB_CONNECTION_CAPABILITY_AP ((u8) (BIT(0)))
+#define MB_CONNECTION_CAPABILITY_PCP ((u8) (BIT(1)))
+#define MB_CONNECTION_CAPABILITY_DLS ((u8) (BIT(2)))
+#define MB_CONNECTION_CAPABILITY_TDLS ((u8) (BIT(3)))
+#define MB_CONNECTION_CAPABILITY_IBSS ((u8) (BIT(4)))
+
+/* IEEE Std 802.11ad-2014 - FST Action field */
+enum fst_action {
+	FST_ACTION_SETUP_REQUEST = 0,
+	FST_ACTION_SETUP_RESPONSE = 1,
+	FST_ACTION_TEAR_DOWN = 2,
+	FST_ACTION_ACK_REQUEST = 3,
+	FST_ACTION_ACK_RESPONSE = 4,
+	FST_ACTION_ON_CHANNEL_TUNNEL = 5,
+};
+
 #endif /* IEEE802_11_DEFS_H */
diff --git a/src/common/privsep_commands.h b/src/common/privsep_commands.h
index 4dc34c4ad322..8dff30382b60 100644
--- a/src/common/privsep_commands.h
+++ b/src/common/privsep_commands.h
@@ -9,6 +9,8 @@
 #ifndef PRIVSEP_COMMANDS_H
 #define PRIVSEP_COMMANDS_H
 
+#include "common/ieee802_11_defs.h"
+
 enum privsep_cmd {
 	PRIVSEP_CMD_REGISTER,
 	PRIVSEP_CMD_UNREGISTER,
@@ -24,12 +26,31 @@ enum privsep_cmd {
 	PRIVSEP_CMD_L2_NOTIFY_AUTH_START,
 	PRIVSEP_CMD_L2_SEND,
 	PRIVSEP_CMD_SET_COUNTRY,
+	PRIVSEP_CMD_AUTHENTICATE,
+};
+
+struct privsep_cmd_authenticate
+{
+	int freq;
+	u8 bssid[ETH_ALEN];
+	u8 ssid[SSID_MAX_LEN];
+	size_t ssid_len;
+	int auth_alg;
+	size_t ie_len;
+	u8 wep_key[4][16];
+	size_t wep_key_len[4];
+	int wep_tx_keyidx;
+	int local_state_change;
+	int p2p;
+	size_t sae_data_len;
+	/* followed by ie_len bytes of ie */
+	/* followed by sae_data_len bytes of sae_data */
 };
 
 struct privsep_cmd_associate
 {
 	u8 bssid[ETH_ALEN];
-	u8 ssid[32];
+	u8 ssid[SSID_MAX_LEN];
 	size_t ssid_len;
 	int hwmode;
 	int freq;
@@ -66,6 +87,18 @@ enum privsep_event {
 	PRIVSEP_EVENT_STKSTART,
 	PRIVSEP_EVENT_FT_RESPONSE,
 	PRIVSEP_EVENT_RX_EAPOL,
+	PRIVSEP_EVENT_SCAN_STARTED,
+	PRIVSEP_EVENT_AUTH,
+};
+
+struct privsep_event_auth {
+	u8 peer[ETH_ALEN];
+	u8 bssid[ETH_ALEN];
+	u16 auth_type;
+	u16 auth_transaction;
+	u16 status_code;
+	size_t ies_len;
+	/* followed by ies_len bytes of ies */
 };
 
 #endif /* PRIVSEP_COMMANDS_H */
diff --git a/src/common/qca-vendor.h b/src/common/qca-vendor.h
index 2117ee7028c1..28985f5194e2 100644
--- a/src/common/qca-vendor.h
+++ b/src/common/qca-vendor.h
@@ -132,7 +132,7 @@ enum qca_nl80211_vendor_subcmds {
 	QCA_NL80211_VENDOR_SUBCMD_KEY_MGMT_SET_KEY = 50,
 	QCA_NL80211_VENDOR_SUBCMD_KEY_MGMT_ROAM_AUTH = 51,
 	QCA_NL80211_VENDOR_SUBCMD_APFIND = 52,
-	/* 53 - reserved for QCA */
+	/* 53 - reserved - was used by QCA, but not in use anymore */
 	QCA_NL80211_VENDOR_SUBCMD_DO_ACS = 54,
 	QCA_NL80211_VENDOR_SUBCMD_GET_FEATURES = 55,
 	QCA_NL80211_VENDOR_SUBCMD_DFS_OFFLOAD_CAC_STARTED = 56,
@@ -142,6 +142,20 @@ enum qca_nl80211_vendor_subcmds {
 	QCA_NL80211_VENDOR_SUBCMD_DFS_OFFLOAD_RADAR_DETECTED = 60,
 	/* 61-90 - reserved for QCA */
 	QCA_NL80211_VENDOR_SUBCMD_DATA_OFFLOAD = 91,
+	QCA_NL80211_VENDOR_SUBCMD_OCB_SET_CONFIG = 92,
+	QCA_NL80211_VENDOR_SUBCMD_OCB_SET_UTC_TIME = 93,
+	QCA_NL80211_VENDOR_SUBCMD_OCB_START_TIMING_ADVERT = 94,
+	QCA_NL80211_VENDOR_SUBCMD_OCB_STOP_TIMING_ADVERT = 95,
+	QCA_NL80211_VENDOR_SUBCMD_OCB_GET_TSF_TIMER = 96,
+	QCA_NL80211_VENDOR_SUBCMD_DCC_GET_STATS = 97,
+	QCA_NL80211_VENDOR_SUBCMD_DCC_CLEAR_STATS = 98,
+	QCA_NL80211_VENDOR_SUBCMD_DCC_UPDATE_NDL = 99,
+	QCA_NL80211_VENDOR_SUBCMD_DCC_STATS_EVENT = 100,
+	QCA_NL80211_VENDOR_SUBCMD_LINK_PROPERTIES = 101,
+	QCA_NL80211_VENDOR_SUBCMD_GW_PARAM_CONFIG = 102,
+	QCA_NL80211_VENDOR_SUBCMD_GET_PREFERRED_FREQ_LIST = 103,
+	QCA_NL80211_VENDOR_SUBCMD_SET_PROBABLE_OPER_CHANNEL = 104,
+	QCA_NL80211_VENDOR_SUBCMD_SETBAND = 105,
 };
 
 
@@ -162,6 +176,15 @@ enum qca_wlan_vendor_attr {
 	/* used by QCA_NL80211_VENDOR_SUBCMD_GET_FEATURES */
 	QCA_WLAN_VENDOR_ATTR_FEATURE_FLAGS = 7,
 	QCA_WLAN_VENDOR_ATTR_TEST = 8,
+	/* used by QCA_NL80211_VENDOR_SUBCMD_GET_FEATURES */
+	/* Unsigned 32-bit value. */
+	QCA_WLAN_VENDOR_ATTR_CONCURRENCY_CAPA = 9,
+	/* Unsigned 32-bit value */
+	QCA_WLAN_VENDOR_ATTR_MAX_CONCURRENT_CHANNELS_2_4_BAND = 10,
+	/* Unsigned 32-bit value */
+	QCA_WLAN_VENDOR_ATTR_MAX_CONCURRENT_CHANNELS_5_0_BAND = 11,
+	/* Unsigned 32-bit value from enum qca_set_band. */
+	QCA_WLAN_VENDOR_ATTR_SETBAND_VALUE = 12,
 	/* keep last */
 	QCA_WLAN_VENDOR_ATTR_AFTER_LAST,
 	QCA_WLAN_VENDOR_ATTR_MAX	= QCA_WLAN_VENDOR_ATTR_AFTER_LAST - 1,
@@ -195,6 +218,12 @@ enum qca_wlan_vendor_attr_acs_offload {
 	QCA_WLAN_VENDOR_ATTR_ACS_HW_MODE,
 	QCA_WLAN_VENDOR_ATTR_ACS_HT_ENABLED,
 	QCA_WLAN_VENDOR_ATTR_ACS_HT40_ENABLED,
+	QCA_WLAN_VENDOR_ATTR_ACS_VHT_ENABLED,
+	QCA_WLAN_VENDOR_ATTR_ACS_CHWIDTH,
+	QCA_WLAN_VENDOR_ATTR_ACS_CH_LIST,
+	QCA_WLAN_VENDOR_ATTR_ACS_VHT_SEG0_CENTER_CHANNEL,
+	QCA_WLAN_VENDOR_ATTR_ACS_VHT_SEG1_CENTER_CHANNEL,
+	QCA_WLAN_VENDOR_ATTR_ACS_FREQ_LIST,
 	/* keep last */
 	QCA_WLAN_VENDOR_ATTR_ACS_AFTER_LAST,
 	QCA_WLAN_VENDOR_ATTR_ACS_MAX =
@@ -206,6 +235,7 @@ enum qca_wlan_vendor_acs_hw_mode {
 	QCA_ACS_MODE_IEEE80211G,
 	QCA_ACS_MODE_IEEE80211A,
 	QCA_ACS_MODE_IEEE80211AD,
+	QCA_ACS_MODE_IEEE80211ANY,
 };
 
 /**
@@ -215,10 +245,13 @@ enum qca_wlan_vendor_acs_hw_mode {
  *	management offload, a mechanism where the station's firmware
  *	does the exchange with the AP to establish the temporal keys
  *	after roaming, rather than having the user space wpa_supplicant do it.
+ * @QCA_WLAN_VENDOR_FEATURE_SUPPORT_HW_MODE_ANY: Device supports automatic
+ *	band selection based on channel selection results.
  * @NUM_QCA_WLAN_VENDOR_FEATURES: Number of assigned feature bits
  */
 enum qca_wlan_vendor_features {
 	QCA_WLAN_VENDOR_FEATURE_KEY_MGMT_OFFLOAD	= 0,
+	QCA_WLAN_VENDOR_FEATURE_SUPPORT_HW_MODE_ANY     = 1,
 	NUM_QCA_WLAN_VENDOR_FEATURES /* keep last */
 };
 
@@ -243,4 +276,82 @@ enum qca_wlan_vendor_attr_data_offload_ind {
 	QCA_WLAN_VENDOR_ATTR_DATA_OFFLOAD_IND_MAX =
 	QCA_WLAN_VENDOR_ATTR_DATA_OFFLOAD_IND_AFTER_LAST - 1
 };
+
+enum qca_vendor_attr_get_preferred_freq_list {
+	QCA_WLAN_VENDOR_ATTR_GET_PREFERRED_FREQ_LIST_INVALID,
+	/* A 32-unsigned value; the interface type/mode for which the preferred
+	 * frequency list is requested (see enum qca_iface_type for possible
+	 * values); used in GET_PREFERRED_FREQ_LIST command from user-space to
+	 * kernel and in the kernel response back to user-space.
+	 */
+	QCA_WLAN_VENDOR_ATTR_GET_PREFERRED_FREQ_LIST_IFACE_TYPE,
+	/* An array of 32-unsigned values; values are frequency (MHz); sent
+	 * from kernel space to user space.
+	 */
+	QCA_WLAN_VENDOR_ATTR_GET_PREFERRED_FREQ_LIST,
+	/* keep last */
+	QCA_WLAN_VENDOR_ATTR_GET_PREFERRED_FREQ_LIST_AFTER_LAST,
+	QCA_WLAN_VENDOR_ATTR_GET_PREFERRED_FREQ_LIST_MAX =
+	QCA_WLAN_VENDOR_ATTR_GET_PREFERRED_FREQ_LIST_AFTER_LAST - 1
+};
+
+enum qca_vendor_attr_probable_oper_channel {
+	QCA_WLAN_VENDOR_ATTR_PROBABLE_OPER_CHANNEL_INVALID,
+	/* 32-bit unsigned value; indicates the connection/iface type likely to
+	 * come on this channel (see enum qca_iface_type).
+	 */
+	QCA_WLAN_VENDOR_ATTR_PROBABLE_OPER_CHANNEL_IFACE_TYPE,
+	/* 32-bit unsigned value; the frequency (MHz) of the probable channel */
+	QCA_WLAN_VENDOR_ATTR_PROBABLE_OPER_CHANNEL_FREQ,
+	/* keep last */
+	QCA_WLAN_VENDOR_ATTR_PROBABLE_OPER_CHANNEL_AFTER_LAST,
+	QCA_WLAN_VENDOR_ATTR_PROBABLE_OPER_CHANNEL_MAX =
+	QCA_WLAN_VENDOR_ATTR_PROBABLE_OPER_CHANNEL_AFTER_LAST - 1
+};
+
+enum qca_iface_type {
+	QCA_IFACE_TYPE_STA,
+	QCA_IFACE_TYPE_AP,
+	QCA_IFACE_TYPE_P2P_CLIENT,
+	QCA_IFACE_TYPE_P2P_GO,
+	QCA_IFACE_TYPE_IBSS,
+	QCA_IFACE_TYPE_TDLS,
+};
+
+enum qca_set_band {
+	QCA_SETBAND_AUTO,
+	QCA_SETBAND_5G,
+	QCA_SETBAND_2G,
+};
+
+/* IEEE 802.11 Vendor Specific elements */
+
+/**
+ * enum qca_vendor_element_id - QCA Vendor Specific element types
+ *
+ * These values are used to identify QCA Vendor Specific elements. The
+ * payload of the element starts with the three octet OUI (OUI_QCA) and
+ * is followed by a single octet type which is defined by this enum.
+ *
+ * @QCA_VENDOR_ELEM_P2P_PREF_CHAN_LIST: P2P preferred channel list.
+ *	This element can be used to specify preference order for supported
+ *	channels. The channels in this list are in preference order (the first
+ *	one has the highest preference) and are described as a pair of
+ *	(global) Operating Class and Channel Number (each one octet) fields.
+ *
+ *	This extends the standard P2P functionality by providing option to have
+ *	more than one preferred operating channel. When this element is present,
+ *	it replaces the preference indicated in the Operating Channel attribute.
+ *	For supporting other implementations, the Operating Channel attribute is
+ *	expected to be used with the highest preference channel. Similarly, all
+ *	the channels included in this Preferred channel list element are
+ *	expected to be included in the Channel List attribute.
+ *
+ *	This vendor element may be included in GO Negotiation Request, P2P
+ *	Invitation Request, and Provision Discovery Request frames.
+ */
+enum qca_vendor_element_id {
+	QCA_VENDOR_ELEM_P2P_PREF_CHAN_LIST = 0,
+};
+
 #endif /* QCA_VENDOR_H */
diff --git a/src/common/sae.c b/src/common/sae.c
index 588895808fde..503fa1d7b9a9 100644
--- a/src/common/sae.c
+++ b/src/common/sae.c
@@ -1,6 +1,6 @@
 /*
  * Simultaneous authentication of equals
- * Copyright (c) 2012-2013, Jouni Malinen <j@w1.fi>
+ * Copyright (c) 2012-2015, Jouni Malinen <j@w1.fi>
  *
  * This software may be distributed under the terms of the BSD license.
  * See README for more details.
@@ -124,9 +124,7 @@ static struct crypto_bignum * sae_get_rand(struct sae_data *sae)
 		return NULL;
 
 	for (;;) {
-		if (iter++ > 100)
-			return NULL;
-		if (random_get_bytes(val, order_len) < 0)
+		if (iter++ > 100 || random_get_bytes(val, order_len) < 0)
 			return NULL;
 		if (order_len_bits % 8)
 			buf_shift_right(val, order_len, 8 - order_len_bits % 8);
@@ -171,17 +169,107 @@ static void sae_pwd_seed_key(const u8 *addr1, const u8 *addr2, u8 *key)
 }
 
 
-static int sae_test_pwd_seed_ecc(struct sae_data *sae, const u8 *pwd_seed,
-				 struct crypto_ec_point *pwe)
+static struct crypto_bignum *
+get_rand_1_to_p_1(const u8 *prime, size_t prime_len, size_t prime_bits,
+		  int *r_odd)
 {
-	u8 pwd_value[SAE_MAX_ECC_PRIME_LEN], prime[SAE_MAX_ECC_PRIME_LEN];
-	struct crypto_bignum *x;
-	int y_bit;
+	for (;;) {
+		struct crypto_bignum *r;
+		u8 tmp[SAE_MAX_ECC_PRIME_LEN];
+
+		if (random_get_bytes(tmp, prime_len) < 0)
+			break;
+		if (prime_bits % 8)
+			buf_shift_right(tmp, prime_len, 8 - prime_bits % 8);
+		if (os_memcmp(tmp, prime, prime_len) >= 0)
+			continue;
+		r = crypto_bignum_init_set(tmp, prime_len);
+		if (!r)
+			break;
+		if (crypto_bignum_is_zero(r)) {
+			crypto_bignum_deinit(r, 0);
+			continue;
+		}
+
+		*r_odd = tmp[prime_len - 1] & 0x01;
+		return r;
+	}
+
+	return NULL;
+}
+
+
+static int is_quadratic_residue_blind(struct sae_data *sae,
+				      const u8 *prime, size_t bits,
+				      const struct crypto_bignum *qr,
+				      const struct crypto_bignum *qnr,
+				      const struct crypto_bignum *y_sqr)
+{
+	struct crypto_bignum *r, *num;
+	int r_odd, check, res = -1;
+
+	/*
+	 * Use the blinding technique to mask y_sqr while determining
+	 * whether it is a quadratic residue modulo p to avoid leaking
+	 * timing information while determining the Legendre symbol.
+	 *
+	 * v = y_sqr
+	 * r = a random number between 1 and p-1, inclusive
+	 * num = (v * r * r) modulo p
+	 */
+	r = get_rand_1_to_p_1(prime, sae->tmp->prime_len, bits, &r_odd);
+	if (!r)
+		return -1;
+
+	num = crypto_bignum_init();
+	if (!num ||
+	    crypto_bignum_mulmod(y_sqr, r, sae->tmp->prime, num) < 0 ||
+	    crypto_bignum_mulmod(num, r, sae->tmp->prime, num) < 0)
+		goto fail;
+
+	if (r_odd) {
+		/*
+		 * num = (num * qr) module p
+		 * LGR(num, p) = 1 ==> quadratic residue
+		 */
+		if (crypto_bignum_mulmod(num, qr, sae->tmp->prime, num) < 0)
+			goto fail;
+		check = 1;
+	} else {
+		/*
+		 * num = (num * qnr) module p
+		 * LGR(num, p) = -1 ==> quadratic residue
+		 */
+		if (crypto_bignum_mulmod(num, qnr, sae->tmp->prime, num) < 0)
+			goto fail;
+		check = -1;
+	}
+
+	res = crypto_bignum_legendre(num, sae->tmp->prime);
+	if (res == -2) {
+		res = -1;
+		goto fail;
+	}
+	res = res == check;
+fail:
+	crypto_bignum_deinit(num, 1);
+	crypto_bignum_deinit(r, 1);
+	return res;
+}
+
+
+static int sae_test_pwd_seed_ecc(struct sae_data *sae, const u8 *pwd_seed,
+				 const u8 *prime,
+				 const struct crypto_bignum *qr,
+				 const struct crypto_bignum *qnr,
+				 struct crypto_bignum **ret_x_cand)
+{
+	u8 pwd_value[SAE_MAX_ECC_PRIME_LEN];
+	struct crypto_bignum *y_sqr, *x_cand;
+	int res;
 	size_t bits;
 
-	if (crypto_bignum_to_bin(sae->tmp->prime, prime, sizeof(prime),
-				 sae->tmp->prime_len) < 0)
-		return -1;
+	*ret_x_cand = NULL;
 
 	wpa_hexdump_key(MSG_DEBUG, "SAE: pwd-seed", pwd_seed, SHA256_MAC_LEN);
 
@@ -197,20 +285,23 @@ static int sae_test_pwd_seed_ecc(struct sae_data *sae, const u8 *pwd_seed,
 	if (os_memcmp(pwd_value, prime, sae->tmp->prime_len) >= 0)
 		return 0;
 
-	y_bit = pwd_seed[SHA256_MAC_LEN - 1] & 0x01;
-
-	x = crypto_bignum_init_set(pwd_value, sae->tmp->prime_len);
-	if (x == NULL)
+	x_cand = crypto_bignum_init_set(pwd_value, sae->tmp->prime_len);
+	if (!x_cand)
+		return -1;
+	y_sqr = crypto_ec_point_compute_y_sqr(sae->tmp->ec, x_cand);
+	if (!y_sqr) {
+		crypto_bignum_deinit(x_cand, 1);
 		return -1;
-	if (crypto_ec_point_solve_y_coord(sae->tmp->ec, pwe, x, y_bit) < 0) {
-		crypto_bignum_deinit(x, 0);
-		wpa_printf(MSG_DEBUG, "SAE: No solution found");
-		return 0;
 	}
-	crypto_bignum_deinit(x, 0);
 
-	wpa_printf(MSG_DEBUG, "SAE: PWE found");
+	res = is_quadratic_residue_blind(sae, prime, bits, qr, qnr, y_sqr);
+	crypto_bignum_deinit(y_sqr, 1);
+	if (res <= 0) {
+		crypto_bignum_deinit(x_cand, 1);
+		return res;
+	}
 
+	*ret_x_cand = x_cand;
 	return 1;
 }
 
@@ -288,24 +379,77 @@ static int sae_test_pwd_seed_ffc(struct sae_data *sae, const u8 *pwd_seed,
 }
 
 
+static int get_random_qr_qnr(const u8 *prime, size_t prime_len,
+			     const struct crypto_bignum *prime_bn,
+			     size_t prime_bits, struct crypto_bignum **qr,
+			     struct crypto_bignum **qnr)
+{
+	*qr = NULL;
+	*qnr = NULL;
+
+	while (!(*qr) || !(*qnr)) {
+		u8 tmp[SAE_MAX_ECC_PRIME_LEN];
+		struct crypto_bignum *q;
+		int res;
+
+		if (random_get_bytes(tmp, prime_len) < 0)
+			break;
+		if (prime_bits % 8)
+			buf_shift_right(tmp, prime_len, 8 - prime_bits % 8);
+		if (os_memcmp(tmp, prime, prime_len) >= 0)
+			continue;
+		q = crypto_bignum_init_set(tmp, prime_len);
+		if (!q)
+			break;
+		res = crypto_bignum_legendre(q, prime_bn);
+
+		if (res == 1 && !(*qr))
+			*qr = q;
+		else if (res == -1 && !(*qnr))
+			*qnr = q;
+		else
+			crypto_bignum_deinit(q, 0);
+	}
+
+	return (*qr && *qnr) ? 0 : -1;
+}
+
+
 static int sae_derive_pwe_ecc(struct sae_data *sae, const u8 *addr1,
 			      const u8 *addr2, const u8 *password,
 			      size_t password_len)
 {
-	u8 counter, k = 4;
+	u8 counter, k = 40;
 	u8 addrs[2 * ETH_ALEN];
 	const u8 *addr[2];
 	size_t len[2];
-	int found = 0;
-	struct crypto_ec_point *pwe_tmp;
+	u8 dummy_password[32];
+	size_t dummy_password_len;
+	int pwd_seed_odd = 0;
+	u8 prime[SAE_MAX_ECC_PRIME_LEN];
+	size_t prime_len;
+	struct crypto_bignum *x = NULL, *qr, *qnr;
+	size_t bits;
+	int res;
 
-	if (sae->tmp->pwe_ecc == NULL) {
-		sae->tmp->pwe_ecc = crypto_ec_point_init(sae->tmp->ec);
-		if (sae->tmp->pwe_ecc == NULL)
-			return -1;
-	}
-	pwe_tmp = crypto_ec_point_init(sae->tmp->ec);
-	if (pwe_tmp == NULL)
+	dummy_password_len = password_len;
+	if (dummy_password_len > sizeof(dummy_password))
+		dummy_password_len = sizeof(dummy_password);
+	if (random_get_bytes(dummy_password, dummy_password_len) < 0)
+		return -1;
+
+	prime_len = sae->tmp->prime_len;
+	if (crypto_bignum_to_bin(sae->tmp->prime, prime, sizeof(prime),
+				 prime_len) < 0)
+		return -1;
+	bits = crypto_ec_prime_len_bits(sae->tmp->ec);
+
+	/*
+	 * Create a random quadratic residue (qr) and quadratic non-residue
+	 * (qnr) modulo p for blinding purposes during the loop.
+	 */
+	if (get_random_qr_qnr(prime, prime_len, sae->tmp->prime, bits,
+			      &qr, &qnr) < 0)
 		return -1;
 
 	wpa_hexdump_ascii_key(MSG_DEBUG, "SAE: password",
@@ -313,8 +457,9 @@ static int sae_derive_pwe_ecc(struct sae_data *sae, const u8 *addr1,
 
 	/*
 	 * H(salt, ikm) = HMAC-SHA256(salt, ikm)
+	 * base = password
 	 * pwd-seed = H(MAX(STA-A-MAC, STA-B-MAC) || MIN(STA-A-MAC, STA-B-MAC),
-	 *              password || counter)
+	 *              base || counter)
 	 */
 	sae_pwd_seed_key(addr1, addr2, addrs);
 
@@ -328,9 +473,9 @@ static int sae_derive_pwe_ecc(struct sae_data *sae, const u8 *addr1,
 	 * attacks that attempt to determine the number of iterations required
 	 * in the loop.
 	 */
-	for (counter = 1; counter < k || !found; counter++) {
+	for (counter = 1; counter <= k || !x; counter++) {
 		u8 pwd_seed[SHA256_MAC_LEN];
-		int res;
+		struct crypto_bignum *x_cand;
 
 		if (counter > 200) {
 			/* This should not happen in practice */
@@ -342,25 +487,58 @@ static int sae_derive_pwe_ecc(struct sae_data *sae, const u8 *addr1,
 		if (hmac_sha256_vector(addrs, sizeof(addrs), 2, addr, len,
 				       pwd_seed) < 0)
 			break;
+
 		res = sae_test_pwd_seed_ecc(sae, pwd_seed,
-					    found ? pwe_tmp :
-					    sae->tmp->pwe_ecc);
+					    prime, qr, qnr, &x_cand);
 		if (res < 0)
-			break;
-		if (res == 0)
-			continue;
-		if (found) {
-			wpa_printf(MSG_DEBUG, "SAE: Ignore this PWE (one was "
-				   "already selected)");
-		} else {
-			wpa_printf(MSG_DEBUG, "SAE: Use this PWE");
-			found = 1;
+			goto fail;
+		if (res > 0 && !x) {
+			wpa_printf(MSG_DEBUG,
+				   "SAE: Selected pwd-seed with counter %u",
+				   counter);
+			x = x_cand;
+			pwd_seed_odd = pwd_seed[SHA256_MAC_LEN - 1] & 0x01;
+			os_memset(pwd_seed, 0, sizeof(pwd_seed));
+
+			/*
+			 * Use a dummy password for the following rounds, if
+			 * any.
+			 */
+			addr[0] = dummy_password;
+			len[0] = dummy_password_len;
+		} else if (res > 0) {
+			crypto_bignum_deinit(x_cand, 1);
 		}
 	}
 
-	crypto_ec_point_deinit(pwe_tmp, 1);
+	if (!x) {
+		wpa_printf(MSG_DEBUG, "SAE: Could not generate PWE");
+		res = -1;
+		goto fail;
+	}
 
-	return found ? 0 : -1;
+	if (!sae->tmp->pwe_ecc)
+		sae->tmp->pwe_ecc = crypto_ec_point_init(sae->tmp->ec);
+	if (!sae->tmp->pwe_ecc)
+		res = -1;
+	else
+		res = crypto_ec_point_solve_y_coord(sae->tmp->ec,
+						    sae->tmp->pwe_ecc, x,
+						    pwd_seed_odd);
+	crypto_bignum_deinit(x, 1);
+	if (res < 0) {
+		/*
+		 * This should not happen since we already checked that there
+		 * is a result.
+		 */
+		wpa_printf(MSG_DEBUG, "SAE: Could not solve y");
+	}
+
+fail:
+	crypto_bignum_deinit(qr, 0);
+	crypto_bignum_deinit(qnr, 0);
+
+	return res;
 }
 
 
@@ -472,27 +650,41 @@ static int sae_derive_commit(struct sae_data *sae)
 {
 	struct crypto_bignum *mask;
 	int ret = -1;
+	unsigned int counter = 0;
 
-	mask = sae_get_rand_and_mask(sae);
-	if (mask == NULL) {
-		wpa_printf(MSG_DEBUG, "SAE: Could not get rand/mask");
-		return -1;
-	}
+	do {
+		counter++;
+		if (counter > 100) {
+			/*
+			 * This cannot really happen in practice if the random
+			 * number generator is working. Anyway, to avoid even a
+			 * theoretical infinite loop, break out after 100
+			 * attemps.
+			 */
+			return -1;
+		}
 
-	/* commit-scalar = (rand + mask) modulo r */
-	if (!sae->tmp->own_commit_scalar) {
-		sae->tmp->own_commit_scalar = crypto_bignum_init();
-		if (!sae->tmp->own_commit_scalar)
-			goto fail;
-	}
-	crypto_bignum_add(sae->tmp->sae_rand, mask,
-			  sae->tmp->own_commit_scalar);
-	crypto_bignum_mod(sae->tmp->own_commit_scalar, sae->tmp->order,
-			  sae->tmp->own_commit_scalar);
+		mask = sae_get_rand_and_mask(sae);
+		if (mask == NULL) {
+			wpa_printf(MSG_DEBUG, "SAE: Could not get rand/mask");
+			return -1;
+		}
 
-	if (sae->tmp->ec && sae_derive_commit_element_ecc(sae, mask) < 0)
-		goto fail;
-	if (sae->tmp->dh && sae_derive_commit_element_ffc(sae, mask) < 0)
+		/* commit-scalar = (rand + mask) modulo r */
+		if (!sae->tmp->own_commit_scalar) {
+			sae->tmp->own_commit_scalar = crypto_bignum_init();
+			if (!sae->tmp->own_commit_scalar)
+				goto fail;
+		}
+		crypto_bignum_add(sae->tmp->sae_rand, mask,
+				  sae->tmp->own_commit_scalar);
+		crypto_bignum_mod(sae->tmp->own_commit_scalar, sae->tmp->order,
+				  sae->tmp->own_commit_scalar);
+	} while (crypto_bignum_is_zero(sae->tmp->own_commit_scalar) ||
+		 crypto_bignum_is_one(sae->tmp->own_commit_scalar));
+
+	if ((sae->tmp->ec && sae_derive_commit_element_ecc(sae, mask) < 0) ||
+	    (sae->tmp->dh && sae_derive_commit_element_ffc(sae, mask) < 0))
 		goto fail;
 
 	ret = 0;
@@ -506,15 +698,12 @@ int sae_prepare_commit(const u8 *addr1, const u8 *addr2,
 		       const u8 *password, size_t password_len,
 		       struct sae_data *sae)
 {
-	if (sae->tmp == NULL)
-		return -1;
-	if (sae->tmp->ec && sae_derive_pwe_ecc(sae, addr1, addr2, password,
-					  password_len) < 0)
-		return -1;
-	if (sae->tmp->dh && sae_derive_pwe_ffc(sae, addr1, addr2, password,
-					  password_len) < 0)
-		return -1;
-	if (sae_derive_commit(sae) < 0)
+	if (sae->tmp == NULL ||
+	    (sae->tmp->ec && sae_derive_pwe_ecc(sae, addr1, addr2, password,
+						password_len) < 0) ||
+	    (sae->tmp->dh && sae_derive_pwe_ffc(sae, addr1, addr2, password,
+						password_len) < 0) ||
+	    sae_derive_commit(sae) < 0)
 		return -1;
 	return 0;
 }
@@ -780,8 +969,9 @@ static u16 sae_parse_commit_scalar(struct sae_data *sae, const u8 **pos,
 		return WLAN_STATUS_UNSPECIFIED_FAILURE;
 	}
 
-	/* 0 < scalar < r */
+	/* 1 < scalar < r */
 	if (crypto_bignum_is_zero(peer_scalar) ||
+	    crypto_bignum_is_one(peer_scalar) ||
 	    crypto_bignum_cmp(peer_scalar, sae->tmp->order) >= 0) {
 		wpa_printf(MSG_DEBUG, "SAE: Invalid peer scalar");
 		crypto_bignum_deinit(peer_scalar, 0);
@@ -847,7 +1037,8 @@ static u16 sae_parse_commit_element_ecc(struct sae_data *sae, const u8 *pos,
 static u16 sae_parse_commit_element_ffc(struct sae_data *sae, const u8 *pos,
 					const u8 *end)
 {
-	struct crypto_bignum *res;
+	struct crypto_bignum *res, *one;
+	const u8 one_bin[1] = { 0x01 };
 
 	if (pos + sae->tmp->prime_len > end) {
 		wpa_printf(MSG_DEBUG, "SAE: Not enough data for "
@@ -862,18 +1053,23 @@ static u16 sae_parse_commit_element_ffc(struct sae_data *sae, const u8 *pos,
 		crypto_bignum_init_set(pos, sae->tmp->prime_len);
 	if (sae->tmp->peer_commit_element_ffc == NULL)
 		return WLAN_STATUS_UNSPECIFIED_FAILURE;
-	if (crypto_bignum_is_zero(sae->tmp->peer_commit_element_ffc) ||
+	/* 1 < element < p - 1 */
+	res = crypto_bignum_init();
+	one = crypto_bignum_init_set(one_bin, sizeof(one_bin));
+	if (!res || !one ||
+	    crypto_bignum_sub(sae->tmp->prime, one, res) ||
+	    crypto_bignum_is_zero(sae->tmp->peer_commit_element_ffc) ||
 	    crypto_bignum_is_one(sae->tmp->peer_commit_element_ffc) ||
-	    crypto_bignum_cmp(sae->tmp->peer_commit_element_ffc,
-			      sae->tmp->prime) >= 0) {
+	    crypto_bignum_cmp(sae->tmp->peer_commit_element_ffc, res) >= 0) {
+		crypto_bignum_deinit(res, 0);
+		crypto_bignum_deinit(one, 0);
 		wpa_printf(MSG_DEBUG, "SAE: Invalid peer element");
 		return WLAN_STATUS_UNSPECIFIED_FAILURE;
 	}
+	crypto_bignum_deinit(one, 0);
 
 	/* scalar-op(r, ELEMENT) = 1 modulo p */
-	res = crypto_bignum_init();
-	if (res == NULL ||
-	    crypto_bignum_exptmod(sae->tmp->peer_commit_element_ffc,
+	if (crypto_bignum_exptmod(sae->tmp->peer_commit_element_ffc,
 				  sae->tmp->order, sae->tmp->prime, res) < 0 ||
 	    !crypto_bignum_is_one(res)) {
 		wpa_printf(MSG_DEBUG, "SAE: Invalid peer element (scalar-op)");
@@ -918,7 +1114,34 @@ u16 sae_parse_commit(struct sae_data *sae, const u8 *data, size_t len,
 		return res;
 
 	/* commit-element */
-	return sae_parse_commit_element(sae, pos, end);
+	res = sae_parse_commit_element(sae, pos, end);
+	if (res != WLAN_STATUS_SUCCESS)
+		return res;
+
+	/*
+	 * Check whether peer-commit-scalar and PEER-COMMIT-ELEMENT are same as
+	 * the values we sent which would be evidence of a reflection attack.
+	 */
+	if (!sae->tmp->own_commit_scalar ||
+	    crypto_bignum_cmp(sae->tmp->own_commit_scalar,
+			      sae->peer_commit_scalar) != 0 ||
+	    (sae->tmp->dh &&
+	     (!sae->tmp->own_commit_element_ffc ||
+	      crypto_bignum_cmp(sae->tmp->own_commit_element_ffc,
+				sae->tmp->peer_commit_element_ffc) != 0)) ||
+	    (sae->tmp->ec &&
+	     (!sae->tmp->own_commit_element_ecc ||
+	      crypto_ec_point_cmp(sae->tmp->ec,
+				  sae->tmp->own_commit_element_ecc,
+				  sae->tmp->peer_commit_element_ecc) != 0)))
+		return WLAN_STATUS_SUCCESS; /* scalars/elements are different */
+
+	/*
+	 * This is a reflection attack - return special value to trigger caller
+	 * to silently discard the frame instead of replying with a specific
+	 * status code.
+	 */
+	return SAE_SILENTLY_DISCARD;
 }
 
 
diff --git a/src/common/sae.h b/src/common/sae.h
index 3ebf40cf4a45..c07026cd497c 100644
--- a/src/common/sae.h
+++ b/src/common/sae.h
@@ -18,6 +18,9 @@
 #define SAE_COMMIT_MAX_LEN (2 + 3 * SAE_MAX_PRIME_LEN)
 #define SAE_CONFIRM_MAX_LEN (2 + SAE_MAX_PRIME_LEN)
 
+/* Special value returned by sae_parse_commit() */
+#define SAE_SILENTLY_DISCARD 65535
+
 struct sae_temporary_data {
 	u8 kck[SAE_KCK_LEN];
 	struct crypto_bignum *own_commit_scalar;
diff --git a/src/common/version.h b/src/common/version.h
index e39a8dbf92e7..a5cc5b7b5bcc 100644
--- a/src/common/version.h
+++ b/src/common/version.h
@@ -5,6 +5,6 @@
 #define VERSION_STR_POSTFIX ""
 #endif /* VERSION_STR_POSTFIX */
 
-#define VERSION_STR "2.4" VERSION_STR_POSTFIX
+#define VERSION_STR "2.5" VERSION_STR_POSTFIX
 
 #endif /* VERSION_H */
diff --git a/src/common/wpa_common.c b/src/common/wpa_common.c
index de81d53694c2..e9d4248d72d4 100644
--- a/src/common/wpa_common.c
+++ b/src/common/wpa_common.c
@@ -170,6 +170,12 @@ int wpa_pmk_to_ptk(const u8 *pmk, size_t pmk_len, const char *label,
 	ptk->tk_len = wpa_cipher_key_len(cipher);
 	ptk_len = ptk->kck_len + ptk->kek_len + ptk->tk_len;
 
+#ifdef CONFIG_SUITEB192
+	if (wpa_key_mgmt_sha384(akmp))
+		sha384_prf(pmk, pmk_len, label, data, sizeof(data),
+			   tmp, ptk_len);
+	else
+#endif /* CONFIG_SUITEB192 */
 #ifdef CONFIG_IEEE80211W
 	if (wpa_key_mgmt_sha256(akmp))
 		sha256_prf(pmk, pmk_len, label, data, sizeof(data),
@@ -207,8 +213,10 @@ int wpa_ft_mic(const u8 *kck, size_t kck_len, const u8 *sta_addr,
 	       const u8 *rsnie, size_t rsnie_len,
 	       const u8 *ric, size_t ric_len, u8 *mic)
 {
-	u8 *buf, *pos;
-	size_t buf_len;
+	const u8 *addr[9];
+	size_t len[9];
+	size_t i, num_elem = 0;
+	u8 zero_mic[16];
 
 	if (kck_len != 16) {
 		wpa_printf(MSG_WARNING, "FT: Unsupported KCK length %u",
@@ -216,48 +224,58 @@ int wpa_ft_mic(const u8 *kck, size_t kck_len, const u8 *sta_addr,
 		return -1;
 	}
 
-	buf_len = 2 * ETH_ALEN + 1 + mdie_len + ftie_len + rsnie_len + ric_len;
-	buf = os_malloc(buf_len);
-	if (buf == NULL)
-		return -1;
+	addr[num_elem] = sta_addr;
+	len[num_elem] = ETH_ALEN;
+	num_elem++;
+
+	addr[num_elem] = ap_addr;
+	len[num_elem] = ETH_ALEN;
+	num_elem++;
+
+	addr[num_elem] = &transaction_seqnum;
+	len[num_elem] = 1;
+	num_elem++;
 
-	pos = buf;
-	os_memcpy(pos, sta_addr, ETH_ALEN);
-	pos += ETH_ALEN;
-	os_memcpy(pos, ap_addr, ETH_ALEN);
-	pos += ETH_ALEN;
-	*pos++ = transaction_seqnum;
 	if (rsnie) {
-		os_memcpy(pos, rsnie, rsnie_len);
-		pos += rsnie_len;
+		addr[num_elem] = rsnie;
+		len[num_elem] = rsnie_len;
+		num_elem++;
 	}
 	if (mdie) {
-		os_memcpy(pos, mdie, mdie_len);
-		pos += mdie_len;
+		addr[num_elem] = mdie;
+		len[num_elem] = mdie_len;
+		num_elem++;
 	}
 	if (ftie) {
-		struct rsn_ftie *_ftie;
-		os_memcpy(pos, ftie, ftie_len);
-		if (ftie_len < 2 + sizeof(*_ftie)) {
-			os_free(buf);
+		if (ftie_len < 2 + sizeof(struct rsn_ftie))
 			return -1;
-		}
-		_ftie = (struct rsn_ftie *) (pos + 2);
-		os_memset(_ftie->mic, 0, sizeof(_ftie->mic));
-		pos += ftie_len;
+
+		/* IE hdr and mic_control */
+		addr[num_elem] = ftie;
+		len[num_elem] = 2 + 2;
+		num_elem++;
+
+		/* MIC field with all zeros */
+		os_memset(zero_mic, 0, sizeof(zero_mic));
+		addr[num_elem] = zero_mic;
+		len[num_elem] = sizeof(zero_mic);
+		num_elem++;
+
+		/* Rest of FTIE */
+		addr[num_elem] = ftie + 2 + 2 + 16;
+		len[num_elem] = ftie_len - (2 + 2 + 16);
+		num_elem++;
 	}
 	if (ric) {
-		os_memcpy(pos, ric, ric_len);
-		pos += ric_len;
+		addr[num_elem] = ric;
+		len[num_elem] = ric_len;
+		num_elem++;
 	}
 
-	wpa_hexdump(MSG_MSGDUMP, "FT: MIC data", buf, pos - buf);
-	if (omac1_aes_128(kck, buf, pos - buf, mic)) {
-		os_free(buf);
+	for (i = 0; i < num_elem; i++)
+		wpa_hexdump(MSG_MSGDUMP, "FT: MIC data", addr[i], len[i]);
+	if (omac1_aes_128_vector(kck, num_elem, addr, len, mic))
 		return -1;
-	}
-
-	os_free(buf);
 
 	return 0;
 }
@@ -344,6 +362,8 @@ int wpa_ft_parse_ies(const u8 *ies, size_t ies_len,
 				parse->rsn_pmkid = data.pmkid;
 			break;
 		case WLAN_EID_MOBILITY_DOMAIN:
+			if (pos[1] < sizeof(struct rsn_mdie))
+				return -1;
 			parse->mdie = pos + 2;
 			parse->mdie_len = pos[1];
 			break;
@@ -356,6 +376,8 @@ int wpa_ft_parse_ies(const u8 *ies, size_t ies_len,
 				return -1;
 			break;
 		case WLAN_EID_TIMEOUT_INTERVAL:
+			if (pos[1] != 5)
+				break;
 			parse->tie = pos + 2;
 			parse->tie_len = pos[1];
 			break;
@@ -416,14 +438,10 @@ static int rsn_selector_to_bitfield(const u8 *s)
 {
 	if (RSN_SELECTOR_GET(s) == RSN_CIPHER_SUITE_NONE)
 		return WPA_CIPHER_NONE;
-	if (RSN_SELECTOR_GET(s) == RSN_CIPHER_SUITE_WEP40)
-		return WPA_CIPHER_WEP40;
 	if (RSN_SELECTOR_GET(s) == RSN_CIPHER_SUITE_TKIP)
 		return WPA_CIPHER_TKIP;
 	if (RSN_SELECTOR_GET(s) == RSN_CIPHER_SUITE_CCMP)
 		return WPA_CIPHER_CCMP;
-	if (RSN_SELECTOR_GET(s) == RSN_CIPHER_SUITE_WEP104)
-		return WPA_CIPHER_WEP104;
 #ifdef CONFIG_IEEE80211W
 	if (RSN_SELECTOR_GET(s) == RSN_CIPHER_SUITE_AES_128_CMAC)
 		return WPA_CIPHER_AES_128_CMAC;
@@ -474,15 +492,15 @@ static int rsn_key_mgmt_to_bitfield(const u8 *s)
 		return WPA_KEY_MGMT_IEEE8021X_SUITE_B;
 	if (RSN_SELECTOR_GET(s) == RSN_AUTH_KEY_MGMT_802_1X_SUITE_B_192)
 		return WPA_KEY_MGMT_IEEE8021X_SUITE_B_192;
+	if (RSN_SELECTOR_GET(s) == RSN_AUTH_KEY_MGMT_OSEN)
+		return WPA_KEY_MGMT_OSEN;
 	return 0;
 }
 
 
-static int wpa_cipher_valid_group(int cipher)
+int wpa_cipher_valid_group(int cipher)
 {
 	return wpa_cipher_valid_pairwise(cipher) ||
-		cipher == WPA_CIPHER_WEP104 ||
-		cipher == WPA_CIPHER_WEP40 ||
 		cipher == WPA_CIPHER_GTK_NOT_USED;
 }
 
@@ -508,7 +526,6 @@ int wpa_cipher_valid_mgmt_group(int cipher)
 int wpa_parse_wpa_ie_rsn(const u8 *rsn_ie, size_t rsn_ie_len,
 			 struct wpa_ie_data *data)
 {
-	const struct rsn_ie_hdr *hdr;
 	const u8 *pos;
 	int left;
 	int i, count;
@@ -538,19 +555,30 @@ int wpa_parse_wpa_ie_rsn(const u8 *rsn_ie, size_t rsn_ie_len,
 		return -1;
 	}
 
-	hdr = (const struct rsn_ie_hdr *) rsn_ie;
+	if (rsn_ie_len >= 6 && rsn_ie[1] >= 4 &&
+	    rsn_ie[1] == rsn_ie_len - 2 &&
+	    WPA_GET_BE32(&rsn_ie[2]) == OSEN_IE_VENDOR_TYPE) {
+		pos = rsn_ie + 6;
+		left = rsn_ie_len - 6;
 
-	if (hdr->elem_id != WLAN_EID_RSN ||
-	    hdr->len != rsn_ie_len - 2 ||
-	    WPA_GET_LE16(hdr->version) != RSN_VERSION) {
-		wpa_printf(MSG_DEBUG, "%s: malformed ie or unknown version",
-			   __func__);
-		return -2;
+		data->proto = WPA_PROTO_OSEN;
+	} else {
+		const struct rsn_ie_hdr *hdr;
+
+		hdr = (const struct rsn_ie_hdr *) rsn_ie;
+
+		if (hdr->elem_id != WLAN_EID_RSN ||
+		    hdr->len != rsn_ie_len - 2 ||
+		    WPA_GET_LE16(hdr->version) != RSN_VERSION) {
+			wpa_printf(MSG_DEBUG, "%s: malformed ie or unknown version",
+				   __func__);
+			return -2;
+		}
+
+		pos = (const u8 *) (hdr + 1);
+		left = rsn_ie_len - sizeof(*hdr);
 	}
 
-	pos = (const u8 *) (hdr + 1);
-	left = rsn_ie_len - sizeof(*hdr);
-
 	if (left >= RSN_SELECTOR_LEN) {
 		data->group_cipher = rsn_selector_to_bitfield(pos);
 		if (!wpa_cipher_valid_group(data->group_cipher)) {
@@ -667,14 +695,10 @@ static int wpa_selector_to_bitfield(const u8 *s)
 {
 	if (RSN_SELECTOR_GET(s) == WPA_CIPHER_SUITE_NONE)
 		return WPA_CIPHER_NONE;
-	if (RSN_SELECTOR_GET(s) == WPA_CIPHER_SUITE_WEP40)
-		return WPA_CIPHER_WEP40;
 	if (RSN_SELECTOR_GET(s) == WPA_CIPHER_SUITE_TKIP)
 		return WPA_CIPHER_TKIP;
 	if (RSN_SELECTOR_GET(s) == WPA_CIPHER_SUITE_CCMP)
 		return WPA_CIPHER_CCMP;
-	if (RSN_SELECTOR_GET(s) == WPA_CIPHER_SUITE_WEP104)
-		return WPA_CIPHER_WEP104;
 	return 0;
 }
 
@@ -709,11 +733,6 @@ int wpa_parse_wpa_ie_wpa(const u8 *wpa_ie, size_t wpa_ie_len,
 	data->num_pmkid = 0;
 	data->mgmt_group_cipher = 0;
 
-	if (wpa_ie_len == 0) {
-		/* No WPA IE - fail silently */
-		return -1;
-	}
-
 	if (wpa_ie_len < sizeof(struct wpa_ie_hdr)) {
 		wpa_printf(MSG_DEBUG, "%s: ie len too short %lu",
 			   __func__, (unsigned long) wpa_ie_len);
@@ -814,7 +833,7 @@ void wpa_derive_pmk_r0(const u8 *xxkey, size_t xxkey_len,
 		       const u8 *mdid, const u8 *r0kh_id, size_t r0kh_id_len,
 		       const u8 *s0kh_id, u8 *pmk_r0, u8 *pmk_r0_name)
 {
-	u8 buf[1 + WPA_MAX_SSID_LEN + MOBILITY_DOMAIN_ID_LEN + 1 +
+	u8 buf[1 + SSID_MAX_LEN + MOBILITY_DOMAIN_ID_LEN + 1 +
 	       FT_R0KH_ID_MAX_LEN + ETH_ALEN];
 	u8 *pos, r0_key_data[48], hash[32];
 	const u8 *addr[2];
@@ -828,7 +847,7 @@ void wpa_derive_pmk_r0(const u8 *xxkey, size_t xxkey_len,
 	 * PMK-R0 = L(R0-Key-Data, 0, 256)
 	 * PMK-R0Name-Salt = L(R0-Key-Data, 256, 128)
 	 */
-	if (ssid_len > WPA_MAX_SSID_LEN || r0kh_id_len > FT_R0KH_ID_MAX_LEN)
+	if (ssid_len > SSID_MAX_LEN || r0kh_id_len > FT_R0KH_ID_MAX_LEN)
 		return;
 	pos = buf;
 	*pos++ = ssid_len;
@@ -1279,6 +1298,9 @@ int wpa_insert_pmkid(u8 *ies, size_t ies_len, const u8 *pmkid)
 		os_memmove(rpos + 2, rpos, end - rpos);
 		*rpos++ = 0;
 		*rpos++ = 0;
+		added += 2;
+		start[1] += 2;
+		rend = rpos;
 	} else {
 		/* Skip RSN Capabilities */
 		rpos += 2;
@@ -1291,7 +1313,7 @@ int wpa_insert_pmkid(u8 *ies, size_t ies_len, const u8 *pmkid)
 
 	if (rpos == rend) {
 		/* No PMKID-Count field included; add it */
-		os_memmove(rpos + 2 + PMKID_LEN, rpos, end - rpos);
+		os_memmove(rpos + 2 + PMKID_LEN, rpos, end + added - rpos);
 		WPA_PUT_LE16(rpos, 1);
 		rpos += 2;
 		os_memcpy(rpos, pmkid, PMKID_LEN);
@@ -1306,7 +1328,7 @@ int wpa_insert_pmkid(u8 *ies, size_t ies_len, const u8 *pmkid)
 		}
 		WPA_PUT_LE16(rpos, 1);
 		rpos += 2;
-		os_memmove(rpos + PMKID_LEN, rpos, end - rpos);
+		os_memmove(rpos + PMKID_LEN, rpos, end + added - rpos);
 		os_memcpy(rpos, pmkid, PMKID_LEN);
 		added += PMKID_LEN;
 		start[1] += PMKID_LEN;
@@ -1335,10 +1357,6 @@ int wpa_cipher_key_len(int cipher)
 		return 16;
 	case WPA_CIPHER_TKIP:
 		return 32;
-	case WPA_CIPHER_WEP104:
-		return 13;
-	case WPA_CIPHER_WEP40:
-		return 5;
 	}
 
 	return 0;
@@ -1354,9 +1372,6 @@ int wpa_cipher_rsc_len(int cipher)
 	case WPA_CIPHER_GCMP:
 	case WPA_CIPHER_TKIP:
 		return 6;
-	case WPA_CIPHER_WEP104:
-	case WPA_CIPHER_WEP40:
-		return 0;
 	}
 
 	return 0;
@@ -1376,9 +1391,6 @@ int wpa_cipher_to_alg(int cipher)
 		return WPA_ALG_GCMP;
 	case WPA_CIPHER_TKIP:
 		return WPA_ALG_TKIP;
-	case WPA_CIPHER_WEP104:
-	case WPA_CIPHER_WEP40:
-		return WPA_ALG_WEP;
 	case WPA_CIPHER_AES_128_CMAC:
 		return WPA_ALG_IGTK;
 	case WPA_CIPHER_BIP_GMAC_128:
@@ -1416,12 +1428,6 @@ u32 wpa_cipher_to_suite(int proto, int cipher)
 	if (cipher & WPA_CIPHER_TKIP)
 		return (proto == WPA_PROTO_RSN ?
 			RSN_CIPHER_SUITE_TKIP : WPA_CIPHER_SUITE_TKIP);
-	if (cipher & WPA_CIPHER_WEP104)
-		return (proto == WPA_PROTO_RSN ?
-			RSN_CIPHER_SUITE_WEP104 : WPA_CIPHER_SUITE_WEP104);
-	if (cipher & WPA_CIPHER_WEP40)
-		return (proto == WPA_PROTO_RSN ?
-			RSN_CIPHER_SUITE_WEP40 : WPA_CIPHER_SUITE_WEP40);
 	if (cipher & WPA_CIPHER_NONE)
 		return (proto == WPA_PROTO_RSN ?
 			RSN_CIPHER_SUITE_NONE : WPA_CIPHER_SUITE_NONE);
@@ -1525,10 +1531,6 @@ int wpa_pick_group_cipher(int ciphers)
 		return WPA_CIPHER_GTK_NOT_USED;
 	if (ciphers & WPA_CIPHER_TKIP)
 		return WPA_CIPHER_TKIP;
-	if (ciphers & WPA_CIPHER_WEP104)
-		return WPA_CIPHER_WEP104;
-	if (ciphers & WPA_CIPHER_WEP40)
-		return WPA_CIPHER_WEP40;
 	return -1;
 }
 
@@ -1626,20 +1628,6 @@ int wpa_write_ciphers(char *start, char *end, int ciphers, const char *delim)
 			return -1;
 		pos += ret;
 	}
-	if (ciphers & WPA_CIPHER_WEP104) {
-		ret = os_snprintf(pos, end - pos, "%sWEP104",
-				  pos == start ? "" : delim);
-		if (os_snprintf_error(end - pos, ret))
-			return -1;
-		pos += ret;
-	}
-	if (ciphers & WPA_CIPHER_WEP40) {
-		ret = os_snprintf(pos, end - pos, "%sWEP40",
-				  pos == start ? "" : delim);
-		if (os_snprintf_error(end - pos, ret))
-			return -1;
-		pos += ret;
-	}
 	if (ciphers & WPA_CIPHER_NONE) {
 		ret = os_snprintf(pos, end - pos, "%sNONE",
 				  pos == start ? "" : delim);
diff --git a/src/common/wpa_common.h b/src/common/wpa_common.h
index 091e317fdd68..c08f6514ab57 100644
--- a/src/common/wpa_common.h
+++ b/src/common/wpa_common.h
@@ -9,8 +9,6 @@
 #ifndef WPA_COMMON_H
 #define WPA_COMMON_H
 
-#define WPA_MAX_SSID_LEN 32
-
 /* IEEE 802.11i */
 #define PMKID_LEN 16
 #define PMK_LEN 32
@@ -24,8 +22,8 @@
 (WPA_CIPHER_CCMP | WPA_CIPHER_GCMP | WPA_CIPHER_TKIP | WPA_CIPHER_NONE | \
 WPA_CIPHER_GCMP_256 | WPA_CIPHER_CCMP_256)
 #define WPA_ALLOWED_GROUP_CIPHERS \
-(WPA_CIPHER_CCMP | WPA_CIPHER_GCMP | WPA_CIPHER_TKIP | WPA_CIPHER_WEP104 | \
-WPA_CIPHER_WEP40 | WPA_CIPHER_GCMP_256 | WPA_CIPHER_CCMP_256 | \
+(WPA_CIPHER_CCMP | WPA_CIPHER_GCMP | WPA_CIPHER_TKIP | \
+WPA_CIPHER_GCMP_256 | WPA_CIPHER_CCMP_256 | \
 WPA_CIPHER_GTK_NOT_USED)
 
 #define WPA_SELECTOR_LEN 4
@@ -42,13 +40,8 @@ WPA_CIPHER_GTK_NOT_USED)
 #define WPA_AUTH_KEY_MGMT_PSK_OVER_802_1X RSN_SELECTOR(0x00, 0x50, 0xf2, 2)
 #define WPA_AUTH_KEY_MGMT_CCKM RSN_SELECTOR(0x00, 0x40, 0x96, 0)
 #define WPA_CIPHER_SUITE_NONE RSN_SELECTOR(0x00, 0x50, 0xf2, 0)
-#define WPA_CIPHER_SUITE_WEP40 RSN_SELECTOR(0x00, 0x50, 0xf2, 1)
 #define WPA_CIPHER_SUITE_TKIP RSN_SELECTOR(0x00, 0x50, 0xf2, 2)
-#if 0
-#define WPA_CIPHER_SUITE_WRAP RSN_SELECTOR(0x00, 0x50, 0xf2, 3)
-#endif
 #define WPA_CIPHER_SUITE_CCMP RSN_SELECTOR(0x00, 0x50, 0xf2, 4)
-#define WPA_CIPHER_SUITE_WEP104 RSN_SELECTOR(0x00, 0x50, 0xf2, 5)
 
 
 #define RSN_AUTH_KEY_MGMT_UNSPEC_802_1X RSN_SELECTOR(0x00, 0x0f, 0xac, 1)
@@ -70,13 +63,11 @@ RSN_SELECTOR(0x00, 0x0f, 0xac, 13)
 #define RSN_AUTH_KEY_MGMT_OSEN RSN_SELECTOR(0x50, 0x6f, 0x9a, 0x01)
 
 #define RSN_CIPHER_SUITE_NONE RSN_SELECTOR(0x00, 0x0f, 0xac, 0)
-#define RSN_CIPHER_SUITE_WEP40 RSN_SELECTOR(0x00, 0x0f, 0xac, 1)
 #define RSN_CIPHER_SUITE_TKIP RSN_SELECTOR(0x00, 0x0f, 0xac, 2)
 #if 0
 #define RSN_CIPHER_SUITE_WRAP RSN_SELECTOR(0x00, 0x0f, 0xac, 3)
 #endif
 #define RSN_CIPHER_SUITE_CCMP RSN_SELECTOR(0x00, 0x0f, 0xac, 4)
-#define RSN_CIPHER_SUITE_WEP104 RSN_SELECTOR(0x00, 0x0f, 0xac, 5)
 #define RSN_CIPHER_SUITE_AES_128_CMAC RSN_SELECTOR(0x00, 0x0f, 0xac, 6)
 #define RSN_CIPHER_SUITE_NO_GROUP_ADDRESSED RSN_SELECTOR(0x00, 0x0f, 0xac, 7)
 #define RSN_CIPHER_SUITE_GCMP RSN_SELECTOR(0x00, 0x0f, 0xac, 8)
@@ -308,7 +299,6 @@ struct wpa_igtk_kde {
 } STRUCT_PACKED;
 #endif /* CONFIG_IEEE80211W */
 
-#ifdef CONFIG_IEEE80211R
 struct rsn_mdie {
 	u8 mobility_domain[MOBILITY_DOMAIN_ID_LEN];
 	u8 ft_capab;
@@ -336,7 +326,6 @@ struct rsn_rdie {
 	le16 status_code;
 } STRUCT_PACKED;
 
-#endif /* CONFIG_IEEE80211R */
 
 #ifdef _MSC_VER
 #pragma pack(pop)
@@ -446,6 +435,7 @@ int wpa_ft_parse_ies(const u8 *ies, size_t ies_len, struct wpa_ft_ies *parse);
 int wpa_cipher_key_len(int cipher);
 int wpa_cipher_rsc_len(int cipher);
 int wpa_cipher_to_alg(int cipher);
+int wpa_cipher_valid_group(int cipher);
 int wpa_cipher_valid_pairwise(int cipher);
 int wpa_cipher_valid_mgmt_group(int cipher);
 u32 wpa_cipher_to_suite(int proto, int cipher);
diff --git a/src/common/wpa_ctrl.c b/src/common/wpa_ctrl.c
index ccaaf1b056b5..5733aa605d18 100644
--- a/src/common/wpa_ctrl.c
+++ b/src/common/wpa_ctrl.c
@@ -21,6 +21,7 @@
 
 #ifdef ANDROID
 #include <dirent.h>
+#include <sys/stat.h>
 #include <cutils/sockets.h>
 #include "private/android_filesystem_config.h"
 #endif /* ANDROID */
@@ -83,6 +84,13 @@ struct wpa_ctrl {
 
 
 struct wpa_ctrl * wpa_ctrl_open(const char *ctrl_path)
+{
+	return wpa_ctrl_open2(ctrl_path, NULL);
+}
+
+
+struct wpa_ctrl * wpa_ctrl_open2(const char *ctrl_path,
+				 const char *cli_path)
 {
 	struct wpa_ctrl *ctrl;
 	static int counter = 0;
@@ -107,10 +115,18 @@ struct wpa_ctrl * wpa_ctrl_open(const char *ctrl_path)
 	ctrl->local.sun_family = AF_UNIX;
 	counter++;
 try_again:
-	ret = os_snprintf(ctrl->local.sun_path, sizeof(ctrl->local.sun_path),
-			  CONFIG_CTRL_IFACE_CLIENT_DIR "/"
-			  CONFIG_CTRL_IFACE_CLIENT_PREFIX "%d-%d",
-			  (int) getpid(), counter);
+	if (cli_path && cli_path[0] == '/') {
+		ret = os_snprintf(ctrl->local.sun_path,
+				  sizeof(ctrl->local.sun_path),
+				  "%s/" CONFIG_CTRL_IFACE_CLIENT_PREFIX "%d-%d",
+				  cli_path, (int) getpid(), counter);
+	} else {
+		ret = os_snprintf(ctrl->local.sun_path,
+				  sizeof(ctrl->local.sun_path),
+				  CONFIG_CTRL_IFACE_CLIENT_DIR "/"
+				  CONFIG_CTRL_IFACE_CLIENT_PREFIX "%d-%d",
+				  (int) getpid(), counter);
+	}
 	if (os_snprintf_error(sizeof(ctrl->local.sun_path), ret)) {
 		close(ctrl->s);
 		os_free(ctrl);
@@ -136,6 +152,8 @@ try_again:
 
 #ifdef ANDROID
 	chmod(ctrl->local.sun_path, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
+	/* Set group even if we do not have privileges to change owner */
+	chown(ctrl->local.sun_path, -1, AID_WIFI);
 	chown(ctrl->local.sun_path, AID_SYSTEM, AID_WIFI);
 
 	if (os_strncmp(ctrl_path, "@android:", 9) == 0) {
diff --git a/src/common/wpa_ctrl.h b/src/common/wpa_ctrl.h
index 1d19fc550df6..3de46823588b 100644
--- a/src/common/wpa_ctrl.h
+++ b/src/common/wpa_ctrl.h
@@ -28,6 +28,8 @@ extern "C" {
 #define WPA_EVENT_DISCONNECTED "CTRL-EVENT-DISCONNECTED "
 /** Association rejected during connection attempt */
 #define WPA_EVENT_ASSOC_REJECT "CTRL-EVENT-ASSOC-REJECT "
+/** Authentication rejected during connection attempt */
+#define WPA_EVENT_AUTH_REJECT "CTRL-EVENT-AUTH-REJECT "
 /** wpa_supplicant is exiting */
 #define WPA_EVENT_TERMINATING "CTRL-EVENT-TERMINATING "
 /** Password change was completed successfully */
@@ -68,6 +70,8 @@ extern "C" {
 #define WPA_EVENT_BSS_ADDED "CTRL-EVENT-BSS-ADDED "
 /** A BSS entry was removed (followed by BSS entry id and BSSID) */
 #define WPA_EVENT_BSS_REMOVED "CTRL-EVENT-BSS-REMOVED "
+/** No suitable network was found */
+#define WPA_EVENT_NETWORK_NOT_FOUND "CTRL-EVENT-NETWORK-NOT-FOUND "
 /** Change in the signal level was reported by the driver */
 #define WPA_EVENT_SIGNAL_CHANGE "CTRL-EVENT-SIGNAL-CHANGE "
 /** Regulatory domain channel */
@@ -227,6 +231,7 @@ extern "C" {
 #define WPS_EVENT_AP_PIN_DISABLED "WPS-AP-PIN-DISABLED "
 #define AP_STA_CONNECTED "AP-STA-CONNECTED "
 #define AP_STA_DISCONNECTED "AP-STA-DISCONNECTED "
+#define AP_STA_POSSIBLE_PSK_MISMATCH "AP-STA-POSSIBLE-PSK-MISMATCH "
 
 #define AP_REJECTED_MAX_STA "AP-REJECTED-MAX-STA "
 #define AP_REJECTED_BLOCKED_STA "AP-REJECTED-BLOCKED-STA "
@@ -276,6 +281,7 @@ extern "C" {
 #define WPA_BSS_MASK_MESH_SCAN		BIT(18)
 #define WPA_BSS_MASK_SNR		BIT(19)
 #define WPA_BSS_MASK_EST_THROUGHPUT	BIT(20)
+#define WPA_BSS_MASK_FST		BIT(21)
 
 
 /* VENDOR_ELEM_* frame id values */
@@ -312,6 +318,20 @@ enum wpa_vendor_elem_frame {
  */
 struct wpa_ctrl * wpa_ctrl_open(const char *ctrl_path);
 
+/**
+ * wpa_ctrl_open2 - Open a control interface to wpa_supplicant/hostapd
+ * @ctrl_path: Path for UNIX domain sockets; ignored if UDP sockets are used.
+ * @cli_path: Path for client UNIX domain sockets; ignored if UDP socket
+ *            is used.
+ * Returns: Pointer to abstract control interface data or %NULL on failure
+ *
+ * This function is used to open a control interface to wpa_supplicant/hostapd
+ * when the socket path for client need to be specified explicitly. Default
+ * ctrl_path is usually /var/run/wpa_supplicant or /var/run/hostapd and client
+ * socket path is /tmp.
+ */
+struct wpa_ctrl * wpa_ctrl_open2(const char *ctrl_path, const char *cli_path);
+
 
 /**
  * wpa_ctrl_close - Close a control interface to wpa_supplicant/hostapd
diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h
index f2d5662ff01e..534c4bd78654 100644
--- a/src/crypto/crypto.h
+++ b/src/crypto/crypto.h
@@ -613,6 +613,15 @@ int crypto_bignum_is_zero(const struct crypto_bignum *a);
  */
 int crypto_bignum_is_one(const struct crypto_bignum *a);
 
+/**
+ * crypto_bignum_legendre - Compute the Legendre symbol (a/p)
+ * @a: Bignum
+ * @p: Bignum
+ * Returns: Legendre symbol -1,0,1 on success; -2 on calculation failure
+ */
+int crypto_bignum_legendre(const struct crypto_bignum *a,
+			   const struct crypto_bignum *p);
+
 /**
  * struct crypto_ec - Elliptic curve context
  *
@@ -757,6 +766,16 @@ int crypto_ec_point_solve_y_coord(struct crypto_ec *e,
 				  struct crypto_ec_point *p,
 				  const struct crypto_bignum *x, int y_bit);
 
+/**
+ * crypto_ec_point_compute_y_sqr - Compute y^2 = x^3 + ax + b
+ * @e: EC context from crypto_ec_init()
+ * @x: x coordinate
+ * Returns: y^2 on success, %NULL failure
+ */
+struct crypto_bignum *
+crypto_ec_point_compute_y_sqr(struct crypto_ec *e,
+			      const struct crypto_bignum *x);
+
 /**
  * crypto_ec_point_is_at_infinity - Check whether EC point is neutral element
  * @e: EC context from crypto_ec_init()
@@ -776,4 +795,15 @@ int crypto_ec_point_is_at_infinity(struct crypto_ec *e,
 int crypto_ec_point_is_on_curve(struct crypto_ec *e,
 				const struct crypto_ec_point *p);
 
+/**
+ * crypto_ec_point_cmp - Compare two EC points
+ * @e: EC context from crypto_ec_init()
+ * @a: EC point
+ * @b: EC point
+ * Returns: 0 on equal, non-zero otherwise
+ */
+int crypto_ec_point_cmp(const struct crypto_ec *e,
+			const struct crypto_ec_point *a,
+			const struct crypto_ec_point *b);
+
 #endif /* CRYPTO_H */
diff --git a/src/crypto/crypto_cryptoapi.c b/src/crypto/crypto_cryptoapi.c
deleted file mode 100644
index 55a069b0d036..000000000000
--- a/src/crypto/crypto_cryptoapi.c
+++ /dev/null
@@ -1,783 +0,0 @@
-/*
- * Crypto wrapper for Microsoft CryptoAPI
- * Copyright (c) 2005-2009, Jouni Malinen <j@w1.fi>
- *
- * This software may be distributed under the terms of the BSD license.
- * See README for more details.
- */
-
-#include "includes.h"
-#include <windows.h>
-#include <wincrypt.h>
-
-#include "common.h"
-#include "crypto.h"
-
-#ifndef MS_ENH_RSA_AES_PROV
-#ifdef UNICODE
-#define MS_ENH_RSA_AES_PROV \
-L"Microsoft Enhanced RSA and AES Cryptographic Provider (Prototype)"
-#else
-#define MS_ENH_RSA_AES_PROV \
-"Microsoft Enhanced RSA and AES Cryptographic Provider (Prototype)"
-#endif
-#endif /* MS_ENH_RSA_AES_PROV */
-
-#ifndef CALG_HMAC
-#define CALG_HMAC (ALG_CLASS_HASH | ALG_TYPE_ANY | ALG_SID_HMAC)
-#endif
-
-#ifdef __MINGW32_VERSION
-/*
- * MinGW does not yet include all the needed definitions for CryptoAPI, so
- * define here whatever extra is needed.
- */
-
-static BOOL WINAPI
-(*CryptImportPublicKeyInfo)(HCRYPTPROV hCryptProv, DWORD dwCertEncodingType,
-			    PCERT_PUBLIC_KEY_INFO pInfo, HCRYPTKEY *phKey)
-= NULL; /* to be loaded from crypt32.dll */
-
-
-static int mingw_load_crypto_func(void)
-{
-	HINSTANCE dll;
-
-	/* MinGW does not yet have full CryptoAPI support, so load the needed
-	 * function here. */
-
-	if (CryptImportPublicKeyInfo)
-		return 0;
-
-	dll = LoadLibrary("crypt32");
-	if (dll == NULL) {
-		wpa_printf(MSG_DEBUG, "CryptoAPI: Could not load crypt32 "
-			   "library");
-		return -1;
-	}
-
-	CryptImportPublicKeyInfo = GetProcAddress(
-		dll, "CryptImportPublicKeyInfo");
-	if (CryptImportPublicKeyInfo == NULL) {
-		wpa_printf(MSG_DEBUG, "CryptoAPI: Could not get "
-			   "CryptImportPublicKeyInfo() address from "
-			   "crypt32 library");
-		return -1;
-	}
-
-	return 0;
-}
-
-#else /* __MINGW32_VERSION */
-
-static int mingw_load_crypto_func(void)
-{
-	return 0;
-}
-
-#endif /* __MINGW32_VERSION */
-
-
-static void cryptoapi_report_error(const char *msg)
-{
-	char *s, *pos;
-	DWORD err = GetLastError();
-
-	if (FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER |
-			  FORMAT_MESSAGE_FROM_SYSTEM,
-			  NULL, err, 0, (LPTSTR) &s, 0, NULL) == 0) {
- 		wpa_printf(MSG_DEBUG, "CryptoAPI: %s: %d", msg, (int) err);
-	}
-
-	pos = s;
-	while (*pos) {
-		if (*pos == '\n' || *pos == '\r') {
-			*pos = '\0';
-			break;
-		}
-		pos++;
-	}
-
-	wpa_printf(MSG_DEBUG, "CryptoAPI: %s: %d: (%s)", msg, (int) err, s);
-	LocalFree(s);
-}
-
-
-int cryptoapi_hash_vector(ALG_ID alg, size_t hash_len, size_t num_elem,
-			  const u8 *addr[], const size_t *len, u8 *mac)
-{
-	HCRYPTPROV prov;
-	HCRYPTHASH hash;
-	size_t i;
-	DWORD hlen;
-	int ret = 0;
-
-	if (!CryptAcquireContext(&prov, NULL, NULL, PROV_RSA_FULL, 0)) {
-		cryptoapi_report_error("CryptAcquireContext");
-		return -1;
-	}
-
-	if (!CryptCreateHash(prov, alg, 0, 0, &hash)) {
-		cryptoapi_report_error("CryptCreateHash");
-		CryptReleaseContext(prov, 0);
-		return -1;
-	}
-
-	for (i = 0; i < num_elem; i++) {
-		if (!CryptHashData(hash, (BYTE *) addr[i], len[i], 0)) {
-			cryptoapi_report_error("CryptHashData");
-			CryptDestroyHash(hash);
-			CryptReleaseContext(prov, 0);
-		}
-	}
-
-	hlen = hash_len;
-	if (!CryptGetHashParam(hash, HP_HASHVAL, mac, &hlen, 0)) {
-		cryptoapi_report_error("CryptGetHashParam");
-		ret = -1;
-	}
-
-	CryptDestroyHash(hash);
-	CryptReleaseContext(prov, 0);
-
-	return ret;
-}
-
-
-int md4_vector(size_t num_elem, const u8 *addr[], const size_t *len, u8 *mac)
-{
-	return cryptoapi_hash_vector(CALG_MD4, 16, num_elem, addr, len, mac);
-}
-
-
-void des_encrypt(const u8 *clear, const u8 *key, u8 *cypher)
-{
-	u8 next, tmp;
-	int i;
-	HCRYPTPROV prov;
-	HCRYPTKEY ckey;
-	DWORD dlen;
-	struct {
-		BLOBHEADER hdr;
-		DWORD len;
-		BYTE key[8];
-	} key_blob;
-	DWORD mode = CRYPT_MODE_ECB;
-
-	key_blob.hdr.bType = PLAINTEXTKEYBLOB;
-	key_blob.hdr.bVersion = CUR_BLOB_VERSION;
-	key_blob.hdr.reserved = 0;
-	key_blob.hdr.aiKeyAlg = CALG_DES;
-	key_blob.len = 8;
-
-	/* Add parity bits to the key */
-	next = 0;
-	for (i = 0; i < 7; i++) {
-		tmp = key[i];
-		key_blob.key[i] = (tmp >> i) | next | 1;
-		next = tmp << (7 - i);
-	}
-	key_blob.key[i] = next | 1;
-
-	if (!CryptAcquireContext(&prov, NULL, MS_ENHANCED_PROV, PROV_RSA_FULL,
-				 CRYPT_VERIFYCONTEXT)) {
- 		wpa_printf(MSG_DEBUG, "CryptoAPI: CryptAcquireContext failed: "
-			   "%d", (int) GetLastError());
-		return;
-	}
-
-	if (!CryptImportKey(prov, (BYTE *) &key_blob, sizeof(key_blob), 0, 0,
-			    &ckey)) {
- 		wpa_printf(MSG_DEBUG, "CryptoAPI: CryptImportKey failed: %d",
-			   (int) GetLastError());
-		CryptReleaseContext(prov, 0);
-		return;
-	}
-
-	if (!CryptSetKeyParam(ckey, KP_MODE, (BYTE *) &mode, 0)) {
- 		wpa_printf(MSG_DEBUG, "CryptoAPI: CryptSetKeyParam(KP_MODE) "
-			   "failed: %d", (int) GetLastError());
-		CryptDestroyKey(ckey);
-		CryptReleaseContext(prov, 0);
-		return;
-	}
-
-	os_memcpy(cypher, clear, 8);
-	dlen = 8;
-	if (!CryptEncrypt(ckey, 0, FALSE, 0, cypher, &dlen, 8)) {
-		wpa_printf(MSG_DEBUG, "CryptoAPI: CryptEncrypt failed: %d",
-			   (int) GetLastError());
-		os_memset(cypher, 0, 8);
-	}
-
-	CryptDestroyKey(ckey);
-	CryptReleaseContext(prov, 0);
-}
-
-
-int md5_vector(size_t num_elem, const u8 *addr[], const size_t *len, u8 *mac)
-{
-	return cryptoapi_hash_vector(CALG_MD5, 16, num_elem, addr, len, mac);
-}
-
-
-int sha1_vector(size_t num_elem, const u8 *addr[], const size_t *len, u8 *mac)
-{
-	return cryptoapi_hash_vector(CALG_SHA, 20, num_elem, addr, len, mac);
-}
-
-
-struct aes_context {
-	HCRYPTPROV prov;
-	HCRYPTKEY ckey;
-};
-
-
-void * aes_encrypt_init(const u8 *key, size_t len)
-{
-	struct aes_context *akey;
-	struct {
-		BLOBHEADER hdr;
-		DWORD len;
-		BYTE key[16];
-	} key_blob;
-	DWORD mode = CRYPT_MODE_ECB;
-
-	if (len != 16)
-		return NULL;
-
-	key_blob.hdr.bType = PLAINTEXTKEYBLOB;
-	key_blob.hdr.bVersion = CUR_BLOB_VERSION;
-	key_blob.hdr.reserved = 0;
-	key_blob.hdr.aiKeyAlg = CALG_AES_128;
-	key_blob.len = len;
-	os_memcpy(key_blob.key, key, len);
-
-	akey = os_zalloc(sizeof(*akey));
-	if (akey == NULL)
-		return NULL;
-
-	if (!CryptAcquireContext(&akey->prov, NULL,
-				 MS_ENH_RSA_AES_PROV, PROV_RSA_AES,
-				 CRYPT_VERIFYCONTEXT)) {
- 		wpa_printf(MSG_DEBUG, "CryptoAPI: CryptAcquireContext failed: "
-			   "%d", (int) GetLastError());
-		os_free(akey);
-		return NULL;
-	}
-
-	if (!CryptImportKey(akey->prov, (BYTE *) &key_blob, sizeof(key_blob),
-			    0, 0, &akey->ckey)) {
- 		wpa_printf(MSG_DEBUG, "CryptoAPI: CryptImportKey failed: %d",
-			   (int) GetLastError());
-		CryptReleaseContext(akey->prov, 0);
-		os_free(akey);
-		return NULL;
-	}
-
-	if (!CryptSetKeyParam(akey->ckey, KP_MODE, (BYTE *) &mode, 0)) {
- 		wpa_printf(MSG_DEBUG, "CryptoAPI: CryptSetKeyParam(KP_MODE) "
-			   "failed: %d", (int) GetLastError());
-		CryptDestroyKey(akey->ckey);
-		CryptReleaseContext(akey->prov, 0);
-		os_free(akey);
-		return NULL;
-	}
-
-	return akey;
-}
-
-
-void aes_encrypt(void *ctx, const u8 *plain, u8 *crypt)
-{
-	struct aes_context *akey = ctx;
-	DWORD dlen;
-
-	os_memcpy(crypt, plain, 16);
-	dlen = 16;
-	if (!CryptEncrypt(akey->ckey, 0, FALSE, 0, crypt, &dlen, 16)) {
-		wpa_printf(MSG_DEBUG, "CryptoAPI: CryptEncrypt failed: %d",
-			   (int) GetLastError());
-		os_memset(crypt, 0, 16);
-	}
-}
-
-
-void aes_encrypt_deinit(void *ctx)
-{
-	struct aes_context *akey = ctx;
-	if (akey) {
-		CryptDestroyKey(akey->ckey);
-		CryptReleaseContext(akey->prov, 0);
-		os_free(akey);
-	}
-}
-
-
-void * aes_decrypt_init(const u8 *key, size_t len)
-{
-	return aes_encrypt_init(key, len);
-}
-
-
-void aes_decrypt(void *ctx, const u8 *crypt, u8 *plain)
-{
-	struct aes_context *akey = ctx;
-	DWORD dlen;
-
-	os_memcpy(plain, crypt, 16);
-	dlen = 16;
-
-	if (!CryptDecrypt(akey->ckey, 0, FALSE, 0, plain, &dlen)) {
-		wpa_printf(MSG_DEBUG, "CryptoAPI: CryptDecrypt failed: %d",
-			   (int) GetLastError());
-	}
-}
-
-
-void aes_decrypt_deinit(void *ctx)
-{
-	aes_encrypt_deinit(ctx);
-}
-
-
-struct crypto_hash {
-	enum crypto_hash_alg alg;
-	int error;
-	HCRYPTPROV prov;
-	HCRYPTHASH hash;
-	HCRYPTKEY key;
-};
-
-struct crypto_hash * crypto_hash_init(enum crypto_hash_alg alg, const u8 *key,
-				      size_t key_len)
-{
-	struct crypto_hash *ctx;
-	ALG_ID calg;
-	struct {
-		BLOBHEADER hdr;
-		DWORD len;
-		BYTE key[32];
-	} key_blob;
-
-	os_memset(&key_blob, 0, sizeof(key_blob));
-	switch (alg) {
-	case CRYPTO_HASH_ALG_MD5:
-		calg = CALG_MD5;
-		break;
-	case CRYPTO_HASH_ALG_SHA1:
-		calg = CALG_SHA;
-		break;
-	case CRYPTO_HASH_ALG_HMAC_MD5:
-	case CRYPTO_HASH_ALG_HMAC_SHA1:
-		calg = CALG_HMAC;
-		key_blob.hdr.bType = PLAINTEXTKEYBLOB;
-		key_blob.hdr.bVersion = CUR_BLOB_VERSION;
-		key_blob.hdr.reserved = 0;
-		/*
-		 * Note: RC2 is not really used, but that can be used to
-		 * import HMAC keys of up to 16 byte long.
-		 * CRYPT_IPSEC_HMAC_KEY flag for CryptImportKey() is needed to
-		 * be able to import longer keys (HMAC-SHA1 uses 20-byte key).
-		 */
-		key_blob.hdr.aiKeyAlg = CALG_RC2;
-		key_blob.len = key_len;
-		if (key_len > sizeof(key_blob.key))
-			return NULL;
-		os_memcpy(key_blob.key, key, key_len);
-		break;
-	default:
-		return NULL;
-	}
-
-	ctx = os_zalloc(sizeof(*ctx));
-	if (ctx == NULL)
-		return NULL;
-
-	ctx->alg = alg;
-
-	if (!CryptAcquireContext(&ctx->prov, NULL, NULL, PROV_RSA_FULL, 0)) {
-		cryptoapi_report_error("CryptAcquireContext");
-		os_free(ctx);
-		return NULL;
-	}
-
-	if (calg == CALG_HMAC) {
-#ifndef CRYPT_IPSEC_HMAC_KEY
-#define CRYPT_IPSEC_HMAC_KEY 0x00000100
-#endif
-		if (!CryptImportKey(ctx->prov, (BYTE *) &key_blob,
-				    sizeof(key_blob), 0, CRYPT_IPSEC_HMAC_KEY,
-				    &ctx->key)) {
-			cryptoapi_report_error("CryptImportKey");
-			CryptReleaseContext(ctx->prov, 0);
-			os_free(ctx);
-			return NULL;
-		}
-	}
-
-	if (!CryptCreateHash(ctx->prov, calg, ctx->key, 0, &ctx->hash)) {
-		cryptoapi_report_error("CryptCreateHash");
-		CryptReleaseContext(ctx->prov, 0);
-		os_free(ctx);
-		return NULL;
-	}
-
-	if (calg == CALG_HMAC) {
-		HMAC_INFO info;
-		os_memset(&info, 0, sizeof(info));
-		switch (alg) {
-		case CRYPTO_HASH_ALG_HMAC_MD5:
-			info.HashAlgid = CALG_MD5;
-			break;
-		case CRYPTO_HASH_ALG_HMAC_SHA1:
-			info.HashAlgid = CALG_SHA;
-			break;
-		default:
-			/* unreachable */
-			break;
-		}
-
-		if (!CryptSetHashParam(ctx->hash, HP_HMAC_INFO, (BYTE *) &info,
-				       0)) {
-			cryptoapi_report_error("CryptSetHashParam");
-			CryptDestroyHash(ctx->hash);
-			CryptReleaseContext(ctx->prov, 0);
-			os_free(ctx);
-			return NULL;
-		}
-	}
-
-	return ctx;
-}
-
-
-void crypto_hash_update(struct crypto_hash *ctx, const u8 *data, size_t len)
-{
-	if (ctx == NULL || ctx->error)
-		return;
-
-	if (!CryptHashData(ctx->hash, (BYTE *) data, len, 0)) {
-		cryptoapi_report_error("CryptHashData");
-		ctx->error = 1;
-	}
-}
-
-
-int crypto_hash_finish(struct crypto_hash *ctx, u8 *mac, size_t *len)
-{
-	int ret = 0;
-	DWORD hlen;
-
-	if (ctx == NULL)
-		return -2;
-
-	if (mac == NULL || len == NULL)
-		goto done;
-
-	if (ctx->error) {
-		ret = -2;
-		goto done;
-	}
-
-	hlen = *len;
-	if (!CryptGetHashParam(ctx->hash, HP_HASHVAL, mac, &hlen, 0)) {
-		cryptoapi_report_error("CryptGetHashParam");
-		ret = -2;
-	}
-	*len = hlen;
-
-done:
-	if (ctx->alg == CRYPTO_HASH_ALG_HMAC_SHA1 ||
-	    ctx->alg == CRYPTO_HASH_ALG_HMAC_MD5)
-		CryptDestroyKey(ctx->key);
-
-	os_free(ctx);
-
-	return ret;
-}
-
-
-struct crypto_cipher {
-	HCRYPTPROV prov;
-	HCRYPTKEY key;
-};
-
-
-struct crypto_cipher * crypto_cipher_init(enum crypto_cipher_alg alg,
-					  const u8 *iv, const u8 *key,
-					  size_t key_len)
-{	
-	struct crypto_cipher *ctx;
-	struct {
-		BLOBHEADER hdr;
-		DWORD len;
-		BYTE key[32];
-	} key_blob;
-	DWORD mode = CRYPT_MODE_CBC;
-
-	key_blob.hdr.bType = PLAINTEXTKEYBLOB;
-	key_blob.hdr.bVersion = CUR_BLOB_VERSION;
-	key_blob.hdr.reserved = 0;
-	key_blob.len = key_len;
-	if (key_len > sizeof(key_blob.key))
-		return NULL;
-	os_memcpy(key_blob.key, key, key_len);
-
-	switch (alg) {
-	case CRYPTO_CIPHER_ALG_AES:
-		if (key_len == 32)
-			key_blob.hdr.aiKeyAlg = CALG_AES_256;
-		else if (key_len == 24)
-			key_blob.hdr.aiKeyAlg = CALG_AES_192;
-		else
-			key_blob.hdr.aiKeyAlg = CALG_AES_128;
-		break;
-	case CRYPTO_CIPHER_ALG_3DES:
-		key_blob.hdr.aiKeyAlg = CALG_3DES;
-		break;
-	case CRYPTO_CIPHER_ALG_DES:
-		key_blob.hdr.aiKeyAlg = CALG_DES;
-		break;
-	case CRYPTO_CIPHER_ALG_RC2:
-		key_blob.hdr.aiKeyAlg = CALG_RC2;
-		break;
-	case CRYPTO_CIPHER_ALG_RC4:
-		key_blob.hdr.aiKeyAlg = CALG_RC4;
-		break;
-	default:
-		return NULL;
-	}
-
-	ctx = os_zalloc(sizeof(*ctx));
-	if (ctx == NULL)
-		return NULL;
-
-	if (!CryptAcquireContext(&ctx->prov, NULL, MS_ENH_RSA_AES_PROV,
-				 PROV_RSA_AES, CRYPT_VERIFYCONTEXT)) {
-		cryptoapi_report_error("CryptAcquireContext");
-		goto fail1;
-	}
-
-	if (!CryptImportKey(ctx->prov, (BYTE *) &key_blob,
-			    sizeof(key_blob), 0, 0, &ctx->key)) {
- 		cryptoapi_report_error("CryptImportKey");
-		goto fail2;
-	}
-
-	if (!CryptSetKeyParam(ctx->key, KP_MODE, (BYTE *) &mode, 0)) {
- 		cryptoapi_report_error("CryptSetKeyParam(KP_MODE)");
-		goto fail3;
-	}
-
-	if (iv && !CryptSetKeyParam(ctx->key, KP_IV, (BYTE *) iv, 0)) {
- 		cryptoapi_report_error("CryptSetKeyParam(KP_IV)");
-		goto fail3;
-	}
-
-	return ctx;
-
-fail3:
-	CryptDestroyKey(ctx->key);
-fail2:
-	CryptReleaseContext(ctx->prov, 0);
-fail1:
-	os_free(ctx);
-	return NULL;
-}
-
-
-int crypto_cipher_encrypt(struct crypto_cipher *ctx, const u8 *plain,
-			  u8 *crypt, size_t len)
-{
-	DWORD dlen;
-
-	os_memcpy(crypt, plain, len);
-	dlen = len;
-	if (!CryptEncrypt(ctx->key, 0, FALSE, 0, crypt, &dlen, len)) {
- 		cryptoapi_report_error("CryptEncrypt");
-		os_memset(crypt, 0, len);
-		return -1;
-	}
-
-	return 0;
-}
-
-
-int crypto_cipher_decrypt(struct crypto_cipher *ctx, const u8 *crypt,
-			  u8 *plain, size_t len)
-{
-	DWORD dlen;
-
-	os_memcpy(plain, crypt, len);
-	dlen = len;
-	if (!CryptDecrypt(ctx->key, 0, FALSE, 0, plain, &dlen)) {
- 		cryptoapi_report_error("CryptDecrypt");
-		return -1;
-	}
-
-	return 0;
-}
-
-
-void crypto_cipher_deinit(struct crypto_cipher *ctx)
-{
-	CryptDestroyKey(ctx->key);
-	CryptReleaseContext(ctx->prov, 0);
-	os_free(ctx);
-}
-
-
-struct crypto_public_key {
-	HCRYPTPROV prov;
-	HCRYPTKEY rsa;
-};
-
-struct crypto_private_key {
-	HCRYPTPROV prov;
-	HCRYPTKEY rsa;
-};
-
-
-struct crypto_public_key * crypto_public_key_import(const u8 *key, size_t len)
-{
-	/* Use crypto_public_key_from_cert() instead. */
-	return NULL;
-}
-
-
-struct crypto_private_key * crypto_private_key_import(const u8 *key,
-						      size_t len,
-						      const char *passwd)
-{
-	/* TODO */
-	return NULL;
-}
-
-
-struct crypto_public_key * crypto_public_key_from_cert(const u8 *buf,
-						       size_t len)
-{
-	struct crypto_public_key *pk;
-	PCCERT_CONTEXT cc;
-
-	pk = os_zalloc(sizeof(*pk));
-	if (pk == NULL)
-		return NULL;
-
-	cc = CertCreateCertificateContext(X509_ASN_ENCODING |
-					  PKCS_7_ASN_ENCODING, buf, len);
-	if (!cc) {
- 		cryptoapi_report_error("CryptCreateCertificateContext");
-		os_free(pk);
-		return NULL;
-	}
-
-	if (!CryptAcquireContext(&pk->prov, NULL, MS_DEF_PROV, PROV_RSA_FULL,
-				 0)) {
- 		cryptoapi_report_error("CryptAcquireContext");
-		os_free(pk);
-		CertFreeCertificateContext(cc);
-		return NULL;
-	}
-
-	if (!CryptImportPublicKeyInfo(pk->prov, X509_ASN_ENCODING |
-				      PKCS_7_ASN_ENCODING,
-				      &cc->pCertInfo->SubjectPublicKeyInfo,
-				      &pk->rsa)) {
- 		cryptoapi_report_error("CryptImportPublicKeyInfo");
-		CryptReleaseContext(pk->prov, 0);
-		os_free(pk);
-		CertFreeCertificateContext(cc);
-		return NULL;
-	}
-
-	CertFreeCertificateContext(cc);
-
-	return pk;
-}
-
-
-int crypto_public_key_encrypt_pkcs1_v15(struct crypto_public_key *key,
-					const u8 *in, size_t inlen,
-					u8 *out, size_t *outlen)
-{
-	DWORD clen;
-	u8 *tmp;
-	size_t i;
-
-	if (*outlen < inlen)
-		return -1;
-	tmp = malloc(*outlen);
-	if (tmp == NULL)
-		return -1;
-
-	os_memcpy(tmp, in, inlen);
-	clen = inlen;
-	if (!CryptEncrypt(key->rsa, 0, TRUE, 0, tmp, &clen, *outlen)) {
-		wpa_printf(MSG_DEBUG, "CryptoAPI: Failed to encrypt using "
-			   "public key: %d", (int) GetLastError());
-		os_free(tmp);
-		return -1;
-	}
-
-	*outlen = clen;
-
-	/* Reverse the output */
-	for (i = 0; i < *outlen; i++)
-		out[i] = tmp[*outlen - 1 - i];
-
-	os_free(tmp);
-
-	return 0;
-}
-
-
-int crypto_private_key_sign_pkcs1(struct crypto_private_key *key,
-				  const u8 *in, size_t inlen,
-				  u8 *out, size_t *outlen)
-{
-	/* TODO */
-	return -1;
-}
-
-
-void crypto_public_key_free(struct crypto_public_key *key)
-{
-	if (key) {
-		CryptDestroyKey(key->rsa);
-		CryptReleaseContext(key->prov, 0);
-		os_free(key);
-	}
-}
-
-
-void crypto_private_key_free(struct crypto_private_key *key)
-{
-	if (key) {
-		CryptDestroyKey(key->rsa);
-		CryptReleaseContext(key->prov, 0);
-		os_free(key);
-	}
-}
-
-
-int crypto_global_init(void)
-{
-	return mingw_load_crypto_func();
-}
-
-
-void crypto_global_deinit(void)
-{
-}
-
-
-int crypto_mod_exp(const u8 *base, size_t base_len,
-		   const u8 *power, size_t power_len,
-		   const u8 *modulus, size_t modulus_len,
-		   u8 *result, size_t *result_len)
-{
-	/* TODO */
-	return -1;
-}
diff --git a/src/crypto/crypto_module_tests.c b/src/crypto/crypto_module_tests.c
index 7137c27d0e8c..581005df3e39 100644
--- a/src/crypto/crypto_module_tests.c
+++ b/src/crypto/crypto_module_tests.c
@@ -161,7 +161,7 @@ struct omac1_test_vector {
 	u8 tag[16];
 };
 
-static struct omac1_test_vector omac1_test_vectors[] =
+static const struct omac1_test_vector omac1_test_vectors[] =
 {
 	{
 		{ 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6,
@@ -210,7 +210,8 @@ static struct omac1_test_vector omac1_test_vectors[] =
 };
 
 
-static int test_omac1_vector(struct omac1_test_vector *tv, unsigned int i)
+static int test_omac1_vector(const struct omac1_test_vector *tv,
+			     unsigned int i)
 {
 	u8 key[] = {
 		0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6,
@@ -515,6 +516,7 @@ static int test_key_wrap(void)
 		0xAE, 0xF3, 0x4B, 0xD8, 0xFB, 0x5A, 0x7B, 0x82,
 		0x9D, 0x3E, 0x86, 0x23, 0x71, 0xD2, 0xCF, 0xE5
 	};
+#ifndef CONFIG_BORINGSSL
 	/* RFC 3394 - Test vector 4.2 */
 	u8 kek42[] = {
 		0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
@@ -530,6 +532,7 @@ static int test_key_wrap(void)
 		0xF9, 0x2B, 0x5B, 0x97, 0xC0, 0x50, 0xAE, 0xD2,
 		0x46, 0x8A, 0xB8, 0xA1, 0x7A, 0xD8, 0x4E, 0x5D
 	};
+#endif /* CONFIG_BORINGSSL */
 	/* RFC 3394 - Test vector 4.3 */
 	u8 kek43[] = {
 		0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
@@ -546,6 +549,7 @@ static int test_key_wrap(void)
 		0x63, 0xE9, 0x77, 0x79, 0x05, 0x81, 0x8A, 0x2A,
 		0x93, 0xC8, 0x19, 0x1E, 0x7D, 0x6E, 0x8A, 0xE7,
 	};
+#ifndef CONFIG_BORINGSSL
 	/* RFC 3394 - Test vector 4.4 */
 	u8 kek44[] = {
 		0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
@@ -563,6 +567,7 @@ static int test_key_wrap(void)
 		0xE1, 0xC6, 0xC7, 0xDD, 0xEE, 0x72, 0x5A, 0x93,
 		0x6B, 0xA8, 0x14, 0x91, 0x5C, 0x67, 0x62, 0xD2
 	};
+#endif /* CONFIG_BORINGSSL */
 	/* RFC 3394 - Test vector 4.5 */
 	u8 kek45[] = {
 		0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
@@ -623,6 +628,7 @@ static int test_key_wrap(void)
 		ret++;
 	}
 
+#ifndef CONFIG_BORINGSSL
 	wpa_printf(MSG_INFO, "RFC 3394 - Test vector 4.2");
 	if (aes_wrap(kek42, sizeof(kek42), sizeof(plain42) / 8, plain42,
 		     result)) {
@@ -642,6 +648,7 @@ static int test_key_wrap(void)
 		wpa_printf(MSG_ERROR, "AES-UNWRAP-192 failed");
 		ret++;
 	}
+#endif /* CONFIG_BORINGSSL */
 
 	wpa_printf(MSG_INFO, "RFC 3394 - Test vector 4.3");
 	if (aes_wrap(kek43, sizeof(kek43), sizeof(plain43) / 8, plain43,
@@ -663,6 +670,7 @@ static int test_key_wrap(void)
 		ret++;
 	}
 
+#ifndef CONFIG_BORINGSSL
 	wpa_printf(MSG_INFO, "RFC 3394 - Test vector 4.4");
 	if (aes_wrap(kek44, sizeof(kek44), sizeof(plain44) / 8, plain44,
 		     result)) {
@@ -682,6 +690,7 @@ static int test_key_wrap(void)
 		wpa_printf(MSG_ERROR, "AES-UNWRAP-192 failed");
 		ret++;
 	}
+#endif /* CONFIG_BORINGSSL */
 
 	wpa_printf(MSG_INFO, "RFC 3394 - Test vector 4.5");
 	if (aes_wrap(kek45, sizeof(kek45), sizeof(plain45) / 8, plain45,
@@ -732,6 +741,7 @@ static int test_key_wrap(void)
 
 static int test_md5(void)
 {
+#ifndef CONFIG_FIPS
 	struct {
 		char *data;
 		char *hash;
@@ -810,6 +820,10 @@ static int test_md5(void)
 		wpa_printf(MSG_INFO, "MD5 test cases passed");
 
 	return errors;
+#else /* CONFIG_FIPS */
+	wpa_printf(MSG_INFO, "MD5 test cases skipped due to CONFIG_FIPS");
+	return 0;
+#endif /* CONFIG_FIPS */
 }
 
 
@@ -841,6 +855,7 @@ static int test_eap_fast(void)
 		0x38, 0x4B, 0x7A, 0x85, 0xBE, 0x16, 0x4D, 0x27,
 		0x33, 0xD5, 0x24, 0x79, 0x87, 0xB1, 0xC5, 0xA2
 	};
+#ifndef CONFIG_FIPS
 	const u8 key_block[] = {
 		0x59, 0x59, 0xBE, 0x8E, 0x41, 0x3A, 0x77, 0x74,
 		0x8B, 0xB2, 0xE5, 0xD3, 0x60, 0xAC, 0x4D, 0x35,
@@ -857,6 +872,7 @@ static int test_eap_fast(void)
 		0x64, 0xC1, 0xC8, 0x0C, 0x96, 0x44, 0x09, 0x98,
 		0xFF, 0x92, 0xA8, 0xB4, 0xC6, 0x42, 0x28, 0x71
 	};
+#endif /* CONFIG_FIPS */
 	const u8 sks[] = {
 		0xD6, 0x4B, 0x7D, 0x72, 0x17, 0x59, 0x28, 0x05,
 		0xAF, 0xF9, 0xB7, 0xFF, 0x66, 0x6D, 0xA1, 0x96,
@@ -931,6 +947,7 @@ static int test_eap_fast(void)
 		errors++;
 	}
 
+#ifndef CONFIG_FIPS
 	wpa_printf(MSG_INFO, "- PRF (TLS, SHA1/MD5) test case / key_block");
 	if (tls_prf_sha1_md5(master_secret, sizeof(master_secret),
 			     "key expansion", seed, sizeof(seed),
@@ -939,6 +956,7 @@ static int test_eap_fast(void)
 		wpa_printf(MSG_INFO, "PRF test - FAILED!");
 		errors++;
 	}
+#endif /* CONFIG_FIPS */
 
 	wpa_printf(MSG_INFO, "- T-PRF (SHA1) test case / IMCK");
 	if (sha1_t_prf(sks, sizeof(sks), "Inner Methods Compound Keys",
@@ -983,14 +1001,14 @@ static int test_eap_fast(void)
 }
 
 
-static u8 key0[] =
+static const u8 key0[] =
 {
 	0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b,
 	0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b,
 	0x0b, 0x0b, 0x0b, 0x0b
 };
-static u8 data0[] = "Hi There";
-static u8 prf0[] =
+static const u8 data0[] = "Hi There";
+static const u8 prf0[] =
 {
 	0xbc, 0xd4, 0xc6, 0x50, 0xb3, 0x0b, 0x96, 0x84,
 	0x95, 0x18, 0x29, 0xe0, 0xd7, 0x5f, 0x9d, 0x54,
@@ -1002,9 +1020,9 @@ static u8 prf0[] =
 	0xdb, 0x83, 0x73, 0x69, 0x83, 0x56, 0xcf, 0x5a
 };
 
-static u8 key1[] = "Jefe";
-static u8 data1[] = "what do ya want for nothing?";
-static u8 prf1[] =
+static const u8 key1[] = "Jefe";
+static const u8 data1[] = "what do ya want for nothing?";
+static const u8 prf1[] =
 {
 	0x51, 0xf4, 0xde, 0x5b, 0x33, 0xf2, 0x49, 0xad,
 	0xf8, 0x1a, 0xeb, 0x71, 0x3a, 0x3c, 0x20, 0xf4,
@@ -1017,13 +1035,13 @@ static u8 prf1[] =
 };
 
 
-static u8 key2[] =
+static const u8 key2[] =
 {
 	0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa,
 	0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa,
 	0xaa, 0xaa, 0xaa, 0xaa
 };
-static u8 data2[] =
+static const u8 data2[] =
 {
 	0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
 	0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
@@ -1033,7 +1051,7 @@ static u8 data2[] =
 	0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
 	0xdd, 0xdd
 };
-static u8 prf2[] =
+static const u8 prf2[] =
 {
 	0xe1, 0xac, 0x54, 0x6e, 0xc4, 0xcb, 0x63, 0x6f,
 	0x99, 0x76, 0x48, 0x7b, 0xe5, 0xc8, 0x6b, 0xe1,
@@ -1052,7 +1070,7 @@ struct passphrase_test {
 	char psk[32];
 };
 
-static struct passphrase_test passphrase_tests[] =
+static const struct passphrase_test passphrase_tests[] =
 {
 	{
 		"password",
@@ -1097,7 +1115,7 @@ struct rfc6070_test {
 	size_t dk_len;
 };
 
-static struct rfc6070_test rfc6070_tests[] =
+static const struct rfc6070_test rfc6070_tests[] =
 {
 	{
 		"password",
@@ -1214,7 +1232,7 @@ static int test_sha1(void)
 	wpa_printf(MSG_INFO, "PBKDF2-SHA1 Passphrase test cases:");
 	for (i = 0; i < NUM_PASSPHRASE_TESTS; i++) {
 		u8 psk[32];
-		struct passphrase_test *test = &passphrase_tests[i];
+		const struct passphrase_test *test = &passphrase_tests[i];
 
 		if (pbkdf2_sha1(test->passphrase,
 				(const u8 *) test->ssid, strlen(test->ssid),
@@ -1230,7 +1248,7 @@ static int test_sha1(void)
 	wpa_printf(MSG_INFO, "PBKDF2-SHA1 test cases (RFC 6070):");
 	for (i = 0; i < NUM_RFC6070_TESTS; i++) {
 		u8 dk[25];
-		struct rfc6070_test *test = &rfc6070_tests[i];
+		const struct rfc6070_test *test = &rfc6070_tests[i];
 
 		if (pbkdf2_sha1(test->p, (const u8 *) test->s, strlen(test->s),
 				test->c, dk, test->dk_len) == 0 &&
@@ -1248,7 +1266,7 @@ static int test_sha1(void)
 }
 
 
-struct {
+const struct {
 	char *data;
 	u8 hash[32];
 } tests[] = {
@@ -1272,7 +1290,7 @@ struct {
 	}
 };
 
-struct hmac_test {
+const struct hmac_test {
 	u8 key[80];
 	size_t key_len;
 	u8 data[128];
@@ -1513,7 +1531,7 @@ static int test_sha256(void)
 	}
 
 	for (i = 0; i < ARRAY_SIZE(hmac_tests); i++) {
-		struct hmac_test *t = &hmac_tests[i];
+		const struct hmac_test *t = &hmac_tests[i];
 
 		wpa_printf(MSG_INFO, "HMAC-SHA256 test case %d:", i + 1);
 
@@ -1563,6 +1581,7 @@ static int test_sha256(void)
 
 static int test_ms_funcs(void)
 {
+#ifndef CONFIG_FIPS
 	/* Test vector from RFC2759 example */
 	char *username = "User";
 	char *password = "clientPass";
@@ -1655,6 +1674,10 @@ static int test_ms_funcs(void)
 		wpa_printf(MSG_INFO, "ms_funcs test cases passed");
 
 	return errors;
+#else /* CONFIG_FIPS */
+	wpa_printf(MSG_INFO, "ms_funcs test cases skipped due to CONFIG_FIPS");
+	return 0;
+#endif /* CONFIG_FIPS */
 }
 
 
diff --git a/src/crypto/crypto_openssl.c b/src/crypto/crypto_openssl.c
index f158ef43a645..6cff75c64ae5 100644
--- a/src/crypto/crypto_openssl.c
+++ b/src/crypto/crypto_openssl.c
@@ -93,10 +93,12 @@ static int openssl_digest_vector(const EVP_MD *type, size_t num_elem,
 }
 
 
+#ifndef CONFIG_FIPS
 int md4_vector(size_t num_elem, const u8 *addr[], const size_t *len, u8 *mac)
 {
 	return openssl_digest_vector(EVP_md4(), num_elem, addr, len, mac);
 }
+#endif /* CONFIG_FIPS */
 
 
 void des_encrypt(const u8 *clear, const u8 *key, u8 *cypher)
@@ -120,6 +122,7 @@ void des_encrypt(const u8 *clear, const u8 *key, u8 *cypher)
 }
 
 
+#ifndef CONFIG_NO_RC4
 int rc4_skip(const u8 *key, size_t keylen, size_t skip,
 	     u8 *data, size_t data_len)
 {
@@ -155,12 +158,15 @@ out:
 	return res;
 #endif /* OPENSSL_NO_RC4 */
 }
+#endif /* CONFIG_NO_RC4 */
 
 
+#ifndef CONFIG_FIPS
 int md5_vector(size_t num_elem, const u8 *addr[], const size_t *len, u8 *mac)
 {
 	return openssl_digest_vector(EVP_md5(), num_elem, addr, len, mac);
 }
+#endif /* CONFIG_FIPS */
 
 
 int sha1_vector(size_t num_elem, const u8 *addr[], const size_t *len, u8 *mac)
@@ -297,6 +303,9 @@ void aes_decrypt_deinit(void *ctx)
 }
 
 
+#ifndef CONFIG_FIPS
+#ifndef CONFIG_OPENSSL_INTERNAL_AES_WRAP
+
 int aes_wrap(const u8 *kek, size_t kek_len, int n, const u8 *plain, u8 *cipher)
 {
 	AES_KEY actx;
@@ -323,6 +332,59 @@ int aes_unwrap(const u8 *kek, size_t kek_len, int n, const u8 *cipher,
 	return res <= 0 ? -1 : 0;
 }
 
+#endif /* CONFIG_OPENSSL_INTERNAL_AES_WRAP */
+#endif /* CONFIG_FIPS */
+
+
+int aes_128_cbc_encrypt(const u8 *key, const u8 *iv, u8 *data, size_t data_len)
+{
+	EVP_CIPHER_CTX ctx;
+	int clen, len;
+	u8 buf[16];
+
+	EVP_CIPHER_CTX_init(&ctx);
+	if (EVP_EncryptInit_ex(&ctx, EVP_aes_128_cbc(), NULL, key, iv) != 1)
+		return -1;
+	EVP_CIPHER_CTX_set_padding(&ctx, 0);
+
+	clen = data_len;
+	if (EVP_EncryptUpdate(&ctx, data, &clen, data, data_len) != 1 ||
+	    clen != (int) data_len)
+		return -1;
+
+	len = sizeof(buf);
+	if (EVP_EncryptFinal_ex(&ctx, buf, &len) != 1 || len != 0)
+		return -1;
+	EVP_CIPHER_CTX_cleanup(&ctx);
+
+	return 0;
+}
+
+
+int aes_128_cbc_decrypt(const u8 *key, const u8 *iv, u8 *data, size_t data_len)
+{
+	EVP_CIPHER_CTX ctx;
+	int plen, len;
+	u8 buf[16];
+
+	EVP_CIPHER_CTX_init(&ctx);
+	if (EVP_DecryptInit_ex(&ctx, EVP_aes_128_cbc(), NULL, key, iv) != 1)
+		return -1;
+	EVP_CIPHER_CTX_set_padding(&ctx, 0);
+
+	plen = data_len;
+	if (EVP_DecryptUpdate(&ctx, data, &plen, data, data_len) != 1 ||
+	    plen != (int) data_len)
+		return -1;
+
+	len = sizeof(buf);
+	if (EVP_DecryptFinal_ex(&ctx, buf, &len) != 1 || len != 0)
+		return -1;
+	EVP_CIPHER_CTX_cleanup(&ctx);
+
+	return 0;
+}
+
 
 int crypto_mod_exp(const u8 *base, size_t base_len,
 		   const u8 *power, size_t power_len,
@@ -380,11 +442,13 @@ struct crypto_cipher * crypto_cipher_init(enum crypto_cipher_alg alg,
 		return NULL;
 
 	switch (alg) {
+#ifndef CONFIG_NO_RC4
 #ifndef OPENSSL_NO_RC4
 	case CRYPTO_CIPHER_ALG_RC4:
 		cipher = EVP_rc4();
 		break;
 #endif /* OPENSSL_NO_RC4 */
+#endif /* CONFIG_NO_RC4 */
 #ifndef OPENSSL_NO_AES
 	case CRYPTO_CIPHER_ALG_AES:
 		switch (key_len) {
@@ -1057,6 +1121,42 @@ int crypto_bignum_is_one(const struct crypto_bignum *a)
 }
 
 
+int crypto_bignum_legendre(const struct crypto_bignum *a,
+			   const struct crypto_bignum *p)
+{
+	BN_CTX *bnctx;
+	BIGNUM *exp = NULL, *tmp = NULL;
+	int res = -2;
+
+	bnctx = BN_CTX_new();
+	if (bnctx == NULL)
+		return -2;
+
+	exp = BN_new();
+	tmp = BN_new();
+	if (!exp || !tmp ||
+	    /* exp = (p-1) / 2 */
+	    !BN_sub(exp, (const BIGNUM *) p, BN_value_one()) ||
+	    !BN_rshift1(exp, exp) ||
+	    !BN_mod_exp(tmp, (const BIGNUM *) a, exp, (const BIGNUM *) p,
+			bnctx))
+		goto fail;
+
+	if (BN_is_word(tmp, 1))
+		res = 1;
+	else if (BN_is_zero(tmp))
+		res = 0;
+	else
+		res = -1;
+
+fail:
+	BN_clear_free(tmp);
+	BN_clear_free(exp);
+	BN_CTX_free(bnctx);
+	return res;
+}
+
+
 #ifdef CONFIG_ECC
 
 struct crypto_ec {
@@ -1064,6 +1164,8 @@ struct crypto_ec {
 	BN_CTX *bnctx;
 	BIGNUM *prime;
 	BIGNUM *order;
+	BIGNUM *a;
+	BIGNUM *b;
 };
 
 struct crypto_ec * crypto_ec_init(int group)
@@ -1088,6 +1190,26 @@ struct crypto_ec * crypto_ec_init(int group)
 	case 26:
 		nid = NID_secp224r1;
 		break;
+#ifdef NID_brainpoolP224r1
+	case 27:
+		nid = NID_brainpoolP224r1;
+		break;
+#endif /* NID_brainpoolP224r1 */
+#ifdef NID_brainpoolP256r1
+	case 28:
+		nid = NID_brainpoolP256r1;
+		break;
+#endif /* NID_brainpoolP256r1 */
+#ifdef NID_brainpoolP384r1
+	case 29:
+		nid = NID_brainpoolP384r1;
+		break;
+#endif /* NID_brainpoolP384r1 */
+#ifdef NID_brainpoolP512r1
+	case 30:
+		nid = NID_brainpoolP512r1;
+		break;
+#endif /* NID_brainpoolP512r1 */
 	default:
 		return NULL;
 	}
@@ -1100,9 +1222,11 @@ struct crypto_ec * crypto_ec_init(int group)
 	e->group = EC_GROUP_new_by_curve_name(nid);
 	e->prime = BN_new();
 	e->order = BN_new();
+	e->a = BN_new();
+	e->b = BN_new();
 	if (e->group == NULL || e->bnctx == NULL || e->prime == NULL ||
-	    e->order == NULL ||
-	    !EC_GROUP_get_curve_GFp(e->group, e->prime, NULL, NULL, e->bnctx) ||
+	    e->order == NULL || e->a == NULL || e->b == NULL ||
+	    !EC_GROUP_get_curve_GFp(e->group, e->prime, e->a, e->b, e->bnctx) ||
 	    !EC_GROUP_get_order(e->group, e->order, e->bnctx)) {
 		crypto_ec_deinit(e);
 		e = NULL;
@@ -1116,6 +1240,8 @@ void crypto_ec_deinit(struct crypto_ec *e)
 {
 	if (e == NULL)
 		return;
+	BN_clear_free(e->b);
+	BN_clear_free(e->a);
 	BN_clear_free(e->order);
 	BN_clear_free(e->prime);
 	EC_GROUP_free(e->group);
@@ -1263,6 +1389,33 @@ int crypto_ec_point_solve_y_coord(struct crypto_ec *e,
 }
 
 
+struct crypto_bignum *
+crypto_ec_point_compute_y_sqr(struct crypto_ec *e,
+			      const struct crypto_bignum *x)
+{
+	BIGNUM *tmp, *tmp2, *y_sqr = NULL;
+
+	tmp = BN_new();
+	tmp2 = BN_new();
+
+	/* y^2 = x^3 + ax + b */
+	if (tmp && tmp2 &&
+	    BN_mod_sqr(tmp, (const BIGNUM *) x, e->prime, e->bnctx) &&
+	    BN_mod_mul(tmp, tmp, (const BIGNUM *) x, e->prime, e->bnctx) &&
+	    BN_mod_mul(tmp2, e->a, (const BIGNUM *) x, e->prime, e->bnctx) &&
+	    BN_mod_add_quick(tmp2, tmp2, tmp, e->prime) &&
+	    BN_mod_add_quick(tmp2, tmp2, e->b, e->prime)) {
+		y_sqr = tmp2;
+		tmp2 = NULL;
+	}
+
+	BN_clear_free(tmp);
+	BN_clear_free(tmp2);
+
+	return (struct crypto_bignum *) y_sqr;
+}
+
+
 int crypto_ec_point_is_at_infinity(struct crypto_ec *e,
 				   const struct crypto_ec_point *p)
 {
@@ -1273,7 +1426,17 @@ int crypto_ec_point_is_at_infinity(struct crypto_ec *e,
 int crypto_ec_point_is_on_curve(struct crypto_ec *e,
 				const struct crypto_ec_point *p)
 {
-	return EC_POINT_is_on_curve(e->group, (const EC_POINT *) p, e->bnctx);
+	return EC_POINT_is_on_curve(e->group, (const EC_POINT *) p,
+				    e->bnctx) == 1;
+}
+
+
+int crypto_ec_point_cmp(const struct crypto_ec *e,
+			const struct crypto_ec_point *a,
+			const struct crypto_ec_point *b)
+{
+	return EC_POINT_cmp(e->group, (const EC_POINT *) a,
+			    (const EC_POINT *) b, e->bnctx);
 }
 
 #endif /* CONFIG_ECC */
diff --git a/src/crypto/dh_groups.c b/src/crypto/dh_groups.c
index d3b263196e2d..3aeb2bbc60af 100644
--- a/src/crypto/dh_groups.c
+++ b/src/crypto/dh_groups.c
@@ -1153,7 +1153,7 @@ dh_group ## id ## _prime, sizeof(dh_group ## id ## _prime), \
 dh_group ## id ## _order, sizeof(dh_group ## id ## _order), safe }
 		
 
-static struct dh_group dh_groups[] = {
+static const struct dh_group dh_groups[] = {
 	DH_GROUP(5, 1),
 #ifdef ALL_DH_GROUPS
 	DH_GROUP(1, 1),
diff --git a/src/crypto/fips_prf_openssl.c b/src/crypto/fips_prf_openssl.c
index d69eceabff1c..fb03efcd4ffc 100644
--- a/src/crypto/fips_prf_openssl.c
+++ b/src/crypto/fips_prf_openssl.c
@@ -13,13 +13,21 @@
 #include "crypto.h"
 
 
-static void sha1_transform(u8 *state, const u8 data[64])
+static void sha1_transform(u32 *state, const u8 data[64])
 {
 	SHA_CTX context;
 	os_memset(&context, 0, sizeof(context));
-	os_memcpy(&context.h0, state, 5 * 4);
+	context.h0 = state[0];
+	context.h1 = state[1];
+	context.h2 = state[2];
+	context.h3 = state[3];
+	context.h4 = state[4];
 	SHA1_Transform(&context, data);
-	os_memcpy(state, &context.h0, 5 * 4);
+	state[0] = context.h0;
+	state[1] = context.h1;
+	state[2] = context.h2;
+	state[3] = context.h3;
+	state[4] = context.h4;
 }
 
 
@@ -53,7 +61,7 @@ int fips186_2_prf(const u8 *seed, size_t seed_len, u8 *x, size_t xlen)
 
 			/* w_i = G(t, XVAL) */
 			os_memcpy(_t, t, 20);
-			sha1_transform((u8 *) _t, xkey);
+			sha1_transform(_t, xkey);
 			_t[0] = host_to_be32(_t[0]);
 			_t[1] = host_to_be32(_t[1]);
 			_t[2] = host_to_be32(_t[2]);
diff --git a/src/crypto/ms_funcs.c b/src/crypto/ms_funcs.c
index 49a5c1c245d6..053d203cb65b 100644
--- a/src/crypto/ms_funcs.c
+++ b/src/crypto/ms_funcs.c
@@ -78,9 +78,8 @@ static int utf8_to_ucs2(const u8 *utf8_string, size_t utf8_string_len,
  * @challenge: 8-octet Challenge (OUT)
  * Returns: 0 on success, -1 on failure
  */
-static int challenge_hash(const u8 *peer_challenge, const u8 *auth_challenge,
-			  const u8 *username, size_t username_len,
-			  u8 *challenge)
+int challenge_hash(const u8 *peer_challenge, const u8 *auth_challenge,
+		   const u8 *username, size_t username_len, u8 *challenge)
 {
 	u8 hash[SHA1_MAC_LEN];
 	const unsigned char *addr[3];
@@ -175,9 +174,8 @@ int generate_nt_response(const u8 *auth_challenge, const u8 *peer_challenge,
 	u8 password_hash[16];
 
 	if (challenge_hash(peer_challenge, auth_challenge, username,
-			   username_len, challenge))
-		return -1;
-	if (nt_password_hash(password, password_len, password_hash))
+			   username_len, challenge) ||
+	    nt_password_hash(password, password_len, password_hash))
 		return -1;
 	challenge_response(challenge, password_hash, response);
 	return 0;
@@ -257,12 +255,9 @@ int generate_authenticator_response_pwhash(
 	addr2[1] = challenge;
 	addr2[2] = magic2;
 
-	if (hash_nt_password_hash(password_hash, password_hash_hash))
-		return -1;
-	if (sha1_vector(3, addr1, len1, response))
-		return -1;
-
-	if (challenge_hash(peer_challenge, auth_challenge, username,
+	if (hash_nt_password_hash(password_hash, password_hash_hash) ||
+	    sha1_vector(3, addr1, len1, response) ||
+	    challenge_hash(peer_challenge, auth_challenge, username,
 			   username_len, challenge))
 		return -1;
 	return sha1_vector(3, addr2, len2, response);
@@ -417,6 +412,8 @@ int get_asymetric_start_key(const u8 *master_key, u8 *session_key,
 }
 
 
+#ifndef CONFIG_NO_RC4
+
 #define PWBLOCK_LEN 516
 
 /**
@@ -436,10 +433,8 @@ int encrypt_pw_block_with_password_hash(
 
 	os_memset(pw_block, 0, PWBLOCK_LEN);
 
-	if (utf8_to_ucs2(password, password_len, pw_block, 512, &ucs2_len) < 0)
-		return -1;
-
-	if (ucs2_len > 256)
+	if (utf8_to_ucs2(password, password_len, pw_block, 512, &ucs2_len) < 0
+	    || ucs2_len > 256)
 		return -1;
 
 	offset = (256 - ucs2_len) * 2;
@@ -484,6 +479,8 @@ int new_password_encrypted_with_old_nt_password_hash(
 	return 0;
 }
 
+#endif /* CONFIG_NO_RC4 */
+
 
 /**
  * nt_password_hash_encrypted_with_block - NtPasswordHashEncryptedWithBlock() - RFC 2759, Sect 8.13
diff --git a/src/crypto/ms_funcs.h b/src/crypto/ms_funcs.h
index bd9bfee95b7c..b5b5918e1a55 100644
--- a/src/crypto/ms_funcs.h
+++ b/src/crypto/ms_funcs.h
@@ -33,6 +33,8 @@ int nt_challenge_response(const u8 *challenge, const u8 *password,
 
 void challenge_response(const u8 *challenge, const u8 *password_hash,
 			u8 *response);
+int challenge_hash(const u8 *peer_challenge, const u8 *auth_challenge,
+		   const u8 *username, size_t username_len, u8 *challenge);
 int nt_password_hash(const u8 *password, size_t password_len,
 		     u8 *password_hash);
 int hash_nt_password_hash(const u8 *password_hash, u8 *password_hash_hash);
diff --git a/src/crypto/random.c b/src/crypto/random.c
index bc758aa57232..3a86a93a46a8 100644
--- a/src/crypto/random.c
+++ b/src/crypto/random.c
@@ -181,6 +181,7 @@ int random_get_bytes(void *buf, size_t len)
 
 #ifdef CONFIG_FIPS
 	/* Mix in additional entropy from the crypto module */
+	bytes = buf;
 	left = len;
 	while (left) {
 		size_t siz, i;
diff --git a/src/crypto/sha1-tlsprf.c b/src/crypto/sha1-tlsprf.c
index 0effd9b76dd8..f9bc0ebf6e3d 100644
--- a/src/crypto/sha1-tlsprf.c
+++ b/src/crypto/sha1-tlsprf.c
@@ -95,5 +95,10 @@ int tls_prf_sha1_md5(const u8 *secret, size_t secret_len, const char *label,
 		SHA1_pos++;
 	}
 
+	os_memset(A_MD5, 0, MD5_MAC_LEN);
+	os_memset(P_MD5, 0, MD5_MAC_LEN);
+	os_memset(A_SHA1, 0, SHA1_MAC_LEN);
+	os_memset(P_SHA1, 0, SHA1_MAC_LEN);
+
 	return 0;
 }
diff --git a/src/crypto/sha1-tprf.c b/src/crypto/sha1-tprf.c
index a52949462f77..562510f8937d 100644
--- a/src/crypto/sha1-tprf.c
+++ b/src/crypto/sha1-tprf.c
@@ -66,5 +66,7 @@ int sha1_t_prf(const u8 *key, size_t key_len, const char *label,
 		len[0] = SHA1_MAC_LEN;
 	}
 
+	os_memset(hash, 0, SHA1_MAC_LEN);
+
 	return 0;
 }
diff --git a/src/crypto/sha256-kdf.c b/src/crypto/sha256-kdf.c
index d8a1beb32e90..e7509ce41aba 100644
--- a/src/crypto/sha256-kdf.c
+++ b/src/crypto/sha256-kdf.c
@@ -61,6 +61,7 @@ int hmac_sha256_kdf(const u8 *secret, size_t secret_len,
 
 		if (iter == 255) {
 			os_memset(out, 0, outlen);
+			os_memset(T, 0, SHA256_MAC_LEN);
 			return -1;
 		}
 		iter++;
@@ -68,9 +69,11 @@ int hmac_sha256_kdf(const u8 *secret, size_t secret_len,
 		if (hmac_sha256_vector(secret, secret_len, 4, addr, len, T) < 0)
 		{
 			os_memset(out, 0, outlen);
+			os_memset(T, 0, SHA256_MAC_LEN);
 			return -1;
 		}
 	}
 
+	os_memset(T, 0, SHA256_MAC_LEN);
 	return 0;
 }
diff --git a/src/crypto/sha384-prf.c b/src/crypto/sha384-prf.c
new file mode 100644
index 000000000000..653920ba283d
--- /dev/null
+++ b/src/crypto/sha384-prf.c
@@ -0,0 +1,100 @@
+/*
+ * SHA384-based KDF (IEEE 802.11ac)
+ * Copyright (c) 2003-2015, Jouni Malinen <j@w1.fi>
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#include "includes.h"
+
+#include "common.h"
+#include "sha384.h"
+#include "crypto.h"
+
+
+/**
+ * sha384_prf - SHA384-based Key derivation function (IEEE 802.11ac, 11.6.1.7.2)
+ * @key: Key for KDF
+ * @key_len: Length of the key in bytes
+ * @label: A unique label for each purpose of the PRF
+ * @data: Extra data to bind into the key
+ * @data_len: Length of the data
+ * @buf: Buffer for the generated pseudo-random key
+ * @buf_len: Number of bytes of key to generate
+ *
+ * This function is used to derive new, cryptographically separate keys from a
+ * given key.
+ */
+void sha384_prf(const u8 *key, size_t key_len, const char *label,
+		const u8 *data, size_t data_len, u8 *buf, size_t buf_len)
+{
+	sha384_prf_bits(key, key_len, label, data, data_len, buf, buf_len * 8);
+}
+
+
+/**
+ * sha384_prf_bits - IEEE Std 802.11ac-2013, 11.6.1.7.2 Key derivation function
+ * @key: Key for KDF
+ * @key_len: Length of the key in bytes
+ * @label: A unique label for each purpose of the PRF
+ * @data: Extra data to bind into the key
+ * @data_len: Length of the data
+ * @buf: Buffer for the generated pseudo-random key
+ * @buf_len: Number of bits of key to generate
+ *
+ * This function is used to derive new, cryptographically separate keys from a
+ * given key. If the requested buf_len is not divisible by eight, the least
+ * significant 1-7 bits of the last octet in the output are not part of the
+ * requested output.
+ */
+void sha384_prf_bits(const u8 *key, size_t key_len, const char *label,
+		     const u8 *data, size_t data_len, u8 *buf,
+		     size_t buf_len_bits)
+{
+	u16 counter = 1;
+	size_t pos, plen;
+	u8 hash[SHA384_MAC_LEN];
+	const u8 *addr[4];
+	size_t len[4];
+	u8 counter_le[2], length_le[2];
+	size_t buf_len = (buf_len_bits + 7) / 8;
+
+	addr[0] = counter_le;
+	len[0] = 2;
+	addr[1] = (u8 *) label;
+	len[1] = os_strlen(label);
+	addr[2] = data;
+	len[2] = data_len;
+	addr[3] = length_le;
+	len[3] = sizeof(length_le);
+
+	WPA_PUT_LE16(length_le, buf_len_bits);
+	pos = 0;
+	while (pos < buf_len) {
+		plen = buf_len - pos;
+		WPA_PUT_LE16(counter_le, counter);
+		if (plen >= SHA384_MAC_LEN) {
+			hmac_sha384_vector(key, key_len, 4, addr, len,
+					   &buf[pos]);
+			pos += SHA384_MAC_LEN;
+		} else {
+			hmac_sha384_vector(key, key_len, 4, addr, len, hash);
+			os_memcpy(&buf[pos], hash, plen);
+			pos += plen;
+			break;
+		}
+		counter++;
+	}
+
+	/*
+	 * Mask out unused bits in the last octet if it does not use all the
+	 * bits.
+	 */
+	if (buf_len_bits % 8) {
+		u8 mask = 0xff << (8 - buf_len_bits % 8);
+		buf[pos - 1] &= mask;
+	}
+
+	os_memset(hash, 0, sizeof(hash));
+}
diff --git a/src/crypto/sha384.h b/src/crypto/sha384.h
index e6a1fe41e1a1..3deafa59ec29 100644
--- a/src/crypto/sha384.h
+++ b/src/crypto/sha384.h
@@ -15,5 +15,10 @@ int hmac_sha384_vector(const u8 *key, size_t key_len, size_t num_elem,
 		       const u8 *addr[], const size_t *len, u8 *mac);
 int hmac_sha384(const u8 *key, size_t key_len, const u8 *data,
 		size_t data_len, u8 *mac);
+void sha384_prf(const u8 *key, size_t key_len, const char *label,
+		const u8 *data, size_t data_len, u8 *buf, size_t buf_len);
+void sha384_prf_bits(const u8 *key, size_t key_len, const char *label,
+		     const u8 *data, size_t data_len, u8 *buf,
+		     size_t buf_len_bits);
 
 #endif /* SHA384_H */
diff --git a/src/crypto/tls.h b/src/crypto/tls.h
index 9ae95a66c9ed..2e562339cc5c 100644
--- a/src/crypto/tls.h
+++ b/src/crypto/tls.h
@@ -11,9 +11,7 @@
 
 struct tls_connection;
 
-struct tls_keys {
-	const u8 *master_key; /* TLS master secret */
-	size_t master_key_len;
+struct tls_random {
 	const u8 *client_random;
 	size_t client_random_len;
 	const u8 *server_random;
@@ -81,6 +79,7 @@ struct tls_config {
 	int fips_mode;
 	int cert_in_cb;
 	const char *openssl_ciphers;
+	unsigned int tls_session_lifetime;
 
 	void (*event_cb)(void *ctx, enum tls_event ev,
 			 union tls_event_data *data);
@@ -95,6 +94,7 @@ struct tls_config {
 #define TLS_CONN_DISABLE_TLSv1_1 BIT(5)
 #define TLS_CONN_DISABLE_TLSv1_2 BIT(6)
 #define TLS_CONN_EAP_FAST BIT(7)
+#define TLS_CONN_DISABLE_TLSv1_0 BIT(8)
 
 /**
  * struct tls_connection_params - Parameters for TLS connection
@@ -255,6 +255,7 @@ int tls_connection_established(void *tls_ctx, struct tls_connection *conn);
 int tls_connection_shutdown(void *tls_ctx, struct tls_connection *conn);
 
 enum {
+	TLS_SET_PARAMS_ENGINE_PRV_BAD_PIN = -4,
 	TLS_SET_PARAMS_ENGINE_PRV_VERIFY_FAILED = -3,
 	TLS_SET_PARAMS_ENGINE_PRV_INIT_FAILED = -2
 };
@@ -265,10 +266,12 @@ enum {
  * @conn: Connection context data from tls_connection_init()
  * @params: Connection parameters
  * Returns: 0 on success, -1 on failure,
- * TLS_SET_PARAMS_ENGINE_PRV_INIT_FAILED (-2) on possible PIN error causing
- * PKCS#11 engine failure, or
+ * TLS_SET_PARAMS_ENGINE_PRV_INIT_FAILED (-2) on error causing PKCS#11 engine
+ * failure, or
  * TLS_SET_PARAMS_ENGINE_PRV_VERIFY_FAILED (-3) on failure to verify the
- * PKCS#11 engine private key.
+ * PKCS#11 engine private key, or
+ * TLS_SET_PARAMS_ENGINE_PRV_BAD_PIN (-4) on PIN error causing PKCS#11 engine
+ * failure.
  */
 int __must_check
 tls_connection_set_params(void *tls_ctx, struct tls_connection *conn,
@@ -279,10 +282,12 @@ tls_connection_set_params(void *tls_ctx, struct tls_connection *conn,
  * @tls_ctx: TLS context data from tls_init()
  * @params: Global TLS parameters
  * Returns: 0 on success, -1 on failure,
- * TLS_SET_PARAMS_ENGINE_PRV_INIT_FAILED (-2) on possible PIN error causing
- * PKCS#11 engine failure, or
+ * TLS_SET_PARAMS_ENGINE_PRV_INIT_FAILED (-2) on error causing PKCS#11 engine
+ * failure, or
  * TLS_SET_PARAMS_ENGINE_PRV_VERIFY_FAILED (-3) on failure to verify the
- * PKCS#11 engine private key.
+ * PKCS#11 engine private key, or
+ * TLS_SET_PARAMS_ENGINE_PRV_BAD_PIN (-4) on PIN error causing PKCS#11 engine
+ * failure.
  */
 int __must_check tls_global_set_params(
 	void *tls_ctx, const struct tls_connection_params *params);
@@ -301,22 +306,28 @@ int __must_check tls_global_set_verify(void *tls_ctx, int check_crl);
  * @tls_ctx: TLS context data from tls_init()
  * @conn: Connection context data from tls_connection_init()
  * @verify_peer: 1 = verify peer certificate
+ * @flags: Connection flags (TLS_CONN_*)
+ * @session_ctx: Session caching context or %NULL to use default
+ * @session_ctx_len: Length of @session_ctx in bytes.
  * Returns: 0 on success, -1 on failure
  */
 int __must_check tls_connection_set_verify(void *tls_ctx,
 					   struct tls_connection *conn,
-					   int verify_peer);
+					   int verify_peer,
+					   unsigned int flags,
+					   const u8 *session_ctx,
+					   size_t session_ctx_len);
 
 /**
- * tls_connection_get_keys - Get master key and random data from TLS connection
+ * tls_connection_get_random - Get random data from TLS connection
  * @tls_ctx: TLS context data from tls_init()
  * @conn: Connection context data from tls_connection_init()
- * @keys: Structure of key/random data (filled on success)
+ * @data: Structure of client/server random data (filled on success)
  * Returns: 0 on success, -1 on failure
  */
-int __must_check tls_connection_get_keys(void *tls_ctx,
+int __must_check tls_connection_get_random(void *tls_ctx,
 					 struct tls_connection *conn,
-					 struct tls_keys *keys);
+					 struct tls_random *data);
 
 /**
  * tls_connection_prf - Use TLS-PRF to derive keying material
@@ -325,23 +336,22 @@ int __must_check tls_connection_get_keys(void *tls_ctx,
  * @label: Label (e.g., description of the key) for PRF
  * @server_random_first: seed is 0 = client_random|server_random,
  * 1 = server_random|client_random
+ * @skip_keyblock: Skip TLS key block from the beginning of PRF output
  * @out: Buffer for output data from TLS-PRF
  * @out_len: Length of the output buffer
  * Returns: 0 on success, -1 on failure
  *
- * This function is optional to implement if tls_connection_get_keys() provides
- * access to master secret and server/client random values. If these values are
- * not exported from the TLS library, tls_connection_prf() is required so that
- * further keying material can be derived from the master secret. If not
- * implemented, the function will still need to be defined, but it can just
- * return -1. Example implementation of this function is in tls_prf_sha1_md5()
- * when it is called with seed set to client_random|server_random (or
- * server_random|client_random).
+ * tls_connection_prf() is required so that further keying material can be
+ * derived from the master secret. Example implementation of this function is in
+ * tls_prf_sha1_md5() when it is called with seed set to
+ * client_random|server_random (or server_random|client_random). For TLSv1.2 and
+ * newer, a different PRF is needed, though.
  */
 int __must_check  tls_connection_prf(void *tls_ctx,
 				     struct tls_connection *conn,
 				     const char *label,
 				     int server_random_first,
+				     int skip_keyblock,
 				     u8 *out, size_t out_len);
 
 /**
@@ -460,6 +470,19 @@ int __must_check tls_connection_set_cipher_list(void *tls_ctx,
 						struct tls_connection *conn,
 						u8 *ciphers);
 
+/**
+ * tls_get_version - Get the current TLS version number
+ * @tls_ctx: TLS context data from tls_init()
+ * @conn: Connection context data from tls_connection_init()
+ * @buf: Buffer for returning the TLS version number
+ * @buflen: buf size
+ * Returns: 0 on success, -1 on failure
+ *
+ * Get the currently used TLS version number.
+ */
+int __must_check tls_get_version(void *tls_ctx, struct tls_connection *conn,
+				 char *buf, size_t buflen);
+
 /**
  * tls_get_cipher - Get current cipher name
  * @tls_ctx: TLS context data from tls_init()
@@ -527,23 +550,6 @@ int tls_connection_get_read_alerts(void *tls_ctx, struct tls_connection *conn);
 int tls_connection_get_write_alerts(void *tls_ctx,
 				    struct tls_connection *conn);
 
-/**
- * tls_connection_get_keyblock_size - Get TLS key_block size
- * @tls_ctx: TLS context data from tls_init()
- * @conn: Connection context data from tls_connection_init()
- * Returns: Size of the key_block for the negotiated cipher suite or -1 on
- * failure
- */
-int tls_connection_get_keyblock_size(void *tls_ctx,
-				     struct tls_connection *conn);
-
-/**
- * tls_capabilities - Get supported TLS capabilities
- * @tls_ctx: TLS context data from tls_init()
- * Returns: Bit field of supported TLS capabilities (TLS_CAPABILITY_*)
- */
-unsigned int tls_capabilities(void *tls_ctx);
-
 typedef int (*tls_session_ticket_cb)
 (void *ctx, const u8 *ticket, size_t len, const u8 *client_random,
  const u8 *server_random, u8 *master_secret);
@@ -569,4 +575,14 @@ void tls_connection_set_test_flags(struct tls_connection *conn, u32 flags);
 
 int tls_get_library_version(char *buf, size_t buf_len);
 
+void tls_connection_set_success_data(struct tls_connection *conn,
+				     struct wpabuf *data);
+
+void tls_connection_set_success_data_resumed(struct tls_connection *conn);
+
+const struct wpabuf *
+tls_connection_get_success_data(struct tls_connection *conn);
+
+void tls_connection_remove_session(struct tls_connection *conn);
+
 #endif /* TLS_H */
diff --git a/src/crypto/tls_gnutls.c b/src/crypto/tls_gnutls.c
index 65db6fcc2565..f994379b16b2 100644
--- a/src/crypto/tls_gnutls.c
+++ b/src/crypto/tls_gnutls.c
@@ -708,7 +708,8 @@ int tls_global_set_verify(void *ssl_ctx, int check_crl)
 
 
 int tls_connection_set_verify(void *ssl_ctx, struct tls_connection *conn,
-			      int verify_peer)
+			      int verify_peer, unsigned int flags,
+			      const u8 *session_ctx, size_t session_ctx_len)
 {
 	if (conn == NULL || conn->session == NULL)
 		return -1;
@@ -722,8 +723,8 @@ int tls_connection_set_verify(void *ssl_ctx, struct tls_connection *conn,
 }
 
 
-int tls_connection_get_keys(void *ssl_ctx, struct tls_connection *conn,
-			    struct tls_keys *keys)
+int tls_connection_get_random(void *ssl_ctx, struct tls_connection *conn,
+			    struct tls_random *keys)
 {
 #if GNUTLS_VERSION_NUMBER >= 0x030012
 	gnutls_datum_t client, server;
@@ -747,9 +748,9 @@ int tls_connection_get_keys(void *ssl_ctx, struct tls_connection *conn,
 
 int tls_connection_prf(void *tls_ctx, struct tls_connection *conn,
 		       const char *label, int server_random_first,
-		       u8 *out, size_t out_len)
+		       int skip_keyblock, u8 *out, size_t out_len)
 {
-	if (conn == NULL || conn->session == NULL)
+	if (conn == NULL || conn->session == NULL || skip_keyblock)
 		return -1;
 
 	return gnutls_prf(conn->session, os_strlen(label), label,
@@ -1426,6 +1427,14 @@ int tls_connection_set_cipher_list(void *tls_ctx, struct tls_connection *conn,
 }
 
 
+int tls_get_version(void *ssl_ctx, struct tls_connection *conn,
+		    char *buf, size_t buflen)
+{
+	/* TODO */
+	return -1;
+}
+
+
 int tls_get_cipher(void *ssl_ctx, struct tls_connection *conn,
 		   char *buf, size_t buflen)
 {
@@ -1476,20 +1485,6 @@ int tls_connection_get_write_alerts(void *ssl_ctx, struct tls_connection *conn)
 }
 
 
-int tls_connection_get_keyblock_size(void *tls_ctx,
-				     struct tls_connection *conn)
-{
-	/* TODO */
-	return -1;
-}
-
-
-unsigned int tls_capabilities(void *tls_ctx)
-{
-	return 0;
-}
-
-
 int tls_connection_set_session_ticket_cb(void *tls_ctx,
 					 struct tls_connection *conn,
 					 tls_session_ticket_cb cb, void *ctx)
@@ -1503,3 +1498,26 @@ int tls_get_library_version(char *buf, size_t buf_len)
 	return os_snprintf(buf, buf_len, "GnuTLS build=%s run=%s",
 			   GNUTLS_VERSION, gnutls_check_version(NULL));
 }
+
+
+void tls_connection_set_success_data(struct tls_connection *conn,
+				     struct wpabuf *data)
+{
+}
+
+
+void tls_connection_set_success_data_resumed(struct tls_connection *conn)
+{
+}
+
+
+const struct wpabuf *
+tls_connection_get_success_data(struct tls_connection *conn)
+{
+	return NULL;
+}
+
+
+void tls_connection_remove_session(struct tls_connection *conn)
+{
+}
diff --git a/src/crypto/tls_internal.c b/src/crypto/tls_internal.c
index 0c955da29f1d..704751d308fc 100644
--- a/src/crypto/tls_internal.c
+++ b/src/crypto/tls_internal.c
@@ -192,26 +192,31 @@ int tls_connection_set_params(void *tls_ctx, struct tls_connection *conn,
 
 	if (params->subject_match) {
 		wpa_printf(MSG_INFO, "TLS: subject_match not supported");
+		tlsv1_cred_free(cred);
 		return -1;
 	}
 
 	if (params->altsubject_match) {
 		wpa_printf(MSG_INFO, "TLS: altsubject_match not supported");
+		tlsv1_cred_free(cred);
 		return -1;
 	}
 
 	if (params->suffix_match) {
 		wpa_printf(MSG_INFO, "TLS: suffix_match not supported");
+		tlsv1_cred_free(cred);
 		return -1;
 	}
 
 	if (params->domain_match) {
 		wpa_printf(MSG_INFO, "TLS: domain_match not supported");
+		tlsv1_cred_free(cred);
 		return -1;
 	}
 
 	if (params->openssl_ciphers) {
-		wpa_printf(MSG_INFO, "GnuTLS: openssl_ciphers not supported");
+		wpa_printf(MSG_INFO, "TLS: openssl_ciphers not supported");
+		tlsv1_cred_free(cred);
 		return -1;
 	}
 
@@ -323,7 +328,8 @@ int tls_global_set_verify(void *tls_ctx, int check_crl)
 
 
 int tls_connection_set_verify(void *tls_ctx, struct tls_connection *conn,
-			      int verify_peer)
+			      int verify_peer, unsigned int flags,
+			      const u8 *session_ctx, size_t session_ctx_len)
 {
 #ifdef CONFIG_TLS_INTERNAL_SERVER
 	if (conn->server)
@@ -333,16 +339,30 @@ int tls_connection_set_verify(void *tls_ctx, struct tls_connection *conn,
 }
 
 
-int tls_connection_get_keys(void *tls_ctx, struct tls_connection *conn,
-			    struct tls_keys *keys)
+int tls_connection_get_random(void *tls_ctx, struct tls_connection *conn,
+			      struct tls_random *data)
 {
 #ifdef CONFIG_TLS_INTERNAL_CLIENT
 	if (conn->client)
-		return tlsv1_client_get_keys(conn->client, keys);
+		return tlsv1_client_get_random(conn->client, data);
 #endif /* CONFIG_TLS_INTERNAL_CLIENT */
 #ifdef CONFIG_TLS_INTERNAL_SERVER
 	if (conn->server)
-		return tlsv1_server_get_keys(conn->server, keys);
+		return tlsv1_server_get_random(conn->server, data);
+#endif /* CONFIG_TLS_INTERNAL_SERVER */
+	return -1;
+}
+
+
+static int tls_get_keyblock_size(struct tls_connection *conn)
+{
+#ifdef CONFIG_TLS_INTERNAL_CLIENT
+	if (conn->client)
+		return tlsv1_client_get_keyblock_size(conn->client);
+#endif /* CONFIG_TLS_INTERNAL_CLIENT */
+#ifdef CONFIG_TLS_INTERNAL_SERVER
+	if (conn->server)
+		return tlsv1_server_get_keyblock_size(conn->server);
 #endif /* CONFIG_TLS_INTERNAL_SERVER */
 	return -1;
 }
@@ -350,23 +370,41 @@ int tls_connection_get_keys(void *tls_ctx, struct tls_connection *conn,
 
 int tls_connection_prf(void *tls_ctx, struct tls_connection *conn,
 		       const char *label, int server_random_first,
-		       u8 *out, size_t out_len)
+		       int skip_keyblock, u8 *out, size_t out_len)
 {
+	int ret = -1, skip = 0;
+	u8 *tmp_out = NULL;
+	u8 *_out = out;
+
+	if (skip_keyblock) {
+		skip = tls_get_keyblock_size(conn);
+		if (skip < 0)
+			return -1;
+		tmp_out = os_malloc(skip + out_len);
+		if (!tmp_out)
+			return -1;
+		_out = tmp_out;
+	}
+
 #ifdef CONFIG_TLS_INTERNAL_CLIENT
 	if (conn->client) {
-		return tlsv1_client_prf(conn->client, label,
-					server_random_first,
-					out, out_len);
+		ret = tlsv1_client_prf(conn->client, label,
+				       server_random_first,
+				       _out, out_len);
 	}
 #endif /* CONFIG_TLS_INTERNAL_CLIENT */
 #ifdef CONFIG_TLS_INTERNAL_SERVER
 	if (conn->server) {
-		return tlsv1_server_prf(conn->server, label,
-					server_random_first,
-					out, out_len);
+		ret = tlsv1_server_prf(conn->server, label,
+				       server_random_first,
+				       _out, out_len);
 	}
 #endif /* CONFIG_TLS_INTERNAL_SERVER */
-	return -1;
+	if (ret == 0 && skip_keyblock)
+		os_memcpy(out, _out + skip, out_len);
+	bin_clear_free(tmp_out, skip);
+
+	return ret;
 }
 
 
@@ -580,6 +618,14 @@ int tls_connection_set_cipher_list(void *tls_ctx, struct tls_connection *conn,
 }
 
 
+int tls_get_version(void *ssl_ctx, struct tls_connection *conn,
+		    char *buf, size_t buflen)
+{
+	/* TODO */
+	return -1;
+}
+
+
 int tls_get_cipher(void *tls_ctx, struct tls_connection *conn,
 		   char *buf, size_t buflen)
 {
@@ -637,27 +683,6 @@ int tls_connection_get_write_alerts(void *tls_ctx,
 }
 
 
-int tls_connection_get_keyblock_size(void *tls_ctx,
-				     struct tls_connection *conn)
-{
-#ifdef CONFIG_TLS_INTERNAL_CLIENT
-	if (conn->client)
-		return tlsv1_client_get_keyblock_size(conn->client);
-#endif /* CONFIG_TLS_INTERNAL_CLIENT */
-#ifdef CONFIG_TLS_INTERNAL_SERVER
-	if (conn->server)
-		return tlsv1_server_get_keyblock_size(conn->server);
-#endif /* CONFIG_TLS_INTERNAL_SERVER */
-	return -1;
-}
-
-
-unsigned int tls_capabilities(void *tls_ctx)
-{
-	return 0;
-}
-
-
 int tls_connection_set_session_ticket_cb(void *tls_ctx,
 					 struct tls_connection *conn,
 					 tls_session_ticket_cb cb,
@@ -683,3 +708,26 @@ int tls_get_library_version(char *buf, size_t buf_len)
 {
 	return os_snprintf(buf, buf_len, "internal");
 }
+
+
+void tls_connection_set_success_data(struct tls_connection *conn,
+				     struct wpabuf *data)
+{
+}
+
+
+void tls_connection_set_success_data_resumed(struct tls_connection *conn)
+{
+}
+
+
+const struct wpabuf *
+tls_connection_get_success_data(struct tls_connection *conn)
+{
+	return NULL;
+}
+
+
+void tls_connection_remove_session(struct tls_connection *conn)
+{
+}
diff --git a/src/crypto/tls_none.c b/src/crypto/tls_none.c
index a6d210afcf0f..ae392ad8aa0f 100644
--- a/src/crypto/tls_none.c
+++ b/src/crypto/tls_none.c
@@ -72,14 +72,15 @@ int tls_global_set_verify(void *tls_ctx, int check_crl)
 
 
 int tls_connection_set_verify(void *tls_ctx, struct tls_connection *conn,
-			      int verify_peer)
+			      int verify_peer, unsigned int flags,
+			      const u8 *session_ctx, size_t session_ctx_len)
 {
 	return -1;
 }
 
 
-int tls_connection_get_keys(void *tls_ctx, struct tls_connection *conn,
-			    struct tls_keys *keys)
+int tls_connection_get_random(void *tls_ctx, struct tls_connection *conn,
+			      struct tls_random *data)
 {
 	return -1;
 }
@@ -87,7 +88,7 @@ int tls_connection_get_keys(void *tls_ctx, struct tls_connection *conn,
 
 int tls_connection_prf(void *tls_ctx, struct tls_connection *conn,
 		       const char *label, int server_random_first,
-		       u8 *out, size_t out_len)
+		       int skip_keyblock, u8 *out, size_t out_len)
 {
 	return -1;
 }
@@ -140,6 +141,13 @@ int tls_connection_set_cipher_list(void *tls_ctx, struct tls_connection *conn,
 }
 
 
+int tls_get_version(void *ssl_ctx, struct tls_connection *conn,
+		    char *buf, size_t buflen)
+{
+	return -1;
+}
+
+
 int tls_get_cipher(void *tls_ctx, struct tls_connection *conn,
 		   char *buf, size_t buflen)
 {
@@ -181,20 +189,30 @@ int tls_connection_get_write_alerts(void *tls_ctx,
 }
 
 
-int tls_connection_get_keyblock_size(void *tls_ctx,
-				     struct tls_connection *conn)
-{
-	return -1;
-}
-
-
-unsigned int tls_capabilities(void *tls_ctx)
-{
-	return 0;
-}
-
-
 int tls_get_library_version(char *buf, size_t buf_len)
 {
 	return os_snprintf(buf, buf_len, "none");
 }
+
+
+void tls_connection_set_success_data(struct tls_connection *conn,
+				     struct wpabuf *data)
+{
+}
+
+
+void tls_connection_set_success_data_resumed(struct tls_connection *conn)
+{
+}
+
+
+const struct wpabuf *
+tls_connection_get_success_data(struct tls_connection *conn)
+{
+	return NULL;
+}
+
+
+void tls_connection_remove_session(struct tls_connection *conn)
+{
+}
diff --git a/src/crypto/tls_openssl.c b/src/crypto/tls_openssl.c
index 52db8fc076ac..8b7b47bc256d 100644
--- a/src/crypto/tls_openssl.c
+++ b/src/crypto/tls_openssl.c
@@ -1,6 +1,6 @@
 /*
  * SSL/TLS interface functions for OpenSSL
- * Copyright (c) 2004-2013, Jouni Malinen <j@w1.fi>
+ * Copyright (c) 2004-2015, Jouni Malinen <j@w1.fi>
  *
  * This software may be distributed under the terms of the BSD license.
  * See README for more details.
@@ -23,15 +23,19 @@
 #ifndef OPENSSL_NO_ENGINE
 #include <openssl/engine.h>
 #endif /* OPENSSL_NO_ENGINE */
+#ifndef OPENSSL_NO_DSA
+#include <openssl/dsa.h>
+#endif
+#ifndef OPENSSL_NO_DH
+#include <openssl/dh.h>
+#endif
 
 #include "common.h"
 #include "crypto.h"
+#include "sha1.h"
+#include "sha256.h"
 #include "tls.h"
 
-#if defined(SSL_CTX_get_app_data) && defined(SSL_CTX_set_app_data)
-#define OPENSSL_SUPPORTS_CTX_APP_DATA
-#endif
-
 #if OPENSSL_VERSION_NUMBER < 0x10000000L
 /* ERR_remove_thread_state replaces ERR_remove_state and the latter is
  * deprecated. However, OpenSSL 0.9.8 doesn't include
@@ -70,6 +74,7 @@ static BIO * BIO_from_keystore(const char *key)
 #endif /* ANDROID */
 
 static int tls_openssl_ref_count = 0;
+static int tls_ex_idx_session = -1;
 
 struct tls_context {
 	void (*event_cb)(void *ctx, enum tls_event ev,
@@ -82,6 +87,11 @@ struct tls_context {
 static struct tls_context *tls_global = NULL;
 
 
+struct tls_data {
+	SSL_CTX *ssl;
+	unsigned int tls_session_lifetime;
+};
+
 struct tls_connection {
 	struct tls_context *context;
 	SSL_CTX *ssl_ctx;
@@ -105,6 +115,7 @@ struct tls_connection {
 	unsigned int cert_probe:1;
 	unsigned int server_cert_only:1;
 	unsigned int invalid_hb_used:1;
+	unsigned int success_data:1;
 
 	u8 srv_cert_hash[32];
 
@@ -113,6 +124,11 @@ struct tls_connection {
 	X509 *peer_cert;
 	X509 *peer_issuer;
 	X509 *peer_issuer_issuer;
+
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+	unsigned char client_random[SSL3_RANDOM_SIZE];
+	unsigned char server_random[SSL3_RANDOM_SIZE];
+#endif
 };
 
 
@@ -735,8 +751,27 @@ static int tls_engine_load_dynamic_opensc(const char *opensc_so_path)
 #endif /* OPENSSL_NO_ENGINE */
 
 
+static void remove_session_cb(SSL_CTX *ctx, SSL_SESSION *sess)
+{
+	struct wpabuf *buf;
+
+	if (tls_ex_idx_session < 0)
+		return;
+	buf = SSL_SESSION_get_ex_data(sess, tls_ex_idx_session);
+	if (!buf)
+		return;
+	wpa_printf(MSG_DEBUG,
+		   "OpenSSL: Free application session data %p (sess %p)",
+		   buf, sess);
+	wpabuf_free(buf);
+
+	SSL_SESSION_set_ex_data(sess, tls_ex_idx_session, NULL);
+}
+
+
 void * tls_init(const struct tls_config *conf)
 {
+	struct tls_data *data;
 	SSL_CTX *ssl;
 	struct tls_context *context;
 	const char *ciphers;
@@ -748,7 +783,9 @@ void * tls_init(const struct tls_config *conf)
 #ifdef CONFIG_FIPS
 #ifdef OPENSSL_FIPS
 		if (conf && conf->fips_mode) {
-			if (!FIPS_mode_set(1)) {
+			static int fips_enabled = 0;
+
+			if (!fips_enabled && !FIPS_mode_set(1)) {
 				wpa_printf(MSG_ERROR, "Failed to enable FIPS "
 					   "mode");
 				ERR_load_crypto_strings();
@@ -756,8 +793,10 @@ void * tls_init(const struct tls_config *conf)
 				os_free(tls_global);
 				tls_global = NULL;
 				return NULL;
-			} else
+			} else {
 				wpa_printf(MSG_INFO, "Running in FIPS mode");
+				fips_enabled = 1;
+			}
 		}
 #else /* OPENSSL_FIPS */
 		if (conf && conf->fips_mode) {
@@ -791,38 +830,58 @@ void * tls_init(const struct tls_config *conf)
 		PKCS12_PBE_add();
 #endif  /* PKCS12_FUNCS */
 	} else {
-#ifdef OPENSSL_SUPPORTS_CTX_APP_DATA
-		/* Newer OpenSSL can store app-data per-SSL */
 		context = tls_context_new(conf);
 		if (context == NULL)
 			return NULL;
-#else /* OPENSSL_SUPPORTS_CTX_APP_DATA */
-		context = tls_global;
-#endif /* OPENSSL_SUPPORTS_CTX_APP_DATA */
 	}
 	tls_openssl_ref_count++;
 
-	ssl = SSL_CTX_new(SSLv23_method());
+	data = os_zalloc(sizeof(*data));
+	if (data)
+		ssl = SSL_CTX_new(SSLv23_method());
+	else
+		ssl = NULL;
 	if (ssl == NULL) {
 		tls_openssl_ref_count--;
-#ifdef OPENSSL_SUPPORTS_CTX_APP_DATA
 		if (context != tls_global)
 			os_free(context);
-#endif /* OPENSSL_SUPPORTS_CTX_APP_DATA */
 		if (tls_openssl_ref_count == 0) {
 			os_free(tls_global);
 			tls_global = NULL;
 		}
 		return NULL;
 	}
+	data->ssl = ssl;
+	if (conf)
+		data->tls_session_lifetime = conf->tls_session_lifetime;
 
 	SSL_CTX_set_options(ssl, SSL_OP_NO_SSLv2);
 	SSL_CTX_set_options(ssl, SSL_OP_NO_SSLv3);
 
 	SSL_CTX_set_info_callback(ssl, ssl_info_cb);
-#ifdef OPENSSL_SUPPORTS_CTX_APP_DATA
 	SSL_CTX_set_app_data(ssl, context);
-#endif /* OPENSSL_SUPPORTS_CTX_APP_DATA */
+	if (data->tls_session_lifetime > 0) {
+		SSL_CTX_set_quiet_shutdown(ssl, 1);
+		/*
+		 * Set default context here. In practice, this will be replaced
+		 * by the per-EAP method context in tls_connection_set_verify().
+		 */
+		SSL_CTX_set_session_id_context(ssl, (u8 *) "hostapd", 7);
+		SSL_CTX_set_session_cache_mode(ssl, SSL_SESS_CACHE_SERVER);
+		SSL_CTX_set_timeout(ssl, data->tls_session_lifetime);
+		SSL_CTX_sess_set_remove_cb(ssl, remove_session_cb);
+	} else {
+		SSL_CTX_set_session_cache_mode(ssl, SSL_SESS_CACHE_OFF);
+	}
+
+	if (tls_ex_idx_session < 0) {
+		tls_ex_idx_session = SSL_SESSION_get_ex_new_index(
+			0, NULL, NULL, NULL, NULL);
+		if (tls_ex_idx_session < 0) {
+			tls_deinit(data);
+			return NULL;
+		}
+	}
 
 #ifndef OPENSSL_NO_ENGINE
 	wpa_printf(MSG_DEBUG, "ENGINE: Loading dynamic engine");
@@ -835,7 +894,7 @@ void * tls_init(const struct tls_config *conf)
 		if (tls_engine_load_dynamic_opensc(conf->opensc_engine_path) ||
 		    tls_engine_load_dynamic_pkcs11(conf->pkcs11_engine_path,
 						   conf->pkcs11_module_path)) {
-			tls_deinit(ssl);
+			tls_deinit(data);
 			return NULL;
 		}
 	}
@@ -849,22 +908,23 @@ void * tls_init(const struct tls_config *conf)
 		wpa_printf(MSG_ERROR,
 			   "OpenSSL: Failed to set cipher string '%s'",
 			   ciphers);
-		tls_deinit(ssl);
+		tls_deinit(data);
 		return NULL;
 	}
 
-	return ssl;
+	return data;
 }
 
 
 void tls_deinit(void *ssl_ctx)
 {
-	SSL_CTX *ssl = ssl_ctx;
-#ifdef OPENSSL_SUPPORTS_CTX_APP_DATA
+	struct tls_data *data = ssl_ctx;
+	SSL_CTX *ssl = data->ssl;
 	struct tls_context *context = SSL_CTX_get_app_data(ssl);
 	if (context != tls_global)
 		os_free(context);
-#endif /* OPENSSL_SUPPORTS_CTX_APP_DATA */
+	if (data->tls_session_lifetime > 0)
+		SSL_CTX_flush_sessions(ssl, 0);
 	SSL_CTX_free(ssl);
 
 	tls_openssl_ref_count--;
@@ -881,9 +941,32 @@ void tls_deinit(void *ssl_ctx)
 		os_free(tls_global);
 		tls_global = NULL;
 	}
+
+	os_free(data);
 }
 
 
+#ifndef OPENSSL_NO_ENGINE
+
+/* Cryptoki return values */
+#define CKR_PIN_INCORRECT 0x000000a0
+#define CKR_PIN_INVALID 0x000000a1
+#define CKR_PIN_LEN_RANGE 0x000000a2
+
+/* libp11 */
+#define ERR_LIB_PKCS11	ERR_LIB_USER
+
+static int tls_is_pin_error(unsigned int err)
+{
+	return ERR_GET_LIB(err) == ERR_LIB_PKCS11 &&
+		(ERR_GET_REASON(err) == CKR_PIN_INCORRECT ||
+		 ERR_GET_REASON(err) == CKR_PIN_INVALID ||
+		 ERR_GET_REASON(err) == CKR_PIN_LEN_RANGE);
+}
+
+#endif /* OPENSSL_NO_ENGINE */
+
+
 static int tls_engine_init(struct tls_connection *conn, const char *engine_id,
 			   const char *pin, const char *key_id,
 			   const char *cert_id, const char *ca_cert_id)
@@ -935,11 +1018,16 @@ static int tls_engine_init(struct tls_connection *conn, const char *engine_id,
 							    key_id, NULL,
 							    &key_cb);
 		if (!conn->private_key) {
+			unsigned long err = ERR_get_error();
+
 			wpa_printf(MSG_ERROR,
 				   "ENGINE: cannot load private key with id '%s' [%s]",
 				   key_id,
-				   ERR_error_string(ERR_get_error(), NULL));
-			ret = TLS_SET_PARAMS_ENGINE_PRV_INIT_FAILED;
+				   ERR_error_string(err, NULL));
+			if (tls_is_pin_error(err))
+				ret = TLS_SET_PARAMS_ENGINE_PRV_BAD_PIN;
+			else
+				ret = TLS_SET_PARAMS_ENGINE_PRV_INIT_FAILED;
 			goto err;
 		}
 	}
@@ -1030,19 +1118,16 @@ static void tls_msg_cb(int write_p, int version, int content_type,
 
 struct tls_connection * tls_connection_init(void *ssl_ctx)
 {
-	SSL_CTX *ssl = ssl_ctx;
+	struct tls_data *data = ssl_ctx;
+	SSL_CTX *ssl = data->ssl;
 	struct tls_connection *conn;
 	long options;
-#ifdef OPENSSL_SUPPORTS_CTX_APP_DATA
 	struct tls_context *context = SSL_CTX_get_app_data(ssl);
-#else /* OPENSSL_SUPPORTS_CTX_APP_DATA */
-	struct tls_context *context = tls_global;
-#endif /* OPENSSL_SUPPORTS_CTX_APP_DATA */
 
 	conn = os_zalloc(sizeof(*conn));
 	if (conn == NULL)
 		return NULL;
-	conn->ssl_ctx = ssl_ctx;
+	conn->ssl_ctx = ssl;
 	conn->ssl = SSL_new(ssl);
 	if (conn->ssl == NULL) {
 		tls_show_errors(MSG_INFO, __func__,
@@ -1091,6 +1176,14 @@ void tls_connection_deinit(void *ssl_ctx, struct tls_connection *conn)
 {
 	if (conn == NULL)
 		return;
+	if (conn->success_data) {
+		/*
+		 * Make sure ssl_clear_bad_session() does not remove this
+		 * session.
+		 */
+		SSL_set_quiet_shutdown(conn->ssl, 1);
+		SSL_shutdown(conn->ssl);
+	}
 	SSL_free(conn->ssl);
 	tls_engine_deinit(conn);
 	os_free(conn->subject_match);
@@ -1118,7 +1211,7 @@ int tls_connection_shutdown(void *ssl_ctx, struct tls_connection *conn)
 	 * and "close notify" shutdown alert would confuse AS. */
 	SSL_set_quiet_shutdown(conn->ssl, 1);
 	SSL_shutdown(conn->ssl);
-	return 0;
+	return SSL_clear(conn->ssl) == 1 ? 0 : -1;
 }
 
 
@@ -1617,9 +1710,9 @@ static int tls_verify_cb(int preverify_ok, X509_STORE_CTX *x509_ctx)
 
 
 #ifndef OPENSSL_NO_STDIO
-static int tls_load_ca_der(void *_ssl_ctx, const char *ca_cert)
+static int tls_load_ca_der(struct tls_data *data, const char *ca_cert)
 {
-	SSL_CTX *ssl_ctx = _ssl_ctx;
+	SSL_CTX *ssl_ctx = data->ssl;
 	X509_LOOKUP *lookup;
 	int ret = 0;
 
@@ -1649,11 +1742,12 @@ static int tls_load_ca_der(void *_ssl_ctx, const char *ca_cert)
 #endif /* OPENSSL_NO_STDIO */
 
 
-static int tls_connection_ca_cert(void *_ssl_ctx, struct tls_connection *conn,
+static int tls_connection_ca_cert(struct tls_data *data,
+				  struct tls_connection *conn,
 				  const char *ca_cert, const u8 *ca_cert_blob,
 				  size_t ca_cert_blob_len, const char *ca_path)
 {
-	SSL_CTX *ssl_ctx = _ssl_ctx;
+	SSL_CTX *ssl_ctx = data->ssl;
 	X509_STORE *store;
 
 	/*
@@ -1788,7 +1882,7 @@ static int tls_connection_ca_cert(void *_ssl_ctx, struct tls_connection *conn,
 			tls_show_errors(MSG_WARNING, __func__,
 					"Failed to load root certificates");
 			if (ca_cert &&
-			    tls_load_ca_der(ssl_ctx, ca_cert) == 0) {
+			    tls_load_ca_der(data, ca_cert) == 0) {
 				wpa_printf(MSG_DEBUG, "OpenSSL: %s - loaded "
 					   "DER format CA certificate",
 					   __func__);
@@ -1797,7 +1891,7 @@ static int tls_connection_ca_cert(void *_ssl_ctx, struct tls_connection *conn,
 		} else {
 			wpa_printf(MSG_DEBUG, "TLS: Trusted root "
 				   "certificate(s) loaded");
-			tls_get_errors(ssl_ctx);
+			tls_get_errors(data);
 		}
 #else /* OPENSSL_NO_STDIO */
 		wpa_printf(MSG_DEBUG, "OpenSSL: %s - OPENSSL_NO_STDIO",
@@ -1814,8 +1908,10 @@ static int tls_connection_ca_cert(void *_ssl_ctx, struct tls_connection *conn,
 }
 
 
-static int tls_global_ca_cert(SSL_CTX *ssl_ctx, const char *ca_cert)
+static int tls_global_ca_cert(struct tls_data *data, const char *ca_cert)
 {
+	SSL_CTX *ssl_ctx = data->ssl;
+
 	if (ca_cert) {
 		if (SSL_CTX_load_verify_locations(ssl_ctx, ca_cert, NULL) != 1)
 		{
@@ -1843,7 +1939,8 @@ int tls_global_set_verify(void *ssl_ctx, int check_crl)
 	int flags;
 
 	if (check_crl) {
-		X509_STORE *cs = SSL_CTX_get_cert_store(ssl_ctx);
+		struct tls_data *data = ssl_ctx;
+		X509_STORE *cs = SSL_CTX_get_cert_store(data->ssl);
 		if (cs == NULL) {
 			tls_show_errors(MSG_INFO, __func__, "Failed to get "
 					"certificate store when enabling "
@@ -1901,10 +1998,44 @@ static int tls_connection_set_subject_match(struct tls_connection *conn,
 }
 
 
+static void tls_set_conn_flags(SSL *ssl, unsigned int flags)
+{
+#ifdef SSL_OP_NO_TICKET
+	if (flags & TLS_CONN_DISABLE_SESSION_TICKET)
+		SSL_set_options(ssl, SSL_OP_NO_TICKET);
+#ifdef SSL_clear_options
+	else
+		SSL_clear_options(ssl, SSL_OP_NO_TICKET);
+#endif /* SSL_clear_options */
+#endif /* SSL_OP_NO_TICKET */
+
+#ifdef SSL_OP_NO_TLSv1
+	if (flags & TLS_CONN_DISABLE_TLSv1_0)
+		SSL_set_options(ssl, SSL_OP_NO_TLSv1);
+	else
+		SSL_clear_options(ssl, SSL_OP_NO_TLSv1);
+#endif /* SSL_OP_NO_TLSv1 */
+#ifdef SSL_OP_NO_TLSv1_1
+	if (flags & TLS_CONN_DISABLE_TLSv1_1)
+		SSL_set_options(ssl, SSL_OP_NO_TLSv1_1);
+	else
+		SSL_clear_options(ssl, SSL_OP_NO_TLSv1_1);
+#endif /* SSL_OP_NO_TLSv1_1 */
+#ifdef SSL_OP_NO_TLSv1_2
+	if (flags & TLS_CONN_DISABLE_TLSv1_2)
+		SSL_set_options(ssl, SSL_OP_NO_TLSv1_2);
+	else
+		SSL_clear_options(ssl, SSL_OP_NO_TLSv1_2);
+#endif /* SSL_OP_NO_TLSv1_2 */
+}
+
+
 int tls_connection_set_verify(void *ssl_ctx, struct tls_connection *conn,
-			      int verify_peer)
+			      int verify_peer, unsigned int flags,
+			      const u8 *session_ctx, size_t session_ctx_len)
 {
 	static int counter = 0;
+	struct tls_data *data = ssl_ctx;
 
 	if (conn == NULL)
 		return -1;
@@ -1919,20 +2050,25 @@ int tls_connection_set_verify(void *ssl_ctx, struct tls_connection *conn,
 		SSL_set_verify(conn->ssl, SSL_VERIFY_NONE, NULL);
 	}
 
+	tls_set_conn_flags(conn->ssl, flags);
+	conn->flags = flags;
+
 	SSL_set_accept_state(conn->ssl);
 
-	/*
-	 * Set session id context in order to avoid fatal errors when client
-	 * tries to resume a session. However, set the context to a unique
-	 * value in order to effectively disable session resumption for now
-	 * since not all areas of the server code are ready for it (e.g.,
-	 * EAP-TTLS needs special handling for Phase 2 after abbreviated TLS
-	 * handshake).
-	 */
-	counter++;
-	SSL_set_session_id_context(conn->ssl,
-				   (const unsigned char *) &counter,
-				   sizeof(counter));
+	if (data->tls_session_lifetime == 0) {
+		/*
+		 * Set session id context to a unique value to make sure
+		 * session resumption cannot be used either through session
+		 * caching or TLS ticket extension.
+		 */
+		counter++;
+		SSL_set_session_id_context(conn->ssl,
+					   (const unsigned char *) &counter,
+					   sizeof(counter));
+	} else if (session_ctx) {
+		SSL_set_session_id_context(conn->ssl, session_ctx,
+					   session_ctx_len);
+	}
 
 	return 0;
 }
@@ -2004,9 +2140,12 @@ static int tls_connection_client_cert(struct tls_connection *conn,
 }
 
 
-static int tls_global_client_cert(SSL_CTX *ssl_ctx, const char *client_cert)
+static int tls_global_client_cert(struct tls_data *data,
+				  const char *client_cert)
 {
 #ifndef OPENSSL_NO_STDIO
+	SSL_CTX *ssl_ctx = data->ssl;
+
 	if (client_cert == NULL)
 		return 0;
 
@@ -2040,7 +2179,7 @@ static int tls_passwd_cb(char *buf, int size, int rwflag, void *password)
 
 
 #ifdef PKCS12_FUNCS
-static int tls_parse_pkcs12(SSL_CTX *ssl_ctx, SSL *ssl, PKCS12 *p12,
+static int tls_parse_pkcs12(struct tls_data *data, SSL *ssl, PKCS12 *p12,
 			    const char *passwd)
 {
 	EVP_PKEY *pkey;
@@ -2052,6 +2191,8 @@ static int tls_parse_pkcs12(SSL_CTX *ssl_ctx, SSL *ssl, PKCS12 *p12,
 	pkey = NULL;
 	cert = NULL;
 	certs = NULL;
+	if (!passwd)
+		passwd = "";
 	if (!PKCS12_parse(p12, passwd, &pkey, &cert, &certs)) {
 		tls_show_errors(MSG_DEBUG, __func__,
 				"Failed to parse PKCS12 file");
@@ -2069,7 +2210,7 @@ static int tls_parse_pkcs12(SSL_CTX *ssl_ctx, SSL *ssl, PKCS12 *p12,
 			if (SSL_use_certificate(ssl, cert) != 1)
 				res = -1;
 		} else {
-			if (SSL_CTX_use_certificate(ssl_ctx, cert) != 1)
+			if (SSL_CTX_use_certificate(data->ssl, cert) != 1)
 				res = -1;
 		}
 		X509_free(cert);
@@ -2081,13 +2222,52 @@ static int tls_parse_pkcs12(SSL_CTX *ssl_ctx, SSL *ssl, PKCS12 *p12,
 			if (SSL_use_PrivateKey(ssl, pkey) != 1)
 				res = -1;
 		} else {
-			if (SSL_CTX_use_PrivateKey(ssl_ctx, pkey) != 1)
+			if (SSL_CTX_use_PrivateKey(data->ssl, pkey) != 1)
 				res = -1;
 		}
 		EVP_PKEY_free(pkey);
 	}
 
 	if (certs) {
+#if OPENSSL_VERSION_NUMBER >= 0x10002000L
+		SSL_clear_chain_certs(ssl);
+		while ((cert = sk_X509_pop(certs)) != NULL) {
+			X509_NAME_oneline(X509_get_subject_name(cert), buf,
+					  sizeof(buf));
+			wpa_printf(MSG_DEBUG, "TLS: additional certificate"
+				   " from PKCS12: subject='%s'", buf);
+			if (SSL_add1_chain_cert(ssl, cert) != 1) {
+				tls_show_errors(MSG_DEBUG, __func__,
+						"Failed to add additional certificate");
+				res = -1;
+				break;
+			}
+		}
+		if (!res) {
+			/* Try to continue anyway */
+		}
+		sk_X509_free(certs);
+#ifndef OPENSSL_IS_BORINGSSL
+		res = SSL_build_cert_chain(ssl,
+					   SSL_BUILD_CHAIN_FLAG_CHECK |
+					   SSL_BUILD_CHAIN_FLAG_IGNORE_ERROR);
+		if (!res) {
+			tls_show_errors(MSG_DEBUG, __func__,
+					"Failed to build certificate chain");
+		} else if (res == 2) {
+			wpa_printf(MSG_DEBUG,
+				   "TLS: Ignore certificate chain verification error when building chain with PKCS#12 extra certificates");
+		}
+#endif /* OPENSSL_IS_BORINGSSL */
+		/*
+		 * Try to continue regardless of result since it is possible for
+		 * the extra certificates not to be required.
+		 */
+		res = 0;
+#else /* OPENSSL_VERSION_NUMBER >= 0x10002000L */
+#if OPENSSL_VERSION_NUMBER >= 0x10001000L
+		SSL_CTX_clear_extra_chain_certs(data->ssl);
+#endif /* OPENSSL_VERSION_NUMBER >= 0x10001000L */
 		while ((cert = sk_X509_pop(certs)) != NULL) {
 			X509_NAME_oneline(X509_get_subject_name(cert), buf,
 					  sizeof(buf));
@@ -2097,26 +2277,28 @@ static int tls_parse_pkcs12(SSL_CTX *ssl_ctx, SSL *ssl, PKCS12 *p12,
 			 * There is no SSL equivalent for the chain cert - so
 			 * always add it to the context...
 			 */
-			if (SSL_CTX_add_extra_chain_cert(ssl_ctx, cert) != 1) {
+			if (SSL_CTX_add_extra_chain_cert(data->ssl, cert) != 1)
+			{
 				res = -1;
 				break;
 			}
 		}
 		sk_X509_free(certs);
+#endif /* OPENSSL_VERSION_NUMBER >= 0x10002000L */
 	}
 
 	PKCS12_free(p12);
 
 	if (res < 0)
-		tls_get_errors(ssl_ctx);
+		tls_get_errors(data);
 
 	return res;
 }
 #endif  /* PKCS12_FUNCS */
 
 
-static int tls_read_pkcs12(SSL_CTX *ssl_ctx, SSL *ssl, const char *private_key,
-			   const char *passwd)
+static int tls_read_pkcs12(struct tls_data *data, SSL *ssl,
+			   const char *private_key, const char *passwd)
 {
 #ifdef PKCS12_FUNCS
 	FILE *f;
@@ -2135,7 +2317,7 @@ static int tls_read_pkcs12(SSL_CTX *ssl_ctx, SSL *ssl, const char *private_key,
 		return -1;
 	}
 
-	return tls_parse_pkcs12(ssl_ctx, ssl, p12, passwd);
+	return tls_parse_pkcs12(data, ssl, p12, passwd);
 
 #else /* PKCS12_FUNCS */
 	wpa_printf(MSG_INFO, "TLS: PKCS12 support disabled - cannot read "
@@ -2145,7 +2327,7 @@ static int tls_read_pkcs12(SSL_CTX *ssl_ctx, SSL *ssl, const char *private_key,
 }
 
 
-static int tls_read_pkcs12_blob(SSL_CTX *ssl_ctx, SSL *ssl,
+static int tls_read_pkcs12_blob(struct tls_data *data, SSL *ssl,
 				const u8 *blob, size_t len, const char *passwd)
 {
 #ifdef PKCS12_FUNCS
@@ -2158,7 +2340,7 @@ static int tls_read_pkcs12_blob(SSL_CTX *ssl_ctx, SSL *ssl,
 		return -1;
 	}
 
-	return tls_parse_pkcs12(ssl_ctx, ssl, p12, passwd);
+	return tls_parse_pkcs12(data, ssl, p12, passwd);
 
 #else /* PKCS12_FUNCS */
 	wpa_printf(MSG_INFO, "TLS: PKCS12 support disabled - cannot parse "
@@ -2183,9 +2365,13 @@ static int tls_engine_get_cert(struct tls_connection *conn,
 
 	if (!ENGINE_ctrl_cmd(conn->engine, "LOAD_CERT_CTRL",
 			     0, &params, NULL, 1)) {
+		unsigned long err = ERR_get_error();
+
 		wpa_printf(MSG_ERROR, "ENGINE: cannot load client cert with id"
 			   " '%s' [%s]", cert_id,
-			   ERR_error_string(ERR_get_error(), NULL));
+			   ERR_error_string(err, NULL));
+		if (tls_is_pin_error(err))
+			return TLS_SET_PARAMS_ENGINE_PRV_BAD_PIN;
 		return TLS_SET_PARAMS_ENGINE_PRV_INIT_FAILED;
 	}
 	if (!params.cert) {
@@ -2225,13 +2411,13 @@ static int tls_connection_engine_client_cert(struct tls_connection *conn,
 }
 
 
-static int tls_connection_engine_ca_cert(void *_ssl_ctx,
+static int tls_connection_engine_ca_cert(struct tls_data *data,
 					 struct tls_connection *conn,
 					 const char *ca_cert_id)
 {
 #ifndef OPENSSL_NO_ENGINE
 	X509 *cert;
-	SSL_CTX *ssl_ctx = _ssl_ctx;
+	SSL_CTX *ssl_ctx = data->ssl;
 	X509_STORE *store;
 
 	if (tls_engine_get_cert(conn, ca_cert_id, &cert))
@@ -2297,14 +2483,14 @@ static int tls_connection_engine_private_key(struct tls_connection *conn)
 }
 
 
-static int tls_connection_private_key(void *_ssl_ctx,
+static int tls_connection_private_key(struct tls_data *data,
 				      struct tls_connection *conn,
 				      const char *private_key,
 				      const char *private_key_passwd,
 				      const u8 *private_key_blob,
 				      size_t private_key_blob_len)
 {
-	SSL_CTX *ssl_ctx = _ssl_ctx;
+	SSL_CTX *ssl_ctx = data->ssl;
 	char *passwd;
 	int ok;
 
@@ -2350,7 +2536,7 @@ static int tls_connection_private_key(void *_ssl_ctx,
 			break;
 		}
 
-		if (tls_read_pkcs12_blob(ssl_ctx, conn->ssl, private_key_blob,
+		if (tls_read_pkcs12_blob(data, conn->ssl, private_key_blob,
 					 private_key_blob_len, passwd) == 0) {
 			wpa_printf(MSG_DEBUG, "OpenSSL: PKCS#12 as blob --> "
 				   "OK");
@@ -2383,7 +2569,7 @@ static int tls_connection_private_key(void *_ssl_ctx,
 			   __func__);
 #endif /* OPENSSL_NO_STDIO */
 
-		if (tls_read_pkcs12(ssl_ctx, conn->ssl, private_key, passwd)
+		if (tls_read_pkcs12(data, conn->ssl, private_key, passwd)
 		    == 0) {
 			wpa_printf(MSG_DEBUG, "OpenSSL: Reading PKCS#12 file "
 				   "--> OK");
@@ -2422,9 +2608,11 @@ static int tls_connection_private_key(void *_ssl_ctx,
 }
 
 
-static int tls_global_private_key(SSL_CTX *ssl_ctx, const char *private_key,
+static int tls_global_private_key(struct tls_data *data,
+				  const char *private_key,
 				  const char *private_key_passwd)
 {
+	SSL_CTX *ssl_ctx = data->ssl;
 	char *passwd;
 
 	if (private_key == NULL)
@@ -2446,7 +2634,7 @@ static int tls_global_private_key(SSL_CTX *ssl_ctx, const char *private_key,
 	    SSL_CTX_use_PrivateKey_file(ssl_ctx, private_key,
 					SSL_FILETYPE_PEM) != 1 &&
 #endif /* OPENSSL_NO_STDIO */
-	    tls_read_pkcs12(ssl_ctx, NULL, private_key, passwd)) {
+	    tls_read_pkcs12(data, NULL, private_key, passwd)) {
 		tls_show_errors(MSG_INFO, __func__,
 				"Failed to load private key");
 		os_free(passwd);
@@ -2541,7 +2729,7 @@ static int tls_connection_dh(struct tls_connection *conn, const char *dh_file)
 }
 
 
-static int tls_global_dh(SSL_CTX *ssl_ctx, const char *dh_file)
+static int tls_global_dh(struct tls_data *data, const char *dh_file)
 {
 #ifdef OPENSSL_NO_DH
 	if (dh_file == NULL)
@@ -2550,6 +2738,7 @@ static int tls_global_dh(SSL_CTX *ssl_ctx, const char *dh_file)
 		   "dh_file specified");
 	return -1;
 #else /* OPENSSL_NO_DH */
+	SSL_CTX *ssl_ctx = data->ssl;
 	DH *dh;
 	BIO *bio;
 
@@ -2615,45 +2804,275 @@ static int tls_global_dh(SSL_CTX *ssl_ctx, const char *dh_file)
 }
 
 
-int tls_connection_get_keys(void *ssl_ctx, struct tls_connection *conn,
-			    struct tls_keys *keys)
+int tls_connection_get_random(void *ssl_ctx, struct tls_connection *conn,
+			      struct tls_random *keys)
+{
+	SSL *ssl;
+
+	if (conn == NULL || keys == NULL)
+		return -1;
+	ssl = conn->ssl;
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+	if (ssl == NULL || ssl->s3 == NULL || ssl->session == NULL)
+		return -1;
+
+	os_memset(keys, 0, sizeof(*keys));
+	keys->client_random = ssl->s3->client_random;
+	keys->client_random_len = SSL3_RANDOM_SIZE;
+	keys->server_random = ssl->s3->server_random;
+	keys->server_random_len = SSL3_RANDOM_SIZE;
+#else
+	if (ssl == NULL)
+		return -1;
+
+	os_memset(keys, 0, sizeof(*keys));
+	keys->client_random = conn->client_random;
+	keys->client_random_len = SSL_get_client_random(
+		ssl, conn->client_random, sizeof(conn->client_random));
+	keys->server_random = conn->server_random;
+	keys->server_random_len = SSL_get_server_random(
+		ssl, conn->server_random, sizeof(conn->server_random));
+#endif
+
+	return 0;
+}
+
+
+#ifndef CONFIG_FIPS
+static int openssl_get_keyblock_size(SSL *ssl)
+{
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+	const EVP_CIPHER *c;
+	const EVP_MD *h;
+	int md_size;
+
+	if (ssl->enc_read_ctx == NULL || ssl->enc_read_ctx->cipher == NULL ||
+	    ssl->read_hash == NULL)
+		return -1;
+
+	c = ssl->enc_read_ctx->cipher;
+#if OPENSSL_VERSION_NUMBER >= 0x00909000L
+	h = EVP_MD_CTX_md(ssl->read_hash);
+#else
+	h = ssl->read_hash;
+#endif
+	if (h)
+		md_size = EVP_MD_size(h);
+#if OPENSSL_VERSION_NUMBER >= 0x10000000L
+	else if (ssl->s3)
+		md_size = ssl->s3->tmp.new_mac_secret_size;
+#endif
+	else
+		return -1;
+
+	wpa_printf(MSG_DEBUG, "OpenSSL: keyblock size: key_len=%d MD_size=%d "
+		   "IV_len=%d", EVP_CIPHER_key_length(c), md_size,
+		   EVP_CIPHER_iv_length(c));
+	return 2 * (EVP_CIPHER_key_length(c) +
+		    md_size +
+		    EVP_CIPHER_iv_length(c));
+#else
+	const SSL_CIPHER *ssl_cipher;
+	int cipher, digest;
+	const EVP_CIPHER *c;
+	const EVP_MD *h;
+
+	ssl_cipher = SSL_get_current_cipher(ssl);
+	if (!ssl_cipher)
+		return -1;
+	cipher = SSL_CIPHER_get_cipher_nid(ssl_cipher);
+	digest = SSL_CIPHER_get_digest_nid(ssl_cipher);
+	wpa_printf(MSG_DEBUG, "OpenSSL: cipher nid %d digest nid %d",
+		   cipher, digest);
+	if (cipher < 0 || digest < 0)
+		return -1;
+	c = EVP_get_cipherbynid(cipher);
+	h = EVP_get_digestbynid(digest);
+	if (!c || !h)
+		return -1;
+
+	wpa_printf(MSG_DEBUG,
+		   "OpenSSL: keyblock size: key_len=%d MD_size=%d IV_len=%d",
+		   EVP_CIPHER_key_length(c), EVP_MD_size(h),
+		   EVP_CIPHER_iv_length(c));
+	return 2 * (EVP_CIPHER_key_length(c) + EVP_MD_size(h) +
+		    EVP_CIPHER_iv_length(c));
+#endif
+}
+#endif /* CONFIG_FIPS */
+
+
+static int openssl_tls_prf(struct tls_connection *conn,
+			   const char *label, int server_random_first,
+			   int skip_keyblock, u8 *out, size_t out_len)
 {
 #ifdef CONFIG_FIPS
 	wpa_printf(MSG_ERROR, "OpenSSL: TLS keys cannot be exported in FIPS "
 		   "mode");
 	return -1;
 #else /* CONFIG_FIPS */
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
 	SSL *ssl;
+	u8 *rnd;
+	int ret = -1;
+	int skip = 0;
+	u8 *tmp_out = NULL;
+	u8 *_out = out;
+	const char *ver;
 
-	if (conn == NULL || keys == NULL)
+	/*
+	 * TLS library did not support key generation, so get the needed TLS
+	 * session parameters and use an internal implementation of TLS PRF to
+	 * derive the key.
+	 */
+
+	if (conn == NULL)
 		return -1;
 	ssl = conn->ssl;
-	if (ssl == NULL || ssl->s3 == NULL || ssl->session == NULL)
+	if (ssl == NULL || ssl->s3 == NULL || ssl->session == NULL ||
+	    ssl->session->master_key_length <= 0)
+		return -1;
+	ver = SSL_get_version(ssl);
+
+	if (skip_keyblock) {
+		skip = openssl_get_keyblock_size(ssl);
+		if (skip < 0)
+			return -1;
+		tmp_out = os_malloc(skip + out_len);
+		if (!tmp_out)
+			return -1;
+		_out = tmp_out;
+	}
+
+	rnd = os_malloc(2 * SSL3_RANDOM_SIZE);
+	if (!rnd) {
+		os_free(tmp_out);
+		return -1;
+	}
+
+	if (server_random_first) {
+		os_memcpy(rnd, ssl->s3->server_random, SSL3_RANDOM_SIZE);
+		os_memcpy(rnd + SSL3_RANDOM_SIZE, ssl->s3->client_random,
+			SSL3_RANDOM_SIZE);
+	} else {
+		os_memcpy(rnd, ssl->s3->client_random, SSL3_RANDOM_SIZE);
+		os_memcpy(rnd + SSL3_RANDOM_SIZE, ssl->s3->server_random,
+			SSL3_RANDOM_SIZE);
+	}
+
+	if (os_strcmp(ver, "TLSv1.2") == 0) {
+		tls_prf_sha256(ssl->session->master_key,
+			       ssl->session->master_key_length,
+			       label, rnd, 2 * SSL3_RANDOM_SIZE,
+			       _out, skip + out_len);
+		ret = 0;
+	} else if (tls_prf_sha1_md5(ssl->session->master_key,
+				    ssl->session->master_key_length,
+				    label, rnd, 2 * SSL3_RANDOM_SIZE,
+				    _out, skip + out_len) == 0) {
+		ret = 0;
+	}
+	os_free(rnd);
+	if (ret == 0 && skip_keyblock)
+		os_memcpy(out, _out + skip, out_len);
+	bin_clear_free(tmp_out, skip);
+
+	return ret;
+#else
+	SSL *ssl;
+	SSL_SESSION *sess;
+	u8 *rnd;
+	int ret = -1;
+	int skip = 0;
+	u8 *tmp_out = NULL;
+	u8 *_out = out;
+	unsigned char client_random[SSL3_RANDOM_SIZE];
+	unsigned char server_random[SSL3_RANDOM_SIZE];
+	unsigned char master_key[64];
+	size_t master_key_len;
+	const char *ver;
+
+	/*
+	 * TLS library did not support key generation, so get the needed TLS
+	 * session parameters and use an internal implementation of TLS PRF to
+	 * derive the key.
+	 */
+
+	if (conn == NULL)
+		return -1;
+	ssl = conn->ssl;
+	if (ssl == NULL)
+		return -1;
+	ver = SSL_get_version(ssl);
+	sess = SSL_get_session(ssl);
+	if (!ver || !sess)
 		return -1;
 
-	os_memset(keys, 0, sizeof(*keys));
-	keys->master_key = ssl->session->master_key;
-	keys->master_key_len = ssl->session->master_key_length;
-	keys->client_random = ssl->s3->client_random;
-	keys->client_random_len = SSL3_RANDOM_SIZE;
-	keys->server_random = ssl->s3->server_random;
-	keys->server_random_len = SSL3_RANDOM_SIZE;
+	if (skip_keyblock) {
+		skip = openssl_get_keyblock_size(ssl);
+		if (skip < 0)
+			return -1;
+		tmp_out = os_malloc(skip + out_len);
+		if (!tmp_out)
+			return -1;
+		_out = tmp_out;
+	}
 
-	return 0;
+	rnd = os_malloc(2 * SSL3_RANDOM_SIZE);
+	if (!rnd) {
+		os_free(tmp_out);
+		return -1;
+	}
+
+	SSL_get_client_random(ssl, client_random, sizeof(client_random));
+	SSL_get_server_random(ssl, server_random, sizeof(server_random));
+	master_key_len = SSL_SESSION_get_master_key(sess, master_key,
+						    sizeof(master_key));
+
+	if (server_random_first) {
+		os_memcpy(rnd, server_random, SSL3_RANDOM_SIZE);
+		os_memcpy(rnd + SSL3_RANDOM_SIZE, client_random,
+			  SSL3_RANDOM_SIZE);
+	} else {
+		os_memcpy(rnd, client_random, SSL3_RANDOM_SIZE);
+		os_memcpy(rnd + SSL3_RANDOM_SIZE, server_random,
+			  SSL3_RANDOM_SIZE);
+	}
+
+	if (os_strcmp(ver, "TLSv1.2") == 0) {
+		tls_prf_sha256(master_key, master_key_len,
+			       label, rnd, 2 * SSL3_RANDOM_SIZE,
+			       _out, skip + out_len);
+		ret = 0;
+	} else if (tls_prf_sha1_md5(master_key, master_key_len,
+				    label, rnd, 2 * SSL3_RANDOM_SIZE,
+				    _out, skip + out_len) == 0) {
+		ret = 0;
+	}
+	os_memset(master_key, 0, sizeof(master_key));
+	os_free(rnd);
+	if (ret == 0 && skip_keyblock)
+		os_memcpy(out, _out + skip, out_len);
+	bin_clear_free(tmp_out, skip);
+
+	return ret;
+#endif
 #endif /* CONFIG_FIPS */
 }
 
 
 int tls_connection_prf(void *tls_ctx, struct tls_connection *conn,
 		       const char *label, int server_random_first,
-		       u8 *out, size_t out_len)
+		       int skip_keyblock, u8 *out, size_t out_len)
 {
 #if OPENSSL_VERSION_NUMBER >= 0x10001000L
 	SSL *ssl;
 	if (conn == NULL)
 		return -1;
-	if (server_random_first)
-		return -1;
+	if (server_random_first || skip_keyblock)
+		return openssl_tls_prf(conn, label,
+				       server_random_first, skip_keyblock,
+				       out, out_len);
 	ssl = conn->ssl;
 	if (SSL_export_keying_material(ssl, out, out_len, label,
 				       os_strlen(label), NULL, 0, 0) == 1) {
@@ -2661,7 +3080,8 @@ int tls_connection_prf(void *tls_ctx, struct tls_connection *conn,
 		return 0;
 	}
 #endif
-	return -1;
+	return openssl_tls_prf(conn, label, server_random_first,
+			       skip_keyblock, out, out_len);
 }
 
 
@@ -2676,7 +3096,7 @@ openssl_handshake(struct tls_connection *conn, const struct wpabuf *in_data,
 	 * Give TLS handshake data from the server (if available) to OpenSSL
 	 * for processing.
 	 */
-	if (in_data &&
+	if (in_data && wpabuf_len(in_data) > 0 &&
 	    BIO_write(conn->ssl_in, wpabuf_head(in_data), wpabuf_len(in_data))
 	    < 0) {
 		tls_show_errors(MSG_INFO, __func__,
@@ -2788,8 +3208,14 @@ openssl_connection_handshake(struct tls_connection *conn,
 		return NULL;
 	}
 
-	if (SSL_is_init_finished(conn->ssl) && appl_data && in_data)
-		*appl_data = openssl_get_appl_data(conn, wpabuf_len(in_data));
+	if (SSL_is_init_finished(conn->ssl)) {
+		wpa_printf(MSG_DEBUG,
+			   "OpenSSL: Handshake finished - resumed=%d",
+			   tls_connection_resumed(conn->ssl_ctx, conn));
+		if (appl_data && in_data)
+			*appl_data = openssl_get_appl_data(conn,
+							   wpabuf_len(in_data));
+	}
 
 	if (conn->invalid_hb_used) {
 		wpa_printf(MSG_INFO, "TLS: Heartbeat attack detected - do not send response");
@@ -2968,6 +3394,21 @@ int tls_connection_set_cipher_list(void *tls_ctx, struct tls_connection *conn,
 
 	wpa_printf(MSG_DEBUG, "OpenSSL: cipher suites: %s", buf + 1);
 
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+#if defined(EAP_FAST) || defined(EAP_FAST_DYNAMIC) || defined(EAP_SERVER_FAST)
+	if (os_strstr(buf, ":ADH-")) {
+		/*
+		 * Need to drop to security level 0 to allow anonymous
+		 * cipher suites for EAP-FAST.
+		 */
+		SSL_set_security_level(conn->ssl, 0);
+	} else if (SSL_get_security_level(conn->ssl) == 0) {
+		/* Force at least security level 1 */
+		SSL_set_security_level(conn->ssl, 1);
+	}
+#endif /* EAP_FAST || EAP_FAST_DYNAMIC || EAP_SERVER_FAST */
+#endif
+
 	if (SSL_set_cipher_list(conn->ssl, buf + 1) != 1) {
 		tls_show_errors(MSG_INFO, __func__,
 				"Cipher suite configuration failed");
@@ -2978,6 +3419,22 @@ int tls_connection_set_cipher_list(void *tls_ctx, struct tls_connection *conn,
 }
 
 
+int tls_get_version(void *ssl_ctx, struct tls_connection *conn,
+		    char *buf, size_t buflen)
+{
+	const char *name;
+	if (conn == NULL || conn->ssl == NULL)
+		return -1;
+
+	name = SSL_get_version(conn->ssl);
+	if (name == NULL)
+		return -1;
+
+	os_strlcpy(buf, name, buflen);
+	return 0;
+}
+
+
 int tls_get_cipher(void *ssl_ctx, struct tls_connection *conn,
 		   char *buf, size_t buflen)
 {
@@ -3300,6 +3757,7 @@ static int ocsp_status_cb(SSL *s, void *arg)
 int tls_connection_set_params(void *tls_ctx, struct tls_connection *conn,
 			      const struct tls_connection_params *params)
 {
+	struct tls_data *data = tls_ctx;
 	int ret;
 	unsigned long err;
 	int can_pkcs11 = 0;
@@ -3341,6 +3799,8 @@ int tls_connection_set_params(void *tls_ctx, struct tls_connection *conn,
 	if (can_pkcs11 == 2 && !engine_id)
 		engine_id = "pkcs11";
 
+#if defined(EAP_FAST) || defined(EAP_FAST_DYNAMIC) || defined(EAP_SERVER_FAST)
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
 	if (params->flags & TLS_CONN_EAP_FAST) {
 		wpa_printf(MSG_DEBUG,
 			   "OpenSSL: Use TLSv1_method() for EAP-FAST");
@@ -3350,6 +3810,8 @@ int tls_connection_set_params(void *tls_ctx, struct tls_connection *conn,
 			return -1;
 		}
 	}
+#endif
+#endif /* EAP_FAST || EAP_FAST_DYNAMIC || EAP_SERVER_FAST */
 
 	while ((err = ERR_get_error())) {
 		wpa_printf(MSG_INFO, "%s: Clearing pending SSL error: %s",
@@ -3371,10 +3833,9 @@ int tls_connection_set_params(void *tls_ctx, struct tls_connection *conn,
 		return -1;
 
 	if (engine_id && ca_cert_id) {
-		if (tls_connection_engine_ca_cert(tls_ctx, conn,
-						  ca_cert_id))
+		if (tls_connection_engine_ca_cert(data, conn, ca_cert_id))
 			return TLS_SET_PARAMS_ENGINE_PRV_VERIFY_FAILED;
-	} else if (tls_connection_ca_cert(tls_ctx, conn, params->ca_cert,
+	} else if (tls_connection_ca_cert(data, conn, params->ca_cert,
 					  params->ca_cert_blob,
 					  params->ca_cert_blob_len,
 					  params->ca_path))
@@ -3392,7 +3853,7 @@ int tls_connection_set_params(void *tls_ctx, struct tls_connection *conn,
 		wpa_printf(MSG_DEBUG, "TLS: Using private key from engine");
 		if (tls_connection_engine_private_key(conn))
 			return TLS_SET_PARAMS_ENGINE_PRV_VERIFY_FAILED;
-	} else if (tls_connection_private_key(tls_ctx, conn,
+	} else if (tls_connection_private_key(data, conn,
 					      params->private_key,
 					      params->private_key_passwd,
 					      params->private_key_blob,
@@ -3416,40 +3877,30 @@ int tls_connection_set_params(void *tls_ctx, struct tls_connection *conn,
 		return -1;
 	}
 
-#ifdef SSL_OP_NO_TICKET
-	if (params->flags & TLS_CONN_DISABLE_SESSION_TICKET)
-		SSL_set_options(conn->ssl, SSL_OP_NO_TICKET);
-#ifdef SSL_clear_options
-	else
-		SSL_clear_options(conn->ssl, SSL_OP_NO_TICKET);
-#endif /* SSL_clear_options */
-#endif /*  SSL_OP_NO_TICKET */
-
-#ifdef SSL_OP_NO_TLSv1_1
-	if (params->flags & TLS_CONN_DISABLE_TLSv1_1)
-		SSL_set_options(conn->ssl, SSL_OP_NO_TLSv1_1);
-	else
-		SSL_clear_options(conn->ssl, SSL_OP_NO_TLSv1_1);
-#endif /* SSL_OP_NO_TLSv1_1 */
-#ifdef SSL_OP_NO_TLSv1_2
-	if (params->flags & TLS_CONN_DISABLE_TLSv1_2)
-		SSL_set_options(conn->ssl, SSL_OP_NO_TLSv1_2);
-	else
-		SSL_clear_options(conn->ssl, SSL_OP_NO_TLSv1_2);
-#endif /* SSL_OP_NO_TLSv1_2 */
+	tls_set_conn_flags(conn->ssl, params->flags);
 
 #ifdef HAVE_OCSP
 	if (params->flags & TLS_CONN_REQUEST_OCSP) {
-		SSL_CTX *ssl_ctx = tls_ctx;
+		SSL_CTX *ssl_ctx = data->ssl;
 		SSL_set_tlsext_status_type(conn->ssl, TLSEXT_STATUSTYPE_ocsp);
 		SSL_CTX_set_tlsext_status_cb(ssl_ctx, ocsp_resp_cb);
 		SSL_CTX_set_tlsext_status_arg(ssl_ctx, conn);
 	}
+#else /* HAVE_OCSP */
+	if (params->flags & TLS_CONN_REQUIRE_OCSP) {
+		wpa_printf(MSG_INFO,
+			   "OpenSSL: No OCSP support included - reject configuration");
+		return -1;
+	}
+	if (params->flags & TLS_CONN_REQUEST_OCSP) {
+		wpa_printf(MSG_DEBUG,
+			   "OpenSSL: No OCSP support included - allow optional OCSP case to continue");
+	}
 #endif /* HAVE_OCSP */
 
 	conn->flags = params->flags;
 
-	tls_get_errors(tls_ctx);
+	tls_get_errors(data);
 
 	return 0;
 }
@@ -3458,7 +3909,8 @@ int tls_connection_set_params(void *tls_ctx, struct tls_connection *conn,
 int tls_global_set_params(void *tls_ctx,
 			  const struct tls_connection_params *params)
 {
-	SSL_CTX *ssl_ctx = tls_ctx;
+	struct tls_data *data = tls_ctx;
+	SSL_CTX *ssl_ctx = data->ssl;
 	unsigned long err;
 
 	while ((err = ERR_get_error())) {
@@ -3466,19 +3918,12 @@ int tls_global_set_params(void *tls_ctx,
 			   __func__, ERR_error_string(err, NULL));
 	}
 
-	if (tls_global_ca_cert(ssl_ctx, params->ca_cert))
-		return -1;
-
-	if (tls_global_client_cert(ssl_ctx, params->client_cert))
-		return -1;
-
-	if (tls_global_private_key(ssl_ctx, params->private_key,
-				   params->private_key_passwd))
-		return -1;
-
-	if (tls_global_dh(ssl_ctx, params->dh_file)) {
-		wpa_printf(MSG_INFO, "TLS: Failed to load DH file '%s'",
-			   params->dh_file);
+	if (tls_global_ca_cert(data, params->ca_cert) ||
+	    tls_global_client_cert(data, params->client_cert) ||
+	    tls_global_private_key(data, params->private_key,
+				   params->private_key_passwd) ||
+	    tls_global_dh(data, params->dh_file)) {
+		wpa_printf(MSG_INFO, "TLS: Failed to set global parameters");
 		return -1;
 	}
 
@@ -3514,49 +3959,6 @@ int tls_global_set_params(void *tls_ctx,
 }
 
 
-int tls_connection_get_keyblock_size(void *tls_ctx,
-				     struct tls_connection *conn)
-{
-	const EVP_CIPHER *c;
-	const EVP_MD *h;
-	int md_size;
-
-	if (conn == NULL || conn->ssl == NULL ||
-	    conn->ssl->enc_read_ctx == NULL ||
-	    conn->ssl->enc_read_ctx->cipher == NULL ||
-	    conn->ssl->read_hash == NULL)
-		return -1;
-
-	c = conn->ssl->enc_read_ctx->cipher;
-#if OPENSSL_VERSION_NUMBER >= 0x00909000L
-	h = EVP_MD_CTX_md(conn->ssl->read_hash);
-#else
-	h = conn->ssl->read_hash;
-#endif
-	if (h)
-		md_size = EVP_MD_size(h);
-#if OPENSSL_VERSION_NUMBER >= 0x10000000L
-	else if (conn->ssl->s3)
-		md_size = conn->ssl->s3->tmp.new_mac_secret_size;
-#endif
-	else
-		return -1;
-
-	wpa_printf(MSG_DEBUG, "OpenSSL: keyblock size: key_len=%d MD_size=%d "
-		   "IV_len=%d", EVP_CIPHER_key_length(c), md_size,
-		   EVP_CIPHER_iv_length(c));
-	return 2 * (EVP_CIPHER_key_length(c) +
-		    md_size +
-		    EVP_CIPHER_iv_length(c));
-}
-
-
-unsigned int tls_capabilities(void *tls_ctx)
-{
-	return 0;
-}
-
-
 #if defined(EAP_FAST) || defined(EAP_FAST_DYNAMIC) || defined(EAP_SERVER_FAST)
 /* Pre-shared secred requires a patch to openssl, so this function is
  * commented out unless explicitly needed for EAP-FAST in order to be able to
@@ -3575,6 +3977,7 @@ static int tls_sess_sec_cb(SSL *s, void *secret, int *secret_len,
 	struct tls_connection *conn = arg;
 	int ret;
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
 	if (conn == NULL || conn->session_ticket_cb == NULL)
 		return 0;
 
@@ -3583,6 +3986,23 @@ static int tls_sess_sec_cb(SSL *s, void *secret, int *secret_len,
 				      conn->session_ticket_len,
 				      s->s3->client_random,
 				      s->s3->server_random, secret);
+#else
+	unsigned char client_random[SSL3_RANDOM_SIZE];
+	unsigned char server_random[SSL3_RANDOM_SIZE];
+
+	if (conn == NULL || conn->session_ticket_cb == NULL)
+		return 0;
+
+	SSL_get_client_random(s, client_random, sizeof(client_random));
+	SSL_get_server_random(s, server_random, sizeof(server_random));
+
+	ret = conn->session_ticket_cb(conn->session_ticket_cb_ctx,
+				      conn->session_ticket,
+				      conn->session_ticket_len,
+				      client_random,
+				      server_random, secret);
+#endif
+
 	os_free(conn->session_ticket);
 	conn->session_ticket = NULL;
 
@@ -3656,3 +4076,70 @@ int tls_get_library_version(char *buf, size_t buf_len)
 			   OPENSSL_VERSION_TEXT,
 			   SSLeay_version(SSLEAY_VERSION));
 }
+
+
+void tls_connection_set_success_data(struct tls_connection *conn,
+				     struct wpabuf *data)
+{
+	SSL_SESSION *sess;
+	struct wpabuf *old;
+
+	if (tls_ex_idx_session < 0)
+		goto fail;
+	sess = SSL_get_session(conn->ssl);
+	if (!sess)
+		goto fail;
+	old = SSL_SESSION_get_ex_data(sess, tls_ex_idx_session);
+	if (old) {
+		wpa_printf(MSG_DEBUG, "OpenSSL: Replacing old success data %p",
+			   old);
+		wpabuf_free(old);
+	}
+	if (SSL_SESSION_set_ex_data(sess, tls_ex_idx_session, data) != 1)
+		goto fail;
+
+	wpa_printf(MSG_DEBUG, "OpenSSL: Stored success data %p", data);
+	conn->success_data = 1;
+	return;
+
+fail:
+	wpa_printf(MSG_INFO, "OpenSSL: Failed to store success data");
+	wpabuf_free(data);
+}
+
+
+void tls_connection_set_success_data_resumed(struct tls_connection *conn)
+{
+	wpa_printf(MSG_DEBUG,
+		   "OpenSSL: Success data accepted for resumed session");
+	conn->success_data = 1;
+}
+
+
+const struct wpabuf *
+tls_connection_get_success_data(struct tls_connection *conn)
+{
+	SSL_SESSION *sess;
+
+	if (tls_ex_idx_session < 0 ||
+	    !(sess = SSL_get_session(conn->ssl)))
+		return NULL;
+	return SSL_SESSION_get_ex_data(sess, tls_ex_idx_session);
+}
+
+
+void tls_connection_remove_session(struct tls_connection *conn)
+{
+	SSL_SESSION *sess;
+
+	sess = SSL_get_session(conn->ssl);
+	if (!sess)
+		return;
+
+	if (SSL_CTX_remove_session(conn->ssl_ctx, sess) != 1)
+		wpa_printf(MSG_DEBUG,
+			   "OpenSSL: Session was not cached");
+	else
+		wpa_printf(MSG_DEBUG,
+			   "OpenSSL: Removed cached session to disable session resumption");
+}
diff --git a/src/crypto/tls_schannel.c b/src/crypto/tls_schannel.c
deleted file mode 100644
index 31a2c946d047..000000000000
--- a/src/crypto/tls_schannel.c
+++ /dev/null
@@ -1,763 +0,0 @@
-/*
- * SSL/TLS interface functions for Microsoft Schannel
- * Copyright (c) 2005-2009, Jouni Malinen <j@w1.fi>
- *
- * This software may be distributed under the terms of the BSD license.
- * See README for more details.
- */
-
-/*
- * FIX: Go through all SSPI functions and verify what needs to be freed
- * FIX: session resumption
- * TODO: add support for server cert chain validation
- * TODO: add support for CA cert validation
- * TODO: add support for EAP-TLS (client cert/key conf)
- */
-
-#include "includes.h"
-#include <windows.h>
-#include <wincrypt.h>
-#include <schannel.h>
-#define SECURITY_WIN32
-#include <security.h>
-#include <sspi.h>
-
-#include "common.h"
-#include "tls.h"
-
-
-struct tls_global {
-	HMODULE hsecurity;
-	PSecurityFunctionTable sspi;
-	HCERTSTORE my_cert_store;
-};
-
-struct tls_connection {
-	int established, start;
-	int failed, read_alerts, write_alerts;
-
-	SCHANNEL_CRED schannel_cred;
-	CredHandle creds;
-	CtxtHandle context;
-
-	u8 eap_tls_prf[128];
-	int eap_tls_prf_set;
-};
-
-
-static int schannel_load_lib(struct tls_global *global)
-{
-	INIT_SECURITY_INTERFACE pInitSecurityInterface;
-
-	global->hsecurity = LoadLibrary(TEXT("Secur32.dll"));
-	if (global->hsecurity == NULL) {
-		wpa_printf(MSG_ERROR, "%s: Could not load Secur32.dll - 0x%x",
-			   __func__, (unsigned int) GetLastError());
-		return -1;
-	}
-
-	pInitSecurityInterface = (INIT_SECURITY_INTERFACE) GetProcAddress(
-		global->hsecurity, "InitSecurityInterfaceA");
-	if (pInitSecurityInterface == NULL) {
-		wpa_printf(MSG_ERROR, "%s: Could not find "
-			   "InitSecurityInterfaceA from Secur32.dll",
-			   __func__);
-		FreeLibrary(global->hsecurity);
-		global->hsecurity = NULL;
-		return -1;
-	}
-
-	global->sspi = pInitSecurityInterface();
-	if (global->sspi == NULL) {
-		wpa_printf(MSG_ERROR, "%s: Could not read security "
-			   "interface - 0x%x",
-			   __func__, (unsigned int) GetLastError());
-		FreeLibrary(global->hsecurity);
-		global->hsecurity = NULL;
-		return -1;
-	}
-
-	return 0;
-}
-
-
-void * tls_init(const struct tls_config *conf)
-{
-	struct tls_global *global;
-
-	global = os_zalloc(sizeof(*global));
-	if (global == NULL)
-		return NULL;
-	if (schannel_load_lib(global)) {
-		os_free(global);
-		return NULL;
-	}
-	return global;
-}
-
-
-void tls_deinit(void *ssl_ctx)
-{
-	struct tls_global *global = ssl_ctx;
-
-	if (global->my_cert_store)
-		CertCloseStore(global->my_cert_store, 0);
-	FreeLibrary(global->hsecurity);
-	os_free(global);
-}
-
-
-int tls_get_errors(void *ssl_ctx)
-{
-	return 0;
-}
-
-
-struct tls_connection * tls_connection_init(void *ssl_ctx)
-{
-	struct tls_connection *conn;
-
-	conn = os_zalloc(sizeof(*conn));
-	if (conn == NULL)
-		return NULL;
-	conn->start = 1;
-
-	return conn;
-}
-
-
-void tls_connection_deinit(void *ssl_ctx, struct tls_connection *conn)
-{
-	if (conn == NULL)
-		return;
-
-	os_free(conn);
-}
-
-
-int tls_connection_established(void *ssl_ctx, struct tls_connection *conn)
-{
-	return conn ? conn->established : 0;
-}
-
-
-int tls_connection_shutdown(void *ssl_ctx, struct tls_connection *conn)
-{
-	struct tls_global *global = ssl_ctx;
-	if (conn == NULL)
-		return -1;
-
-	conn->eap_tls_prf_set = 0;
-	conn->established = conn->failed = 0;
-	conn->read_alerts = conn->write_alerts = 0;
-	global->sspi->DeleteSecurityContext(&conn->context);
-	/* FIX: what else needs to be reseted? */
-
-	return 0;
-}
-
-
-int tls_global_set_params(void *tls_ctx,
-			  const struct tls_connection_params *params)
-{
-	return -1;
-}
-
-
-int tls_global_set_verify(void *ssl_ctx, int check_crl)
-{
-	return -1;
-}
-
-
-int tls_connection_set_verify(void *ssl_ctx, struct tls_connection *conn,
-			      int verify_peer)
-{
-	return -1;
-}
-
-
-int tls_connection_get_keys(void *ssl_ctx, struct tls_connection *conn,
-			    struct tls_keys *keys)
-{
-	/* Schannel does not export master secret or client/server random. */
-	return -1;
-}
-
-
-int tls_connection_prf(void *tls_ctx, struct tls_connection *conn,
-		       const char *label, int server_random_first,
-		       u8 *out, size_t out_len)
-{
-	/*
-	 * Cannot get master_key from Schannel, but EapKeyBlock can be used to
-	 * generate session keys for EAP-TLS and EAP-PEAPv0. EAP-PEAPv2 and
-	 * EAP-TTLS cannot use this, though, since they are using different
-	 * labels. The only option could be to implement TLSv1 completely here
-	 * and just use Schannel or CryptoAPI for low-level crypto
-	 * functionality..
-	 */
-
-	if (conn == NULL || !conn->eap_tls_prf_set || server_random_first ||
-	    os_strcmp(label, "client EAP encryption") != 0 ||
-	    out_len > sizeof(conn->eap_tls_prf))
-		return -1;
-
-	os_memcpy(out, conn->eap_tls_prf, out_len);
-
-	return 0;
-}
-
-
-static struct wpabuf * tls_conn_hs_clienthello(struct tls_global *global,
-					       struct tls_connection *conn)
-{
-	DWORD sspi_flags, sspi_flags_out;
-	SecBufferDesc outbuf;
-	SecBuffer outbufs[1];
-	SECURITY_STATUS status;
-	TimeStamp ts_expiry;
-
-	sspi_flags = ISC_REQ_REPLAY_DETECT |
-		ISC_REQ_CONFIDENTIALITY |
-		ISC_RET_EXTENDED_ERROR |
-		ISC_REQ_ALLOCATE_MEMORY |
-		ISC_REQ_MANUAL_CRED_VALIDATION;
-
-	wpa_printf(MSG_DEBUG, "%s: Generating ClientHello", __func__);
-
-	outbufs[0].pvBuffer = NULL;
-	outbufs[0].BufferType = SECBUFFER_TOKEN;
-	outbufs[0].cbBuffer = 0;
-
-	outbuf.cBuffers = 1;
-	outbuf.pBuffers = outbufs;
-	outbuf.ulVersion = SECBUFFER_VERSION;
-
-#ifdef UNICODE
-	status = global->sspi->InitializeSecurityContextW(
-		&conn->creds, NULL, NULL /* server name */, sspi_flags, 0,
-		SECURITY_NATIVE_DREP, NULL, 0, &conn->context,
-		&outbuf, &sspi_flags_out, &ts_expiry);
-#else /* UNICODE */
-	status = global->sspi->InitializeSecurityContextA(
-		&conn->creds, NULL, NULL /* server name */, sspi_flags, 0,
-		SECURITY_NATIVE_DREP, NULL, 0, &conn->context,
-		&outbuf, &sspi_flags_out, &ts_expiry);
-#endif /* UNICODE */
-	if (status != SEC_I_CONTINUE_NEEDED) {
-		wpa_printf(MSG_ERROR, "%s: InitializeSecurityContextA "
-			   "failed - 0x%x",
-			   __func__, (unsigned int) status);
-		return NULL;
-	}
-
-	if (outbufs[0].cbBuffer != 0 && outbufs[0].pvBuffer) {
-		struct wpabuf *buf;
-		wpa_hexdump(MSG_MSGDUMP, "SChannel - ClientHello",
-			    outbufs[0].pvBuffer, outbufs[0].cbBuffer);
-		conn->start = 0;
-		buf = wpabuf_alloc_copy(outbufs[0].pvBuffer,
-					outbufs[0].cbBuffer);
-		if (buf == NULL)
-			return NULL;
-		global->sspi->FreeContextBuffer(outbufs[0].pvBuffer);
-		return buf;
-	}
-
-	wpa_printf(MSG_ERROR, "SChannel: Failed to generate ClientHello");
-
-	return NULL;
-}
-
-
-#ifndef SECPKG_ATTR_EAP_KEY_BLOCK
-#define SECPKG_ATTR_EAP_KEY_BLOCK 0x5b
-
-typedef struct _SecPkgContext_EapKeyBlock {
-	BYTE rgbKeys[128];
-	BYTE rgbIVs[64];
-} SecPkgContext_EapKeyBlock, *PSecPkgContext_EapKeyBlock;
-#endif /* !SECPKG_ATTR_EAP_KEY_BLOCK */
-
-static int tls_get_eap(struct tls_global *global, struct tls_connection *conn)
-{
-	SECURITY_STATUS status;
-	SecPkgContext_EapKeyBlock kb;
-
-	/* Note: Windows NT and Windows Me/98/95 do not support getting
-	 * EapKeyBlock */
-
-	status = global->sspi->QueryContextAttributes(
-		&conn->context, SECPKG_ATTR_EAP_KEY_BLOCK, &kb);
-	if (status != SEC_E_OK) {
-		wpa_printf(MSG_DEBUG, "%s: QueryContextAttributes("
-			   "SECPKG_ATTR_EAP_KEY_BLOCK) failed (%d)",
-			   __func__, (int) status);
-		return -1;
-	}
-
-	wpa_hexdump_key(MSG_MSGDUMP, "Schannel - EapKeyBlock - rgbKeys",
-			kb.rgbKeys, sizeof(kb.rgbKeys));
-	wpa_hexdump_key(MSG_MSGDUMP, "Schannel - EapKeyBlock - rgbIVs",
-			kb.rgbIVs, sizeof(kb.rgbIVs));
-
-	os_memcpy(conn->eap_tls_prf, kb.rgbKeys, sizeof(kb.rgbKeys));
-	conn->eap_tls_prf_set = 1;
-	return 0;
-}
-
-
-struct wpabuf * tls_connection_handshake(void *tls_ctx,
-					 struct tls_connection *conn,
-					 const struct wpabuf *in_data,
-					 struct wpabuf **appl_data)
-{
-	struct tls_global *global = tls_ctx;
-	DWORD sspi_flags, sspi_flags_out;
-	SecBufferDesc inbuf, outbuf;
-	SecBuffer inbufs[2], outbufs[1];
-	SECURITY_STATUS status;
-	TimeStamp ts_expiry;
-	struct wpabuf *out_buf = NULL;
-
-	if (appl_data)
-		*appl_data = NULL;
-
-	if (conn->start)
-		return tls_conn_hs_clienthello(global, conn);
-
-	wpa_printf(MSG_DEBUG, "SChannel: %d bytes handshake data to process",
-		   (int) wpabuf_len(in_data));
-
-	sspi_flags = ISC_REQ_REPLAY_DETECT |
-		ISC_REQ_CONFIDENTIALITY |
-		ISC_RET_EXTENDED_ERROR |
-		ISC_REQ_ALLOCATE_MEMORY |
-		ISC_REQ_MANUAL_CRED_VALIDATION;
-
-	/* Input buffer for Schannel */
-	inbufs[0].pvBuffer = (u8 *) wpabuf_head(in_data);
-	inbufs[0].cbBuffer = wpabuf_len(in_data);
-	inbufs[0].BufferType = SECBUFFER_TOKEN;
-
-	/* Place for leftover data from Schannel */
-	inbufs[1].pvBuffer = NULL;
-	inbufs[1].cbBuffer = 0;
-	inbufs[1].BufferType = SECBUFFER_EMPTY;
-
-	inbuf.cBuffers = 2;
-	inbuf.pBuffers = inbufs;
-	inbuf.ulVersion = SECBUFFER_VERSION;
-
-	/* Output buffer for Schannel */
-	outbufs[0].pvBuffer = NULL;
-	outbufs[0].cbBuffer = 0;
-	outbufs[0].BufferType = SECBUFFER_TOKEN;
-
-	outbuf.cBuffers = 1;
-	outbuf.pBuffers = outbufs;
-	outbuf.ulVersion = SECBUFFER_VERSION;
-
-#ifdef UNICODE
-	status = global->sspi->InitializeSecurityContextW(
-		&conn->creds, &conn->context, NULL, sspi_flags, 0,
-		SECURITY_NATIVE_DREP, &inbuf, 0, NULL,
-		&outbuf, &sspi_flags_out, &ts_expiry);
-#else /* UNICODE */
-	status = global->sspi->InitializeSecurityContextA(
-		&conn->creds, &conn->context, NULL, sspi_flags, 0,
-		SECURITY_NATIVE_DREP, &inbuf, 0, NULL,
-		&outbuf, &sspi_flags_out, &ts_expiry);
-#endif /* UNICODE */
-
-	wpa_printf(MSG_MSGDUMP, "Schannel: InitializeSecurityContext -> "
-		   "status=%d inlen[0]=%d intype[0]=%d inlen[1]=%d "
-		   "intype[1]=%d outlen[0]=%d",
-		   (int) status, (int) inbufs[0].cbBuffer,
-		   (int) inbufs[0].BufferType, (int) inbufs[1].cbBuffer,
-		   (int) inbufs[1].BufferType,
-		   (int) outbufs[0].cbBuffer);
-	if (status == SEC_E_OK || status == SEC_I_CONTINUE_NEEDED ||
-	    (FAILED(status) && (sspi_flags_out & ISC_RET_EXTENDED_ERROR))) {
-		if (outbufs[0].cbBuffer != 0 && outbufs[0].pvBuffer) {
-			wpa_hexdump(MSG_MSGDUMP, "SChannel - output",
-				    outbufs[0].pvBuffer, outbufs[0].cbBuffer);
-			out_buf = wpabuf_alloc_copy(outbufs[0].pvBuffer,
-						    outbufs[0].cbBuffer);
-			global->sspi->FreeContextBuffer(outbufs[0].pvBuffer);
-			outbufs[0].pvBuffer = NULL;
-			if (out_buf == NULL)
-				return NULL;
-		}
-	}
-
-	switch (status) {
-	case SEC_E_INCOMPLETE_MESSAGE:
-		wpa_printf(MSG_DEBUG, "Schannel: SEC_E_INCOMPLETE_MESSAGE");
-		break;
-	case SEC_I_CONTINUE_NEEDED:
-		wpa_printf(MSG_DEBUG, "Schannel: SEC_I_CONTINUE_NEEDED");
-		break;
-	case SEC_E_OK:
-		/* TODO: verify server certificate chain */
-		wpa_printf(MSG_DEBUG, "Schannel: SEC_E_OK - Handshake "
-			   "completed successfully");
-		conn->established = 1;
-		tls_get_eap(global, conn);
-
-		/* Need to return something to get final TLS ACK. */
-		if (out_buf == NULL)
-			out_buf = wpabuf_alloc(0);
-
-		if (inbufs[1].BufferType == SECBUFFER_EXTRA) {
-			wpa_hexdump(MSG_MSGDUMP, "SChannel - Encrypted "
-				    "application data",
-				    inbufs[1].pvBuffer, inbufs[1].cbBuffer);
-			if (appl_data) {
-				*appl_data = wpabuf_alloc_copy(
-					outbufs[1].pvBuffer,
-					outbufs[1].cbBuffer);
-			}
-			global->sspi->FreeContextBuffer(inbufs[1].pvBuffer);
-			inbufs[1].pvBuffer = NULL;
-		}
-		break;
-	case SEC_I_INCOMPLETE_CREDENTIALS:
-		wpa_printf(MSG_DEBUG,
-			   "Schannel: SEC_I_INCOMPLETE_CREDENTIALS");
-		break;
-	case SEC_E_WRONG_PRINCIPAL:
-		wpa_printf(MSG_DEBUG, "Schannel: SEC_E_WRONG_PRINCIPAL");
-		break;
-	case SEC_E_INTERNAL_ERROR:
-		wpa_printf(MSG_DEBUG, "Schannel: SEC_E_INTERNAL_ERROR");
-		break;
-	}
-
-	if (FAILED(status)) {
-		wpa_printf(MSG_DEBUG, "Schannel: Handshake failed "
-			   "(out_buf=%p)", out_buf);
-		conn->failed++;
-		global->sspi->DeleteSecurityContext(&conn->context);
-		return out_buf;
-	}
-
-	if (inbufs[1].BufferType == SECBUFFER_EXTRA) {
-		/* TODO: Can this happen? What to do with this data? */
-		wpa_hexdump(MSG_MSGDUMP, "SChannel - Leftover data",
-			    inbufs[1].pvBuffer, inbufs[1].cbBuffer);
-		global->sspi->FreeContextBuffer(inbufs[1].pvBuffer);
-		inbufs[1].pvBuffer = NULL;
-	}
-
-	return out_buf;
-}
-
-
-struct wpabuf * tls_connection_server_handshake(void *tls_ctx,
-						struct tls_connection *conn,
-						const struct wpabuf *in_data,
-						struct wpabuf **appl_data)
-{
-	return NULL;
-}
-
-
-struct wpabuf * tls_connection_encrypt(void *tls_ctx,
-				       struct tls_connection *conn,
-				       const struct wpabuf *in_data)
-{
-	struct tls_global *global = tls_ctx;
-	SECURITY_STATUS status;
-	SecBufferDesc buf;
-	SecBuffer bufs[4];
-	SecPkgContext_StreamSizes sizes;
-	int i;
-	struct wpabuf *out;
-
-	status = global->sspi->QueryContextAttributes(&conn->context,
-						      SECPKG_ATTR_STREAM_SIZES,
-						      &sizes);
-	if (status != SEC_E_OK) {
-		wpa_printf(MSG_DEBUG, "%s: QueryContextAttributes failed",
-			   __func__);
-		return NULL;
-	}
-	wpa_printf(MSG_DEBUG, "%s: Stream sizes: header=%u trailer=%u",
-		   __func__,
-		   (unsigned int) sizes.cbHeader,
-		   (unsigned int) sizes.cbTrailer);
-
-	out = wpabuf_alloc(sizes.cbHeader + wpabuf_len(in_data) +
-			   sizes.cbTrailer);
-
-	os_memset(&bufs, 0, sizeof(bufs));
-	bufs[0].pvBuffer = wpabuf_put(out, sizes.cbHeader);
-	bufs[0].cbBuffer = sizes.cbHeader;
-	bufs[0].BufferType = SECBUFFER_STREAM_HEADER;
-
-	bufs[1].pvBuffer = wpabuf_put(out, 0);
-	wpabuf_put_buf(out, in_data);
-	bufs[1].cbBuffer = wpabuf_len(in_data);
-	bufs[1].BufferType = SECBUFFER_DATA;
-
-	bufs[2].pvBuffer = wpabuf_put(out, sizes.cbTrailer);
-	bufs[2].cbBuffer = sizes.cbTrailer;
-	bufs[2].BufferType = SECBUFFER_STREAM_TRAILER;
-
-	buf.ulVersion = SECBUFFER_VERSION;
-	buf.cBuffers = 3;
-	buf.pBuffers = bufs;
-
-	status = global->sspi->EncryptMessage(&conn->context, 0, &buf, 0);
-
-	wpa_printf(MSG_MSGDUMP, "Schannel: EncryptMessage -> "
-		   "status=%d len[0]=%d type[0]=%d len[1]=%d type[1]=%d "
-		   "len[2]=%d type[2]=%d",
-		   (int) status,
-		   (int) bufs[0].cbBuffer, (int) bufs[0].BufferType,
-		   (int) bufs[1].cbBuffer, (int) bufs[1].BufferType,
-		   (int) bufs[2].cbBuffer, (int) bufs[2].BufferType);
-	wpa_printf(MSG_MSGDUMP, "Schannel: EncryptMessage pointers: "
-		   "out_data=%p bufs %p %p %p",
-		   wpabuf_head(out), bufs[0].pvBuffer, bufs[1].pvBuffer,
-		   bufs[2].pvBuffer);
-
-	for (i = 0; i < 3; i++) {
-		if (bufs[i].pvBuffer && bufs[i].BufferType != SECBUFFER_EMPTY)
-		{
-			wpa_hexdump(MSG_MSGDUMP, "SChannel: bufs",
-				    bufs[i].pvBuffer, bufs[i].cbBuffer);
-		}
-	}
-
-	if (status == SEC_E_OK) {
-		wpa_printf(MSG_DEBUG, "%s: SEC_E_OK", __func__);
-		wpa_hexdump_buf_key(MSG_MSGDUMP, "Schannel: Encrypted data "
-				    "from EncryptMessage", out);
-		return out;
-	}
-
-	wpa_printf(MSG_DEBUG, "%s: Failed - status=%d",
-		   __func__, (int) status);
-	wpabuf_free(out);
-	return NULL;
-}
-
-
-struct wpabuf * tls_connection_decrypt(void *tls_ctx,
-				       struct tls_connection *conn,
-				       const struct wpabuf *in_data)
-{
-	struct tls_global *global = tls_ctx;
-	SECURITY_STATUS status;
-	SecBufferDesc buf;
-	SecBuffer bufs[4];
-	int i;
-	struct wpabuf *out, *tmp;
-
-	wpa_hexdump_buf(MSG_MSGDUMP,
-			"Schannel: Encrypted data to DecryptMessage", in_data);
-	os_memset(&bufs, 0, sizeof(bufs));
-	tmp = wpabuf_dup(in_data);
-	if (tmp == NULL)
-		return NULL;
-	bufs[0].pvBuffer = wpabuf_mhead(tmp);
-	bufs[0].cbBuffer = wpabuf_len(in_data);
-	bufs[0].BufferType = SECBUFFER_DATA;
-
-	bufs[1].BufferType = SECBUFFER_EMPTY;
-	bufs[2].BufferType = SECBUFFER_EMPTY;
-	bufs[3].BufferType = SECBUFFER_EMPTY;
-
-	buf.ulVersion = SECBUFFER_VERSION;
-	buf.cBuffers = 4;
-	buf.pBuffers = bufs;
-
-	status = global->sspi->DecryptMessage(&conn->context, &buf, 0,
-						    NULL);
-	wpa_printf(MSG_MSGDUMP, "Schannel: DecryptMessage -> "
-		   "status=%d len[0]=%d type[0]=%d len[1]=%d type[1]=%d "
-		   "len[2]=%d type[2]=%d len[3]=%d type[3]=%d",
-		   (int) status,
-		   (int) bufs[0].cbBuffer, (int) bufs[0].BufferType,
-		   (int) bufs[1].cbBuffer, (int) bufs[1].BufferType,
-		   (int) bufs[2].cbBuffer, (int) bufs[2].BufferType,
-		   (int) bufs[3].cbBuffer, (int) bufs[3].BufferType);
-	wpa_printf(MSG_MSGDUMP, "Schannel: DecryptMessage pointers: "
-		   "out_data=%p bufs %p %p %p %p",
-		   wpabuf_head(tmp), bufs[0].pvBuffer, bufs[1].pvBuffer,
-		   bufs[2].pvBuffer, bufs[3].pvBuffer);
-
-	switch (status) {
-	case SEC_E_INCOMPLETE_MESSAGE:
-		wpa_printf(MSG_DEBUG, "%s: SEC_E_INCOMPLETE_MESSAGE",
-			   __func__);
-		break;
-	case SEC_E_OK:
-		wpa_printf(MSG_DEBUG, "%s: SEC_E_OK", __func__);
-		for (i = 0; i < 4; i++) {
-			if (bufs[i].BufferType == SECBUFFER_DATA)
-				break;
-		}
-		if (i == 4) {
-			wpa_printf(MSG_DEBUG, "%s: No output data from "
-				   "DecryptMessage", __func__);
-			wpabuf_free(tmp);
-			return NULL;
-		}
-		wpa_hexdump_key(MSG_MSGDUMP, "Schannel: Decrypted data from "
-				"DecryptMessage",
-				bufs[i].pvBuffer, bufs[i].cbBuffer);
-		out = wpabuf_alloc_copy(bufs[i].pvBuffer, bufs[i].cbBuffer);
-		wpabuf_free(tmp);
-		return out;
-	}
-
-	wpa_printf(MSG_DEBUG, "%s: Failed - status=%d",
-		   __func__, (int) status);
-	wpabuf_free(tmp);
-	return NULL;
-}
-
-
-int tls_connection_resumed(void *ssl_ctx, struct tls_connection *conn)
-{
-	return 0;
-}
-
-
-int tls_connection_set_cipher_list(void *tls_ctx, struct tls_connection *conn,
-				   u8 *ciphers)
-{
-	return -1;
-}
-
-
-int tls_get_cipher(void *ssl_ctx, struct tls_connection *conn,
-		   char *buf, size_t buflen)
-{
-	return -1;
-}
-
-
-int tls_connection_enable_workaround(void *ssl_ctx,
-				     struct tls_connection *conn)
-{
-	return 0;
-}
-
-
-int tls_connection_client_hello_ext(void *ssl_ctx, struct tls_connection *conn,
-				    int ext_type, const u8 *data,
-				    size_t data_len)
-{
-	return -1;
-}
-
-
-int tls_connection_get_failed(void *ssl_ctx, struct tls_connection *conn)
-{
-	if (conn == NULL)
-		return -1;
-	return conn->failed;
-}
-
-
-int tls_connection_get_read_alerts(void *ssl_ctx, struct tls_connection *conn)
-{
-	if (conn == NULL)
-		return -1;
-	return conn->read_alerts;
-}
-
-
-int tls_connection_get_write_alerts(void *ssl_ctx, struct tls_connection *conn)
-{
-	if (conn == NULL)
-		return -1;
-	return conn->write_alerts;
-}
-
-
-int tls_connection_set_params(void *tls_ctx, struct tls_connection *conn,
-			      const struct tls_connection_params *params)
-{
-	struct tls_global *global = tls_ctx;
-	ALG_ID algs[1];
-	SECURITY_STATUS status;
-	TimeStamp ts_expiry;
-
-	if (conn == NULL)
-		return -1;
-
-	if (params->subject_match) {
-		wpa_printf(MSG_INFO, "TLS: subject_match not supported");
-		return -1;
-	}
-
-	if (params->altsubject_match) {
-		wpa_printf(MSG_INFO, "TLS: altsubject_match not supported");
-		return -1;
-	}
-
-	if (params->suffix_match) {
-		wpa_printf(MSG_INFO, "TLS: suffix_match not supported");
-		return -1;
-	}
-
-	if (params->domain_match) {
-		wpa_printf(MSG_INFO, "TLS: domain_match not supported");
-		return -1;
-	}
-
-	if (params->openssl_ciphers) {
-		wpa_printf(MSG_INFO, "GnuTLS: openssl_ciphers not supported");
-		return -1;
-	}
-
-	if (global->my_cert_store == NULL &&
-	    (global->my_cert_store = CertOpenSystemStore(0, TEXT("MY"))) ==
-	    NULL) {
-		wpa_printf(MSG_ERROR, "%s: CertOpenSystemStore failed - 0x%x",
-			   __func__, (unsigned int) GetLastError());
-		return -1;
-	}
-
-	os_memset(&conn->schannel_cred, 0, sizeof(conn->schannel_cred));
-	conn->schannel_cred.dwVersion = SCHANNEL_CRED_VERSION;
-	conn->schannel_cred.grbitEnabledProtocols = SP_PROT_TLS1;
-	algs[0] = CALG_RSA_KEYX;
-	conn->schannel_cred.cSupportedAlgs = 1;
-	conn->schannel_cred.palgSupportedAlgs = algs;
-	conn->schannel_cred.dwFlags |= SCH_CRED_NO_DEFAULT_CREDS;
-#ifdef UNICODE
-	status = global->sspi->AcquireCredentialsHandleW(
-		NULL, UNISP_NAME_W, SECPKG_CRED_OUTBOUND, NULL,
-		&conn->schannel_cred, NULL, NULL, &conn->creds, &ts_expiry);
-#else /* UNICODE */
-	status = global->sspi->AcquireCredentialsHandleA(
-		NULL, UNISP_NAME_A, SECPKG_CRED_OUTBOUND, NULL,
-		&conn->schannel_cred, NULL, NULL, &conn->creds, &ts_expiry);
-#endif /* UNICODE */
-	if (status != SEC_E_OK) {
-		wpa_printf(MSG_DEBUG, "%s: AcquireCredentialsHandleA failed - "
-			   "0x%x", __func__, (unsigned int) status);
-		return -1;
-	}
-
-	return 0;
-}
-
-
-unsigned int tls_capabilities(void *tls_ctx)
-{
-	return 0;
-}
-
-
-int tls_get_library_version(char *buf, size_t buf_len)
-{
-	return os_snprintf(buf, buf_len, "schannel");
-}
diff --git a/src/drivers/driver.h b/src/drivers/driver.h
index 03bd1a79a14c..3cdab5a7a87d 100644
--- a/src/drivers/driver.h
+++ b/src/drivers/driver.h
@@ -20,6 +20,7 @@
 #define WPA_SUPPLICANT_DRIVER_VERSION 4
 
 #include "common/defs.h"
+#include "common/ieee802_11_defs.h"
 #include "utils/list.h"
 
 #define HOSTAPD_CHAN_DISABLED 0x00000001
@@ -341,7 +342,7 @@ struct wpa_driver_scan_params {
 	 * is not needed anymore.
 	 */
 	struct wpa_driver_scan_filter {
-		u8 ssid[32];
+		u8 ssid[SSID_MAX_LEN];
 		size_t ssid_len;
 	} *filter_ssids;
 
@@ -1211,6 +1212,8 @@ struct wpa_driver_capa {
 #define WPA_DRIVER_FLAGS_HT_IBSS		0x0000001000000000ULL
 /** Driver supports IBSS with VHT datarates */
 #define WPA_DRIVER_FLAGS_VHT_IBSS		0x0000002000000000ULL
+/** Driver supports automatic band selection */
+#define WPA_DRIVER_FLAGS_SUPPORT_HW_MODE_ANY	0x0000004000000000ULL
 	u64 flags;
 
 #define WPA_DRIVER_SMPS_MODE_STATIC			0x00000001
@@ -1294,6 +1297,13 @@ struct wpa_driver_capa {
  */
 #define WPA_DRIVER_FLAGS_TX_POWER_INSERTION		0x00000008
 	u32 rrm_flags;
+
+	/* Driver concurrency capabilities */
+	unsigned int conc_capab;
+	/* Maximum number of concurrent channels on 2.4 GHz */
+	unsigned int max_conc_chan_2_4;
+	/* Maximum number of concurrent channels on 5 GHz */
+	unsigned int max_conc_chan_5_0;
 };
 
 
@@ -1394,6 +1404,16 @@ enum wpa_driver_if_type {
 	 * WPA_IF_MESH - Mesh interface
 	 */
 	WPA_IF_MESH,
+
+	/*
+	 * WPA_IF_TDLS - TDLS offchannel interface (used for pref freq only)
+	 */
+	WPA_IF_TDLS,
+
+	/*
+	 * WPA_IF_IBSS - IBSS interface (used for pref freq only)
+	 */
+	WPA_IF_IBSS,
 };
 
 struct wpa_init_params {
@@ -1477,6 +1497,7 @@ struct wpa_signal_info {
 	int above_threshold;
 	int current_signal;
 	int avg_signal;
+	int avg_beacon_signal;
 	int current_noise;
 	int current_txrate;
 	enum chan_width chanwidth;
@@ -1576,6 +1597,7 @@ enum drv_br_port_attr {
 
 enum drv_br_net_param {
 	DRV_BR_NET_PARAM_GARP_ACCEPT,
+	DRV_BR_MULTICAST_SNOOPING,
 };
 
 struct drv_acs_params {
@@ -1587,6 +1609,17 @@ struct drv_acs_params {
 
 	/* Indicates whether HT40 is enabled */
 	int ht40_enabled;
+
+	/* Indicates whether VHT is enabled */
+	int vht_enabled;
+
+	/* Configured ACS channel width */
+	u16 ch_width;
+
+	/* ACS channel list info */
+	unsigned int ch_list_len;
+	const u8 *ch_list;
+	const int *freq_list;
 };
 
 
@@ -1925,10 +1958,12 @@ struct wpa_driver_ops {
 	 * @data: IEEE 802.11 management frame with IEEE 802.11 header
 	 * @data_len: Size of the management frame
 	 * @noack: Do not wait for this frame to be acked (disable retries)
+	 * @freq: Frequency (in MHz) to send the frame on, or 0 to let the
+	 * driver decide
 	 * Returns: 0 on success, -1 on failure
 	 */
 	int (*send_mlme)(void *priv, const u8 *data, size_t data_len,
-			 int noack);
+			 int noack, unsigned int freq);
 
 	/**
 	 * update_ft_ies - Update FT (IEEE 802.11r) IEs
@@ -2332,7 +2367,8 @@ struct wpa_driver_ops {
 	 * Returns: 0 on success, -1 on failure
 	 */
 	int (*sta_set_flags)(void *priv, const u8 *addr,
-			     int total_flags, int flags_or, int flags_and);
+			     unsigned int total_flags, unsigned int flags_or,
+			     unsigned int flags_and);
 
 	/**
 	 * set_tx_queue_params - Set TX queue parameters
@@ -2655,18 +2691,6 @@ struct wpa_driver_ops {
 	int (*send_frame)(void *priv, const u8 *data, size_t data_len,
 			  int encrypt);
 
-	/**
-	 * shared_freq - Get operating frequency of shared interface(s)
-	 * @priv: Private driver interface data
-	 * Returns: Operating frequency in MHz, 0 if no shared operation in
-	 * use, or -1 on failure
-	 *
-	 * This command can be used to request the current operating frequency
-	 * of any virtual interface that shares the same radio to provide
-	 * information for channel selection for other virtual interfaces.
-	 */
-	int (*shared_freq)(void *priv);
-
 	/**
 	 * get_noa - Get current Notice of Absence attribute payload
 	 * @priv: Private driver interface data
@@ -3381,6 +3405,40 @@ struct wpa_driver_ops {
 	 * indicates support for such offloading (WPA_DRIVER_FLAGS_ACS_OFFLOAD).
 	 */
 	int (*do_acs)(void *priv, struct drv_acs_params *params);
+
+	/**
+	 * set_band - Notify driver of band selection
+	 * @priv: Private driver interface data
+	 * @band: The selected band(s)
+	 * Returns 0 on success, -1 on failure
+	 */
+	int (*set_band)(void *priv, enum set_band band);
+
+	/**
+	 * get_pref_freq_list - Get preferred frequency list for an interface
+	 * @priv: Private driver interface data
+	 * @if_type: Interface type
+	 * @num: Number of channels
+	 * @freq_list: Preferred channel frequency list encoded in MHz values
+	 * Returns 0 on success, -1 on failure
+	 *
+	 * This command can be used to query the preferred frequency list from
+	 * the driver specific to a particular interface type.
+	 */
+	int (*get_pref_freq_list)(void *priv, enum wpa_driver_if_type if_type,
+				  unsigned int *num, unsigned int *freq_list);
+
+	/**
+	 * set_prob_oper_freq - Indicate probable P2P operating channel
+	 * @priv: Private driver interface data
+	 * @freq: Channel frequency in MHz
+	 * Returns 0 on success, -1 on failure
+	 *
+	 * This command can be used to inform the driver of the operating
+	 * frequency that an ongoing P2P group formation is likely to come up
+	 * on. Local device is assuming P2P Client role.
+	 */
+	int (*set_prob_oper_freq)(void *priv, unsigned int freq);
 };
 
 
@@ -4557,10 +4615,20 @@ union wpa_event_data {
 	 * struct acs_selected_channels - Data for EVENT_ACS_CHANNEL_SELECTED
 	 * @pri_channel: Selected primary channel
 	 * @sec_channel: Selected secondary channel
+	 * @vht_seg0_center_ch: VHT mode Segment0 center channel
+	 * @vht_seg1_center_ch: VHT mode Segment1 center channel
+	 * @ch_width: Selected Channel width by driver. Driver may choose to
+	 *	change hostapd configured ACS channel width due driver internal
+	 *	channel restrictions.
+	 * hw_mode: Selected band (used with hw_mode=any)
 	 */
 	struct acs_selected_channels {
 		u8 pri_channel;
 		u8 sec_channel;
+		u8 vht_seg0_center_ch;
+		u8 vht_seg1_center_ch;
+		u16 ch_width;
+		enum hostapd_hw_mode hw_mode;
 	} acs_selected_channels;
 };
 
@@ -4631,6 +4699,6 @@ wpa_get_wowlan_triggers(const char *wowlan_triggers,
 			const struct wpa_driver_capa *capa);
 
 /* NULL terminated array of linked in driver wrappers */
-extern struct wpa_driver_ops *wpa_drivers[];
+extern const struct wpa_driver_ops *const wpa_drivers[];
 
 #endif /* DRIVER_H */
diff --git a/src/drivers/driver_atheros.c b/src/drivers/driver_atheros.c
index f46442163d93..ef140934973a 100644
--- a/src/drivers/driver_atheros.c
+++ b/src/drivers/driver_atheros.c
@@ -55,6 +55,10 @@
 #include "netlink.h"
 #include "linux_ioctl.h"
 
+#if defined(CONFIG_IEEE80211W) || defined(CONFIG_IEEE80211R) || defined(CONFIG_HS20) || defined(CONFIG_WNM) || defined(CONFIG_WPS)
+#define ATHEROS_USE_RAW_RECEIVE
+#endif
+
 
 struct atheros_driver_data {
 	struct hostapd_data *hapd;		/* back pointer */
@@ -430,7 +434,8 @@ atheros_set_sta_authorized(void *priv, const u8 *addr, int authorized)
 
 static int
 atheros_sta_set_flags(void *priv, const u8 *addr,
-		      int total_flags, int flags_or, int flags_and)
+		      unsigned int total_flags, unsigned int flags_or,
+		      unsigned int flags_and)
 {
 	/* For now, only support setting Authorized flag */
 	if (flags_or & WPA_STA_AUTHORIZED)
@@ -823,7 +828,7 @@ static int atheros_set_qos_map(void *ctx, const u8 *qos_map_set,
 	return 0;
 }
 
-#if defined(CONFIG_WPS) || defined(CONFIG_IEEE80211R) || defined(CONFIG_WNM) || defined(CONFIG_HS20)
+#ifdef ATHEROS_USE_RAW_RECEIVE
 static void atheros_raw_receive(void *ctx, const u8 *src_addr, const u8 *buf,
 				size_t len)
 {
@@ -911,7 +916,7 @@ static void atheros_raw_receive(void *ctx, const u8 *src_addr, const u8 *buf,
 		break;
 	}
 }
-#endif
+#endif /* ATHEROS_USE_RAW_RECEIVE */
 
 static int atheros_receive_pkt(struct atheros_driver_data *drv)
 {
@@ -923,11 +928,11 @@ static int atheros_receive_pkt(struct atheros_driver_data *drv)
 #ifdef CONFIG_WPS
 	filt.app_filterype |= IEEE80211_FILTER_TYPE_PROBE_REQ;
 #endif /* CONFIG_WPS */
-#ifdef CONFIG_IEEE80211R
+#if defined(CONFIG_IEEE80211W) || defined(CONFIG_IEEE80211R)
 	filt.app_filterype |= (IEEE80211_FILTER_TYPE_ASSOC_REQ |
 			       IEEE80211_FILTER_TYPE_AUTH |
 			       IEEE80211_FILTER_TYPE_ACTION);
-#endif
+#endif /* CONFIG_IEEE80211R || CONFIG_IEEE80211W */
 #ifdef CONFIG_WNM
 	filt.app_filterype |= IEEE80211_FILTER_TYPE_ACTION;
 #endif /* CONFIG_WNM */
@@ -1026,7 +1031,7 @@ atheros_set_ap_wps_ie(void *priv, const struct wpabuf *beacon,
 #define atheros_set_ap_wps_ie NULL
 #endif /* CONFIG_WPS */
 
-#ifdef CONFIG_IEEE80211R
+#if defined(CONFIG_IEEE80211R) || defined(CONFIG_IEEE80211W)
 static int
 atheros_sta_auth(void *priv, const u8 *own_addr, const u8 *addr, u16 seq,
 		 u16 status_code, const u8 *ie, size_t len)
@@ -1102,7 +1107,7 @@ atheros_sta_assoc(void *priv, const u8 *own_addr, const u8 *addr,
 	}
 	return ret;
 }
-#endif /* CONFIG_IEEE80211R */
+#endif /* CONFIG_IEEE80211R || CONFIG_IEEE80211W */
 
 static void
 atheros_new_sta(struct atheros_driver_data *drv, u8 addr[IEEE80211_ADDR_LEN])
@@ -1177,6 +1182,7 @@ static void
 atheros_wireless_event_wireless_custom(struct atheros_driver_data *drv,
 				       char *custom, char *end)
 {
+#define MGMT_FRAM_TAG_SIZE 30 /* hardcoded in driver */
 	wpa_printf(MSG_DEBUG, "Custom wireless event: '%s'", custom);
 
 	if (strncmp(custom, "MLME-MICHAELMICFAILURE.indication", 33) == 0) {
@@ -1232,9 +1238,6 @@ atheros_wireless_event_wireless_custom(struct atheros_driver_data *drv,
 		 * so all are enabled for WPS... ugh.
 		 */
 		wpa_supplicant_event(drv->hapd, EVENT_WPS_BUTTON_PUSHED, NULL);
-#endif /* CONFIG_WPS */
-#if defined(CONFIG_WPS) || defined(CONFIG_IEEE80211R) || defined(CONFIG_HS20)
-#define MGMT_FRAM_TAG_SIZE 30 /* hardcoded in driver */
 	} else if (strncmp(custom, "Manage.prob_req ", 16) == 0) {
 		/*
 		 * Atheros driver uses a hack to pass Probe Request frames as a
@@ -1250,40 +1253,46 @@ atheros_wireless_event_wireless_custom(struct atheros_driver_data *drv,
 		}
 		atheros_raw_receive(drv, NULL,
 				    (u8 *) custom + MGMT_FRAM_TAG_SIZE, len);
+#endif /* CONFIG_WPS */
+#if defined(CONFIG_IEEE80211R) || defined(CONFIG_IEEE80211W)
 	} else if (strncmp(custom, "Manage.assoc_req ", 17) == 0) {
 		/* Format: "Manage.assoc_req <frame len>" | zero padding |
 		 * frame */
 		int len = atoi(custom + 17);
 		if (len < 0 || custom + MGMT_FRAM_TAG_SIZE + len > end) {
-			wpa_printf(MSG_DEBUG, "Invalid Manage.prob_req/"
-				   "assoc_req/auth event length %d", len);
+			wpa_printf(MSG_DEBUG,
+				   "Invalid Manage.assoc_req event length %d",
+				   len);
 			return;
 		}
 		atheros_raw_receive(drv, NULL,
 				    (u8 *) custom + MGMT_FRAM_TAG_SIZE, len);
-	} else if (strncmp(custom, "Manage.action ", 14) == 0) {
-		/* Format: "Manage.assoc_req <frame len>" | zero padding |
-		 * frame */
+		} else if (strncmp(custom, "Manage.auth ", 12) == 0) {
+		/* Format: "Manage.auth <frame len>" | zero padding | frame */
+		int len = atoi(custom + 12);
+			if (len < 0 ||
+			    custom + MGMT_FRAM_TAG_SIZE + len > end) {
+			wpa_printf(MSG_DEBUG,
+				   "Invalid Manage.auth event length %d", len);
+			return;
+		}
+		atheros_raw_receive(drv, NULL,
+				    (u8 *) custom + MGMT_FRAM_TAG_SIZE, len);
+#endif /* CONFIG_IEEE80211W || CONFIG_IEEE80211R */
+#ifdef ATHEROS_USE_RAW_RECEIVE
+		} else if (strncmp(custom, "Manage.action ", 14) == 0) {
+		/* Format: "Manage.assoc_req <frame len>" | zero padding | frame
+		 */
 		int len = atoi(custom + 14);
 		if (len < 0 || custom + MGMT_FRAM_TAG_SIZE + len > end) {
-			wpa_printf(MSG_DEBUG, "Invalid Manage.prob_req/"
-				   "assoc_req/auth event length %d", len);
+			wpa_printf(MSG_DEBUG,
+				   "Invalid Manage.action event length %d",
+				   len);
 			return;
 		}
 		atheros_raw_receive(drv, NULL,
 				    (u8 *) custom + MGMT_FRAM_TAG_SIZE, len);
-	} else if (strncmp(custom, "Manage.auth ", 12) == 0) {
-		/* Format: "Manage.auth <frame len>" | zero padding | frame
-		 */
-		int len = atoi(custom + 12);
-		if (len < 0 || custom + MGMT_FRAM_TAG_SIZE + len > end) {
-			wpa_printf(MSG_DEBUG, "Invalid Manage.prob_req/"
-				   "assoc_req/auth event length %d", len);
-			return;
-		}
-		atheros_raw_receive(drv, NULL,
-				    (u8 *) custom + MGMT_FRAM_TAG_SIZE, len);
-#endif /* CONFIG_WPS or CONFIG_IEEE80211R */
+#endif /* ATHEROS_USE_RAW_RECEIVE */
 	}
 }
 
@@ -1682,8 +1691,7 @@ bad:
 		l2_packet_deinit(drv->sock_xmit);
 	if (drv->ioctl_sock >= 0)
 		close(drv->ioctl_sock);
-	if (drv != NULL)
-		free(drv);
+	os_free(drv);
 	return NULL;
 }
 
@@ -1694,6 +1702,13 @@ atheros_deinit(void *priv)
 	struct atheros_driver_data *drv = priv;
 
 	atheros_reset_appfilter(drv);
+
+	if (drv->wpa_ie || drv->wps_beacon_ie || drv->wps_probe_resp_ie) {
+		wpabuf_free(drv->wpa_ie);
+		wpabuf_free(drv->wps_beacon_ie);
+		wpabuf_free(drv->wps_probe_resp_ie);
+		atheros_set_opt_ie(priv, NULL, 0);
+	}
 	netlink_deinit(drv->netlink);
 	(void) linux_set_iface_flags(drv->ioctl_sock, drv->iface, 0);
 	if (drv->ioctl_sock >= 0)
@@ -1704,10 +1719,7 @@ atheros_deinit(void *priv)
 		l2_packet_deinit(drv->sock_xmit);
 	if (drv->sock_raw)
 		l2_packet_deinit(drv->sock_raw);
-	wpabuf_free(drv->wpa_ie);
-	wpabuf_free(drv->wps_beacon_ie);
-	wpabuf_free(drv->wps_probe_resp_ie);
-	free(drv);
+	os_free(drv);
 }
 
 static int
@@ -1833,10 +1845,10 @@ static int atheros_set_ap(void *priv, struct wpa_driver_ap_params *params)
 }
 
 
-#ifdef CONFIG_IEEE80211R
+#if defined(CONFIG_IEEE80211R) || defined(CONFIG_IEEE80211W)
 
 static int atheros_send_mgmt(void *priv, const u8 *frm, size_t data_len,
-			     int noack)
+			     int noack, unsigned int freq)
 {
 	struct atheros_driver_data *drv = priv;
 	u8 buf[1510];
@@ -1858,8 +1870,11 @@ static int atheros_send_mgmt(void *priv, const u8 *frm, size_t data_len,
 	return set80211priv(drv, IEEE80211_IOCTL_SEND_MGMT, mgmt_frm,
 			    sizeof(struct ieee80211req_mgmtbuf) + data_len);
 }
+#endif /* CONFIG_IEEE80211R || CONFIG_IEEE80211W */
 
 
+#ifdef CONFIG_IEEE80211R
+
 static int atheros_add_tspec(void *priv, const u8 *addr, u8 *tspec_ie,
 			     size_t tspec_ielen)
 {
@@ -2139,10 +2154,12 @@ const struct wpa_driver_ops wpa_driver_atheros_ops = {
 	.set_ap_wps_ie		= atheros_set_ap_wps_ie,
 	.set_authmode		= atheros_set_authmode,
 	.set_ap			= atheros_set_ap,
-#ifdef CONFIG_IEEE80211R
+#if defined(CONFIG_IEEE80211R) || defined(CONFIG_IEEE80211W)
 	.sta_assoc              = atheros_sta_assoc,
 	.sta_auth               = atheros_sta_auth,
 	.send_mlme       	= atheros_send_mgmt,
+#endif /* CONFIG_IEEE80211R || CONFIG_IEEE80211W */
+#ifdef CONFIG_IEEE80211R
 	.add_tspec      	= atheros_add_tspec,
 	.add_sta_node    	= atheros_add_sta_node,
 #endif /* CONFIG_IEEE80211R */
diff --git a/src/drivers/driver_bsd.c b/src/drivers/driver_bsd.c
index 0f1a0f60f27d..bab1f031d24c 100644
--- a/src/drivers/driver_bsd.c
+++ b/src/drivers/driver_bsd.c
@@ -861,8 +861,7 @@ bad:
 	if (drv->sock >= 0)
 		close(drv->sock);
 	os_free(drv->event_buf);
-	if (drv != NULL)
-		os_free(drv);
+	os_free(drv);
 	return NULL;
 }
 
@@ -895,7 +894,8 @@ bsd_commit(void *priv)
 
 static int
 bsd_set_sta_authorized(void *priv, const u8 *addr,
-		       int total_flags, int flags_or, int flags_and)
+		       unsigned int total_flags, unsigned int flags_or,
+		       unsigned int flags_and)
 {
 	int authorized = -1;
 
diff --git a/src/drivers/driver_hostap.c b/src/drivers/driver_hostap.c
index 84b98fb8c87f..a7aa5eff00bd 100644
--- a/src/drivers/driver_hostap.c
+++ b/src/drivers/driver_hostap.c
@@ -140,7 +140,7 @@ static void handle_tx_callback(struct hostap_driver_data *drv, u8 *buf,
 static void handle_frame(struct hostap_driver_data *drv, u8 *buf, size_t len)
 {
 	struct ieee80211_hdr *hdr;
-	u16 fc, extra_len, type, stype;
+	u16 fc, type, stype;
 	size_t data_len = len;
 	int ver;
 	union wpa_event_data event;
@@ -165,19 +165,10 @@ static void handle_frame(struct hostap_driver_data *drv, u8 *buf, size_t len)
 
 	ver = fc & WLAN_FC_PVER;
 
-	/* protocol version 3 is reserved for indicating extra data after the
-	 * payload, version 2 for indicating ACKed frame (TX callbacks), and
-	 * version 1 for indicating failed frame (no ACK, TX callbacks) */
-	if (ver == 3) {
-		u8 *pos = buf + len - 2;
-		extra_len = WPA_GET_LE16(pos);
-		printf("extra data in frame (elen=%d)\n", extra_len);
-		if ((size_t) extra_len + 2 > len) {
-			printf("  extra data overflow\n");
-			return;
-		}
-		len -= extra_len + 2;
-	} else if (ver == 1 || ver == 2) {
+	/* protocol version 2 is reserved for indicating ACKed frame (TX
+	 * callbacks), and version 1 for indicating failed frame (no ACK, TX
+	 * callbacks) */
+	if (ver == 1 || ver == 2) {
 		handle_tx_callback(drv, buf, data_len, ver == 2 ? 1 : 0);
 		return;
 	} else if (ver != 0) {
@@ -266,7 +257,8 @@ static int hostap_init_sockets(struct hostap_driver_data *drv, u8 *own_addr)
 }
 
 
-static int hostap_send_mlme(void *priv, const u8 *msg, size_t len, int noack)
+static int hostap_send_mlme(void *priv, const u8 *msg, size_t len, int noack,
+			    unsigned int freq)
 {
 	struct hostap_driver_data *drv = priv;
 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) msg;
@@ -315,20 +307,21 @@ static int hostap_send_eapol(void *priv, const u8 *addr, const u8 *data,
 	pos += 2;
 	memcpy(pos, data, data_len);
 
-	res = hostap_send_mlme(drv, (u8 *) hdr, len, 0);
+	res = hostap_send_mlme(drv, (u8 *) hdr, len, 0, 0);
 	if (res < 0) {
 		wpa_printf(MSG_ERROR, "hostap_send_eapol - packet len: %lu - "
 			   "failed: %d (%s)",
 			   (unsigned long) len, errno, strerror(errno));
 	}
-	free(hdr);
+	os_free(hdr);
 
 	return res;
 }
 
 
 static int hostap_sta_set_flags(void *priv, const u8 *addr,
-				int total_flags, int flags_or, int flags_and)
+				unsigned int total_flags, unsigned int flags_or,
+				unsigned int flags_and)
 {
 	struct hostap_driver_data *drv = priv;
 	struct prism2_hostapd_param param;
@@ -470,18 +463,18 @@ static int hostap_get_seqnum(const char *ifname, void *priv, const u8 *addr,
 	param = (struct prism2_hostapd_param *) buf;
 	param->cmd = PRISM2_GET_ENCRYPTION;
 	if (addr == NULL)
-		memset(param->sta_addr, 0xff, ETH_ALEN);
+		os_memset(param->sta_addr, 0xff, ETH_ALEN);
 	else
-		memcpy(param->sta_addr, addr, ETH_ALEN);
+		os_memcpy(param->sta_addr, addr, ETH_ALEN);
 	param->u.crypt.idx = idx;
 
 	if (hostapd_ioctl(drv, param, blen)) {
 		printf("Failed to get encryption.\n");
 		ret = -1;
 	} else {
-		memcpy(seq, param->u.crypt.seq, 8);
+		os_memcpy(seq, param->u.crypt.seq, 8);
 	}
-	free(buf);
+	os_free(buf);
 
 	return ret;
 }
@@ -1052,7 +1045,7 @@ static int hostap_sta_deauth(void *priv, const u8 *own_addr, const u8 *addr,
 	memcpy(mgmt.bssid, own_addr, ETH_ALEN);
 	mgmt.u.deauth.reason_code = host_to_le16(reason);
 	return hostap_send_mlme(drv, (u8 *) &mgmt, IEEE80211_HDRLEN +
-				sizeof(mgmt.u.deauth), 0);
+				sizeof(mgmt.u.deauth), 0, 0);
 }
 
 
@@ -1090,7 +1083,7 @@ static int hostap_sta_disassoc(void *priv, const u8 *own_addr, const u8 *addr,
 	memcpy(mgmt.bssid, own_addr, ETH_ALEN);
 	mgmt.u.disassoc.reason_code = host_to_le16(reason);
 	return  hostap_send_mlme(drv, (u8 *) &mgmt, IEEE80211_HDRLEN +
-				 sizeof(mgmt.u.disassoc), 0);
+				 sizeof(mgmt.u.disassoc), 0, 0);
 }
 
 
@@ -1168,7 +1161,7 @@ static void wpa_driver_hostap_poll_client(void *priv, const u8 *own_addr,
 	os_memcpy(hdr.IEEE80211_BSSID_FROMDS, own_addr, ETH_ALEN);
 	os_memcpy(hdr.IEEE80211_SA_FROMDS, own_addr, ETH_ALEN);
 
-	hostap_send_mlme(priv, (u8 *)&hdr, sizeof(hdr), 0);
+	hostap_send_mlme(priv, (u8 *)&hdr, sizeof(hdr), 0, 0);
 }
 
 
diff --git a/src/drivers/driver_hostap.h b/src/drivers/driver_hostap.h
index a9d3e76cbe8f..4c1e6d69f0a7 100644
--- a/src/drivers/driver_hostap.h
+++ b/src/drivers/driver_hostap.h
@@ -192,7 +192,7 @@ struct prism2_hostapd_param {
 		} mlme;
 		struct {
 			u8 ssid_len;
-			u8 ssid[32];
+			u8 ssid[SSID_MAX_LEN];
 		} scan_req;
 	} u;
 };
diff --git a/src/drivers/driver_ndis.c b/src/drivers/driver_ndis.c
index 4953af6a16b2..669f1b813c43 100644
--- a/src/drivers/driver_ndis.c
+++ b/src/drivers/driver_ndis.c
@@ -709,11 +709,11 @@ static int wpa_driver_ndis_radio_off(struct wpa_driver_ndis_data *drv)
 /* Disconnect by setting SSID to random (i.e., likely not used). */
 static int wpa_driver_ndis_disconnect(struct wpa_driver_ndis_data *drv)
 {
-	char ssid[32];
+	char ssid[SSID_MAX_LEN];
 	int i;
-	for (i = 0; i < 32; i++)
+	for (i = 0; i < SSID_MAX_LEN; i++)
 		ssid[i] = rand() & 0xff;
-	return wpa_driver_ndis_set_ssid(drv, (u8 *) ssid, 32);
+	return wpa_driver_ndis_set_ssid(drv, (u8 *) ssid, SSID_MAX_LEN);
 }
 
 
@@ -806,7 +806,7 @@ static struct wpa_scan_res * wpa_driver_ndis_add_scan_ssid(
 	if (wpa_scan_get_ie(r, WLAN_EID_SSID))
 		return r; /* SSID IE already present */
 
-	if (ssid->SsidLength == 0 || ssid->SsidLength > 32)
+	if (ssid->SsidLength == 0 || ssid->SsidLength > SSID_MAX_LEN)
 		return r; /* No valid SSID inside scan data */
 
 	nr = os_realloc(r, sizeof(*r) + r->ie_len + 2 + ssid->SsidLength);
diff --git a/src/drivers/driver_nl80211.c b/src/drivers/driver_nl80211.c
index d7438683d8f0..00b173f3f85a 100644
--- a/src/drivers/driver_nl80211.c
+++ b/src/drivers/driver_nl80211.c
@@ -87,7 +87,6 @@ static void nl80211_handle_destroy(struct nl_handle *handle)
 #undef nl_socket_set_nonblocking
 #define nl_socket_set_nonblocking(h) android_nl_socket_set_nonblocking(h)
 
-#define genl_ctrl_resolve android_genl_ctrl_resolve
 #endif /* ANDROID */
 
 
@@ -1187,6 +1186,7 @@ static int get_link_signal(struct nl_msg *msg, void *arg)
 	static struct nla_policy policy[NL80211_STA_INFO_MAX + 1] = {
 		[NL80211_STA_INFO_SIGNAL] = { .type = NLA_U8 },
 		[NL80211_STA_INFO_SIGNAL_AVG] = { .type = NLA_U8 },
+		[NL80211_STA_INFO_BEACON_SIGNAL_AVG] = { .type = NLA_U8 },
 	};
 	struct nlattr *rinfo[NL80211_RATE_INFO_MAX + 1];
 	static struct nla_policy rate_policy[NL80211_RATE_INFO_MAX + 1] = {
@@ -1215,6 +1215,13 @@ static int get_link_signal(struct nl_msg *msg, void *arg)
 	else
 		sig_change->avg_signal = 0;
 
+	if (sinfo[NL80211_STA_INFO_BEACON_SIGNAL_AVG])
+		sig_change->avg_beacon_signal =
+			(s8)
+			nla_get_u8(sinfo[NL80211_STA_INFO_BEACON_SIGNAL_AVG]);
+	else
+		sig_change->avg_beacon_signal = 0;
+
 	if (sinfo[NL80211_STA_INFO_TX_BITRATE]) {
 		if (nla_parse_nested(rinfo, NL80211_RATE_INFO_MAX,
 				     sinfo[NL80211_STA_INFO_TX_BITRATE],
@@ -1871,6 +1878,11 @@ static int nl80211_mgmt_subscribe_non_ap(struct i802_bss *bss)
 			ret = -1;
 	}
 #endif /* CONFIG_TDLS */
+#ifdef CONFIG_FST
+	/* FST Action frames */
+	if (nl80211_register_action_frame(bss, (u8 *) "\x12", 1) < 0)
+		ret = -1;
+#endif /* CONFIG_FST */
 
 	/* FT Action frames */
 	if (nl80211_register_action_frame(bss, (u8 *) "\x06", 1) < 0)
@@ -2493,7 +2505,7 @@ static int wpa_driver_nl80211_set_key(const char *ifname, struct i802_bss *bss,
 {
 	struct wpa_driver_nl80211_data *drv = bss->drv;
 	int ifindex;
-	struct nl_msg *msg;
+	struct nl_msg *msg = NULL;
 	int ret;
 	int tdls = 0;
 
@@ -2526,11 +2538,15 @@ static int wpa_driver_nl80211_set_key(const char *ifname, struct i802_bss *bss,
 		if (!msg)
 			return -ENOBUFS;
 	} else {
+		u32 suite;
+
+		suite = wpa_alg_to_cipher_suite(alg, key_len);
+		if (!suite)
+			goto fail;
 		msg = nl80211_ifindex_msg(drv, ifindex, 0, NL80211_CMD_NEW_KEY);
 		if (!msg ||
 		    nla_put(msg, NL80211_ATTR_KEY_DATA, key_len, key) ||
-		    nla_put_u32(msg, NL80211_ATTR_KEY_CIPHER,
-				wpa_alg_to_cipher_suite(alg, key_len)))
+		    nla_put_u32(msg, NL80211_ATTR_KEY_CIPHER, suite))
 			goto fail;
 		wpa_hexdump_key(MSG_DEBUG, "nl80211: KEY_DATA", key, key_len);
 	}
@@ -2632,9 +2648,15 @@ static int nl_add_key(struct nl_msg *msg, enum wpa_alg alg,
 		      const u8 *key, size_t key_len)
 {
 	struct nlattr *key_attr = nla_nest_start(msg, NL80211_ATTR_KEY);
+	u32 suite;
+
 	if (!key_attr)
 		return -1;
 
+	suite = wpa_alg_to_cipher_suite(alg, key_len);
+	if (!suite)
+		return -1;
+
 	if (defkey && alg == WPA_ALG_IGTK) {
 		if (nla_put_flag(msg, NL80211_KEY_DEFAULT_MGMT))
 			return -1;
@@ -2644,8 +2666,7 @@ static int nl_add_key(struct nl_msg *msg, enum wpa_alg alg,
 	}
 
 	if (nla_put_u8(msg, NL80211_KEY_IDX, key_idx) ||
-	    nla_put_u32(msg, NL80211_KEY_CIPHER,
-			wpa_alg_to_cipher_suite(alg, key_len)) ||
+	    nla_put_u32(msg, NL80211_KEY_CIPHER, suite) ||
 	    (seq && seq_len &&
 	     nla_put(msg, NL80211_KEY_SEQ, seq_len, seq)) ||
 	    nla_put(msg, NL80211_KEY_DATA, key_len, key))
@@ -3237,7 +3258,7 @@ static int wpa_driver_nl80211_set_acl(void *priv,
 	struct i802_bss *bss = priv;
 	struct wpa_driver_nl80211_data *drv = bss->drv;
 	struct nl_msg *msg;
-	struct nlattr *acl;
+	struct nl_msg *acl;
 	unsigned int i;
 	int ret;
 
@@ -3250,23 +3271,26 @@ static int wpa_driver_nl80211_set_acl(void *priv,
 	wpa_printf(MSG_DEBUG, "nl80211: Set %s ACL (num_mac_acl=%u)",
 		   params->acl_policy ? "Accept" : "Deny", params->num_mac_acl);
 
-	if (!(msg = nl80211_drv_msg(drv, 0, NL80211_CMD_SET_MAC_ACL)) ||
-	    nla_put_u32(msg, NL80211_ATTR_ACL_POLICY, params->acl_policy ?
-			NL80211_ACL_POLICY_DENY_UNLESS_LISTED :
-			NL80211_ACL_POLICY_ACCEPT_UNLESS_LISTED) ||
-	    (acl = nla_nest_start(msg, NL80211_ATTR_MAC_ADDRS)) == NULL) {
-		nlmsg_free(msg);
+	acl = nlmsg_alloc();
+	if (!acl)
 		return -ENOMEM;
-	}
-
 	for (i = 0; i < params->num_mac_acl; i++) {
-		if (nla_put(msg, i + 1, ETH_ALEN, params->mac_acl[i].addr)) {
-			nlmsg_free(msg);
+		if (nla_put(acl, i + 1, ETH_ALEN, params->mac_acl[i].addr)) {
+			nlmsg_free(acl);
 			return -ENOMEM;
 		}
 	}
 
-	nla_nest_end(msg, acl);
+	if (!(msg = nl80211_drv_msg(drv, 0, NL80211_CMD_SET_MAC_ACL)) ||
+	    nla_put_u32(msg, NL80211_ATTR_ACL_POLICY, params->acl_policy ?
+			NL80211_ACL_POLICY_DENY_UNLESS_LISTED :
+			NL80211_ACL_POLICY_ACCEPT_UNLESS_LISTED) ||
+	    nla_put_nested(msg, NL80211_ATTR_MAC_ADDRS, acl)) {
+		nlmsg_free(msg);
+		nlmsg_free(acl);
+		return -ENOMEM;
+	}
+	nlmsg_free(acl);
 
 	ret = send_and_recv_msgs(drv, msg, NULL, NULL);
 	if (ret) {
@@ -4233,8 +4257,9 @@ static int wpa_driver_nl80211_hapd_send_eapol(
 
 
 static int wpa_driver_nl80211_sta_set_flags(void *priv, const u8 *addr,
-					    int total_flags,
-					    int flags_or, int flags_and)
+					    unsigned int total_flags,
+					    unsigned int flags_or,
+					    unsigned int flags_and)
 {
 	struct i802_bss *bss = priv;
 	struct nl_msg *msg;
@@ -5658,8 +5683,8 @@ static void *i802_init(struct hostapd_data *hapd,
 	struct wpa_driver_nl80211_data *drv;
 	struct i802_bss *bss;
 	size_t i;
-	char brname[IFNAMSIZ];
-	int ifindex, br_ifindex;
+	char master_ifname[IFNAMSIZ];
+	int ifindex, br_ifindex = 0;
 	int br_added = 0;
 
 	bss = wpa_driver_nl80211_drv_init(hapd, params->ifname,
@@ -5670,15 +5695,21 @@ static void *i802_init(struct hostapd_data *hapd,
 
 	drv = bss->drv;
 
-	if (linux_br_get(brname, params->ifname) == 0) {
+	if (linux_br_get(master_ifname, params->ifname) == 0) {
 		wpa_printf(MSG_DEBUG, "nl80211: Interface %s is in bridge %s",
-			   params->ifname, brname);
-		br_ifindex = if_nametoindex(brname);
-		os_strlcpy(bss->brname, brname, IFNAMSIZ);
+			   params->ifname, master_ifname);
+		br_ifindex = if_nametoindex(master_ifname);
+		os_strlcpy(bss->brname, master_ifname, IFNAMSIZ);
+	} else if ((params->num_bridge == 0 || !params->bridge[0]) &&
+		   linux_master_get(master_ifname, params->ifname) == 0) {
+		wpa_printf(MSG_DEBUG, "nl80211: Interface %s is in master %s",
+			params->ifname, master_ifname);
+		/* start listening for EAPOL on the master interface */
+		add_ifidx(drv, if_nametoindex(master_ifname));
 	} else {
-		brname[0] = '\0';
-		br_ifindex = 0;
+		master_ifname[0] = '\0';
 	}
+
 	bss->br_ifindex = br_ifindex;
 
 	for (i = 0; i < params->num_bridge; i++) {
@@ -5698,7 +5729,7 @@ static void *i802_init(struct hostapd_data *hapd,
 		if (i802_check_bridge(drv, bss, params->bridge[0],
 				      params->ifname) < 0)
 			goto failed;
-		if (os_strcmp(params->bridge[0], brname) != 0)
+		if (os_strcmp(params->bridge[0], master_ifname) != 0)
 			br_added = 1;
 	}
 
@@ -5776,13 +5807,12 @@ static enum nl80211_iftype wpa_driver_nl80211_if_type(
 		return NL80211_IFTYPE_P2P_DEVICE;
 	case WPA_IF_MESH:
 		return NL80211_IFTYPE_MESH_POINT;
+	default:
+		return -1;
 	}
-	return -1;
 }
 
 
-#if defined(CONFIG_P2P) || defined(CONFIG_MESH)
-
 static int nl80211_addr_in_use(struct nl80211_global *global, const u8 *addr)
 {
 	struct wpa_driver_nl80211_data *drv;
@@ -5818,8 +5848,6 @@ static int nl80211_vif_addr(struct wpa_driver_nl80211_data *drv, u8 *new_addr)
 	return 0;
 }
 
-#endif /* CONFIG_P2P || CONFIG_MESH */
-
 
 struct wdev_info {
 	u64 wdev_id;
@@ -5895,21 +5923,21 @@ static int wpa_driver_nl80211_if_add(void *priv, enum wpa_driver_if_type type,
 	}
 
 	if (!addr) {
-		if (drv->nlmode == NL80211_IFTYPE_P2P_DEVICE)
+		if (nlmode == NL80211_IFTYPE_P2P_DEVICE)
 			os_memcpy(if_addr, bss->addr, ETH_ALEN);
 		else if (linux_get_ifhwaddr(drv->global->ioctl_sock,
-					    bss->ifname, if_addr) < 0) {
+					    ifname, if_addr) < 0) {
 			if (added)
 				nl80211_remove_iface(drv, ifidx);
 			return -1;
 		}
 	}
 
-#if defined(CONFIG_P2P) || defined(CONFIG_MESH)
 	if (!addr &&
 	    (type == WPA_IF_P2P_CLIENT || type == WPA_IF_P2P_GROUP ||
-	     type == WPA_IF_P2P_GO || type == WPA_IF_MESH)) {
-		/* Enforce unique P2P Interface Address */
+	     type == WPA_IF_P2P_GO || type == WPA_IF_MESH ||
+	     type == WPA_IF_STATION)) {
+		/* Enforce unique address */
 		u8 new_addr[ETH_ALEN];
 
 		if (linux_get_ifhwaddr(drv->global->ioctl_sock, ifname,
@@ -5920,8 +5948,7 @@ static int wpa_driver_nl80211_if_add(void *priv, enum wpa_driver_if_type type,
 		}
 		if (nl80211_addr_in_use(drv->global, new_addr)) {
 			wpa_printf(MSG_DEBUG, "nl80211: Allocate new address "
-				   "for %s interface", type == WPA_IF_MESH ?
-				   "mesh" : "P2P group");
+				   "for interface %s type %d", ifname, type);
 			if (nl80211_vif_addr(drv, new_addr) < 0) {
 				if (added)
 					nl80211_remove_iface(drv, ifidx);
@@ -5936,7 +5963,6 @@ static int wpa_driver_nl80211_if_add(void *priv, enum wpa_driver_if_type type,
 		}
 		os_memcpy(if_addr, new_addr, ETH_ALEN);
 	}
-#endif /* CONFIG_P2P || CONFIG_MESH */
 
 	if (type == WPA_IF_AP_BSS) {
 		struct i802_bss *new_bss = os_zalloc(sizeof(*new_bss));
@@ -6514,47 +6540,6 @@ static int nl80211_signal_poll(void *priv, struct wpa_signal_info *si)
 }
 
 
-static int wpa_driver_nl80211_shared_freq(void *priv)
-{
-	struct i802_bss *bss = priv;
-	struct wpa_driver_nl80211_data *drv = bss->drv;
-	struct wpa_driver_nl80211_data *driver;
-	int freq = 0;
-
-	/*
-	 * If the same PHY is in connected state with some other interface,
-	 * then retrieve the assoc freq.
-	 */
-	wpa_printf(MSG_DEBUG, "nl80211: Get shared freq for PHY %s",
-		   drv->phyname);
-
-	dl_list_for_each(driver, &drv->global->interfaces,
-			 struct wpa_driver_nl80211_data, list) {
-		if (drv == driver ||
-		    os_strcmp(drv->phyname, driver->phyname) != 0 ||
-		    !driver->associated)
-			continue;
-
-		wpa_printf(MSG_DEBUG, "nl80211: Found a match for PHY %s - %s "
-			   MACSTR,
-			   driver->phyname, driver->first_bss->ifname,
-			   MAC2STR(driver->first_bss->addr));
-		if (is_ap_interface(driver->nlmode))
-			freq = driver->first_bss->freq;
-		else
-			freq = nl80211_get_assoc_freq(driver);
-		wpa_printf(MSG_DEBUG, "nl80211: Shared freq for PHY %s: %d",
-			   drv->phyname, freq);
-	}
-
-	if (!freq)
-		wpa_printf(MSG_DEBUG, "nl80211: No shared interface for "
-			   "PHY (%s) in associated state", drv->phyname);
-
-	return freq;
-}
-
-
 static int nl80211_send_frame(void *priv, const u8 *data, size_t data_len,
 			      int encrypt)
 {
@@ -7275,11 +7260,12 @@ static int driver_nl80211_if_remove(void *priv, enum wpa_driver_if_type type,
 
 
 static int driver_nl80211_send_mlme(void *priv, const u8 *data,
-				    size_t data_len, int noack)
+				    size_t data_len, int noack,
+				    unsigned int freq)
 {
 	struct i802_bss *bss = priv;
 	return wpa_driver_nl80211_send_mlme(bss, data, data_len, noack,
-					    0, 0, 0, 0);
+					    freq, 0, 0, 0);
 }
 
 
@@ -7506,7 +7492,10 @@ static int wpa_driver_nl80211_status(void *priv, char *buf, size_t buflen)
 				  "capa.max_acl_mac_addrs=%u\n"
 				  "capa.num_multichan_concurrent=%u\n"
 				  "capa.mac_addr_rand_sched_scan_supported=%d\n"
-				  "capa.mac_addr_rand_scan_supported=%d\n",
+				  "capa.mac_addr_rand_scan_supported=%d\n"
+				  "capa.conc_capab=%u\n"
+				  "capa.max_conc_chan_2_4=%u\n"
+				  "capa.max_conc_chan_5_0=%u\n",
 				  drv->capa.key_mgmt,
 				  drv->capa.enc,
 				  drv->capa.auth,
@@ -7522,7 +7511,10 @@ static int wpa_driver_nl80211_status(void *priv, char *buf, size_t buflen)
 				  drv->capa.max_acl_mac_addrs,
 				  drv->capa.num_multichan_concurrent,
 				  drv->capa.mac_addr_rand_sched_scan_supported,
-				  drv->capa.mac_addr_rand_scan_supported);
+				  drv->capa.mac_addr_rand_scan_supported,
+				  drv->capa.conc_capab,
+				  drv->capa.max_conc_chan_2_4,
+				  drv->capa.max_conc_chan_5_0);
 		if (os_snprintf_error(end - pos, res))
 			return pos - buf;
 		pos += res;
@@ -7837,7 +7829,7 @@ static int nl80211_set_wowlan(void *priv,
 
 	wpa_printf(MSG_DEBUG, "nl80211: Setting wowlan");
 
-	if (!(msg = nl80211_drv_msg(drv, 0, NL80211_CMD_SET_WOWLAN)) ||
+	if (!(msg = nl80211_cmd_msg(bss, 0, NL80211_CMD_SET_WOWLAN)) ||
 	    !(wowlan_triggers = nla_nest_start(msg,
 					       NL80211_ATTR_WOWLAN_TRIGGERS)) ||
 	    (triggers->any &&
@@ -8340,9 +8332,9 @@ static const char * drv_br_net_param_str(enum drv_br_net_param param)
 	switch (param) {
 	case DRV_BR_NET_PARAM_GARP_ACCEPT:
 		return "arp_accept";
+	default:
+		return NULL;
 	}
-
-	return NULL;
 }
 
 
@@ -8354,6 +8346,13 @@ static int wpa_driver_br_set_net_param(void *priv, enum drv_br_net_param param,
 	const char *param_txt;
 	int ip_version = 4;
 
+	if (param == DRV_BR_MULTICAST_SNOOPING) {
+		os_snprintf(path, sizeof(path),
+			    "/sys/devices/virtual/net/%s/bridge/multicast_snooping",
+			    bss->brname);
+		goto set_val;
+	}
+
 	param_txt = drv_br_net_param_str(param);
 	if (param_txt == NULL)
 		return -EINVAL;
@@ -8369,6 +8368,7 @@ static int wpa_driver_br_set_net_param(void *priv, enum drv_br_net_param param,
 	os_snprintf(path, sizeof(path), "/proc/sys/net/ipv%d/conf/%s/%s",
 		    ip_version, bss->brname, param_txt);
 
+set_val:
 	if (linux_write_system_file(path, val))
 		return -1;
 
@@ -8387,12 +8387,34 @@ static int hw_mode_to_qca_acs(enum hostapd_hw_mode hw_mode)
 		return QCA_ACS_MODE_IEEE80211A;
 	case HOSTAPD_MODE_IEEE80211AD:
 		return QCA_ACS_MODE_IEEE80211AD;
+	case HOSTAPD_MODE_IEEE80211ANY:
+		return QCA_ACS_MODE_IEEE80211ANY;
 	default:
 		return -1;
 	}
 }
 
 
+static int add_acs_freq_list(struct nl_msg *msg, const int *freq_list)
+{
+	int i, len, ret;
+	u32 *freqs;
+
+	if (!freq_list)
+		return 0;
+	len = int_array_len(freq_list);
+	freqs = os_malloc(sizeof(u32) * len);
+	if (!freqs)
+		return -1;
+	for (i = 0; i < len; i++)
+		freqs[i] = freq_list[i];
+	ret = nla_put(msg, QCA_WLAN_VENDOR_ATTR_ACS_FREQ_LIST,
+		      sizeof(u32) * len, freqs);
+	os_free(freqs);
+	return ret;
+}
+
+
 static int wpa_driver_do_acs(void *priv, struct drv_acs_params *params)
 {
 	struct i802_bss *bss = priv;
@@ -8415,12 +8437,25 @@ static int wpa_driver_do_acs(void *priv, struct drv_acs_params *params)
 	    (params->ht_enabled &&
 	     nla_put_flag(msg, QCA_WLAN_VENDOR_ATTR_ACS_HT_ENABLED)) ||
 	    (params->ht40_enabled &&
-	     nla_put_flag(msg, QCA_WLAN_VENDOR_ATTR_ACS_HT40_ENABLED))) {
+	     nla_put_flag(msg, QCA_WLAN_VENDOR_ATTR_ACS_HT40_ENABLED)) ||
+	    (params->vht_enabled &&
+	     nla_put_flag(msg, QCA_WLAN_VENDOR_ATTR_ACS_VHT_ENABLED)) ||
+	    nla_put_u16(msg, QCA_WLAN_VENDOR_ATTR_ACS_CHWIDTH,
+			params->ch_width) ||
+	    (params->ch_list_len &&
+	     nla_put(msg, QCA_WLAN_VENDOR_ATTR_ACS_CH_LIST, params->ch_list_len,
+		     params->ch_list)) ||
+	    add_acs_freq_list(msg, params->freq_list)) {
 		nlmsg_free(msg);
 		return -ENOBUFS;
 	}
 	nla_nest_end(msg, data);
 
+	wpa_printf(MSG_DEBUG,
+		   "nl80211: ACS Params: HW_MODE: %d HT: %d HT40: %d VHT: %d BW: %d CH_LIST_LEN: %u",
+		   params->hw_mode, params->ht_enabled, params->ht40_enabled,
+		   params->vht_enabled, params->ch_width, params->ch_list_len);
+
 	ret = send_and_recv_msgs(drv, msg, NULL, NULL);
 	if (ret) {
 		wpa_printf(MSG_DEBUG,
@@ -8431,6 +8466,240 @@ static int wpa_driver_do_acs(void *priv, struct drv_acs_params *params)
 }
 
 
+static int nl80211_set_band(void *priv, enum set_band band)
+{
+	struct i802_bss *bss = priv;
+	struct wpa_driver_nl80211_data *drv = bss->drv;
+	struct nl_msg *msg;
+	struct nlattr *data;
+	int ret;
+	enum qca_set_band qca_band;
+
+	if (!drv->setband_vendor_cmd_avail)
+		return -1;
+
+	switch (band) {
+	case WPA_SETBAND_AUTO:
+		qca_band = QCA_SETBAND_AUTO;
+		break;
+	case WPA_SETBAND_5G:
+		qca_band = QCA_SETBAND_5G;
+		break;
+	case WPA_SETBAND_2G:
+		qca_band = QCA_SETBAND_2G;
+		break;
+	default:
+		return -1;
+	}
+
+	if (!(msg = nl80211_drv_msg(drv, 0, NL80211_CMD_VENDOR)) ||
+	    nla_put_u32(msg, NL80211_ATTR_VENDOR_ID, OUI_QCA) ||
+	    nla_put_u32(msg, NL80211_ATTR_VENDOR_SUBCMD,
+			QCA_NL80211_VENDOR_SUBCMD_SETBAND) ||
+	    !(data = nla_nest_start(msg, NL80211_ATTR_VENDOR_DATA)) ||
+	    nla_put_u32(msg, QCA_WLAN_VENDOR_ATTR_SETBAND_VALUE, qca_band)) {
+		nlmsg_free(msg);
+		return -ENOBUFS;
+	}
+	nla_nest_end(msg, data);
+
+	ret = send_and_recv_msgs(drv, msg, NULL, NULL);
+	if (ret) {
+		wpa_printf(MSG_DEBUG,
+			   "nl80211: Driver setband function failed: %s",
+			   strerror(errno));
+	}
+	return ret;
+}
+
+
+struct nl80211_pcl {
+	unsigned int num;
+	unsigned int *freq_list;
+};
+
+static int preferred_freq_info_handler(struct nl_msg *msg, void *arg)
+{
+	struct nlattr *tb[NL80211_ATTR_MAX + 1];
+	struct genlmsghdr *gnlh = nlmsg_data(nlmsg_hdr(msg));
+	struct nl80211_pcl *param = arg;
+	struct nlattr *nl_vend, *attr;
+	enum qca_iface_type iface_type;
+	struct nlattr *tb_vendor[QCA_WLAN_VENDOR_ATTR_MAX + 1];
+	unsigned int num, max_num;
+	u32 *freqs;
+
+	nla_parse(tb, NL80211_ATTR_MAX, genlmsg_attrdata(gnlh, 0),
+		  genlmsg_attrlen(gnlh, 0), NULL);
+
+	nl_vend = tb[NL80211_ATTR_VENDOR_DATA];
+	if (!nl_vend)
+		return NL_SKIP;
+
+	nla_parse(tb_vendor, QCA_WLAN_VENDOR_ATTR_MAX,
+		  nla_data(nl_vend), nla_len(nl_vend), NULL);
+
+	attr = tb_vendor[
+		QCA_WLAN_VENDOR_ATTR_GET_PREFERRED_FREQ_LIST_IFACE_TYPE];
+	if (!attr) {
+		wpa_printf(MSG_ERROR, "nl80211: iface_type couldn't be found");
+		param->num = 0;
+		return NL_SKIP;
+	}
+
+	iface_type = (enum qca_iface_type) nla_get_u32(attr);
+	wpa_printf(MSG_DEBUG, "nl80211: Driver returned iface_type=%d",
+		   iface_type);
+
+	attr = tb_vendor[QCA_WLAN_VENDOR_ATTR_GET_PREFERRED_FREQ_LIST];
+	if (!attr) {
+		wpa_printf(MSG_ERROR,
+			   "nl80211: preferred_freq_list couldn't be found");
+		param->num = 0;
+		return NL_SKIP;
+	}
+
+	/*
+	 * param->num has the maximum number of entries for which there
+	 * is room in the freq_list provided by the caller.
+	 */
+	freqs = nla_data(attr);
+	max_num = nla_len(attr) / sizeof(u32);
+	if (max_num > param->num)
+		max_num = param->num;
+	for (num = 0; num < max_num; num++)
+		param->freq_list[num] = freqs[num];
+	param->num = num;
+
+	return NL_SKIP;
+}
+
+
+static int nl80211_get_pref_freq_list(void *priv,
+				      enum wpa_driver_if_type if_type,
+				      unsigned int *num,
+				      unsigned int *freq_list)
+{
+	struct i802_bss *bss = priv;
+	struct wpa_driver_nl80211_data *drv = bss->drv;
+	struct nl_msg *msg;
+	int ret;
+	unsigned int i;
+	struct nlattr *params;
+	struct nl80211_pcl param;
+	enum qca_iface_type iface_type;
+
+	if (!drv->get_pref_freq_list)
+		return -1;
+
+	switch (if_type) {
+	case WPA_IF_STATION:
+		iface_type = QCA_IFACE_TYPE_STA;
+		break;
+	case WPA_IF_AP_BSS:
+		iface_type = QCA_IFACE_TYPE_AP;
+		break;
+	case WPA_IF_P2P_GO:
+		iface_type = QCA_IFACE_TYPE_P2P_GO;
+		break;
+	case WPA_IF_P2P_CLIENT:
+		iface_type = QCA_IFACE_TYPE_P2P_CLIENT;
+		break;
+	case WPA_IF_IBSS:
+		iface_type = QCA_IFACE_TYPE_IBSS;
+		break;
+	case WPA_IF_TDLS:
+		iface_type = QCA_IFACE_TYPE_TDLS;
+		break;
+	default:
+		return -1;
+	}
+
+	param.num = *num;
+	param.freq_list = freq_list;
+
+	if (!(msg = nl80211_drv_msg(drv, 0, NL80211_CMD_VENDOR)) ||
+	    nla_put_u32(msg, NL80211_ATTR_IFINDEX, drv->ifindex) ||
+	    nla_put_u32(msg, NL80211_ATTR_VENDOR_ID, OUI_QCA) ||
+	    nla_put_u32(msg, NL80211_ATTR_VENDOR_SUBCMD,
+			QCA_NL80211_VENDOR_SUBCMD_GET_PREFERRED_FREQ_LIST) ||
+	    !(params = nla_nest_start(msg, NL80211_ATTR_VENDOR_DATA)) ||
+	    nla_put_u32(msg,
+			QCA_WLAN_VENDOR_ATTR_GET_PREFERRED_FREQ_LIST_IFACE_TYPE,
+			iface_type)) {
+		wpa_printf(MSG_ERROR,
+			   "%s: err in adding vendor_cmd and vendor_data",
+			   __func__);
+		nlmsg_free(msg);
+		return -1;
+	}
+	nla_nest_end(msg, params);
+
+	os_memset(freq_list, 0, *num * sizeof(freq_list[0]));
+	ret = send_and_recv_msgs(drv, msg, preferred_freq_info_handler, &param);
+	if (ret) {
+		wpa_printf(MSG_ERROR,
+			   "%s: err in send_and_recv_msgs", __func__);
+		return ret;
+	}
+
+	*num = param.num;
+
+	for (i = 0; i < *num; i++) {
+		wpa_printf(MSG_DEBUG, "nl80211: preferred_channel_list[%d]=%d",
+			   i, freq_list[i]);
+	}
+
+	return 0;
+}
+
+
+static int nl80211_set_prob_oper_freq(void *priv, unsigned int freq)
+{
+	struct i802_bss *bss = priv;
+	struct wpa_driver_nl80211_data *drv = bss->drv;
+	struct nl_msg *msg;
+	int ret;
+	struct nlattr *params;
+
+	if (!drv->set_prob_oper_freq)
+		return -1;
+
+	wpa_printf(MSG_DEBUG,
+		   "nl80211: Set P2P probable operating freq %u for ifindex %d",
+		   freq, bss->ifindex);
+
+	if (!(msg = nl80211_drv_msg(drv, 0, NL80211_CMD_VENDOR)) ||
+	    nla_put_u32(msg, NL80211_ATTR_VENDOR_ID, OUI_QCA) ||
+	    nla_put_u32(msg, NL80211_ATTR_VENDOR_SUBCMD,
+			QCA_NL80211_VENDOR_SUBCMD_SET_PROBABLE_OPER_CHANNEL) ||
+	    !(params = nla_nest_start(msg, NL80211_ATTR_VENDOR_DATA)) ||
+	    nla_put_u32(msg,
+			QCA_WLAN_VENDOR_ATTR_PROBABLE_OPER_CHANNEL_IFACE_TYPE,
+			QCA_IFACE_TYPE_P2P_CLIENT) ||
+	    nla_put_u32(msg,
+			QCA_WLAN_VENDOR_ATTR_PROBABLE_OPER_CHANNEL_FREQ,
+			freq)) {
+		wpa_printf(MSG_ERROR,
+			   "%s: err in adding vendor_cmd and vendor_data",
+			   __func__);
+		nlmsg_free(msg);
+		return -1;
+	}
+	nla_nest_end(msg, params);
+
+	ret = send_and_recv_msgs(drv, msg, NULL, NULL);
+	msg = NULL;
+	if (ret) {
+		wpa_printf(MSG_ERROR, "%s: err in send_and_recv_msgs",
+			   __func__);
+		return ret;
+	}
+	nlmsg_free(msg);
+	return 0;
+}
+
+
 const struct wpa_driver_ops wpa_driver_nl80211_ops = {
 	.name = "nl80211",
 	.desc = "Linux nl80211/cfg80211",
@@ -8490,7 +8759,6 @@ const struct wpa_driver_ops wpa_driver_nl80211_ops = {
 	.signal_monitor = nl80211_signal_monitor,
 	.signal_poll = nl80211_signal_poll,
 	.send_frame = nl80211_send_frame,
-	.shared_freq = wpa_driver_nl80211_shared_freq,
 	.set_param = nl80211_set_param,
 	.get_radio_name = nl80211_get_radio_name,
 	.add_pmkid = nl80211_add_pmkid,
@@ -8518,7 +8786,9 @@ const struct wpa_driver_ops wpa_driver_nl80211_ops = {
 	.set_ap_wps_ie = wpa_driver_set_ap_wps_p2p_ie,
 #endif /* ANDROID_P2P */
 #ifdef ANDROID
+#ifndef ANDROID_LIB_STUB
 	.driver_cmd = wpa_driver_nl80211_driver_cmd,
+#endif /* !ANDROID_LIB_STUB */
 #endif /* ANDROID */
 	.vendor_cmd = nl80211_vendor_cmd,
 	.set_qos_map = nl80211_set_qos_map,
@@ -8537,4 +8807,7 @@ const struct wpa_driver_ops wpa_driver_nl80211_ops = {
 	.add_tx_ts = nl80211_add_ts,
 	.del_tx_ts = nl80211_del_ts,
 	.do_acs = wpa_driver_do_acs,
+	.set_band = nl80211_set_band,
+	.get_pref_freq_list = nl80211_get_pref_freq_list,
+	.set_prob_oper_freq = nl80211_set_prob_oper_freq,
 };
diff --git a/src/drivers/driver_nl80211.h b/src/drivers/driver_nl80211.h
index 802589aa7581..5c21e0faf55c 100644
--- a/src/drivers/driver_nl80211.h
+++ b/src/drivers/driver_nl80211.h
@@ -110,7 +110,7 @@ struct wpa_driver_nl80211_data {
 	u8 bssid[ETH_ALEN];
 	u8 prev_bssid[ETH_ALEN];
 	int associated;
-	u8 ssid[32];
+	u8 ssid[SSID_MAX_LEN];
 	size_t ssid_len;
 	enum nl80211_iftype nlmode;
 	enum nl80211_iftype ap_scan_as_station;
@@ -145,6 +145,9 @@ struct wpa_driver_nl80211_data {
 	unsigned int get_features_vendor_cmd_avail:1;
 	unsigned int set_rekey_offload:1;
 	unsigned int p2p_go_ctwindow_supported:1;
+	unsigned int setband_vendor_cmd_avail:1;
+	unsigned int get_pref_freq_list:1;
+	unsigned int set_prob_oper_freq:1;
 
 	u64 remain_on_chan_cookie;
 	u64 send_action_cookie;
@@ -169,7 +172,7 @@ struct wpa_driver_nl80211_data {
 	/* From failed authentication command */
 	int auth_freq;
 	u8 auth_bssid_[ETH_ALEN];
-	u8 auth_ssid[32];
+	u8 auth_ssid[SSID_MAX_LEN];
 	size_t auth_ssid_len;
 	int auth_alg;
 	u8 *auth_ie;
@@ -232,7 +235,6 @@ int process_bss_event(struct nl_msg *msg, void *arg);
 
 #ifdef ANDROID
 int android_nl_socket_set_nonblocking(struct nl_handle *handle);
-int android_genl_ctrl_resolve(struct nl_handle *handle, const char *name);
 int android_pno_start(struct i802_bss *bss,
 		      struct wpa_driver_scan_params *params);
 int android_pno_stop(struct i802_bss *bss);
@@ -270,5 +272,6 @@ int wpa_driver_nl80211_sched_scan(void *priv,
 int wpa_driver_nl80211_stop_sched_scan(void *priv);
 struct wpa_scan_results * wpa_driver_nl80211_get_scan_results(void *priv);
 void nl80211_dump_scan(struct wpa_driver_nl80211_data *drv);
+const u8 * nl80211_get_ie(const u8 *ies, size_t ies_len, u8 ie);
 
 #endif /* DRIVER_NL80211_H */
diff --git a/src/drivers/driver_nl80211_android.c b/src/drivers/driver_nl80211_android.c
index 3cc9a65867f6..ba47888843bb 100644
--- a/src/drivers/driver_nl80211_android.c
+++ b/src/drivers/driver_nl80211_android.c
@@ -151,7 +151,7 @@ int android_pno_stop(struct i802_bss *bss)
 
 
 #ifdef ANDROID_P2P
-#ifdef ANDROID_P2P_STUB
+#ifdef ANDROID_LIB_STUB
 
 int wpa_driver_set_p2p_noa(void *priv, u8 count, int start, int duration)
 {
@@ -178,7 +178,7 @@ int wpa_driver_set_ap_wps_p2p_ie(void *priv, const struct wpabuf *beacon,
 	return 0;
 }
 
-#endif /* ANDROID_P2P_STUB */
+#endif /* ANDROID_LIB_STUB */
 #endif /* ANDROID_P2P */
 
 
@@ -188,33 +188,3 @@ int android_nl_socket_set_nonblocking(struct nl_handle *handle)
 }
 
 
-int android_genl_ctrl_resolve(struct nl_handle *handle, const char *name)
-{
-	/*
-	 * Android ICS has very minimal genl_ctrl_resolve() implementation, so
-	 * need to work around that.
-	 */
-	struct nl_cache *cache = NULL;
-	struct genl_family *nl80211 = NULL;
-	int id = -1;
-
-	if (genl_ctrl_alloc_cache(handle, &cache) < 0) {
-		wpa_printf(MSG_ERROR, "nl80211: Failed to allocate generic "
-			   "netlink cache");
-		goto fail;
-	}
-
-	nl80211 = genl_ctrl_search_by_name(cache, name);
-	if (nl80211 == NULL)
-		goto fail;
-
-	id = genl_family_get_id(nl80211);
-
-fail:
-	if (nl80211)
-		genl_family_put(nl80211);
-	if (cache)
-		nl_cache_free(cache);
-
-	return id;
-}
diff --git a/src/drivers/driver_nl80211_capa.c b/src/drivers/driver_nl80211_capa.c
index e0d1d233e2ad..4cf31238aeb7 100644
--- a/src/drivers/driver_nl80211_capa.c
+++ b/src/drivers/driver_nl80211_capa.c
@@ -335,6 +335,33 @@ static void wiphy_info_tdls(struct wpa_driver_capa *capa, struct nlattr *tdls,
 }
 
 
+static int ext_feature_isset(const u8 *ext_features, int ext_features_len,
+			     enum nl80211_ext_feature_index ftidx)
+{
+	u8 ft_byte;
+
+	if ((int) ftidx / 8 >= ext_features_len)
+		return 0;
+
+	ft_byte = ext_features[ftidx / 8];
+	return (ft_byte & BIT(ftidx % 8)) != 0;
+}
+
+
+static void wiphy_info_ext_feature_flags(struct wiphy_info_data *info,
+					 struct nlattr *tb)
+{
+	struct wpa_driver_capa *capa = info->capa;
+
+	if (tb == NULL)
+		return;
+
+	if (ext_feature_isset(nla_data(tb), nla_len(tb),
+			      NL80211_EXT_FEATURE_VHT_IBSS))
+		capa->flags |= WPA_DRIVER_FLAGS_VHT_IBSS;
+}
+
+
 static void wiphy_info_feature_flags(struct wiphy_info_data *info,
 				     struct nlattr *tb)
 {
@@ -509,6 +536,7 @@ static int wiphy_info_handler(struct nl_msg *msg, void *arg)
 		info->device_ap_sme = 1;
 
 	wiphy_info_feature_flags(info, tb[NL80211_ATTR_FEATURE_FLAGS]);
+	wiphy_info_ext_feature_flags(info, tb[NL80211_ATTR_EXT_FEATURES]);
 	wiphy_info_probe_resp_offload(capa,
 				      tb[NL80211_ATTR_PROBE_RESP_OFFLOAD]);
 
@@ -547,22 +575,34 @@ static int wiphy_info_handler(struct nl_msg *msg, void *arg)
 				continue;
 			}
 			vinfo = nla_data(nl);
-			switch (vinfo->subcmd) {
-			case QCA_NL80211_VENDOR_SUBCMD_TEST:
-				drv->vendor_cmd_test_avail = 1;
-				break;
-			case QCA_NL80211_VENDOR_SUBCMD_ROAMING:
-				drv->roaming_vendor_cmd_avail = 1;
-				break;
-			case QCA_NL80211_VENDOR_SUBCMD_DFS_CAPABILITY:
-				drv->dfs_vendor_cmd_avail = 1;
-				break;
-			case QCA_NL80211_VENDOR_SUBCMD_GET_FEATURES:
-				drv->get_features_vendor_cmd_avail = 1;
-				break;
-			case QCA_NL80211_VENDOR_SUBCMD_DO_ACS:
-				drv->capa.flags |= WPA_DRIVER_FLAGS_ACS_OFFLOAD;
-				break;
+			if (vinfo->vendor_id == OUI_QCA) {
+				switch (vinfo->subcmd) {
+				case QCA_NL80211_VENDOR_SUBCMD_TEST:
+					drv->vendor_cmd_test_avail = 1;
+					break;
+				case QCA_NL80211_VENDOR_SUBCMD_ROAMING:
+					drv->roaming_vendor_cmd_avail = 1;
+					break;
+				case QCA_NL80211_VENDOR_SUBCMD_DFS_CAPABILITY:
+					drv->dfs_vendor_cmd_avail = 1;
+					break;
+				case QCA_NL80211_VENDOR_SUBCMD_GET_FEATURES:
+					drv->get_features_vendor_cmd_avail = 1;
+					break;
+				case QCA_NL80211_VENDOR_SUBCMD_GET_PREFERRED_FREQ_LIST:
+					drv->get_pref_freq_list = 1;
+					break;
+				case QCA_NL80211_VENDOR_SUBCMD_SET_PROBABLE_OPER_CHANNEL:
+					drv->set_prob_oper_freq = 1;
+					break;
+				case QCA_NL80211_VENDOR_SUBCMD_DO_ACS:
+					drv->capa.flags |=
+						WPA_DRIVER_FLAGS_ACS_OFFLOAD;
+					break;
+				case QCA_NL80211_VENDOR_SUBCMD_SETBAND:
+					drv->setband_vendor_cmd_avail = 1;
+					break;
+				}
 			}
 
 			wpa_printf(MSG_DEBUG, "nl80211: Supported vendor command: vendor_id=0x%x subcmd=%u",
@@ -717,6 +757,7 @@ static void qca_nl80211_check_dfs_capa(struct wpa_driver_nl80211_data *drv)
 struct features_info {
 	u8 *flags;
 	size_t flags_len;
+	struct wpa_driver_capa *capa;
 };
 
 
@@ -742,6 +783,19 @@ static int features_info_handler(struct nl_msg *msg, void *arg)
 			info->flags = nla_data(attr);
 			info->flags_len = nla_len(attr);
 		}
+		attr = tb_vendor[QCA_WLAN_VENDOR_ATTR_CONCURRENCY_CAPA];
+		if (attr)
+			info->capa->conc_capab = nla_get_u32(attr);
+
+		attr = tb_vendor[
+			QCA_WLAN_VENDOR_ATTR_MAX_CONCURRENT_CHANNELS_2_4_BAND];
+		if (attr)
+			info->capa->max_conc_chan_2_4 = nla_get_u32(attr);
+
+		attr = tb_vendor[
+			QCA_WLAN_VENDOR_ATTR_MAX_CONCURRENT_CHANNELS_5_0_BAND];
+		if (attr)
+			info->capa->max_conc_chan_5_0 = nla_get_u32(attr);
 	}
 
 	return NL_SKIP;
@@ -776,12 +830,16 @@ static void qca_nl80211_get_features(struct wpa_driver_nl80211_data *drv)
 	}
 
 	os_memset(&info, 0, sizeof(info));
+	info.capa = &drv->capa;
 	ret = send_and_recv_msgs(drv, msg, features_info_handler, &info);
 	if (ret || !info.flags)
 		return;
 
 	if (check_feature(QCA_WLAN_VENDOR_FEATURE_KEY_MGMT_OFFLOAD, &info))
 		drv->capa.flags |= WPA_DRIVER_FLAGS_KEY_MGMT_OFFLOAD;
+
+	if (check_feature(QCA_WLAN_VENDOR_FEATURE_SUPPORT_HW_MODE_ANY, &info))
+		drv->capa.flags |= WPA_DRIVER_FLAGS_SUPPORT_HW_MODE_ANY;
 }
 
 
diff --git a/src/drivers/driver_nl80211_event.c b/src/drivers/driver_nl80211_event.c
index 87e412dc596a..7b0f721e6584 100644
--- a/src/drivers/driver_nl80211_event.c
+++ b/src/drivers/driver_nl80211_event.c
@@ -271,6 +271,7 @@ static void mlme_event_connect(struct wpa_driver_nl80211_data *drv,
 			       struct nlattr *ptk_kek)
 {
 	union wpa_event_data event;
+	const u8 *ssid;
 	u16 status_code;
 
 	if (drv->capa.flags & WPA_DRIVER_FLAGS_SME) {
@@ -331,6 +332,16 @@ static void mlme_event_connect(struct wpa_driver_nl80211_data *drv,
 	if (req_ie) {
 		event.assoc_info.req_ies = nla_data(req_ie);
 		event.assoc_info.req_ies_len = nla_len(req_ie);
+
+		if (cmd == NL80211_CMD_ROAM) {
+			ssid = nl80211_get_ie(event.assoc_info.req_ies,
+					      event.assoc_info.req_ies_len,
+					      WLAN_EID_SSID);
+			if (ssid && ssid[1] > 0 && ssid[1] <= 32) {
+				drv->ssid_len = ssid[1];
+				os_memcpy(drv->ssid, ssid + 2, ssid[1]);
+			}
+		}
 	}
 	if (resp_ie) {
 		event.assoc_info.resp_ies = nla_data(resp_ie);
@@ -1480,6 +1491,25 @@ static void qca_nl80211_avoid_freq(struct wpa_driver_nl80211_data *drv,
 }
 
 
+static enum hostapd_hw_mode get_qca_hw_mode(u8 hw_mode)
+{
+	switch (hw_mode) {
+	case QCA_ACS_MODE_IEEE80211B:
+		return HOSTAPD_MODE_IEEE80211B;
+	case QCA_ACS_MODE_IEEE80211G:
+		return HOSTAPD_MODE_IEEE80211G;
+	case QCA_ACS_MODE_IEEE80211A:
+		return HOSTAPD_MODE_IEEE80211A;
+	case QCA_ACS_MODE_IEEE80211AD:
+		return HOSTAPD_MODE_IEEE80211AD;
+	case QCA_ACS_MODE_IEEE80211ANY:
+		return HOSTAPD_MODE_IEEE80211ANY;
+	default:
+		return NUM_HOSTAPD_MODES;
+	}
+}
+
+
 static void qca_nl80211_acs_select_ch(struct wpa_driver_nl80211_data *drv,
 				   const u8 *data, size_t len)
 {
@@ -1500,6 +1530,39 @@ static void qca_nl80211_acs_select_ch(struct wpa_driver_nl80211_data *drv,
 		nla_get_u8(tb[QCA_WLAN_VENDOR_ATTR_ACS_PRIMARY_CHANNEL]);
 	event.acs_selected_channels.sec_channel =
 		nla_get_u8(tb[QCA_WLAN_VENDOR_ATTR_ACS_SECONDARY_CHANNEL]);
+	if (tb[QCA_WLAN_VENDOR_ATTR_ACS_VHT_SEG0_CENTER_CHANNEL])
+		event.acs_selected_channels.vht_seg0_center_ch =
+			nla_get_u8(tb[QCA_WLAN_VENDOR_ATTR_ACS_VHT_SEG0_CENTER_CHANNEL]);
+	if (tb[QCA_WLAN_VENDOR_ATTR_ACS_VHT_SEG0_CENTER_CHANNEL])
+		event.acs_selected_channels.vht_seg1_center_ch =
+			nla_get_u8(tb[QCA_WLAN_VENDOR_ATTR_ACS_VHT_SEG1_CENTER_CHANNEL]);
+	if (tb[QCA_WLAN_VENDOR_ATTR_ACS_CHWIDTH])
+		event.acs_selected_channels.ch_width =
+			nla_get_u16(tb[QCA_WLAN_VENDOR_ATTR_ACS_CHWIDTH]);
+	if (tb[QCA_WLAN_VENDOR_ATTR_ACS_HW_MODE]) {
+		u8 hw_mode = nla_get_u8(tb[QCA_WLAN_VENDOR_ATTR_ACS_HW_MODE]);
+
+		event.acs_selected_channels.hw_mode = get_qca_hw_mode(hw_mode);
+		if (event.acs_selected_channels.hw_mode == NUM_HOSTAPD_MODES ||
+		    event.acs_selected_channels.hw_mode ==
+		    HOSTAPD_MODE_IEEE80211ANY) {
+			wpa_printf(MSG_DEBUG,
+				   "nl80211: Invalid hw_mode %d in ACS selection event",
+				   hw_mode);
+			return;
+		}
+	}
+
+	wpa_printf(MSG_INFO,
+		   "nl80211: ACS Results: PCH: %d SCH: %d BW: %d VHT0: %d VHT1: %d HW_MODE: %d",
+		   event.acs_selected_channels.pri_channel,
+		   event.acs_selected_channels.sec_channel,
+		   event.acs_selected_channels.ch_width,
+		   event.acs_selected_channels.vht_seg0_center_ch,
+		   event.acs_selected_channels.vht_seg1_center_ch,
+		   event.acs_selected_channels.hw_mode);
+
+	/* Ignore ACS channel list check for backwards compatibility */
 
 	wpa_supplicant_event(drv->ctx, EVENT_ACS_CHANNEL_SELECTED, &event);
 }
diff --git a/src/drivers/driver_nl80211_scan.c b/src/drivers/driver_nl80211_scan.c
index 3911f485f72e..4b762eafbe8a 100644
--- a/src/drivers/driver_nl80211_scan.c
+++ b/src/drivers/driver_nl80211_scan.c
@@ -221,6 +221,9 @@ int wpa_driver_nl80211_scan(struct i802_bss *bss,
 	wpa_dbg(drv->ctx, MSG_DEBUG, "nl80211: scan request");
 	drv->scan_for_auth = 0;
 
+	if (TEST_FAIL())
+		return -1;
+
 	msg = nl80211_scan_common(bss, NL80211_CMD_TRIGGER_SCAN, params);
 	if (!msg)
 		return -1;
@@ -433,7 +436,7 @@ int wpa_driver_nl80211_stop_sched_scan(void *priv)
 }
 
 
-static const u8 * nl80211_get_ie(const u8 *ies, size_t ies_len, u8 ie)
+const u8 * nl80211_get_ie(const u8 *ies, size_t ies_len, u8 ie)
 {
 	const u8 *end, *pos;
 
@@ -583,6 +586,11 @@ int bss_info_handler(struct nl_msg *msg, void *arg)
 		r->flags |= WPA_SCAN_LEVEL_INVALID | WPA_SCAN_QUAL_INVALID;
 	if (bss[NL80211_BSS_TSF])
 		r->tsf = nla_get_u64(bss[NL80211_BSS_TSF]);
+	if (bss[NL80211_BSS_BEACON_TSF]) {
+		u64 tsf = nla_get_u64(bss[NL80211_BSS_BEACON_TSF]);
+		if (tsf > r->tsf)
+			r->tsf = tsf;
+	}
 	if (bss[NL80211_BSS_SEEN_MS_AGO])
 		r->age = nla_get_u32(bss[NL80211_BSS_SEEN_MS_AGO]);
 	r->ie_len = ie_len;
diff --git a/src/drivers/driver_privsep.c b/src/drivers/driver_privsep.c
index de23fbd2b9fe..1f1676a20ac5 100644
--- a/src/drivers/driver_privsep.c
+++ b/src/drivers/driver_privsep.c
@@ -220,6 +220,56 @@ static int wpa_driver_privsep_set_key(const char *ifname, void *priv,
 }
 
 
+static int wpa_driver_privsep_authenticate(
+	void *priv, struct wpa_driver_auth_params *params)
+{
+	struct wpa_driver_privsep_data *drv = priv;
+	struct privsep_cmd_authenticate *data;
+	int i, res;
+	size_t buflen;
+	u8 *pos;
+
+	wpa_printf(MSG_DEBUG, "%s: priv=%p freq=%d bssid=" MACSTR
+		   " auth_alg=%d local_state_change=%d p2p=%d",
+		   __func__, priv, params->freq, MAC2STR(params->bssid),
+		   params->auth_alg, params->local_state_change, params->p2p);
+
+	buflen = sizeof(*data) + params->ie_len + params->sae_data_len;
+	data = os_zalloc(buflen);
+	if (data == NULL)
+		return -1;
+
+	data->freq = params->freq;
+	os_memcpy(data->bssid, params->bssid, ETH_ALEN);
+	os_memcpy(data->ssid, params->ssid, params->ssid_len);
+	data->ssid_len = params->ssid_len;
+	data->auth_alg = params->auth_alg;
+	data->ie_len = params->ie_len;
+	for (i = 0; i < 4; i++) {
+		if (params->wep_key[i])
+			os_memcpy(data->wep_key[i], params->wep_key[i],
+				  params->wep_key_len[i]);
+		data->wep_key_len[i] = params->wep_key_len[i];
+	}
+	data->wep_tx_keyidx = params->wep_tx_keyidx;
+	data->local_state_change = params->local_state_change;
+	data->p2p = params->p2p;
+	pos = (u8 *) (data + 1);
+	if (params->ie_len) {
+		os_memcpy(pos, params->ie, params->ie_len);
+		pos += params->ie_len;
+	}
+	if (params->sae_data_len)
+		os_memcpy(pos, params->sae_data, params->sae_data_len);
+
+	res = wpa_priv_cmd(drv, PRIVSEP_CMD_AUTHENTICATE, data, buflen,
+			   NULL, NULL);
+	os_free(data);
+
+	return res;
+}
+
+
 static int wpa_driver_privsep_associate(
 	void *priv, struct wpa_driver_associate_params *params)
 {
@@ -281,14 +331,15 @@ static int wpa_driver_privsep_get_ssid(void *priv, u8 *ssid)
 {
 	struct wpa_driver_privsep_data *drv = priv;
 	int res, ssid_len;
-	u8 reply[sizeof(int) + 32];
+	u8 reply[sizeof(int) + SSID_MAX_LEN];
 	size_t len = sizeof(reply);
 
 	res = wpa_priv_cmd(drv, PRIVSEP_CMD_GET_SSID, NULL, 0, reply, &len);
 	if (res < 0 || len < sizeof(int))
 		return -1;
 	os_memcpy(&ssid_len, reply, sizeof(int));
-	if (ssid_len < 0 || ssid_len > 32 || sizeof(int) + ssid_len > len) {
+	if (ssid_len < 0 || ssid_len > SSID_MAX_LEN ||
+	    sizeof(int) + ssid_len > len) {
 		wpa_printf(MSG_DEBUG, "privsep: Invalid get SSID reply");
 		return -1;
 	}
@@ -308,6 +359,32 @@ static int wpa_driver_privsep_deauthenticate(void *priv, const u8 *addr,
 }
 
 
+static void wpa_driver_privsep_event_auth(void *ctx, u8 *buf, size_t len)
+{
+	union wpa_event_data data;
+	struct privsep_event_auth *auth;
+
+	os_memset(&data, 0, sizeof(data));
+	if (len < sizeof(*auth))
+		return;
+	auth = (struct privsep_event_auth *) buf;
+	if (len < sizeof(*auth) + auth->ies_len)
+		return;
+
+	os_memcpy(data.auth.peer, auth->peer, ETH_ALEN);
+	os_memcpy(data.auth.bssid, auth->bssid, ETH_ALEN);
+	data.auth.auth_type = auth->auth_type;
+	data.auth.auth_transaction = auth->auth_transaction;
+	data.auth.status_code = auth->status_code;
+	if (auth->ies_len) {
+		data.auth.ies = (u8 *) (auth + 1);
+		data.auth.ies_len = auth->ies_len;
+	}
+
+	wpa_supplicant_event(ctx, EVENT_AUTH, &data);
+}
+
+
 static void wpa_driver_privsep_event_assoc(void *ctx,
 					   enum wpa_event_type event,
 					   u8 *buf, size_t len)
@@ -467,6 +544,9 @@ static void wpa_driver_privsep_receive(int sock, void *eloop_ctx,
 	case PRIVSEP_EVENT_SCAN_RESULTS:
 		wpa_supplicant_event(drv->ctx, EVENT_SCAN_RESULTS, NULL);
 		break;
+	case PRIVSEP_EVENT_SCAN_STARTED:
+		wpa_supplicant_event(drv->ctx, EVENT_SCAN_STARTED, NULL);
+		break;
 	case PRIVSEP_EVENT_ASSOC:
 		wpa_driver_privsep_event_assoc(drv->ctx, EVENT_ASSOC,
 					       event_buf, event_len);
@@ -502,6 +582,9 @@ static void wpa_driver_privsep_receive(int sock, void *eloop_ctx,
 		wpa_driver_privsep_event_rx_eapol(drv->ctx, event_buf,
 						  event_len);
 		break;
+	case PRIVSEP_EVENT_AUTH:
+		wpa_driver_privsep_event_auth(drv->ctx, event_buf, event_len);
+		break;
 	}
 
 	os_free(buf);
@@ -702,6 +785,10 @@ static int wpa_driver_privsep_get_capa(void *priv,
 	res = wpa_priv_cmd(drv, PRIVSEP_CMD_GET_CAPA, NULL, 0, capa, &len);
 	if (res < 0 || len != sizeof(*capa))
 		return -1;
+	/* For now, no support for passing extended_capa pointers */
+	capa->extended_capa = NULL;
+	capa->extended_capa_mask = NULL;
+	capa->extended_capa_len = 0;
 	return 0;
 }
 
@@ -734,6 +821,7 @@ struct wpa_driver_ops wpa_driver_privsep_ops = {
 	.set_param = wpa_driver_privsep_set_param,
 	.scan2 = wpa_driver_privsep_scan,
 	.deauthenticate = wpa_driver_privsep_deauthenticate,
+	.authenticate = wpa_driver_privsep_authenticate,
 	.associate = wpa_driver_privsep_associate,
 	.get_capa = wpa_driver_privsep_get_capa,
 	.get_mac_addr = wpa_driver_privsep_get_mac_addr,
@@ -742,7 +830,7 @@ struct wpa_driver_ops wpa_driver_privsep_ops = {
 };
 
 
-struct wpa_driver_ops *wpa_drivers[] =
+const struct wpa_driver_ops *const wpa_drivers[] =
 {
 	&wpa_driver_privsep_ops,
 	NULL
diff --git a/src/drivers/driver_wext.c b/src/drivers/driver_wext.c
index a1581b8c9cb1..01defdff4f15 100644
--- a/src/drivers/driver_wext.c
+++ b/src/drivers/driver_wext.c
@@ -1,6 +1,6 @@
 /*
  * Driver interaction with generic Linux Wireless Extensions
- * Copyright (c) 2003-2010, Jouni Malinen <j@w1.fi>
+ * Copyright (c) 2003-2015, Jouni Malinen <j@w1.fi>
  *
  * This software may be distributed under the terms of the BSD license.
  * See README for more details.
@@ -18,6 +18,7 @@
 #include <sys/stat.h>
 #include <fcntl.h>
 #include <net/if_arp.h>
+#include <dirent.h>
 
 #include "linux_wext.h"
 #include "common.h"
@@ -131,7 +132,7 @@ int wpa_driver_wext_get_ssid(void *priv, u8 *ssid)
 	os_memset(&iwr, 0, sizeof(iwr));
 	os_strlcpy(iwr.ifr_name, drv->ifname, IFNAMSIZ);
 	iwr.u.essid.pointer = (caddr_t) ssid;
-	iwr.u.essid.length = 32;
+	iwr.u.essid.length = SSID_MAX_LEN;
 
 	if (ioctl(drv->ioctl_sock, SIOCGIWESSID, &iwr) < 0) {
 		wpa_printf(MSG_ERROR, "ioctl[SIOCGIWESSID]: %s",
@@ -139,8 +140,8 @@ int wpa_driver_wext_get_ssid(void *priv, u8 *ssid)
 		ret = -1;
 	} else {
 		ret = iwr.u.essid.length;
-		if (ret > 32)
-			ret = 32;
+		if (ret > SSID_MAX_LEN)
+			ret = SSID_MAX_LEN;
 		/* Some drivers include nul termination in the SSID, so let's
 		 * remove it here before further processing. WE-21 changes this
 		 * to explicitly require the length _not_ to include nul
@@ -168,7 +169,7 @@ int wpa_driver_wext_set_ssid(void *priv, const u8 *ssid, size_t ssid_len)
 	int ret = 0;
 	char buf[33];
 
-	if (ssid_len > 32)
+	if (ssid_len > SSID_MAX_LEN)
 		return -1;
 
 	os_memset(&iwr, 0, sizeof(iwr));
@@ -874,6 +875,105 @@ static void wpa_driver_wext_send_rfkill(void *eloop_ctx, void *timeout_ctx)
 }
 
 
+static int wext_hostap_ifname(struct wpa_driver_wext_data *drv,
+			      const char *ifname)
+{
+	char buf[200], *res;
+	int type;
+	FILE *f;
+
+	if (strcmp(ifname, ".") == 0 || strcmp(ifname, "..") == 0)
+		return -1;
+
+	snprintf(buf, sizeof(buf), "/sys/class/net/%s/device/net/%s/type",
+		 drv->ifname, ifname);
+
+	f = fopen(buf, "r");
+	if (!f)
+		return -1;
+	res = fgets(buf, sizeof(buf), f);
+	fclose(f);
+
+	type = res ? atoi(res) : -1;
+	wpa_printf(MSG_DEBUG, "WEXT: hostap ifname %s type %d", ifname, type);
+
+	if (type == ARPHRD_IEEE80211) {
+		wpa_printf(MSG_DEBUG,
+			   "WEXT: Found hostap driver wifi# interface (%s)",
+			   ifname);
+		wpa_driver_wext_alternative_ifindex(drv, ifname);
+		return 0;
+	}
+	return -1;
+}
+
+
+static int wext_add_hostap(struct wpa_driver_wext_data *drv)
+{
+	char buf[200];
+	int n;
+	struct dirent **names;
+	int ret = -1;
+
+	snprintf(buf, sizeof(buf), "/sys/class/net/%s/device/net", drv->ifname);
+	n = scandir(buf, &names, NULL, alphasort);
+	if (n < 0)
+		return -1;
+
+	while (n--) {
+		if (ret < 0 && wext_hostap_ifname(drv, names[n]->d_name) == 0)
+			ret = 0;
+		free(names[n]);
+	}
+	free(names);
+
+	return ret;
+}
+
+
+static void wext_check_hostap(struct wpa_driver_wext_data *drv)
+{
+	char buf[200], *pos;
+	ssize_t res;
+
+	/*
+	 * Host AP driver may use both wlan# and wifi# interface in wireless
+	 * events. Since some of the versions included WE-18 support, let's add
+	 * the alternative ifindex also from driver_wext.c for the time being.
+	 * This may be removed at some point once it is believed that old
+	 * versions of the driver are not in use anymore. However, it looks like
+	 * the wifi# interface is still used in the current kernel tree, so it
+	 * may not really be possible to remove this before the Host AP driver
+	 * gets removed from the kernel.
+	 */
+
+	/* First, try to see if driver information is available from sysfs */
+	snprintf(buf, sizeof(buf), "/sys/class/net/%s/device/driver",
+		 drv->ifname);
+	res = readlink(buf, buf, sizeof(buf) - 1);
+	if (res > 0) {
+		buf[res] = '\0';
+		pos = strrchr(buf, '/');
+		if (pos)
+			pos++;
+		else
+			pos = buf;
+		wpa_printf(MSG_DEBUG, "WEXT: Driver: %s", pos);
+		if (os_strncmp(pos, "hostap", 6) == 0 &&
+		    wext_add_hostap(drv) == 0)
+			return;
+	}
+
+	/* Second, use the old design with hardcoded ifname */
+	if (os_strncmp(drv->ifname, "wlan", 4) == 0) {
+		char ifname2[IFNAMSIZ + 1];
+		os_strlcpy(ifname2, drv->ifname, sizeof(ifname2));
+		os_memcpy(ifname2, "wifi", 4);
+		wpa_driver_wext_alternative_ifindex(drv, ifname2);
+	}
+}
+
+
 static int wpa_driver_wext_finish_drv_init(struct wpa_driver_wext_data *drv)
 {
 	int send_rfkill_event = 0;
@@ -914,20 +1014,7 @@ static int wpa_driver_wext_finish_drv_init(struct wpa_driver_wext_data *drv)
 
 	drv->ifindex = if_nametoindex(drv->ifname);
 
-	if (os_strncmp(drv->ifname, "wlan", 4) == 0) {
-		/*
-		 * Host AP driver may use both wlan# and wifi# interface in
-		 * wireless events. Since some of the versions included WE-18
-		 * support, let's add the alternative ifindex also from
-		 * driver_wext.c for the time being. This may be removed at
-		 * some point once it is believed that old versions of the
-		 * driver are not in use anymore.
-		 */
-		char ifname2[IFNAMSIZ + 1];
-		os_strlcpy(ifname2, drv->ifname, sizeof(ifname2));
-		os_memcpy(ifname2, "wifi", 4);
-		wpa_driver_wext_alternative_ifindex(drv, ifname2);
-	}
+	wext_check_hostap(drv);
 
 	netlink_send_oper_ifla(drv->netlink, drv->ifindex,
 			       1, IF_OPER_DORMANT);
@@ -1112,7 +1199,7 @@ struct wext_scan_data {
 	struct wpa_scan_res res;
 	u8 *ie;
 	size_t ie_len;
-	u8 ssid[32];
+	u8 ssid[SSID_MAX_LEN];
 	size_t ssid_len;
 	int maxrate;
 };
@@ -1865,7 +1952,7 @@ static void wpa_driver_wext_disconnect(struct wpa_driver_wext_data *drv)
 {
 	struct iwreq iwr;
 	const u8 null_bssid[ETH_ALEN] = { 0, 0, 0, 0, 0, 0 };
-	u8 ssid[32];
+	u8 ssid[SSID_MAX_LEN];
 	int i;
 
 	/*
@@ -1907,9 +1994,9 @@ static void wpa_driver_wext_disconnect(struct wpa_driver_wext_data *drv)
 		 * SIOCSIWMLME commands (or tries to associate automatically
 		 * after deauth/disassoc).
 		 */
-		for (i = 0; i < 32; i++)
+		for (i = 0; i < SSID_MAX_LEN; i++)
 			ssid[i] = rand() & 0xFF;
-		if (wpa_driver_wext_set_ssid(drv, ssid, 32) < 0) {
+		if (wpa_driver_wext_set_ssid(drv, ssid, SSID_MAX_LEN) < 0) {
 			wpa_printf(MSG_DEBUG, "WEXT: Failed to set bogus "
 				   "SSID to disconnect");
 		}
diff --git a/src/drivers/drivers.c b/src/drivers/drivers.c
index f0c3bb3c63ba..a98af9ac7d71 100644
--- a/src/drivers/drivers.c
+++ b/src/drivers/drivers.c
@@ -47,7 +47,7 @@ extern struct wpa_driver_ops wpa_driver_none_ops; /* driver_none.c */
 #endif /* CONFIG_DRIVER_NONE */
 
 
-struct wpa_driver_ops *wpa_drivers[] =
+const struct wpa_driver_ops *const wpa_drivers[] =
 {
 #ifdef CONFIG_DRIVER_NL80211
 	&wpa_driver_nl80211_ops,
diff --git a/src/drivers/drivers.mak b/src/drivers/drivers.mak
index 943407875701..3dd43c73803e 100644
--- a/src/drivers/drivers.mak
+++ b/src/drivers/drivers.mak
@@ -54,7 +54,9 @@ else
   ifdef CONFIG_LIBNL_TINY
     DRV_LIBS += -lnl-tiny
   else
-    DRV_LIBS += -lnl
+    ifndef CONFIG_OSX
+      DRV_LIBS += -lnl
+    endif
   endif
 
   ifdef CONFIG_LIBNL20
diff --git a/src/drivers/linux_ioctl.c b/src/drivers/linux_ioctl.c
index 837971d25c99..e21147af1bcd 100644
--- a/src/drivers/linux_ioctl.c
+++ b/src/drivers/linux_ioctl.c
@@ -219,3 +219,26 @@ int linux_br_get(char *brname, const char *ifname)
 	os_strlcpy(brname, pos, IFNAMSIZ);
 	return 0;
 }
+
+
+int linux_master_get(char *master_ifname, const char *ifname)
+{
+	char buf[128], masterlink[128], *pos;
+	ssize_t res;
+
+	/* check whether there is a master */
+	os_snprintf(buf, sizeof(buf), "/sys/class/net/%s/master", ifname);
+
+	res = readlink(buf, masterlink, sizeof(masterlink));
+	if (res < 0 || (size_t) res >= sizeof(masterlink))
+		return -1;
+
+	masterlink[res] = '\0';
+
+	pos = os_strrchr(masterlink, '/');
+	if (pos == NULL)
+		return -1;
+	pos++;
+	os_strlcpy(master_ifname, pos, IFNAMSIZ);
+	return 0;
+}
diff --git a/src/drivers/linux_ioctl.h b/src/drivers/linux_ioctl.h
index c03fe6e9a3fd..6de4d9b88275 100644
--- a/src/drivers/linux_ioctl.h
+++ b/src/drivers/linux_ioctl.h
@@ -18,5 +18,6 @@ int linux_br_del(int sock, const char *brname);
 int linux_br_add_if(int sock, const char *brname, const char *ifname);
 int linux_br_del_if(int sock, const char *brname, const char *ifname);
 int linux_br_get(char *brname, const char *ifname);
+int linux_master_get(char *master_ifname, const char *ifname);
 
 #endif /* LINUX_IOCTL_H */
diff --git a/src/drivers/nl80211_copy.h b/src/drivers/nl80211_copy.h
index b37bd5a1cb82..ae16ba9cb1e3 100644
--- a/src/drivers/nl80211_copy.h
+++ b/src/drivers/nl80211_copy.h
@@ -25,10 +25,30 @@
  *
  */
 
+/*
+ * This header file defines the userspace API to the wireless stack. Please
+ * be careful not to break things - i.e. don't move anything around or so
+ * unless you can demonstrate that it breaks neither API nor ABI.
+ *
+ * Additions to the API should be accompanied by actual implementations in
+ * an upstream driver, so that example implementations exist in case there
+ * are ever concerns about the precise semantics of the API or changes are
+ * needed, and to ensure that code for dead (no longer implemented) API
+ * can actually be identified and removed.
+ * Nonetheless, semantics should also be documented carefully in this file.
+ */
+
 #include <linux/types.h>
 
 #define NL80211_GENL_NAME "nl80211"
 
+#define NL80211_MULTICAST_GROUP_CONFIG		"config"
+#define NL80211_MULTICAST_GROUP_SCAN		"scan"
+#define NL80211_MULTICAST_GROUP_REG		"regulatory"
+#define NL80211_MULTICAST_GROUP_MLME		"mlme"
+#define NL80211_MULTICAST_GROUP_VENDOR		"vendor"
+#define NL80211_MULTICAST_GROUP_TESTMODE	"testmode"
+
 /**
  * DOC: Station handling
  *
@@ -173,8 +193,8 @@
  *	%NL80211_ATTR_WIPHY and %NL80211_ATTR_WIPHY_NAME.
  *
  * @NL80211_CMD_GET_INTERFACE: Request an interface's configuration;
- *	either a dump request on a %NL80211_ATTR_WIPHY or a specific get
- *	on an %NL80211_ATTR_IFINDEX is supported.
+ *	either a dump request for all interfaces or a specific get with a
+ *	single %NL80211_ATTR_IFINDEX is supported.
  * @NL80211_CMD_SET_INTERFACE: Set type of a virtual interface, requires
  *	%NL80211_ATTR_IFINDEX and %NL80211_ATTR_IFTYPE.
  * @NL80211_CMD_NEW_INTERFACE: Newly created virtual interface or response
@@ -252,7 +272,18 @@
  *	%NL80211_ATTR_IFINDEX.
  *
  * @NL80211_CMD_GET_REG: ask the wireless core to send us its currently set
- * 	regulatory domain.
+ *	regulatory domain. If %NL80211_ATTR_WIPHY is specified and the device
+ *	has a private regulatory domain, it will be returned. Otherwise, the
+ *	global regdomain will be returned.
+ *	A device will have a private regulatory domain if it uses the
+ *	regulatory_hint() API. Even when a private regdomain is used the channel
+ *	information will still be mended according to further hints from
+ *	the regulatory core to help with compliance. A dump version of this API
+ *	is now available which will returns the global regdomain as well as
+ *	all private regdomains of present wiphys (for those that have it).
+ *	If a wiphy is self-managed (%NL80211_ATTR_WIPHY_SELF_MANAGED_REG), then
+ *	its private regdomain is the only valid one for it. The regulatory
+ *	core is not used to help with compliance in this case.
  * @NL80211_CMD_SET_REG: Set current regulatory domain. CRDA sends this command
  *	after being queried by the kernel. CRDA replies by sending a regulatory
  *	domain structure which consists of %NL80211_ATTR_REG_ALPHA set to our
@@ -306,7 +337,9 @@
  *	if passed, define which channels should be scanned; if not
  *	passed, all channels allowed for the current regulatory domain
  *	are used.  Extra IEs can also be passed from the userspace by
- *	using the %NL80211_ATTR_IE attribute.
+ *	using the %NL80211_ATTR_IE attribute.  The first cycle of the
+ *	scheduled scan can be delayed by %NL80211_ATTR_SCHED_SCAN_DELAY
+ *	is supplied.
  * @NL80211_CMD_STOP_SCHED_SCAN: stop a scheduled scan. Returns -ENOENT if
  *	scheduled scan is not running. The caller may assume that as soon
  *	as the call returns, it is safe to start a new scheduled scan again.
@@ -774,6 +807,10 @@
  *	peer given by %NL80211_ATTR_MAC. Both peers must be on the base channel
  *	when this command completes.
  *
+ * @NL80211_CMD_WIPHY_REG_CHANGE: Similar to %NL80211_CMD_REG_CHANGE, but used
+ *	as an event to indicate changes for devices with wiphy-specific regdom
+ *	management.
+ *
  * @NL80211_CMD_MAX: highest used command number
  * @__NL80211_CMD_AFTER_LAST: internal use
  */
@@ -958,6 +995,8 @@ enum nl80211_commands {
 	NL80211_CMD_TDLS_CHANNEL_SWITCH,
 	NL80211_CMD_TDLS_CANCEL_CHANNEL_SWITCH,
 
+	NL80211_CMD_WIPHY_REG_CHANGE,
+
 	/* add new commands above here */
 
 	/* used to define NL80211_CMD_MAX below */
@@ -1655,6 +1694,13 @@ enum nl80211_commands {
  * @NL80211_ATTR_SOCKET_OWNER: Flag attribute, if set during interface
  *	creation then the new interface will be owned by the netlink socket
  *	that created it and will be destroyed when the socket is closed.
+ *	If set during scheduled scan start then the new scan req will be
+ *	owned by the netlink socket that created it and the scheduled scan will
+ *	be stopped when the socket is closed.
+ *	If set during configuration of regulatory indoor operation then the
+ *	regulatory indoor configuration would be owned by the netlink socket
+ *	that configured the indoor setting, and the indoor operation would be
+ *	cleared when the socket is closed.
  *
  * @NL80211_ATTR_TDLS_INITIATOR: flag attribute indicating the current end is
  *	the TDLS link initiator.
@@ -1688,6 +1734,32 @@ enum nl80211_commands {
  *
  * @NL80211_ATTR_MAC_MASK: MAC address mask
  *
+ * @NL80211_ATTR_WIPHY_SELF_MANAGED_REG: flag attribute indicating this device
+ *	is self-managing its regulatory information and any regulatory domain
+ *	obtained from it is coming from the device's wiphy and not the global
+ *	cfg80211 regdomain.
+ *
+ * @NL80211_ATTR_EXT_FEATURES: extended feature flags contained in a byte
+ *	array. The feature flags are identified by their bit index (see &enum
+ *	nl80211_ext_feature_index). The bit index is ordered starting at the
+ *	least-significant bit of the first byte in the array, ie. bit index 0
+ *	is located at bit 0 of byte 0. bit index 25 would be located at bit 1
+ *	of byte 3 (u8 array).
+ *
+ * @NL80211_ATTR_SURVEY_RADIO_STATS: Request overall radio statistics to be
+ *	returned along with other survey data. If set, @NL80211_CMD_GET_SURVEY
+ *	may return a survey entry without a channel indicating global radio
+ *	statistics (only some values are valid and make sense.)
+ *	For devices that don't return such an entry even then, the information
+ *	should be contained in the result as the sum of the respective counters
+ *	over all channels.
+ *
+ * @NL80211_ATTR_SCHED_SCAN_DELAY: delay before a scheduled scan (or a
+ *	WoWLAN net-detect scan) is started, u32 in seconds.
+
+ * @NL80211_ATTR_REG_INDOOR: flag attribute, if set indicates that the device
+ *      is operating in an indoor environment.
+ *
  * @NUM_NL80211_ATTR: total number of nl80211_attrs available
  * @NL80211_ATTR_MAX: highest attribute number currently defined
  * @__NL80211_ATTR_AFTER_LAST: internal use
@@ -2045,6 +2117,18 @@ enum nl80211_attrs {
 
 	NL80211_ATTR_MAC_MASK,
 
+	NL80211_ATTR_WIPHY_SELF_MANAGED_REG,
+
+	NL80211_ATTR_EXT_FEATURES,
+
+	NL80211_ATTR_SURVEY_RADIO_STATS,
+
+	NL80211_ATTR_NETNS_FD,
+
+	NL80211_ATTR_SCHED_SCAN_DELAY,
+
+	NL80211_ATTR_REG_INDOOR,
+
 	/* add attributes here, update the policy in nl80211.c */
 
 	__NL80211_ATTR_AFTER_LAST,
@@ -2085,7 +2169,7 @@ enum nl80211_attrs {
 
 #define NL80211_MAX_SUPP_RATES			32
 #define NL80211_MAX_SUPP_HT_RATES		77
-#define NL80211_MAX_SUPP_REG_RULES		32
+#define NL80211_MAX_SUPP_REG_RULES		64
 #define NL80211_TKIP_DATA_OFFSET_ENCR_KEY	0
 #define NL80211_TKIP_DATA_OFFSET_TX_MIC_KEY	16
 #define NL80211_TKIP_DATA_OFFSET_RX_MIC_KEY	24
@@ -2225,8 +2309,15 @@ struct nl80211_sta_flag_update {
  * @NL80211_RATE_INFO_VHT_MCS: MCS index for VHT (u8)
  * @NL80211_RATE_INFO_VHT_NSS: number of streams in VHT (u8)
  * @NL80211_RATE_INFO_80_MHZ_WIDTH: 80 MHz VHT rate
- * @NL80211_RATE_INFO_80P80_MHZ_WIDTH: 80+80 MHz VHT rate
+ * @NL80211_RATE_INFO_80P80_MHZ_WIDTH: unused - 80+80 is treated the
+ *	same as 160 for purposes of the bitrates
  * @NL80211_RATE_INFO_160_MHZ_WIDTH: 160 MHz VHT rate
+ * @NL80211_RATE_INFO_10_MHZ_WIDTH: 10 MHz width - note that this is
+ *	a legacy rate and will be reported as the actual bitrate, i.e.
+ *	half the base (20 MHz) rate
+ * @NL80211_RATE_INFO_5_MHZ_WIDTH: 5 MHz width - note that this is
+ *	a legacy rate and will be reported as the actual bitrate, i.e.
+ *	a quarter of the base (20 MHz) rate
  * @__NL80211_RATE_INFO_AFTER_LAST: internal use
  */
 enum nl80211_rate_info {
@@ -2241,6 +2332,8 @@ enum nl80211_rate_info {
 	NL80211_RATE_INFO_80_MHZ_WIDTH,
 	NL80211_RATE_INFO_80P80_MHZ_WIDTH,
 	NL80211_RATE_INFO_160_MHZ_WIDTH,
+	NL80211_RATE_INFO_10_MHZ_WIDTH,
+	NL80211_RATE_INFO_5_MHZ_WIDTH,
 
 	/* keep last */
 	__NL80211_RATE_INFO_AFTER_LAST,
@@ -2285,18 +2378,24 @@ enum nl80211_sta_bss_param {
  *
  * @__NL80211_STA_INFO_INVALID: attribute number 0 is reserved
  * @NL80211_STA_INFO_INACTIVE_TIME: time since last activity (u32, msecs)
- * @NL80211_STA_INFO_RX_BYTES: total received bytes (u32, from this station)
- * @NL80211_STA_INFO_TX_BYTES: total transmitted bytes (u32, to this station)
- * @NL80211_STA_INFO_RX_BYTES64: total received bytes (u64, from this station)
- * @NL80211_STA_INFO_TX_BYTES64: total transmitted bytes (u64, to this station)
+ * @NL80211_STA_INFO_RX_BYTES: total received bytes (MPDU length)
+ *	(u32, from this station)
+ * @NL80211_STA_INFO_TX_BYTES: total transmitted bytes (MPDU length)
+ *	(u32, to this station)
+ * @NL80211_STA_INFO_RX_BYTES64: total received bytes (MPDU length)
+ *	(u64, from this station)
+ * @NL80211_STA_INFO_TX_BYTES64: total transmitted bytes (MPDU length)
+ *	(u64, to this station)
  * @NL80211_STA_INFO_SIGNAL: signal strength of last received PPDU (u8, dBm)
  * @NL80211_STA_INFO_TX_BITRATE: current unicast tx rate, nested attribute
  * 	containing info as possible, see &enum nl80211_rate_info
- * @NL80211_STA_INFO_RX_PACKETS: total received packet (u32, from this station)
- * @NL80211_STA_INFO_TX_PACKETS: total transmitted packets (u32, to this
- *	station)
- * @NL80211_STA_INFO_TX_RETRIES: total retries (u32, to this station)
- * @NL80211_STA_INFO_TX_FAILED: total failed packets (u32, to this station)
+ * @NL80211_STA_INFO_RX_PACKETS: total received packet (MSDUs and MMPDUs)
+ *	(u32, from this station)
+ * @NL80211_STA_INFO_TX_PACKETS: total transmitted packets (MSDUs and MMPDUs)
+ *	(u32, to this station)
+ * @NL80211_STA_INFO_TX_RETRIES: total retries (MPDUs) (u32, to this station)
+ * @NL80211_STA_INFO_TX_FAILED: total failed packets (MPDUs)
+ *	(u32, to this station)
  * @NL80211_STA_INFO_SIGNAL_AVG: signal strength average (u8, dBm)
  * @NL80211_STA_INFO_LLID: the station's mesh LLID
  * @NL80211_STA_INFO_PLID: the station's mesh PLID
@@ -2320,6 +2419,16 @@ enum nl80211_sta_bss_param {
  *	Same format as NL80211_STA_INFO_CHAIN_SIGNAL.
  * @NL80211_STA_EXPECTED_THROUGHPUT: expected throughput considering also the
  *	802.11 header (u32, kbps)
+ * @NL80211_STA_INFO_RX_DROP_MISC: RX packets dropped for unspecified reasons
+ *	(u64)
+ * @NL80211_STA_INFO_BEACON_RX: number of beacons received from this peer (u64)
+ * @NL80211_STA_INFO_BEACON_SIGNAL_AVG: signal strength average
+ *	for beacons only (u8, dBm)
+ * @NL80211_STA_INFO_TID_STATS: per-TID statistics (see &enum nl80211_tid_stats)
+ *	This is a nested attribute where each the inner attribute number is the
+ *	TID+1 and the special TID 16 (i.e. value 17) is used for non-QoS frames;
+ *	each one of those is again nested with &enum nl80211_tid_stats
+ *	attributes carrying the actual values.
  * @__NL80211_STA_INFO_AFTER_LAST: internal
  * @NL80211_STA_INFO_MAX: highest possible station info attribute
  */
@@ -2352,12 +2461,41 @@ enum nl80211_sta_info {
 	NL80211_STA_INFO_CHAIN_SIGNAL,
 	NL80211_STA_INFO_CHAIN_SIGNAL_AVG,
 	NL80211_STA_INFO_EXPECTED_THROUGHPUT,
+	NL80211_STA_INFO_RX_DROP_MISC,
+	NL80211_STA_INFO_BEACON_RX,
+	NL80211_STA_INFO_BEACON_SIGNAL_AVG,
+	NL80211_STA_INFO_TID_STATS,
 
 	/* keep last */
 	__NL80211_STA_INFO_AFTER_LAST,
 	NL80211_STA_INFO_MAX = __NL80211_STA_INFO_AFTER_LAST - 1
 };
 
+/**
+ * enum nl80211_tid_stats - per TID statistics attributes
+ * @__NL80211_TID_STATS_INVALID: attribute number 0 is reserved
+ * @NL80211_TID_STATS_RX_MSDU: number of MSDUs received (u64)
+ * @NL80211_TID_STATS_TX_MSDU: number of MSDUs transmitted (or
+ *	attempted to transmit; u64)
+ * @NL80211_TID_STATS_TX_MSDU_RETRIES: number of retries for
+ *	transmitted MSDUs (not counting the first attempt; u64)
+ * @NL80211_TID_STATS_TX_MSDU_FAILED: number of failed transmitted
+ *	MSDUs (u64)
+ * @NUM_NL80211_TID_STATS: number of attributes here
+ * @NL80211_TID_STATS_MAX: highest numbered attribute here
+ */
+enum nl80211_tid_stats {
+	__NL80211_TID_STATS_INVALID,
+	NL80211_TID_STATS_RX_MSDU,
+	NL80211_TID_STATS_TX_MSDU,
+	NL80211_TID_STATS_TX_MSDU_RETRIES,
+	NL80211_TID_STATS_TX_MSDU_FAILED,
+
+	/* keep last */
+	NUM_NL80211_TID_STATS,
+	NL80211_TID_STATS_MAX = NUM_NL80211_TID_STATS - 1
+};
+
 /**
  * enum nl80211_mpath_flags - nl80211 mesh path flags
  *
@@ -2772,16 +2910,18 @@ enum nl80211_user_reg_hint_type {
  * @NL80211_SURVEY_INFO_FREQUENCY: center frequency of channel
  * @NL80211_SURVEY_INFO_NOISE: noise level of channel (u8, dBm)
  * @NL80211_SURVEY_INFO_IN_USE: channel is currently being used
- * @NL80211_SURVEY_INFO_CHANNEL_TIME: amount of time (in ms) that the radio
- *	spent on this channel
- * @NL80211_SURVEY_INFO_CHANNEL_TIME_BUSY: amount of the time the primary
+ * @NL80211_SURVEY_INFO_TIME: amount of time (in ms) that the radio
+ *	was turned on (on channel or globally)
+ * @NL80211_SURVEY_INFO_TIME_BUSY: amount of the time the primary
  *	channel was sensed busy (either due to activity or energy detect)
- * @NL80211_SURVEY_INFO_CHANNEL_TIME_EXT_BUSY: amount of time the extension
+ * @NL80211_SURVEY_INFO_TIME_EXT_BUSY: amount of time the extension
  *	channel was sensed busy
- * @NL80211_SURVEY_INFO_CHANNEL_TIME_RX: amount of time the radio spent
- *	receiving data
- * @NL80211_SURVEY_INFO_CHANNEL_TIME_TX: amount of time the radio spent
- *	transmitting data
+ * @NL80211_SURVEY_INFO_TIME_RX: amount of time the radio spent
+ *	receiving data (on channel or globally)
+ * @NL80211_SURVEY_INFO_TIME_TX: amount of time the radio spent
+ *	transmitting data (on channel or globally)
+ * @NL80211_SURVEY_INFO_TIME_SCAN: time the radio spent for scan
+ *	(on this channel or globally)
  * @NL80211_SURVEY_INFO_MAX: highest survey info attribute number
  *	currently defined
  * @__NL80211_SURVEY_INFO_AFTER_LAST: internal use
@@ -2791,17 +2931,25 @@ enum nl80211_survey_info {
 	NL80211_SURVEY_INFO_FREQUENCY,
 	NL80211_SURVEY_INFO_NOISE,
 	NL80211_SURVEY_INFO_IN_USE,
-	NL80211_SURVEY_INFO_CHANNEL_TIME,
-	NL80211_SURVEY_INFO_CHANNEL_TIME_BUSY,
-	NL80211_SURVEY_INFO_CHANNEL_TIME_EXT_BUSY,
-	NL80211_SURVEY_INFO_CHANNEL_TIME_RX,
-	NL80211_SURVEY_INFO_CHANNEL_TIME_TX,
+	NL80211_SURVEY_INFO_TIME,
+	NL80211_SURVEY_INFO_TIME_BUSY,
+	NL80211_SURVEY_INFO_TIME_EXT_BUSY,
+	NL80211_SURVEY_INFO_TIME_RX,
+	NL80211_SURVEY_INFO_TIME_TX,
+	NL80211_SURVEY_INFO_TIME_SCAN,
 
 	/* keep last */
 	__NL80211_SURVEY_INFO_AFTER_LAST,
 	NL80211_SURVEY_INFO_MAX = __NL80211_SURVEY_INFO_AFTER_LAST - 1
 };
 
+/* keep old names for compatibility */
+#define NL80211_SURVEY_INFO_CHANNEL_TIME		NL80211_SURVEY_INFO_TIME
+#define NL80211_SURVEY_INFO_CHANNEL_TIME_BUSY		NL80211_SURVEY_INFO_TIME_BUSY
+#define NL80211_SURVEY_INFO_CHANNEL_TIME_EXT_BUSY	NL80211_SURVEY_INFO_TIME_EXT_BUSY
+#define NL80211_SURVEY_INFO_CHANNEL_TIME_RX		NL80211_SURVEY_INFO_TIME_RX
+#define NL80211_SURVEY_INFO_CHANNEL_TIME_TX		NL80211_SURVEY_INFO_TIME_TX
+
 /**
  * enum nl80211_mntr_flags - monitor configuration flags
  *
@@ -2966,7 +3114,8 @@ enum nl80211_mesh_power_mode {
  *
  * @NL80211_MESHCONF_PLINK_TIMEOUT: If no tx activity is seen from a STA we've
  *	established peering with for longer than this time (in seconds), then
- *	remove it from the STA's list of peers.  Default is 30 minutes.
+ *	remove it from the STA's list of peers. You may set this to 0 to disable
+ *	the removal of the STA. Default is 30 minutes.
  *
  * @__NL80211_MESHCONF_ATTR_AFTER_LAST: internal use
  */
@@ -3238,6 +3387,9 @@ enum nl80211_bss {
 /**
  * enum nl80211_bss_status - BSS "status"
  * @NL80211_BSS_STATUS_AUTHENTICATED: Authenticated with this BSS.
+ *	Note that this is no longer used since cfg80211 no longer
+ *	keeps track of whether or not authentication was done with
+ *	a given BSS.
  * @NL80211_BSS_STATUS_ASSOCIATED: Associated with this BSS.
  * @NL80211_BSS_STATUS_IBSS_JOINED: Joined to this IBSS.
  *
@@ -3565,6 +3717,8 @@ struct nl80211_pattern_support {
  * @NL80211_WOWLAN_TRIG_ANY: wake up on any activity, do not really put
  *	the chip into a special state -- works best with chips that have
  *	support for low-power operation already (flag)
+ *	Note that this mode is incompatible with all of the others, if
+ *	any others are even supported by the device.
  * @NL80211_WOWLAN_TRIG_DISCONNECT: wake up on disconnect, the way disconnect
  *	is detected is implementation-specific (flag)
  * @NL80211_WOWLAN_TRIG_MAGIC_PKT: wake up on magic packet (6x 0xff, followed
@@ -3621,9 +3775,12 @@ struct nl80211_pattern_support {
  * @NL80211_WOWLAN_TRIG_NET_DETECT: wake up when a configured network
  *	is detected.  This is a nested attribute that contains the
  *	same attributes used with @NL80211_CMD_START_SCHED_SCAN.  It
- *	specifies how the scan is performed (e.g. the interval and the
- *	channels to scan) as well as the scan results that will
- *	trigger a wake (i.e. the matchsets).
+ *	specifies how the scan is performed (e.g. the interval, the
+ *	channels to scan and the initial delay) as well as the scan
+ *	results that will trigger a wake (i.e. the matchsets).  This
+ *	attribute is also sent in a response to
+ *	@NL80211_CMD_GET_WIPHY, indicating the number of match sets
+ *	supported by the driver (u32).
  * @NL80211_WOWLAN_TRIG_NET_DETECT_RESULTS: nested attribute
  *	containing an array with information about what triggered the
  *	wake up.  If no elements are present in the array, it means
@@ -4193,6 +4350,21 @@ enum nl80211_feature_flags {
 	NL80211_FEATURE_ND_RANDOM_MAC_ADDR		= 1 << 31,
 };
 
+/**
+ * enum nl80211_ext_feature_index - bit index of extended features.
+ * @NL80211_EXT_FEATURE_VHT_IBSS: This driver supports IBSS with VHT datarates.
+ *
+ * @NUM_NL80211_EXT_FEATURES: number of extended features.
+ * @MAX_NL80211_EXT_FEATURES: highest extended feature index.
+ */
+enum nl80211_ext_feature_index {
+	NL80211_EXT_FEATURE_VHT_IBSS,
+
+	/* add new features before the definition below */
+	NUM_NL80211_EXT_FEATURES,
+	MAX_NL80211_EXT_FEATURES = NUM_NL80211_EXT_FEATURES - 1
+};
+
 /**
  * enum nl80211_probe_resp_offload_support_attr - optional supported
  *	protocols for probe-response offloading by the driver/FW.
diff --git a/src/eap_common/Makefile b/src/eap_common/Makefile
index adfd3dfd5b9b..f00b438c6188 100644
--- a/src/eap_common/Makefile
+++ b/src/eap_common/Makefile
@@ -1,8 +1,31 @@
-all:
-	@echo Nothing to be made.
+all: libeap_common.a
 
 clean:
-	rm -f *~ *.o *.d *.gcno *.gcda *.gcov
+	rm -f *~ *.o *.d *.gcno *.gcda *.gcov libeap_common.a
 
 install:
 	@echo Nothing to be made.
+
+include ../lib.rules
+
+LIB_OBJS= \
+	chap.o \
+	eap_common.o \
+	eap_eke_common.o \
+	eap_eke_common.o \
+	eap_fast_common.o \
+	eap_gpsk_common.o \
+	eap_ikev2_common.o \
+	eap_pax_common.o \
+	eap_peap_common.o \
+	eap_psk_common.o \
+	eap_pwd_common.o \
+	eap_sake_common.o \
+	eap_sim_common.o \
+	eap_wsc_common.o \
+	ikev2_common.o
+
+libeap_common.a: $(LIB_OBJS)
+	$(AR) crT $@ $?
+
+-include $(OBJS:%.o=%.d)
diff --git a/src/eap_common/eap_common.c b/src/eap_common/eap_common.c
index 1de13281c511..51a15d75bc93 100644
--- a/src/eap_common/eap_common.c
+++ b/src/eap_common/eap_common.c
@@ -192,7 +192,7 @@ u8 eap_get_id(const struct wpabuf *msg)
 
 
 /**
- * eap_get_id - Get EAP Type from wpabuf
+ * eap_get_type - Get EAP Type from wpabuf
  * @msg: Buffer starting with an EAP header
  * Returns: The EAP Type after the EAP header
  */
diff --git a/src/eap_common/eap_fast_common.c b/src/eap_common/eap_fast_common.c
index fceb1b0adc1c..151cc7859c5d 100644
--- a/src/eap_common/eap_fast_common.c
+++ b/src/eap_common/eap_fast_common.c
@@ -96,49 +96,18 @@ void eap_fast_derive_master_secret(const u8 *pac_key, const u8 *server_random,
 u8 * eap_fast_derive_key(void *ssl_ctx, struct tls_connection *conn,
 			 const char *label, size_t len)
 {
-	struct tls_keys keys;
-	u8 *rnd = NULL, *out;
-	int block_size;
+	u8 *out;
 
-	block_size = tls_connection_get_keyblock_size(ssl_ctx, conn);
-	if (block_size < 0)
-		return NULL;
-
-	out = os_malloc(block_size + len);
+	out = os_malloc(len);
 	if (out == NULL)
 		return NULL;
 
-	if (tls_connection_prf(ssl_ctx, conn, label, 1, out, block_size + len)
-	    == 0) {
-		os_memmove(out, out + block_size, len);
-		return out;
+	if (tls_connection_prf(ssl_ctx, conn, label, 1, 1, out, len)) {
+		os_free(out);
+		return NULL;
 	}
 
-	if (tls_connection_get_keys(ssl_ctx, conn, &keys))
-		goto fail;
-
-	rnd = os_malloc(keys.client_random_len + keys.server_random_len);
-	if (rnd == NULL)
-		goto fail;
-
-	os_memcpy(rnd, keys.server_random, keys.server_random_len);
-	os_memcpy(rnd + keys.server_random_len, keys.client_random,
-		  keys.client_random_len);
-
-	wpa_hexdump_key(MSG_MSGDUMP, "EAP-FAST: master_secret for key "
-			"expansion", keys.master_key, keys.master_key_len);
-	if (tls_prf_sha1_md5(keys.master_key, keys.master_key_len,
-			     label, rnd, keys.client_random_len +
-			     keys.server_random_len, out, block_size + len))
-		goto fail;
-	os_free(rnd);
-	os_memmove(out, out + block_size, len);
 	return out;
-
-fail:
-	os_free(rnd);
-	os_free(out);
-	return NULL;
 }
 
 
diff --git a/src/eap_common/eap_pwd_common.c b/src/eap_common/eap_pwd_common.c
index 631c363fb7c9..4d27623f87bf 100644
--- a/src/eap_common/eap_pwd_common.c
+++ b/src/eap_common/eap_pwd_common.c
@@ -86,9 +86,10 @@ static int eap_pwd_kdf(const u8 *key, size_t keylen, const u8 *label,
  * on the password and identities.
  */
 int compute_password_element(EAP_PWD_group *grp, u16 num,
-			     u8 *password, int password_len,
-			     u8 *id_server, int id_server_len,
-			     u8 *id_peer, int id_peer_len, u8 *token)
+			     const u8 *password, size_t password_len,
+			     const u8 *id_server, size_t id_server_len,
+			     const u8 *id_peer, size_t id_peer_len,
+			     const u8 *token)
 {
 	BIGNUM *x_candidate = NULL, *rnd = NULL, *cofactor = NULL;
 	struct crypto_hash *hash;
@@ -283,10 +284,10 @@ int compute_password_element(EAP_PWD_group *grp, u16 num,
 }
 
 
-int compute_keys(EAP_PWD_group *grp, BN_CTX *bnctx, BIGNUM *k,
-		 BIGNUM *peer_scalar, BIGNUM *server_scalar,
-		 u8 *confirm_peer, u8 *confirm_server,
-		 u32 *ciphersuite, u8 *msk, u8 *emsk, u8 *session_id)
+int compute_keys(EAP_PWD_group *grp, BN_CTX *bnctx, const BIGNUM *k,
+		 const BIGNUM *peer_scalar, const BIGNUM *server_scalar,
+		 const u8 *confirm_peer, const u8 *confirm_server,
+		 const u32 *ciphersuite, u8 *msk, u8 *emsk, u8 *session_id)
 {
 	struct crypto_hash *hash;
 	u8 mk[SHA256_MAC_LEN], *cruft;
@@ -306,7 +307,7 @@ int compute_keys(EAP_PWD_group *grp, BN_CTX *bnctx, BIGNUM *k,
 		os_free(cruft);
 		return -1;
 	}
-	eap_pwd_h_update(hash, (u8 *) ciphersuite, sizeof(u32));
+	eap_pwd_h_update(hash, (const u8 *) ciphersuite, sizeof(u32));
 	offset = BN_num_bytes(grp->order) - BN_num_bytes(peer_scalar);
 	os_memset(cruft, 0, BN_num_bytes(grp->prime));
 	BN_bn2bin(peer_scalar, cruft + offset);
diff --git a/src/eap_common/eap_pwd_common.h b/src/eap_common/eap_pwd_common.h
index c54c4414f11f..a0d717edfe7a 100644
--- a/src/eap_common/eap_pwd_common.h
+++ b/src/eap_common/eap_pwd_common.h
@@ -56,10 +56,15 @@ struct eap_pwd_id {
 } STRUCT_PACKED;
 
 /* common routines */
-int compute_password_element(EAP_PWD_group *, u16, u8 *, int, u8 *, int, u8 *,
-			     int, u8 *);
-int compute_keys(EAP_PWD_group *, BN_CTX *, BIGNUM *, BIGNUM *, BIGNUM *,
-		 u8 *, u8 *, u32 *, u8 *, u8 *, u8 *);
+int compute_password_element(EAP_PWD_group *grp, u16 num,
+			     const u8 *password, size_t password_len,
+			     const u8 *id_server, size_t id_server_len,
+			     const u8 *id_peer, size_t id_peer_len,
+			     const u8 *token);
+int compute_keys(EAP_PWD_group *grp, BN_CTX *bnctx, const BIGNUM *k,
+		 const BIGNUM *peer_scalar, const BIGNUM *server_scalar,
+		 const u8 *confirm_peer, const u8 *confirm_server,
+		 const u32 *ciphersuite, u8 *msk, u8 *emsk, u8 *session_id);
 struct crypto_hash * eap_pwd_h_init(void);
 void eap_pwd_h_update(struct crypto_hash *hash, const u8 *data, size_t len);
 void eap_pwd_h_final(struct crypto_hash *hash, u8 *digest);
diff --git a/src/eap_common/eap_sake_common.c b/src/eap_common/eap_sake_common.c
index a76253d00f60..c22e43ed84b6 100644
--- a/src/eap_common/eap_sake_common.c
+++ b/src/eap_common/eap_sake_common.c
@@ -16,99 +16,99 @@
 
 
 static int eap_sake_parse_add_attr(struct eap_sake_parse_attr *attr,
-				   const u8 *pos)
+				   u8 attr_id, u8 len, const u8 *data)
 {
 	size_t i;
 
-	switch (pos[0]) {
+	switch (attr_id) {
 	case EAP_SAKE_AT_RAND_S:
 		wpa_printf(MSG_DEBUG, "EAP-SAKE: Parse: AT_RAND_S");
-		if (pos[1] != 2 + EAP_SAKE_RAND_LEN) {
+		if (len != EAP_SAKE_RAND_LEN) {
 			wpa_printf(MSG_DEBUG, "EAP-SAKE: AT_RAND_S with "
-				   "invalid length %d", pos[1]);
+				   "invalid payload length %d", len);
 			return -1;
 		}
-		attr->rand_s = pos + 2;
+		attr->rand_s = data;
 		break;
 	case EAP_SAKE_AT_RAND_P:
 		wpa_printf(MSG_DEBUG, "EAP-SAKE: Parse: AT_RAND_P");
-		if (pos[1] != 2 + EAP_SAKE_RAND_LEN) {
+		if (len != EAP_SAKE_RAND_LEN) {
 			wpa_printf(MSG_DEBUG, "EAP-SAKE: AT_RAND_P with "
-				   "invalid length %d", pos[1]);
+				   "invalid payload length %d", len);
 			return -1;
 		}
-		attr->rand_p = pos + 2;
+		attr->rand_p = data;
 		break;
 	case EAP_SAKE_AT_MIC_S:
 		wpa_printf(MSG_DEBUG, "EAP-SAKE: Parse: AT_MIC_S");
-		if (pos[1] != 2 + EAP_SAKE_MIC_LEN) {
+		if (len != EAP_SAKE_MIC_LEN) {
 			wpa_printf(MSG_DEBUG, "EAP-SAKE: AT_MIC_S with "
-				   "invalid length %d", pos[1]);
+				   "invalid payload length %d", len);
 			return -1;
 		}
-		attr->mic_s = pos + 2;
+		attr->mic_s = data;
 		break;
 	case EAP_SAKE_AT_MIC_P:
 		wpa_printf(MSG_DEBUG, "EAP-SAKE: Parse: AT_MIC_P");
-		if (pos[1] != 2 + EAP_SAKE_MIC_LEN) {
+		if (len != EAP_SAKE_MIC_LEN) {
 			wpa_printf(MSG_DEBUG, "EAP-SAKE: AT_MIC_P with "
-				   "invalid length %d", pos[1]);
+				   "invalid payload length %d", len);
 			return -1;
 		}
-		attr->mic_p = pos + 2;
+		attr->mic_p = data;
 		break;
 	case EAP_SAKE_AT_SERVERID:
 		wpa_printf(MSG_DEBUG, "EAP-SAKE: Parse: AT_SERVERID");
-		attr->serverid = pos + 2;
-		attr->serverid_len = pos[1] - 2;
+		attr->serverid = data;
+		attr->serverid_len = len;
 		break;
 	case EAP_SAKE_AT_PEERID:
 		wpa_printf(MSG_DEBUG, "EAP-SAKE: Parse: AT_PEERID");
-		attr->peerid = pos + 2;
-		attr->peerid_len = pos[1] - 2;
+		attr->peerid = data;
+		attr->peerid_len = len;
 		break;
 	case EAP_SAKE_AT_SPI_S:
 		wpa_printf(MSG_DEBUG, "EAP-SAKE: Parse: AT_SPI_S");
-		attr->spi_s = pos + 2;
-		attr->spi_s_len = pos[1] - 2;
+		attr->spi_s = data;
+		attr->spi_s_len = len;
 		break;
 	case EAP_SAKE_AT_SPI_P:
 		wpa_printf(MSG_DEBUG, "EAP-SAKE: Parse: AT_SPI_P");
-		attr->spi_p = pos + 2;
-		attr->spi_p_len = pos[1] - 2;
+		attr->spi_p = data;
+		attr->spi_p_len = len;
 		break;
 	case EAP_SAKE_AT_ANY_ID_REQ:
 		wpa_printf(MSG_DEBUG, "EAP-SAKE: Parse: AT_ANY_ID_REQ");
-		if (pos[1] != 4) {
+		if (len != 2) {
 			wpa_printf(MSG_DEBUG, "EAP-SAKE: Invalid AT_ANY_ID_REQ"
-				   " length %d", pos[1]);
+				   " payload length %d", len);
 			return -1;
 		}
-		attr->any_id_req = pos + 2;
+		attr->any_id_req = data;
 		break;
 	case EAP_SAKE_AT_PERM_ID_REQ:
 		wpa_printf(MSG_DEBUG, "EAP-SAKE: Parse: AT_PERM_ID_REQ");
-		if (pos[1] != 4) {
+		if (len != 2) {
 			wpa_printf(MSG_DEBUG, "EAP-SAKE: Invalid "
-				   "AT_PERM_ID_REQ length %d", pos[1]);
+				   "AT_PERM_ID_REQ payload length %d", len);
 			return -1;
 		}
-		attr->perm_id_req = pos + 2;
+		attr->perm_id_req = data;
 		break;
 	case EAP_SAKE_AT_ENCR_DATA:
 		wpa_printf(MSG_DEBUG, "EAP-SAKE: Parse: AT_ENCR_DATA");
-		attr->encr_data = pos + 2;
-		attr->encr_data_len = pos[1] - 2;
+		attr->encr_data = data;
+		attr->encr_data_len = len;
 		break;
 	case EAP_SAKE_AT_IV:
 		wpa_printf(MSG_DEBUG, "EAP-SAKE: Parse: AT_IV");
-		attr->iv = pos + 2;
-		attr->iv_len = pos[1] - 2;
+		attr->iv = data;
+		attr->iv_len = len;
 		break;
 	case EAP_SAKE_AT_PADDING:
 		wpa_printf(MSG_DEBUG, "EAP-SAKE: Parse: AT_PADDING");
-		for (i = 2; i < pos[1]; i++) {
-			if (pos[i]) {
+		for (i = 0; i < len; i++) {
+			if (data[i]) {
 				wpa_printf(MSG_DEBUG, "EAP-SAKE: AT_PADDING "
 					   "with non-zero pad byte");
 				return -1;
@@ -117,26 +117,26 @@ static int eap_sake_parse_add_attr(struct eap_sake_parse_attr *attr,
 		break;
 	case EAP_SAKE_AT_NEXT_TMPID:
 		wpa_printf(MSG_DEBUG, "EAP-SAKE: Parse: AT_NEXT_TMPID");
-		attr->next_tmpid = pos + 2;
-		attr->next_tmpid_len = pos[1] - 2;
+		attr->next_tmpid = data;
+		attr->next_tmpid_len = len;
 		break;
 	case EAP_SAKE_AT_MSK_LIFE:
 		wpa_printf(MSG_DEBUG, "EAP-SAKE: Parse: AT_IV");
-		if (pos[1] != 6) {
+		if (len != 4) {
 			wpa_printf(MSG_DEBUG, "EAP-SAKE: Invalid "
-				   "AT_MSK_LIFE length %d", pos[1]);
+				   "AT_MSK_LIFE payload length %d", len);
 			return -1;
 		}
-		attr->msk_life = pos + 2;
+		attr->msk_life = data;
 		break;
 	default:
-		if (pos[0] < 128) {
+		if (attr_id < 128) {
 			wpa_printf(MSG_DEBUG, "EAP-SAKE: Unknown non-skippable"
-				   " attribute %d", pos[0]);
+				   " attribute %d", attr_id);
 			return -1;
 		}
 		wpa_printf(MSG_DEBUG, "EAP-SAKE: Ignoring unknown skippable "
-			   "attribute %d", pos[0]);
+			   "attribute %d", attr_id);
 		break;
 	}
 
@@ -180,7 +180,7 @@ int eap_sake_parse_attributes(const u8 *buf, size_t len,
 			return -1;
 		}
 
-		if (eap_sake_parse_add_attr(attr, pos))
+		if (eap_sake_parse_add_attr(attr, pos[0], pos[1] - 2, pos + 2))
 			return -1;
 
 		pos += pos[1];
diff --git a/src/eap_common/ikev2_common.c b/src/eap_common/ikev2_common.c
index 4f9e64eced08..d60358c733f0 100644
--- a/src/eap_common/ikev2_common.c
+++ b/src/eap_common/ikev2_common.c
@@ -16,7 +16,7 @@
 #include "ikev2_common.h"
 
 
-static struct ikev2_integ_alg ikev2_integ_algs[] = {
+static const struct ikev2_integ_alg ikev2_integ_algs[] = {
 	{ AUTH_HMAC_SHA1_96, 20, 12 },
 	{ AUTH_HMAC_MD5_96, 16, 12 }
 };
@@ -24,7 +24,7 @@ static struct ikev2_integ_alg ikev2_integ_algs[] = {
 #define NUM_INTEG_ALGS ARRAY_SIZE(ikev2_integ_algs)
 
 
-static struct ikev2_prf_alg ikev2_prf_algs[] = {
+static const struct ikev2_prf_alg ikev2_prf_algs[] = {
 	{ PRF_HMAC_SHA1, 20, 20 },
 	{ PRF_HMAC_MD5, 16, 16 }
 };
@@ -32,7 +32,7 @@ static struct ikev2_prf_alg ikev2_prf_algs[] = {
 #define NUM_PRF_ALGS ARRAY_SIZE(ikev2_prf_algs)
 
 
-static struct ikev2_encr_alg ikev2_encr_algs[] = {
+static const struct ikev2_encr_alg ikev2_encr_algs[] = {
 	{ ENCR_AES_CBC, 16, 16 }, /* only 128-bit keys supported for now */
 	{ ENCR_3DES, 24, 8 }
 };
diff --git a/src/eap_peer/Makefile b/src/eap_peer/Makefile
index f79519b71849..6531ccd5dac0 100644
--- a/src/eap_peer/Makefile
+++ b/src/eap_peer/Makefile
@@ -1,11 +1,23 @@
-all:
-	@echo Nothing to be made.
+all: libeap_peer.a
 
 clean:
-	rm -f *~ *.o *.so *.d *.gcno *.gcda *.gcov
+	rm -f *~ *.o *.so *.d *.gcno *.gcda *.gcov libeap_peer.a
 
 install:
 	if ls *.so >/dev/null 2>&1; then \
 		install -d $(DESTDIR)$(LIBDIR)/wpa_supplicant && \
 		cp *.so $(DESTDIR)$(LIBDIR)/wpa_supplicant \
 	; fi
+
+include ../lib.rules
+
+CFLAGS += -DIEEE8021X_EAPOL
+
+LIB_OBJS= \
+	eap.o \
+	eap_methods.o
+
+libeap_peer.a: $(LIB_OBJS)
+	$(AR) crT $@ $?
+
+-include $(OBJS:%.o=%.d)
diff --git a/src/eap_peer/eap.c b/src/eap_peer/eap.c
index 35433f3bd8e4..56c24b550320 100644
--- a/src/eap_peer/eap.c
+++ b/src/eap_peer/eap.c
@@ -584,7 +584,7 @@ static int eap_peer_erp_reauth_start(struct eap_sm *sm,
 	wpa_printf(MSG_DEBUG, "EAP: Valid ERP key found %s (SEQ=%u)",
 		   erp->keyname_nai, erp->next_seq);
 
-	msg = eap_msg_alloc(EAP_VENDOR_IETF, EAP_ERP_TYPE_REAUTH,
+	msg = eap_msg_alloc(EAP_VENDOR_IETF, (EapType) EAP_ERP_TYPE_REAUTH,
 			    1 + 2 + 2 + os_strlen(erp->keyname_nai) + 1 + 16,
 			    EAP_CODE_INITIATE, hdr->identifier);
 	if (msg == NULL)
@@ -708,7 +708,7 @@ SM_STATE(EAP, SEND_RESPONSE)
 	wpabuf_free(sm->lastRespData);
 	if (sm->eapRespData) {
 		if (sm->workaround)
-			os_memcpy(sm->last_md5, sm->req_md5, 16);
+			os_memcpy(sm->last_sha1, sm->req_sha1, 20);
 		sm->lastId = sm->reqId;
 		sm->lastRespData = wpabuf_dup(sm->eapRespData);
 		eapol_set_bool(sm, EAPOL_eapResp, TRUE);
@@ -914,12 +914,12 @@ static int eap_peer_req_is_duplicate(struct eap_sm *sm)
 
 	duplicate = (sm->reqId == sm->lastId) && sm->rxReq;
 	if (sm->workaround && duplicate &&
-	    os_memcmp(sm->req_md5, sm->last_md5, 16) != 0) {
+	    os_memcmp(sm->req_sha1, sm->last_sha1, 20) != 0) {
 		/*
 		 * RFC 4137 uses (reqId == lastId) as the only verification for
 		 * duplicate EAP requests. However, this misses cases where the
 		 * AS is incorrectly using the same id again; and
-		 * unfortunately, such implementations exist. Use MD5 hash as
+		 * unfortunately, such implementations exist. Use SHA1 hash as
 		 * an extra verification for the packets being duplicate to
 		 * workaround these issues.
 		 */
@@ -1765,7 +1765,7 @@ static void eap_sm_parseEapReq(struct eap_sm *sm, const struct wpabuf *req)
 	if (sm->workaround) {
 		const u8 *addr[1];
 		addr[0] = wpabuf_head(req);
-		md5_vector(1, addr, &plen, sm->req_md5);
+		sha1_vector(1, addr, &plen, sm->req_sha1);
 	}
 
 	switch (hdr->code) {
@@ -1911,7 +1911,7 @@ static void eap_peer_sm_tls_event(void *ctx, enum tls_event ev,
  * structure remains alive while the EAP state machine is active.
  */
 struct eap_sm * eap_peer_sm_init(void *eapol_ctx,
-				 struct eapol_callbacks *eapol_cb,
+				 const struct eapol_callbacks *eapol_cb,
 				 void *msg_ctx, struct eap_config *conf)
 {
 	struct eap_sm *sm;
@@ -2400,7 +2400,7 @@ static int eap_allowed_phase2_type(int vendor, int type)
 u32 eap_get_phase2_type(const char *name, int *vendor)
 {
 	int v;
-	u8 type = eap_peer_get_type(name, &v);
+	u32 type = eap_peer_get_type(name, &v);
 	if (eap_allowed_phase2_type(v, type)) {
 		*vendor = v;
 		return type;
diff --git a/src/eap_peer/eap.h b/src/eap_peer/eap.h
index 702463b9d514..1a645af8b200 100644
--- a/src/eap_peer/eap.h
+++ b/src/eap_peer/eap.h
@@ -307,7 +307,7 @@ struct eap_config {
 };
 
 struct eap_sm * eap_peer_sm_init(void *eapol_ctx,
-				 struct eapol_callbacks *eapol_cb,
+				 const struct eapol_callbacks *eapol_cb,
 				 void *msg_ctx, struct eap_config *conf);
 void eap_peer_sm_deinit(struct eap_sm *sm);
 int eap_peer_sm_step(struct eap_sm *sm);
diff --git a/src/eap_peer/eap_aka.c b/src/eap_peer/eap_aka.c
index 0662ae738367..dc9e8cc34d4a 100644
--- a/src/eap_peer/eap_aka.c
+++ b/src/eap_peer/eap_aka.c
@@ -1296,7 +1296,7 @@ static struct wpabuf * eap_aka_process(struct eap_sm *sm, void *priv,
 
 	pos = eap_hdr_validate(EAP_VENDOR_IETF, data->eap_method, reqData,
 			       &len);
-	if (pos == NULL || len < 1) {
+	if (pos == NULL || len < 3) {
 		ret->ignore = TRUE;
 		return NULL;
 	}
diff --git a/src/eap_peer/eap_eke.c b/src/eap_peer/eap_eke.c
index 9fec66c06867..dfbda5644f6f 100644
--- a/src/eap_peer/eap_eke.c
+++ b/src/eap_peer/eap_eke.c
@@ -195,15 +195,14 @@ static int eap_eke_supp_mac(u8 mac)
 
 static struct wpabuf * eap_eke_build_fail(struct eap_eke_data *data,
 					  struct eap_method_ret *ret,
-					  const struct wpabuf *reqData,
-					  u32 failure_code)
+					  u8 id, u32 failure_code)
 {
 	struct wpabuf *resp;
 
 	wpa_printf(MSG_DEBUG, "EAP-EKE: Sending EAP-EKE-Failure/Response - code=0x%x",
 		   failure_code);
 
-	resp = eap_eke_build_msg(data, eap_get_id(reqData), 4, EAP_EKE_FAILURE);
+	resp = eap_eke_build_msg(data, id, 4, EAP_EKE_FAILURE);
 	if (resp)
 		wpabuf_put_be32(resp, failure_code);
 
@@ -230,9 +229,10 @@ static struct wpabuf * eap_eke_process_id(struct eap_eke_data *data,
 	const u8 *pos, *end;
 	const u8 *prop = NULL;
 	u8 idtype;
+	u8 id = eap_get_id(reqData);
 
 	if (data->state != IDENTITY) {
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PROTO_ERROR);
 	}
 
@@ -240,7 +240,7 @@ static struct wpabuf * eap_eke_process_id(struct eap_eke_data *data,
 
 	if (payload_len < 2 + 4) {
 		wpa_printf(MSG_DEBUG, "EAP-EKE: Too short ID/Request Data");
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PROTO_ERROR);
 	}
 
@@ -253,7 +253,7 @@ static struct wpabuf * eap_eke_process_id(struct eap_eke_data *data,
 	if (pos + num_prop * 4 > end) {
 		wpa_printf(MSG_DEBUG, "EAP-EKE: Too short ID/Request Data (num_prop=%u)",
 			   num_prop);
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PROTO_ERROR);
 	}
 
@@ -293,7 +293,7 @@ static struct wpabuf * eap_eke_process_id(struct eap_eke_data *data,
 
 	if (prop == NULL) {
 		wpa_printf(MSG_DEBUG, "EAP-EKE: No acceptable proposal found");
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_NO_PROPOSAL_CHOSEN);
 	}
 
@@ -301,7 +301,7 @@ static struct wpabuf * eap_eke_process_id(struct eap_eke_data *data,
 
 	if (pos == end) {
 		wpa_printf(MSG_DEBUG, "EAP-EKE: Too short ID/Request Data to include IDType/Identity");
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PROTO_ERROR);
 	}
 
@@ -312,7 +312,7 @@ static struct wpabuf * eap_eke_process_id(struct eap_eke_data *data,
 	os_free(data->serverid);
 	data->serverid = os_malloc(end - pos);
 	if (data->serverid == NULL) {
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PRIVATE_INTERNAL_ERROR);
 	}
 	os_memcpy(data->serverid, pos, end - pos);
@@ -320,11 +320,11 @@ static struct wpabuf * eap_eke_process_id(struct eap_eke_data *data,
 
 	wpa_printf(MSG_DEBUG, "EAP-EKE: Sending EAP-EKE-ID/Response");
 
-	resp = eap_eke_build_msg(data, eap_get_id(reqData),
+	resp = eap_eke_build_msg(data, id,
 				 2 + 4 + 1 + data->peerid_len,
 				 EAP_EKE_ID);
 	if (resp == NULL) {
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PRIVATE_INTERNAL_ERROR);
 	}
 
@@ -339,7 +339,7 @@ static struct wpabuf * eap_eke_process_id(struct eap_eke_data *data,
 	data->msgs = wpabuf_alloc(wpabuf_len(reqData) + wpabuf_len(resp));
 	if (data->msgs == NULL) {
 		wpabuf_free(resp);
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PRIVATE_INTERNAL_ERROR);
 	}
 	wpabuf_put_buf(data->msgs, reqData);
@@ -366,10 +366,11 @@ static struct wpabuf * eap_eke_process_commit(struct eap_sm *sm,
 	u8 pub[EAP_EKE_MAX_DH_LEN];
 	const u8 *password;
 	size_t password_len;
+	u8 id = eap_get_id(reqData);
 
 	if (data->state != COMMIT) {
 		wpa_printf(MSG_DEBUG, "EAP-EKE: EAP-EKE-Commit/Request received in unexpected state (%d)", data->state);
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PROTO_ERROR);
 	}
 
@@ -378,7 +379,7 @@ static struct wpabuf * eap_eke_process_commit(struct eap_sm *sm,
 	password = eap_get_config_password(sm, &password_len);
 	if (password == NULL) {
 		wpa_printf(MSG_INFO, "EAP-EKE: No password configured!");
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PASSWD_NOT_FOUND);
 	}
 
@@ -387,7 +388,7 @@ static struct wpabuf * eap_eke_process_commit(struct eap_sm *sm,
 
 	if (pos + data->sess.dhcomp_len > end) {
 		wpa_printf(MSG_DEBUG, "EAP-EKE: Too short EAP-EKE-Commit");
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PROTO_ERROR);
 	}
 
@@ -405,7 +406,7 @@ static struct wpabuf * eap_eke_process_commit(struct eap_sm *sm,
 			       data->serverid, data->serverid_len,
 			       data->peerid, data->peerid_len, key) < 0) {
 		wpa_printf(MSG_INFO, "EAP-EKE: Failed to derive key");
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PRIVATE_INTERNAL_ERROR);
 	}
 
@@ -416,7 +417,7 @@ static struct wpabuf * eap_eke_process_commit(struct eap_sm *sm,
 	if (eap_eke_dh_init(data->sess.dhgroup, data->dh_priv, pub) < 0) {
 		wpa_printf(MSG_INFO, "EAP-EKE: Failed to initialize DH");
 		os_memset(key, 0, sizeof(key));
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PRIVATE_INTERNAL_ERROR);
 	}
 
@@ -424,7 +425,7 @@ static struct wpabuf * eap_eke_process_commit(struct eap_sm *sm,
 	{
 		wpa_printf(MSG_INFO, "EAP-EKE: Failed to derive shared secret");
 		os_memset(key, 0, sizeof(key));
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PRIVATE_INTERNAL_ERROR);
 	}
 
@@ -433,18 +434,18 @@ static struct wpabuf * eap_eke_process_commit(struct eap_sm *sm,
 				 data->peerid, data->peerid_len) < 0) {
 		wpa_printf(MSG_INFO, "EAP-EKE: Failed to derive Ke/Ki");
 		os_memset(key, 0, sizeof(key));
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PRIVATE_INTERNAL_ERROR);
 	}
 
 	wpa_printf(MSG_DEBUG, "EAP-EKE: Sending EAP-EKE-Commit/Response");
 
-	resp = eap_eke_build_msg(data, eap_get_id(reqData),
+	resp = eap_eke_build_msg(data, id,
 				 data->sess.dhcomp_len + data->sess.pnonce_len,
 				 EAP_EKE_COMMIT);
 	if (resp == NULL) {
 		os_memset(key, 0, sizeof(key));
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PRIVATE_INTERNAL_ERROR);
 	}
 
@@ -453,7 +454,7 @@ static struct wpabuf * eap_eke_process_commit(struct eap_sm *sm,
 	if (eap_eke_dhcomp(&data->sess, key, pub, rpos) < 0) {
 		wpa_printf(MSG_INFO, "EAP-EKE: Failed to build DHComponent_P");
 		os_memset(key, 0, sizeof(key));
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PRIVATE_INTERNAL_ERROR);
 	}
 	os_memset(key, 0, sizeof(key));
@@ -463,7 +464,7 @@ static struct wpabuf * eap_eke_process_commit(struct eap_sm *sm,
 
 	if (random_get_bytes(data->nonce_p, data->sess.nonce_len)) {
 		wpabuf_free(resp);
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PRIVATE_INTERNAL_ERROR);
 	}
 	wpa_hexdump_key(MSG_DEBUG, "EAP-EKE: Nonce_P",
@@ -472,7 +473,7 @@ static struct wpabuf * eap_eke_process_commit(struct eap_sm *sm,
 	if (eap_eke_prot(&data->sess, data->nonce_p, data->sess.nonce_len,
 			 wpabuf_put(resp, 0), &prot_len) < 0) {
 		wpabuf_free(resp);
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PRIVATE_INTERNAL_ERROR);
 	}
 	wpa_hexdump(MSG_DEBUG, "EAP-EKE: PNonce_P",
@@ -484,7 +485,7 @@ static struct wpabuf * eap_eke_process_commit(struct eap_sm *sm,
 	if (wpabuf_resize(&data->msgs, wpabuf_len(reqData) + wpabuf_len(resp))
 	    < 0) {
 		wpabuf_free(resp);
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PRIVATE_INTERNAL_ERROR);
 	}
 	wpabuf_put_buf(data->msgs, reqData);
@@ -509,11 +510,12 @@ static struct wpabuf * eap_eke_process_confirm(struct eap_eke_data *data,
 	u8 auth_s[EAP_EKE_MAX_HASH_LEN];
 	size_t decrypt_len;
 	u8 *auth;
+	u8 id = eap_get_id(reqData);
 
 	if (data->state != CONFIRM) {
 		wpa_printf(MSG_DEBUG, "EAP-EKE: EAP-EKE-Confirm/Request received in unexpected state (%d)",
 			   data->state);
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PROTO_ERROR);
 	}
 
@@ -524,7 +526,7 @@ static struct wpabuf * eap_eke_process_confirm(struct eap_eke_data *data,
 
 	if (pos + data->sess.pnonce_ps_len + data->sess.prf_len > end) {
 		wpa_printf(MSG_DEBUG, "EAP-EKE: Too short EAP-EKE-Confirm");
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PROTO_ERROR);
 	}
 
@@ -532,19 +534,19 @@ static struct wpabuf * eap_eke_process_confirm(struct eap_eke_data *data,
 	if (eap_eke_decrypt_prot(&data->sess, pos, data->sess.pnonce_ps_len,
 				 nonces, &decrypt_len) < 0) {
 		wpa_printf(MSG_INFO, "EAP-EKE: Failed to decrypt PNonce_PS");
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_AUTHENTICATION_FAIL);
 	}
 	if (decrypt_len != (size_t) 2 * data->sess.nonce_len) {
 		wpa_printf(MSG_INFO, "EAP-EKE: PNonce_PS protected data length does not match length of Nonce_P and Nonce_S");
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_AUTHENTICATION_FAIL);
 	}
 	wpa_hexdump_key(MSG_DEBUG, "EAP-EKE: Received Nonce_P | Nonce_S",
 			nonces, 2 * data->sess.nonce_len);
 	if (os_memcmp(data->nonce_p, nonces, data->sess.nonce_len) != 0) {
 		wpa_printf(MSG_INFO, "EAP-EKE: Received Nonce_P does not match transmitted Nonce_P");
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_AUTHENTICATION_FAIL);
 	}
 
@@ -556,30 +558,30 @@ static struct wpabuf * eap_eke_process_confirm(struct eap_eke_data *data,
 	if (eap_eke_derive_ka(&data->sess, data->serverid, data->serverid_len,
 			      data->peerid, data->peerid_len,
 			      data->nonce_p, data->nonce_s) < 0) {
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PRIVATE_INTERNAL_ERROR);
 	}
 
 	if (eap_eke_auth(&data->sess, "EAP-EKE server", data->msgs, auth_s) < 0)
 	{
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PRIVATE_INTERNAL_ERROR);
 	}
 	wpa_hexdump(MSG_DEBUG, "EAP-EKE: Auth_S", auth_s, data->sess.prf_len);
 	if (os_memcmp_const(auth_s, pos + data->sess.pnonce_ps_len,
 			    data->sess.prf_len) != 0) {
 		wpa_printf(MSG_INFO, "EAP-EKE: Auth_S does not match");
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_AUTHENTICATION_FAIL);
 	}
 
 	wpa_printf(MSG_DEBUG, "EAP-EKE: Sending EAP-EKE-Confirm/Response");
 
-	resp = eap_eke_build_msg(data, eap_get_id(reqData),
+	resp = eap_eke_build_msg(data, id,
 				 data->sess.pnonce_len + data->sess.prf_len,
 				 EAP_EKE_CONFIRM);
 	if (resp == NULL) {
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PRIVATE_INTERNAL_ERROR);
 	}
 
@@ -587,7 +589,7 @@ static struct wpabuf * eap_eke_process_confirm(struct eap_eke_data *data,
 	if (eap_eke_prot(&data->sess, data->nonce_s, data->sess.nonce_len,
 			 wpabuf_put(resp, 0), &prot_len) < 0) {
 		wpabuf_free(resp);
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PRIVATE_INTERNAL_ERROR);
 	}
 	wpabuf_put(resp, prot_len);
@@ -595,7 +597,7 @@ static struct wpabuf * eap_eke_process_confirm(struct eap_eke_data *data,
 	auth = wpabuf_put(resp, data->sess.prf_len);
 	if (eap_eke_auth(&data->sess, "EAP-EKE peer", data->msgs, auth) < 0) {
 		wpabuf_free(resp);
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PRIVATE_INTERNAL_ERROR);
 	}
 	wpa_hexdump(MSG_DEBUG, "EAP-EKE: Auth_P", auth, data->sess.prf_len);
@@ -606,7 +608,7 @@ static struct wpabuf * eap_eke_process_confirm(struct eap_eke_data *data,
 			       data->msk, data->emsk) < 0) {
 		wpa_printf(MSG_INFO, "EAP-EKE: Failed to derive MSK/EMSK");
 		wpabuf_free(resp);
-		return eap_eke_build_fail(data, ret, reqData,
+		return eap_eke_build_fail(data, ret, id,
 					  EAP_EKE_FAIL_PRIVATE_INTERNAL_ERROR);
 	}
 
@@ -638,7 +640,8 @@ static struct wpabuf * eap_eke_process_failure(struct eap_eke_data *data,
 		wpa_printf(MSG_INFO, "EAP-EKE: Failure-Code 0x%x", code);
 	}
 
-	return eap_eke_build_fail(data, ret, reqData, EAP_EKE_FAIL_NO_ERROR);
+	return eap_eke_build_fail(data, ret, eap_get_id(reqData),
+				  EAP_EKE_FAIL_NO_ERROR);
 }
 
 
@@ -741,6 +744,29 @@ static u8 * eap_eke_get_emsk(struct eap_sm *sm, void *priv, size_t *len)
 }
 
 
+static u8 * eap_eke_get_session_id(struct eap_sm *sm, void *priv, size_t *len)
+{
+	struct eap_eke_data *data = priv;
+	u8 *sid;
+	size_t sid_len;
+
+	if (data->state != SUCCESS)
+		return NULL;
+
+	sid_len = 1 + 2 * data->sess.nonce_len;
+	sid = os_malloc(sid_len);
+	if (sid == NULL)
+		return NULL;
+	sid[0] = EAP_TYPE_EKE;
+	os_memcpy(sid + 1, data->nonce_p, data->sess.nonce_len);
+	os_memcpy(sid + 1 + data->sess.nonce_len, data->nonce_s,
+		  data->sess.nonce_len);
+	*len = sid_len;
+
+	return sid;
+}
+
+
 int eap_peer_eke_register(void)
 {
 	struct eap_method *eap;
@@ -757,6 +783,7 @@ int eap_peer_eke_register(void)
 	eap->isKeyAvailable = eap_eke_isKeyAvailable;
 	eap->getKey = eap_eke_getKey;
 	eap->get_emsk = eap_eke_get_emsk;
+	eap->getSessionId = eap_eke_get_session_id;
 
 	ret = eap_peer_method_register(eap);
 	if (ret)
diff --git a/src/eap_peer/eap_fast.c b/src/eap_peer/eap_fast.c
index 68d7fba8892e..4cbe3bacb0a6 100644
--- a/src/eap_peer/eap_fast.c
+++ b/src/eap_peer/eap_fast.c
@@ -267,8 +267,8 @@ static int eap_fast_derive_msk(struct eap_fast_data *data)
 }
 
 
-static void eap_fast_derive_key_auth(struct eap_sm *sm,
-				     struct eap_fast_data *data)
+static int eap_fast_derive_key_auth(struct eap_sm *sm,
+				    struct eap_fast_data *data)
 {
 	u8 *sks;
 
@@ -281,7 +281,7 @@ static void eap_fast_derive_key_auth(struct eap_sm *sm,
 	if (sks == NULL) {
 		wpa_printf(MSG_DEBUG, "EAP-FAST: Failed to derive "
 			   "session_key_seed");
-		return;
+		return -1;
 	}
 
 	/*
@@ -294,11 +294,12 @@ static void eap_fast_derive_key_auth(struct eap_sm *sm,
 	data->simck_idx = 0;
 	os_memcpy(data->simck, sks, EAP_FAST_SIMCK_LEN);
 	os_free(sks);
+	return 0;
 }
 
 
-static void eap_fast_derive_key_provisioning(struct eap_sm *sm,
-					     struct eap_fast_data *data)
+static int eap_fast_derive_key_provisioning(struct eap_sm *sm,
+					    struct eap_fast_data *data)
 {
 	os_free(data->key_block_p);
 	data->key_block_p = (struct eap_fast_key_block_provisioning *)
@@ -307,7 +308,7 @@ static void eap_fast_derive_key_provisioning(struct eap_sm *sm,
 				    sizeof(*data->key_block_p));
 	if (data->key_block_p == NULL) {
 		wpa_printf(MSG_DEBUG, "EAP-FAST: Failed to derive key block");
-		return;
+		return -1;
 	}
 	/*
 	 * RFC 4851, Section 5.2:
@@ -326,15 +327,19 @@ static void eap_fast_derive_key_provisioning(struct eap_sm *sm,
 	wpa_hexdump_key(MSG_DEBUG, "EAP-FAST: client_challenge",
 			data->key_block_p->client_challenge,
 			sizeof(data->key_block_p->client_challenge));
+	return 0;
 }
 
 
-static void eap_fast_derive_keys(struct eap_sm *sm, struct eap_fast_data *data)
+static int eap_fast_derive_keys(struct eap_sm *sm, struct eap_fast_data *data)
 {
+	int res;
+
 	if (data->anon_provisioning)
-		eap_fast_derive_key_provisioning(sm, data);
+		res = eap_fast_derive_key_provisioning(sm, data);
 	else
-		eap_fast_derive_key_auth(sm, data);
+		res = eap_fast_derive_key_auth(sm, data);
+	return res;
 }
 
 
@@ -1172,7 +1177,7 @@ static struct wpabuf * eap_fast_pac_request(void)
 static int eap_fast_process_decrypted(struct eap_sm *sm,
 				      struct eap_fast_data *data,
 				      struct eap_method_ret *ret,
-				      const struct eap_hdr *req,
+				      u8 identifier,
 				      struct wpabuf *decrypted,
 				      struct wpabuf **out_data)
 {
@@ -1184,18 +1189,18 @@ static int eap_fast_process_decrypted(struct eap_sm *sm,
 		return 0;
 	if (resp)
 		return eap_fast_encrypt_response(sm, data, resp,
-						 req->identifier, out_data);
+						 identifier, out_data);
 
 	if (tlv.result == EAP_TLV_RESULT_FAILURE) {
 		resp = eap_fast_tlv_result(EAP_TLV_RESULT_FAILURE, 0);
 		return eap_fast_encrypt_response(sm, data, resp,
-						 req->identifier, out_data);
+						 identifier, out_data);
 	}
 
 	if (tlv.iresult == EAP_TLV_RESULT_FAILURE) {
 		resp = eap_fast_tlv_result(EAP_TLV_RESULT_FAILURE, 1);
 		return eap_fast_encrypt_response(sm, data, resp,
-						 req->identifier, out_data);
+						 identifier, out_data);
 	}
 
 	if (tlv.crypto_binding) {
@@ -1277,14 +1282,13 @@ static int eap_fast_process_decrypted(struct eap_sm *sm,
 		resp = wpabuf_alloc(1);
 	}
 
-	return eap_fast_encrypt_response(sm, data, resp, req->identifier,
+	return eap_fast_encrypt_response(sm, data, resp, identifier,
 					 out_data);
 }
 
 
 static int eap_fast_decrypt(struct eap_sm *sm, struct eap_fast_data *data,
-			    struct eap_method_ret *ret,
-			    const struct eap_hdr *req,
+			    struct eap_method_ret *ret, u8 identifier,
 			    const struct wpabuf *in_data,
 			    struct wpabuf **out_data)
 {
@@ -1309,7 +1313,7 @@ static int eap_fast_decrypt(struct eap_sm *sm, struct eap_fast_data *data,
 		/* Received TLS ACK - requesting more fragments */
 		return eap_peer_tls_encrypt(sm, &data->ssl, EAP_TYPE_FAST,
 					    data->fast_version,
-					    req->identifier, NULL, out_data);
+					    identifier, NULL, out_data);
 	}
 
 	res = eap_peer_tls_decrypt(sm, &data->ssl, in_data, &in_decrypted);
@@ -1328,7 +1332,7 @@ continue_req:
 		return -1;
 	}
 
-	res = eap_fast_process_decrypted(sm, data, ret, req,
+	res = eap_fast_process_decrypted(sm, data, ret, identifier,
 					 in_decrypted, out_data);
 
 	wpabuf_free(in_decrypted);
@@ -1340,7 +1344,7 @@ continue_req:
 static const u8 * eap_fast_get_a_id(const u8 *buf, size_t len, size_t *id_len)
 {
 	const u8 *a_id;
-	struct pac_tlv_hdr *hdr;
+	const struct pac_tlv_hdr *hdr;
 
 	/*
 	 * Parse authority identity (A-ID) from the EAP-FAST/Start. This
@@ -1350,13 +1354,13 @@ static const u8 * eap_fast_get_a_id(const u8 *buf, size_t len, size_t *id_len)
 	*id_len = len;
 	if (len > sizeof(*hdr)) {
 		int tlen;
-		hdr = (struct pac_tlv_hdr *) buf;
+		hdr = (const struct pac_tlv_hdr *) buf;
 		tlen = be_to_host16(hdr->len);
 		if (be_to_host16(hdr->type) == PAC_TYPE_A_ID &&
 		    sizeof(*hdr) + tlen <= len) {
 			wpa_printf(MSG_DEBUG, "EAP-FAST: A-ID was in TLV "
 				   "(Start)");
-			a_id = (u8 *) (hdr + 1);
+			a_id = (const u8 *) (hdr + 1);
 			*id_len = tlen;
 		}
 	}
@@ -1529,6 +1533,7 @@ static struct wpabuf * eap_fast_process(struct eap_sm *sm, void *priv,
 	struct wpabuf *resp;
 	const u8 *pos;
 	struct eap_fast_data *data = priv;
+	struct wpabuf msg;
 
 	pos = eap_peer_tls_process_init(sm, &data->ssl, EAP_TYPE_FAST, ret,
 					reqData, &left, &flags);
@@ -1545,13 +1550,13 @@ static struct wpabuf * eap_fast_process(struct eap_sm *sm, void *priv,
 		left = 0; /* A-ID is not used in further packet processing */
 	}
 
+	wpabuf_set(&msg, pos, left);
+
 	resp = NULL;
 	if (tls_connection_established(sm->ssl_ctx, data->ssl.conn) &&
 	    !data->resuming) {
 		/* Process tunneled (encrypted) phase 2 data. */
-		struct wpabuf msg;
-		wpabuf_set(&msg, pos, left);
-		res = eap_fast_decrypt(sm, data, ret, req, &msg, &resp);
+		res = eap_fast_decrypt(sm, data, ret, id, &msg, &resp);
 		if (res < 0) {
 			ret->methodState = METHOD_DONE;
 			ret->decision = DECISION_FAIL;
@@ -1565,8 +1570,15 @@ static struct wpabuf * eap_fast_process(struct eap_sm *sm, void *priv,
 		/* Continue processing TLS handshake (phase 1). */
 		res = eap_peer_tls_process_helper(sm, &data->ssl,
 						  EAP_TYPE_FAST,
-						  data->fast_version, id, pos,
-						  left, &resp);
+						  data->fast_version, id, &msg,
+						  &resp);
+		if (res < 0) {
+			wpa_printf(MSG_DEBUG,
+				   "EAP-FAST: TLS processing failed");
+			ret->methodState = METHOD_DONE;
+			ret->decision = DECISION_FAIL;
+			return resp;
+		}
 
 		if (tls_connection_established(sm->ssl_ctx, data->ssl.conn)) {
 			char cipher[80];
@@ -1586,20 +1598,24 @@ static struct wpabuf * eap_fast_process(struct eap_sm *sm, void *priv,
 			} else
 				data->anon_provisioning = 0;
 			data->resuming = 0;
-			eap_fast_derive_keys(sm, data);
+			if (eap_fast_derive_keys(sm, data) < 0) {
+				wpa_printf(MSG_DEBUG,
+					   "EAP-FAST: Could not derive keys");
+				ret->methodState = METHOD_DONE;
+				ret->decision = DECISION_FAIL;
+				wpabuf_free(resp);
+				return NULL;
+			}
 		}
 
 		if (res == 2) {
-			struct wpabuf msg;
 			/*
 			 * Application data included in the handshake message.
 			 */
 			wpabuf_free(data->pending_phase2_req);
 			data->pending_phase2_req = resp;
 			resp = NULL;
-			wpabuf_set(&msg, pos, left);
-			res = eap_fast_decrypt(sm, data, ret, req, &msg,
-					       &resp);
+			res = eap_fast_decrypt(sm, data, ret, id, &msg, &resp);
 		}
 	}
 
diff --git a/src/eap_peer/eap_gpsk.c b/src/eap_peer/eap_gpsk.c
index c54bf116477c..902b4ba26d6e 100644
--- a/src/eap_peer/eap_gpsk.c
+++ b/src/eap_peer/eap_gpsk.c
@@ -274,7 +274,7 @@ static const u8 * eap_gpsk_process_csuite_list(struct eap_sm *sm,
 static struct wpabuf * eap_gpsk_process_gpsk_1(struct eap_sm *sm,
 					       struct eap_gpsk_data *data,
 					       struct eap_method_ret *ret,
-					       const struct wpabuf *reqData,
+					       u8 identifier,
 					       const u8 *payload,
 					       size_t payload_len)
 {
@@ -301,7 +301,7 @@ static struct wpabuf * eap_gpsk_process_gpsk_1(struct eap_sm *sm,
 		return NULL;
 	}
 
-	resp = eap_gpsk_send_gpsk_2(data, eap_get_id(reqData),
+	resp = eap_gpsk_send_gpsk_2(data, identifier,
 				    csuite_list, csuite_list_len);
 	if (resp == NULL)
 		return NULL;
@@ -583,7 +583,7 @@ static const u8 * eap_gpsk_validate_gpsk_3_mic(struct eap_gpsk_data *data,
 static struct wpabuf * eap_gpsk_process_gpsk_3(struct eap_sm *sm,
 					       struct eap_gpsk_data *data,
 					       struct eap_method_ret *ret,
-					       const struct wpabuf *reqData,
+					       u8 identifier,
 					       const u8 *payload,
 					       size_t payload_len)
 {
@@ -615,7 +615,7 @@ static struct wpabuf * eap_gpsk_process_gpsk_3(struct eap_sm *sm,
 			   (unsigned long) (end - pos));
 	}
 
-	resp = eap_gpsk_send_gpsk_4(data, eap_get_id(reqData));
+	resp = eap_gpsk_send_gpsk_4(data, identifier);
 	if (resp == NULL)
 		return NULL;
 
@@ -670,6 +670,7 @@ static struct wpabuf * eap_gpsk_process(struct eap_sm *sm, void *priv,
 	struct wpabuf *resp;
 	const u8 *pos;
 	size_t len;
+	u8 opcode, id;
 
 	pos = eap_hdr_validate(EAP_VENDOR_IETF, EAP_TYPE_GPSK, reqData, &len);
 	if (pos == NULL || len < 1) {
@@ -677,25 +678,27 @@ static struct wpabuf * eap_gpsk_process(struct eap_sm *sm, void *priv,
 		return NULL;
 	}
 
-	wpa_printf(MSG_DEBUG, "EAP-GPSK: Received frame: opcode %d", *pos);
+	id = eap_get_id(reqData);
+	opcode = *pos++;
+	len--;
+	wpa_printf(MSG_DEBUG, "EAP-GPSK: Received frame: opcode %d", opcode);
 
 	ret->ignore = FALSE;
 	ret->methodState = METHOD_MAY_CONT;
 	ret->decision = DECISION_FAIL;
 	ret->allowNotifications = FALSE;
 
-	switch (*pos) {
+	switch (opcode) {
 	case EAP_GPSK_OPCODE_GPSK_1:
-		resp = eap_gpsk_process_gpsk_1(sm, data, ret, reqData,
-					       pos + 1, len - 1);
+		resp = eap_gpsk_process_gpsk_1(sm, data, ret, id, pos, len);
 		break;
 	case EAP_GPSK_OPCODE_GPSK_3:
-		resp = eap_gpsk_process_gpsk_3(sm, data, ret, reqData,
-					       pos + 1, len - 1);
+		resp = eap_gpsk_process_gpsk_3(sm, data, ret, id, pos, len);
 		break;
 	default:
-		wpa_printf(MSG_DEBUG, "EAP-GPSK: Ignoring message with "
-			   "unknown opcode %d", *pos);
+		wpa_printf(MSG_DEBUG,
+			   "EAP-GPSK: Ignoring message with unknown opcode %d",
+			   opcode);
 		ret->ignore = TRUE;
 		return NULL;
 	}
diff --git a/src/eap_peer/eap_i.h b/src/eap_peer/eap_i.h
index 2d7fdea22774..99b44dae4e34 100644
--- a/src/eap_peer/eap_i.h
+++ b/src/eap_peer/eap_i.h
@@ -328,7 +328,7 @@ struct eap_sm {
 	/* not defined in RFC 4137 */
 	Boolean changed;
 	void *eapol_ctx;
-	struct eapol_callbacks *eapol_cb;
+	const struct eapol_callbacks *eapol_cb;
 	void *eap_method_priv;
 	int init_phase2;
 	int fast_reauth;
@@ -338,9 +338,9 @@ struct eap_sm {
 	Boolean rxResp /* LEAP only */;
 	Boolean leap_done;
 	Boolean peap_done;
-	u8 req_md5[16]; /* MD5() of the current EAP packet */
-	u8 last_md5[16]; /* MD5() of the previously received EAP packet; used
-			  * in duplicate request detection. */
+	u8 req_sha1[20]; /* SHA1() of the current EAP packet */
+	u8 last_sha1[20]; /* SHA1() of the previously received EAP packet; used
+			   * in duplicate request detection. */
 
 	void *msg_ctx;
 	void *scard_ctx;
diff --git a/src/eap_peer/eap_mschapv2.c b/src/eap_peer/eap_mschapv2.c
index 9e486e7d18fa..6acf1e8ad390 100644
--- a/src/eap_peer/eap_mschapv2.c
+++ b/src/eap_peer/eap_mschapv2.c
@@ -511,6 +511,11 @@ static struct wpabuf * eap_mschapv2_change_password(
 	struct eap_sm *sm, struct eap_mschapv2_data *data,
 	struct eap_method_ret *ret, const struct eap_mschapv2_hdr *req, u8 id)
 {
+#ifdef CONFIG_NO_RC4
+	wpa_printf(MSG_ERROR,
+		"EAP-MSCHAPV2: RC4 not support in the build - cannot change password");
+	return NULL;
+#else /* CONFIG_NO_RC4 */
 	struct wpabuf *resp;
 	int ms_len;
 	const u8 *username, *password, *new_password;
@@ -628,6 +633,7 @@ static struct wpabuf * eap_mschapv2_change_password(
 fail:
 	wpabuf_free(resp);
 	return NULL;
+#endif /* CONFIG_NO_RC4 */
 }
 
 
diff --git a/src/eap_peer/eap_pax.c b/src/eap_peer/eap_pax.c
index 6d1ff208ac7e..c920bcd3182f 100644
--- a/src/eap_peer/eap_pax.c
+++ b/src/eap_peer/eap_pax.c
@@ -333,7 +333,7 @@ static struct wpabuf * eap_pax_process(struct eap_sm *sm, void *priv,
 	u16 flen, mlen;
 
 	pos = eap_hdr_validate(EAP_VENDOR_IETF, EAP_TYPE_PAX, reqData, &len);
-	if (pos == NULL || len < EAP_PAX_ICV_LEN) {
+	if (pos == NULL || len < sizeof(*req) + EAP_PAX_ICV_LEN) {
 		ret->ignore = TRUE;
 		return NULL;
 	}
diff --git a/src/eap_peer/eap_peap.c b/src/eap_peer/eap_peap.c
index 86a18bb866de..98a48a6cf5d3 100644
--- a/src/eap_peer/eap_peap.c
+++ b/src/eap_peer/eap_peap.c
@@ -968,6 +968,7 @@ static struct wpabuf * eap_peap_process(struct eap_sm *sm, void *priv,
 	struct wpabuf *resp;
 	const u8 *pos;
 	struct eap_peap_data *data = priv;
+	struct wpabuf msg;
 
 	pos = eap_peer_tls_process_init(sm, &data->ssl, EAP_TYPE_PEAP, ret,
 					reqData, &left, &flags);
@@ -998,18 +999,25 @@ static struct wpabuf * eap_peap_process(struct eap_sm *sm, void *priv,
 			   * should always be, anyway */
 	}
 
+	wpabuf_set(&msg, pos, left);
+
 	resp = NULL;
 	if (tls_connection_established(sm->ssl_ctx, data->ssl.conn) &&
 	    !data->resuming) {
-		struct wpabuf msg;
-		wpabuf_set(&msg, pos, left);
 		res = eap_peap_decrypt(sm, data, ret, req, &msg, &resp);
 	} else {
 		res = eap_peer_tls_process_helper(sm, &data->ssl,
 						  EAP_TYPE_PEAP,
-						  data->peap_version, id, pos,
-						  left, &resp);
+						  data->peap_version, id, &msg,
+						  &resp);
 
+		if (res < 0) {
+			wpa_printf(MSG_DEBUG,
+				   "EAP-PEAP: TLS processing failed");
+			ret->methodState = METHOD_DONE;
+			ret->decision = DECISION_FAIL;
+			return resp;
+		}
 		if (tls_connection_established(sm->ssl_ctx, data->ssl.conn)) {
 			char *label;
 			wpa_printf(MSG_DEBUG,
@@ -1077,14 +1085,12 @@ static struct wpabuf * eap_peap_process(struct eap_sm *sm, void *priv,
 		}
 
 		if (res == 2) {
-			struct wpabuf msg;
 			/*
 			 * Application data included in the handshake message.
 			 */
 			wpabuf_free(data->pending_phase2_req);
 			data->pending_phase2_req = resp;
 			resp = NULL;
-			wpabuf_set(&msg, pos, left);
 			res = eap_peap_decrypt(sm, data, ret, req, &msg,
 					       &resp);
 		}
diff --git a/src/eap_peer/eap_pwd.c b/src/eap_peer/eap_pwd.c
index 059bbeecb72d..1f785443ee5a 100644
--- a/src/eap_peer/eap_pwd.c
+++ b/src/eap_peer/eap_pwd.c
@@ -10,6 +10,7 @@
 
 #include "common.h"
 #include "crypto/sha256.h"
+#include "crypto/ms_funcs.h"
 #include "eap_peer/eap_i.h"
 #include "eap_common/eap_pwd_common.h"
 
@@ -25,6 +26,7 @@ struct eap_pwd_data {
 	size_t id_server_len;
 	u8 *password;
 	size_t password_len;
+	int password_hash;
 	u16 group_num;
 	EAP_PWD_group *grp;
 
@@ -86,8 +88,9 @@ static void * eap_pwd_init(struct eap_sm *sm)
 	const u8 *identity, *password;
 	size_t identity_len, password_len;
 	int fragment_size;
+	int pwhash;
 
-	password = eap_get_config_password(sm, &password_len);
+	password = eap_get_config_password2(sm, &password_len, &pwhash);
 	if (password == NULL) {
 		wpa_printf(MSG_INFO, "EAP-PWD: No password configured!");
 		return NULL;
@@ -129,6 +132,7 @@ static void * eap_pwd_init(struct eap_sm *sm)
 	}
 	os_memcpy(data->password, password, password_len);
 	data->password_len = password_len;
+	data->password_hash = pwhash;
 
 	data->out_frag_pos = data->in_frag_pos = 0;
 	data->inbuf = data->outbuf = NULL;
@@ -216,6 +220,10 @@ eap_pwd_perform_id_exchange(struct eap_sm *sm, struct eap_pwd_data *data,
 			    const u8 *payload, size_t payload_len)
 {
 	struct eap_pwd_id *id;
+	const u8 *password;
+	size_t password_len;
+	u8 pwhashhash[16];
+	int res;
 
 	if (data->state != PWD_ID_Req) {
 		ret->ignore = TRUE;
@@ -231,6 +239,9 @@ eap_pwd_perform_id_exchange(struct eap_sm *sm, struct eap_pwd_data *data,
 
 	id = (struct eap_pwd_id *) payload;
 	data->group_num = be_to_host16(id->group_num);
+	wpa_printf(MSG_DEBUG,
+		   "EAP-PWD: Server EAP-pwd-ID proposal: group=%u random=%u prf=%u prep=%u",
+		   data->group_num, id->random_function, id->prf, id->prep);
 	if ((id->random_function != EAP_PWD_DEFAULT_RAND_FUNC) ||
 	    (id->prf != EAP_PWD_DEFAULT_PRF)) {
 		ret->ignore = TRUE;
@@ -238,6 +249,22 @@ eap_pwd_perform_id_exchange(struct eap_sm *sm, struct eap_pwd_data *data,
 		return;
 	}
 
+	if (id->prep != EAP_PWD_PREP_NONE &&
+	    id->prep != EAP_PWD_PREP_MS) {
+		wpa_printf(MSG_DEBUG,
+			   "EAP-PWD: Unsupported password pre-processing technique (Prep=%u)",
+			   id->prep);
+		eap_pwd_state(data, FAILURE);
+		return;
+	}
+
+	if (id->prep == EAP_PWD_PREP_NONE && data->password_hash) {
+		wpa_printf(MSG_DEBUG,
+			   "EAP-PWD: Unhashed password not available");
+		eap_pwd_state(data, FAILURE);
+		return;
+	}
+
 	wpa_printf(MSG_DEBUG, "EAP-PWD (peer): using group %d",
 		   data->group_num);
 
@@ -260,12 +287,46 @@ eap_pwd_perform_id_exchange(struct eap_sm *sm, struct eap_pwd_data *data,
 		return;
 	}
 
+	if (id->prep == EAP_PWD_PREP_MS) {
+#ifdef CONFIG_FIPS
+		wpa_printf(MSG_ERROR,
+			   "EAP-PWD (peer): MS password hash not supported in FIPS mode");
+		eap_pwd_state(data, FAILURE);
+		return;
+#else /* CONFIG_FIPS */
+		if (data->password_hash) {
+			res = hash_nt_password_hash(data->password, pwhashhash);
+		} else {
+			u8 pwhash[16];
+
+			res = nt_password_hash(data->password,
+					       data->password_len, pwhash);
+			if (res == 0)
+				res = hash_nt_password_hash(pwhash, pwhashhash);
+			os_memset(pwhash, 0, sizeof(pwhash));
+		}
+
+		if (res) {
+			eap_pwd_state(data, FAILURE);
+			return;
+		}
+
+		password = pwhashhash;
+		password_len = sizeof(pwhashhash);
+#endif /* CONFIG_FIPS */
+	} else {
+		password = data->password;
+		password_len = data->password_len;
+	}
+
 	/* compute PWE */
-	if (compute_password_element(data->grp, data->group_num,
-				     data->password, data->password_len,
-				     data->id_server, data->id_server_len,
-				     data->id_peer, data->id_peer_len,
-				     id->token)) {
+	res = compute_password_element(data->grp, data->group_num,
+				       password, password_len,
+				       data->id_server, data->id_server_len,
+				       data->id_peer, data->id_peer_len,
+				       id->token);
+	os_memset(pwhashhash, 0, sizeof(pwhashhash));
+	if (res) {
 		wpa_printf(MSG_INFO, "EAP-PWD (peer): unable to compute PWE");
 		eap_pwd_state(data, FAILURE);
 		return;
@@ -301,6 +362,23 @@ eap_pwd_perform_commit_exchange(struct eap_sm *sm, struct eap_pwd_data *data,
 	BIGNUM *mask = NULL, *x = NULL, *y = NULL, *cofactor = NULL;
 	u16 offset;
 	u8 *ptr, *scalar = NULL, *element = NULL;
+	size_t prime_len, order_len;
+
+	if (data->state != PWD_Commit_Req) {
+		ret->ignore = TRUE;
+		goto fin;
+	}
+
+	prime_len = BN_num_bytes(data->grp->prime);
+	order_len = BN_num_bytes(data->grp->order);
+
+	if (payload_len != 2 * prime_len + order_len) {
+		wpa_printf(MSG_INFO,
+			   "EAP-pwd: Unexpected Commit payload length %u (expected %u)",
+			   (unsigned int) payload_len,
+			   (unsigned int) (2 * prime_len + order_len));
+		goto fin;
+	}
 
 	if (((data->private_value = BN_new()) == NULL) ||
 	    ((data->my_element = EC_POINT_new(data->grp->group)) == NULL) ||
@@ -500,6 +578,18 @@ eap_pwd_perform_confirm_exchange(struct eap_sm *sm, struct eap_pwd_data *data,
 	u8 conf[SHA256_MAC_LEN], *cruft = NULL, *ptr;
 	int offset;
 
+	if (data->state != PWD_Confirm_Req) {
+		ret->ignore = TRUE;
+		goto fin;
+	}
+
+	if (payload_len != SHA256_MAC_LEN) {
+		wpa_printf(MSG_INFO,
+			   "EAP-pwd: Unexpected Confirm payload length %u (expected %u)",
+			   (unsigned int) payload_len, SHA256_MAC_LEN);
+		goto fin;
+	}
+
 	/*
 	 * first build up the ciphersuite which is group | random_function |
 	 *	prf
@@ -783,17 +873,30 @@ eap_pwd_process(struct eap_sm *sm, void *priv, struct eap_method_ret *ret,
 	 * if it's the first fragment there'll be a length field
 	 */
 	if (EAP_PWD_GET_LENGTH_BIT(lm_exch)) {
+		if (len < 2) {
+			wpa_printf(MSG_DEBUG,
+				   "EAP-pwd: Frame too short to contain Total-Length field");
+			ret->ignore = TRUE;
+			return NULL;
+		}
 		tot_len = WPA_GET_BE16(pos);
 		wpa_printf(MSG_DEBUG, "EAP-pwd: Incoming fragments whose "
 			   "total length = %d", tot_len);
 		if (tot_len > 15000)
 			return NULL;
+		if (data->inbuf) {
+			wpa_printf(MSG_DEBUG,
+				   "EAP-pwd: Unexpected new fragment start when previous fragment is still in use");
+			ret->ignore = TRUE;
+			return NULL;
+		}
 		data->inbuf = wpabuf_alloc(tot_len);
 		if (data->inbuf == NULL) {
 			wpa_printf(MSG_INFO, "Out of memory to buffer "
 				   "fragments!");
 			return NULL;
 		}
+		data->in_frag_pos = 0;
 		pos += sizeof(u16);
 		len -= sizeof(u16);
 	}
@@ -873,6 +976,7 @@ eap_pwd_process(struct eap_sm *sm, void *priv, struct eap_method_ret *ret,
 	/*
 	 * we have output! Do we need to fragment it?
 	 */
+	lm_exch = EAP_PWD_GET_EXCHANGE(lm_exch);
 	len = wpabuf_len(data->outbuf);
 	if ((len + EAP_PWD_HDR_SIZE) > data->mtu) {
 		resp = eap_msg_alloc(EAP_VENDOR_IETF, EAP_TYPE_PWD, data->mtu,
diff --git a/src/eap_peer/eap_sake.c b/src/eap_peer/eap_sake.c
index 7d14907433e5..c4f9843febb3 100644
--- a/src/eap_peer/eap_sake.c
+++ b/src/eap_peer/eap_sake.c
@@ -141,7 +141,7 @@ static struct wpabuf * eap_sake_build_msg(struct eap_sake_data *data,
 static struct wpabuf * eap_sake_process_identity(struct eap_sm *sm,
 						 struct eap_sake_data *data,
 						 struct eap_method_ret *ret,
-						 const struct wpabuf *reqData,
+						 u8 id,
 						 const u8 *payload,
 						 size_t payload_len)
 {
@@ -166,8 +166,7 @@ static struct wpabuf * eap_sake_process_identity(struct eap_sm *sm,
 
 	wpa_printf(MSG_DEBUG, "EAP-SAKE: Sending Response/Identity");
 
-	resp = eap_sake_build_msg(data, eap_get_id(reqData),
-				  2 + data->peerid_len,
+	resp = eap_sake_build_msg(data, id, 2 + data->peerid_len,
 				  EAP_SAKE_SUBTYPE_IDENTITY);
 	if (resp == NULL)
 		return NULL;
@@ -185,7 +184,7 @@ static struct wpabuf * eap_sake_process_identity(struct eap_sm *sm,
 static struct wpabuf * eap_sake_process_challenge(struct eap_sm *sm,
 						  struct eap_sake_data *data,
 						  struct eap_method_ret *ret,
-						  const struct wpabuf *reqData,
+						  u8 id,
 						  const u8 *payload,
 						  size_t payload_len)
 {
@@ -247,8 +246,7 @@ static struct wpabuf * eap_sake_process_challenge(struct eap_sm *sm,
 	rlen = 2 + EAP_SAKE_RAND_LEN + 2 + EAP_SAKE_MIC_LEN;
 	if (data->peerid)
 		rlen += 2 + data->peerid_len;
-	resp = eap_sake_build_msg(data, eap_get_id(reqData), rlen,
-				  EAP_SAKE_SUBTYPE_CHALLENGE);
+	resp = eap_sake_build_msg(data, id, rlen, EAP_SAKE_SUBTYPE_CHALLENGE);
 	if (resp == NULL)
 		return NULL;
 
@@ -285,6 +283,7 @@ static struct wpabuf * eap_sake_process_challenge(struct eap_sm *sm,
 static struct wpabuf * eap_sake_process_confirm(struct eap_sm *sm,
 						struct eap_sake_data *data,
 						struct eap_method_ret *ret,
+						u8 id,
 						const struct wpabuf *reqData,
 						const u8 *payload,
 						size_t payload_len)
@@ -323,14 +322,13 @@ static struct wpabuf * eap_sake_process_confirm(struct eap_sm *sm,
 		ret->allowNotifications = FALSE;
 		wpa_printf(MSG_DEBUG, "EAP-SAKE: Sending "
 			   "Response/Auth-Reject");
-		return eap_sake_build_msg(data, eap_get_id(reqData), 0,
+		return eap_sake_build_msg(data, id, 0,
 					  EAP_SAKE_SUBTYPE_AUTH_REJECT);
 	}
 
 	wpa_printf(MSG_DEBUG, "EAP-SAKE: Sending Response/Confirm");
 
-	resp = eap_sake_build_msg(data, eap_get_id(reqData),
-				  2 + EAP_SAKE_MIC_LEN,
+	resp = eap_sake_build_msg(data, id, 2 + EAP_SAKE_MIC_LEN,
 				  EAP_SAKE_SUBTYPE_CONFIRM);
 	if (resp == NULL)
 		return NULL;
@@ -367,7 +365,7 @@ static struct wpabuf * eap_sake_process(struct eap_sm *sm, void *priv,
 	struct wpabuf *resp;
 	const u8 *pos, *end;
 	size_t len;
-	u8 subtype, session_id;
+	u8 subtype, session_id, id;
 
 	pos = eap_hdr_validate(EAP_VENDOR_IETF, EAP_TYPE_SAKE, reqData, &len);
 	if (pos == NULL || len < sizeof(struct eap_sake_hdr)) {
@@ -377,6 +375,7 @@ static struct wpabuf * eap_sake_process(struct eap_sm *sm, void *priv,
 
 	req = (const struct eap_sake_hdr *) pos;
 	end = pos + len;
+	id = eap_get_id(reqData);
 	subtype = req->subtype;
 	session_id = req->session_id;
 	pos = (const u8 *) (req + 1);
@@ -402,15 +401,15 @@ static struct wpabuf * eap_sake_process(struct eap_sm *sm, void *priv,
 
 	switch (subtype) {
 	case EAP_SAKE_SUBTYPE_IDENTITY:
-		resp = eap_sake_process_identity(sm, data, ret, reqData,
+		resp = eap_sake_process_identity(sm, data, ret, id,
 						 pos, end - pos);
 		break;
 	case EAP_SAKE_SUBTYPE_CHALLENGE:
-		resp = eap_sake_process_challenge(sm, data, ret, reqData,
+		resp = eap_sake_process_challenge(sm, data, ret, id,
 						  pos, end - pos);
 		break;
 	case EAP_SAKE_SUBTYPE_CONFIRM:
-		resp = eap_sake_process_confirm(sm, data, ret, reqData,
+		resp = eap_sake_process_confirm(sm, data, ret, id, reqData,
 						pos, end - pos);
 		break;
 	default:
diff --git a/src/eap_peer/eap_sim.c b/src/eap_peer/eap_sim.c
index bd06df78db4c..99a2816ce61e 100644
--- a/src/eap_peer/eap_sim.c
+++ b/src/eap_peer/eap_sim.c
@@ -1042,7 +1042,7 @@ static struct wpabuf * eap_sim_process(struct eap_sm *sm, void *priv,
 	}
 
 	pos = eap_hdr_validate(EAP_VENDOR_IETF, EAP_TYPE_SIM, reqData, &len);
-	if (pos == NULL || len < 1) {
+	if (pos == NULL || len < 3) {
 		ret->ignore = TRUE;
 		return NULL;
 	}
diff --git a/src/eap_peer/eap_tls.c b/src/eap_peer/eap_tls.c
index 5aa3fd591256..66a027a626e0 100644
--- a/src/eap_peer/eap_tls.c
+++ b/src/eap_peer/eap_tls.c
@@ -156,20 +156,6 @@ static struct wpabuf * eap_tls_failure(struct eap_sm *sm,
 	ret->methodState = METHOD_DONE;
 	ret->decision = DECISION_FAIL;
 
-	if (res == -1) {
-		struct eap_peer_config *config = eap_get_config(sm);
-		if (config) {
-			/*
-			 * The TLS handshake failed. So better forget the old
-			 * PIN. It may be wrong, we cannot be sure but trying
-			 * the wrong one again might block it on the card--so
-			 * better ask the user again.
-			 */
-			os_free(config->pin);
-			config->pin = NULL;
-		}
-	}
-
 	if (resp) {
 		/*
 		 * This is likely an alert message, so send it instead of just
@@ -228,6 +214,7 @@ static struct wpabuf * eap_tls_process(struct eap_sm *sm, void *priv,
 	u8 flags, id;
 	const u8 *pos;
 	struct eap_tls_data *data = priv;
+	struct wpabuf msg;
 
 	pos = eap_peer_tls_process_init(sm, &data->ssl, data->eap_type, ret,
 					reqData, &left, &flags);
@@ -242,8 +229,9 @@ static struct wpabuf * eap_tls_process(struct eap_sm *sm, void *priv,
 	}
 
 	resp = NULL;
+	wpabuf_set(&msg, pos, left);
 	res = eap_peer_tls_process_helper(sm, &data->ssl, data->eap_type, 0,
-					  id, pos, left, &resp);
+					  id, &msg, &resp);
 
 	if (res < 0) {
 		return eap_tls_failure(sm, data, ret, res, resp, id);
diff --git a/src/eap_peer/eap_tls_common.c b/src/eap_peer/eap_tls_common.c
index 8710781618ef..af2b7541d701 100644
--- a/src/eap_peer/eap_tls_common.c
+++ b/src/eap_peer/eap_tls_common.c
@@ -68,6 +68,10 @@ static void eap_tls_params_flags(struct tls_connection_params *params,
 		params->flags |= TLS_CONN_DISABLE_SESSION_TICKET;
 	if (os_strstr(txt, "tls_disable_session_ticket=0"))
 		params->flags &= ~TLS_CONN_DISABLE_SESSION_TICKET;
+	if (os_strstr(txt, "tls_disable_tlsv1_0=1"))
+		params->flags |= TLS_CONN_DISABLE_TLSv1_0;
+	if (os_strstr(txt, "tls_disable_tlsv1_0=0"))
+		params->flags &= ~TLS_CONN_DISABLE_TLSv1_0;
 	if (os_strstr(txt, "tls_disable_tlsv1_1=1"))
 		params->flags |= TLS_CONN_DISABLE_TLSv1_1;
 	if (os_strstr(txt, "tls_disable_tlsv1_1=0"))
@@ -196,28 +200,25 @@ static int eap_tls_init_connection(struct eap_sm *sm,
 	}
 
 	res = tls_connection_set_params(data->ssl_ctx, data->conn, params);
-	if (res == TLS_SET_PARAMS_ENGINE_PRV_INIT_FAILED) {
+	if (res == TLS_SET_PARAMS_ENGINE_PRV_BAD_PIN) {
 		/*
-		 * At this point with the pkcs11 engine the PIN might be wrong.
-		 * We reset the PIN in the configuration to be sure to not use
-		 * it again and the calling function must request a new one.
-		 */
-		os_free(config->pin);
-		config->pin = NULL;
-	} else if (res == TLS_SET_PARAMS_ENGINE_PRV_VERIFY_FAILED) {
-		wpa_printf(MSG_INFO, "TLS: Failed to load private key");
-		/*
-		 * We do not know exactly but maybe the PIN was wrong,
-		 * so ask for a new one.
+		 * At this point with the pkcs11 engine the PIN is wrong. We
+		 * reset the PIN in the configuration to be sure to not use it
+		 * again and the calling function must request a new one.
 		 */
+		wpa_printf(MSG_INFO,
+			   "TLS: Bad PIN provided, requesting a new one");
 		os_free(config->pin);
 		config->pin = NULL;
 		eap_sm_request_pin(sm);
 		sm->ignore = TRUE;
-		tls_connection_deinit(data->ssl_ctx, data->conn);
-		data->conn = NULL;
-		return -1;
-	} else if (res) {
+	} else if (res == TLS_SET_PARAMS_ENGINE_PRV_INIT_FAILED) {
+		wpa_printf(MSG_INFO, "TLS: Failed to initialize engine");
+	} else if (res == TLS_SET_PARAMS_ENGINE_PRV_VERIFY_FAILED) {
+		wpa_printf(MSG_INFO, "TLS: Failed to load private key");
+		sm->ignore = TRUE;
+	}
+	if (res) {
 		wpa_printf(MSG_INFO, "TLS: Failed to set TLS connection "
 			   "parameters");
 		tls_connection_deinit(data->ssl_ctx, data->conn);
@@ -313,53 +314,19 @@ void eap_peer_tls_ssl_deinit(struct eap_sm *sm, struct eap_ssl_data *data)
 u8 * eap_peer_tls_derive_key(struct eap_sm *sm, struct eap_ssl_data *data,
 			     const char *label, size_t len)
 {
-#ifndef CONFIG_FIPS
-	struct tls_keys keys;
-#endif /* CONFIG_FIPS */
-	u8 *rnd = NULL, *out;
+	u8 *out;
 
 	out = os_malloc(len);
 	if (out == NULL)
 		return NULL;
 
-	/* First, try to use TLS library function for PRF, if available. */
-	if (tls_connection_prf(data->ssl_ctx, data->conn, label, 0, out, len)
-	    == 0)
-		return out;
+	if (tls_connection_prf(data->ssl_ctx, data->conn, label, 0, 0,
+			       out, len)) {
+		os_free(out);
+		return NULL;
+	}
 
-#ifndef CONFIG_FIPS
-	/*
-	 * TLS library did not support key generation, so get the needed TLS
-	 * session parameters and use an internal implementation of TLS PRF to
-	 * derive the key.
-	 */
-	if (tls_connection_get_keys(data->ssl_ctx, data->conn, &keys))
-		goto fail;
-
-	if (keys.client_random == NULL || keys.server_random == NULL ||
-	    keys.master_key == NULL)
-		goto fail;
-
-	rnd = os_malloc(keys.client_random_len + keys.server_random_len);
-	if (rnd == NULL)
-		goto fail;
-	os_memcpy(rnd, keys.client_random, keys.client_random_len);
-	os_memcpy(rnd + keys.client_random_len, keys.server_random,
-		  keys.server_random_len);
-
-	if (tls_prf_sha1_md5(keys.master_key, keys.master_key_len,
-			     label, rnd, keys.client_random_len +
-			     keys.server_random_len, out, len))
-		goto fail;
-
-	os_free(rnd);
 	return out;
-
-fail:
-#endif /* CONFIG_FIPS */
-	os_free(out);
-	os_free(rnd);
-	return NULL;
 }
 
 
@@ -380,10 +347,10 @@ u8 * eap_peer_tls_derive_session_id(struct eap_sm *sm,
 				    struct eap_ssl_data *data, u8 eap_type,
 				    size_t *len)
 {
-	struct tls_keys keys;
+	struct tls_random keys;
 	u8 *out;
 
-	if (tls_connection_get_keys(sm->ssl_ctx, data->conn, &keys))
+	if (tls_connection_get_random(sm->ssl_ctx, data->conn, &keys))
 		return NULL;
 
 	if (keys.client_random == NULL || keys.server_random == NULL)
@@ -514,22 +481,19 @@ static const struct wpabuf * eap_peer_tls_data_reassemble(
  * @sm: Pointer to EAP state machine allocated with eap_peer_sm_init()
  * @data: Data for TLS processing
  * @in_data: Message received from the server
- * @in_len: Length of in_data
  * @out_data: Buffer for returning a pointer to application data (if available)
  * Returns: 0 on success, 1 if more input data is needed, 2 if application data
  * is available, -1 on failure
  */
 static int eap_tls_process_input(struct eap_sm *sm, struct eap_ssl_data *data,
-				 const u8 *in_data, size_t in_len,
+				 const struct wpabuf *in_data,
 				 struct wpabuf **out_data)
 {
 	const struct wpabuf *msg;
 	int need_more_input;
 	struct wpabuf *appl_data;
-	struct wpabuf buf;
 
-	wpabuf_set(&buf, in_data, in_len);
-	msg = eap_peer_tls_data_reassemble(data, &buf, &need_more_input);
+	msg = eap_peer_tls_data_reassemble(data, in_data, &need_more_input);
 	if (msg == NULL)
 		return need_more_input ? 1 : -1;
 
@@ -649,7 +613,6 @@ static int eap_tls_process_output(struct eap_ssl_data *data, EapType eap_type,
  * @peap_version: Version number for EAP-PEAP/TTLS
  * @id: EAP identifier for the response
  * @in_data: Message received from the server
- * @in_len: Length of in_data
  * @out_data: Buffer for returning a pointer to the response message
  * Returns: 0 on success, 1 if more input data is needed, 2 if application data
  * is available, or -1 on failure
@@ -672,14 +635,15 @@ static int eap_tls_process_output(struct eap_ssl_data *data, EapType eap_type,
  */
 int eap_peer_tls_process_helper(struct eap_sm *sm, struct eap_ssl_data *data,
 				EapType eap_type, int peap_version,
-				u8 id, const u8 *in_data, size_t in_len,
+				u8 id, const struct wpabuf *in_data,
 				struct wpabuf **out_data)
 {
 	int ret = 0;
 
 	*out_data = NULL;
 
-	if (data->tls_out && wpabuf_len(data->tls_out) > 0 && in_len > 0) {
+	if (data->tls_out && wpabuf_len(data->tls_out) > 0 &&
+	    wpabuf_len(in_data) > 0) {
 		wpa_printf(MSG_DEBUG, "SSL: Received non-ACK when output "
 			   "fragments are waiting to be sent out");
 		return -1;
@@ -690,8 +654,7 @@ int eap_peer_tls_process_helper(struct eap_sm *sm, struct eap_ssl_data *data,
 		 * No more data to send out - expect to receive more data from
 		 * the AS.
 		 */
-		int res = eap_tls_process_input(sm, data, in_data, in_len,
-						out_data);
+		int res = eap_tls_process_input(sm, data, in_data, out_data);
 		if (res) {
 			/*
 			 * Input processing failed (res = -1) or more data is
@@ -719,12 +682,18 @@ int eap_peer_tls_process_helper(struct eap_sm *sm, struct eap_ssl_data *data,
 	if (tls_connection_get_failed(data->ssl_ctx, data->conn)) {
 		/* TLS processing has failed - return error */
 		wpa_printf(MSG_DEBUG, "SSL: Failed - tls_out available to "
-			   "report error");
+			   "report error (len=%u)",
+			   (unsigned int) wpabuf_len(data->tls_out));
 		ret = -1;
 		/* TODO: clean pin if engine used? */
+		if (wpabuf_len(data->tls_out) == 0) {
+			wpabuf_free(data->tls_out);
+			data->tls_out = NULL;
+			return -1;
+		}
 	}
 
-	if (data->tls_out == NULL || wpabuf_len(data->tls_out) == 0) {
+	if (wpabuf_len(data->tls_out) == 0) {
 		/*
 		 * TLS negotiation should now be complete since all other cases
 		 * needing more data should have been caught above based on
@@ -790,20 +759,24 @@ int eap_peer_tls_reauth_init(struct eap_sm *sm, struct eap_ssl_data *data)
 int eap_peer_tls_status(struct eap_sm *sm, struct eap_ssl_data *data,
 			char *buf, size_t buflen, int verbose)
 {
-	char name[128];
+	char version[20], name[128];
 	int len = 0, ret;
 
-	if (tls_get_cipher(data->ssl_ctx, data->conn, name, sizeof(name)) == 0)
-	{
-		ret = os_snprintf(buf + len, buflen - len,
-				  "EAP TLS cipher=%s\n"
-				  "tls_session_reused=%d\n",
-				  name, tls_connection_resumed(data->ssl_ctx,
-							       data->conn));
-		if (os_snprintf_error(buflen - len, ret))
-			return len;
-		len += ret;
-	}
+	if (tls_get_version(data->ssl_ctx, data->conn, version,
+			    sizeof(version)) < 0)
+		version[0] = '\0';
+	if (tls_get_cipher(data->ssl_ctx, data->conn, name, sizeof(name)) < 0)
+		name[0] = '\0';
+
+	ret = os_snprintf(buf + len, buflen - len,
+			  "eap_tls_version=%s\n"
+			  "EAP TLS cipher=%s\n"
+			  "tls_session_reused=%d\n",
+			  version, name,
+			  tls_connection_resumed(data->ssl_ctx, data->conn));
+	if (os_snprintf_error(buflen - len, ret))
+		return len;
+	len += ret;
 
 	return len;
 }
@@ -1032,7 +1005,7 @@ int eap_peer_select_phase2_methods(struct eap_peer_config *config,
 {
 	char *start, *pos, *buf;
 	struct eap_method_type *methods = NULL, *_methods;
-	u8 method;
+	u32 method;
 	size_t num_methods = 0, prefix_len;
 
 	if (config == NULL || config->phase2 == NULL)
diff --git a/src/eap_peer/eap_tls_common.h b/src/eap_peer/eap_tls_common.h
index 390c2165927c..acd2b783617f 100644
--- a/src/eap_peer/eap_tls_common.h
+++ b/src/eap_peer/eap_tls_common.h
@@ -100,7 +100,7 @@ u8 * eap_peer_tls_derive_session_id(struct eap_sm *sm,
 				    size_t *len);
 int eap_peer_tls_process_helper(struct eap_sm *sm, struct eap_ssl_data *data,
 				EapType eap_type, int peap_version,
-				u8 id, const u8 *in_data, size_t in_len,
+				u8 id, const struct wpabuf *in_data,
 				struct wpabuf **out_data);
 struct wpabuf * eap_peer_tls_build_ack(u8 id, EapType eap_type,
 				       int peap_version);
diff --git a/src/eap_peer/eap_ttls.c b/src/eap_peer/eap_ttls.c
index b5c028b5276d..b186c9156a74 100644
--- a/src/eap_peer/eap_ttls.c
+++ b/src/eap_peer/eap_ttls.c
@@ -175,7 +175,8 @@ static u8 * eap_ttls_avp_hdr(u8 *avphdr, u32 avp_code, u32 vendor_id,
 	}
 
 	avp->avp_code = host_to_be32(avp_code);
-	avp->avp_length = host_to_be32((flags << 24) | (u32) (hdrlen + len));
+	avp->avp_length = host_to_be32(((u32) flags << 24) |
+				       (u32) (hdrlen + len));
 
 	return avphdr + hdrlen;
 }
@@ -253,11 +254,13 @@ static int eap_ttls_v0_derive_key(struct eap_sm *sm,
 }
 
 
+#ifndef CONFIG_FIPS
 static u8 * eap_ttls_implicit_challenge(struct eap_sm *sm,
 					struct eap_ttls_data *data, size_t len)
 {
 	return eap_peer_tls_derive_key(sm, &data->ssl, "ttls challenge", len);
 }
+#endif /* CONFIG_FIPS */
 
 
 static void eap_ttls_phase2_select_eap_method(struct eap_ttls_data *data,
@@ -428,6 +431,10 @@ static int eap_ttls_phase2_request_mschapv2(struct eap_sm *sm,
 					    struct eap_method_ret *ret,
 					    struct wpabuf **resp)
 {
+#ifdef CONFIG_FIPS
+	wpa_printf(MSG_ERROR, "EAP-TTLS: MSCHAPV2 not supported in FIPS build");
+	return -1;
+#else /* CONFIG_FIPS */
 #ifdef EAP_MSCHAPv2
 	struct wpabuf *msg;
 	u8 *buf, *pos, *challenge, *peer_challenge;
@@ -510,6 +517,7 @@ static int eap_ttls_phase2_request_mschapv2(struct eap_sm *sm,
 	wpa_printf(MSG_ERROR, "EAP-TTLS: MSCHAPv2 not included in the build");
 	return -1;
 #endif /* EAP_MSCHAPv2 */
+#endif /* CONFIG_FIPS */
 }
 
 
@@ -518,6 +526,10 @@ static int eap_ttls_phase2_request_mschap(struct eap_sm *sm,
 					  struct eap_method_ret *ret,
 					  struct wpabuf **resp)
 {
+#ifdef CONFIG_FIPS
+	wpa_printf(MSG_ERROR, "EAP-TTLS: MSCHAP not supported in FIPS build");
+	return -1;
+#else /* CONFIG_FIPS */
 	struct wpabuf *msg;
 	u8 *buf, *pos, *challenge;
 	const u8 *identity, *password;
@@ -592,6 +604,7 @@ static int eap_ttls_phase2_request_mschap(struct eap_sm *sm,
 	ret->decision = DECISION_COND_SUCC;
 
 	return 0;
+#endif /* CONFIG_FIPS */
 }
 
 
@@ -654,6 +667,10 @@ static int eap_ttls_phase2_request_chap(struct eap_sm *sm,
 					struct eap_method_ret *ret,
 					struct wpabuf **resp)
 {
+#ifdef CONFIG_FIPS
+	wpa_printf(MSG_ERROR, "EAP-TTLS: CHAP not supported in FIPS build");
+	return -1;
+#else /* CONFIG_FIPS */
 	struct wpabuf *msg;
 	u8 *buf, *pos, *challenge;
 	const u8 *identity, *password;
@@ -722,6 +739,7 @@ static int eap_ttls_phase2_request_chap(struct eap_sm *sm,
 	ret->decision = DECISION_COND_SUCC;
 
 	return 0;
+#endif /* CONFIG_FIPS */
 }
 
 
@@ -1385,14 +1403,20 @@ static int eap_ttls_process_handshake(struct eap_sm *sm,
 				      struct eap_ttls_data *data,
 				      struct eap_method_ret *ret,
 				      u8 identifier,
-				      const u8 *in_data, size_t in_len,
+				      const struct wpabuf *in_data,
 				      struct wpabuf **out_data)
 {
 	int res;
 
 	res = eap_peer_tls_process_helper(sm, &data->ssl, EAP_TYPE_TTLS,
 					  data->ttls_version, identifier,
-					  in_data, in_len, out_data);
+					  in_data, out_data);
+	if (res < 0) {
+		wpa_printf(MSG_DEBUG, "EAP-TTLS: TLS processing failed");
+		ret->methodState = METHOD_DONE;
+		ret->decision = DECISION_FAIL;
+		return -1;
+	}
 
 	if (tls_connection_established(sm->ssl_ctx, data->ssl.conn)) {
 		wpa_printf(MSG_DEBUG, "EAP-TTLS: TLS done, proceed to "
@@ -1419,15 +1443,13 @@ static int eap_ttls_process_handshake(struct eap_sm *sm,
 	}
 
 	if (res == 2) {
-		struct wpabuf msg;
 		/*
 		 * Application data included in the handshake message.
 		 */
 		wpabuf_free(data->pending_phase2_req);
 		data->pending_phase2_req = *out_data;
 		*out_data = NULL;
-		wpabuf_set(&msg, in_data, in_len);
-		res = eap_ttls_decrypt(sm, data, ret, identifier, &msg,
+		res = eap_ttls_decrypt(sm, data, ret, identifier, in_data,
 				       out_data);
 	}
 
@@ -1477,6 +1499,7 @@ static struct wpabuf * eap_ttls_process(struct eap_sm *sm, void *priv,
 	struct wpabuf *resp;
 	const u8 *pos;
 	struct eap_ttls_data *data = priv;
+	struct wpabuf msg;
 
 	pos = eap_peer_tls_process_init(sm, &data->ssl, EAP_TYPE_TTLS, ret,
 					reqData, &left, &flags);
@@ -1497,15 +1520,15 @@ static struct wpabuf * eap_ttls_process(struct eap_sm *sm, void *priv,
 		left = 0;
 	}
 
+	wpabuf_set(&msg, pos, left);
+
 	resp = NULL;
 	if (tls_connection_established(sm->ssl_ctx, data->ssl.conn) &&
 	    !data->resuming) {
-		struct wpabuf msg;
-		wpabuf_set(&msg, pos, left);
 		res = eap_ttls_decrypt(sm, data, ret, id, &msg, &resp);
 	} else {
 		res = eap_ttls_process_handshake(sm, data, ret, id,
-						 pos, left, &resp);
+						 &msg, &resp);
 	}
 
 	eap_ttls_check_auth_status(sm, data, ret);
diff --git a/src/eap_peer/eap_wsc.c b/src/eap_peer/eap_wsc.c
index 7ce0a53d0b29..7ac99c7ce727 100644
--- a/src/eap_peer/eap_wsc.c
+++ b/src/eap_peer/eap_wsc.c
@@ -557,6 +557,9 @@ send_msg:
 		if (data->out_buf == NULL) {
 			wpa_printf(MSG_DEBUG, "EAP-WSC: Failed to receive "
 				   "message from WPS");
+			eap_wsc_state(data, FAIL);
+			ret->methodState = METHOD_DONE;
+			ret->decision = DECISION_FAIL;
 			return NULL;
 		}
 		data->out_used = 0;
diff --git a/src/eap_server/Makefile b/src/eap_server/Makefile
index adfd3dfd5b9b..1172b72466d2 100644
--- a/src/eap_server/Makefile
+++ b/src/eap_server/Makefile
@@ -1,8 +1,21 @@
-all:
-	@echo Nothing to be made.
+all: libeap_server.a
 
 clean:
-	rm -f *~ *.o *.d *.gcno *.gcda *.gcov
+	rm -f *~ *.o *.d *.gcno *.gcda *.gcov libeap_server.a
 
 install:
 	@echo Nothing to be made.
+
+include ../lib.rules
+
+CFLAGS += -DCONFIG_HS20
+
+LIB_OBJS= \
+	eap_server.o \
+	eap_server_identity.o \
+	eap_server_methods.o
+
+libeap_server.a: $(LIB_OBJS)
+	$(AR) crT $@ $?
+
+-include $(OBJS:%.o=%.d)
diff --git a/src/eap_server/eap.h b/src/eap_server/eap.h
index 9de6cb62f517..69eaab8de946 100644
--- a/src/eap_server/eap.h
+++ b/src/eap_server/eap.h
@@ -131,6 +131,7 @@ struct eap_config {
 	const u8 *server_id;
 	size_t server_id_len;
 	int erp;
+	unsigned int tls_session_lifetime;
 
 #ifdef CONFIG_TESTING_OPTIONS
 	u32 tls_test_flags;
@@ -139,7 +140,7 @@ struct eap_config {
 
 
 struct eap_sm * eap_server_sm_init(void *eapol_ctx,
-				   struct eapol_callbacks *eapol_cb,
+				   const struct eapol_callbacks *eapol_cb,
 				   struct eap_config *eap_conf);
 void eap_server_sm_deinit(struct eap_sm *sm);
 int eap_server_sm_step(struct eap_sm *sm);
@@ -149,5 +150,8 @@ int eap_sm_method_pending(struct eap_sm *sm);
 const u8 * eap_get_identity(struct eap_sm *sm, size_t *len);
 struct eap_eapol_interface * eap_get_interface(struct eap_sm *sm);
 void eap_server_clear_identity(struct eap_sm *sm);
+void eap_server_mschap_rx_callback(struct eap_sm *sm, const char *source,
+				   const u8 *username, size_t username_len,
+				   const u8 *challenge, const u8 *response);
 
 #endif /* EAP_H */
diff --git a/src/eap_server/eap_i.h b/src/eap_server/eap_i.h
index 7d723091ffb5..c90443d19cb9 100644
--- a/src/eap_server/eap_i.h
+++ b/src/eap_server/eap_i.h
@@ -155,7 +155,7 @@ struct eap_sm {
 	/* not defined in RFC 4137 */
 	Boolean changed;
 	void *eapol_ctx, *msg_ctx;
-	struct eapol_callbacks *eapol_cb;
+	const struct eapol_callbacks *eapol_cb;
 	void *eap_method_priv;
 	u8 *identity;
 	size_t identity_len;
@@ -210,6 +210,7 @@ struct eap_sm {
 	Boolean initiate_reauth_start_sent;
 	Boolean try_initiate_reauth;
 	int erp;
+	unsigned int tls_session_lifetime;
 
 #ifdef CONFIG_TESTING_OPTIONS
 	u32 tls_test_flags;
diff --git a/src/eap_server/eap_server.c b/src/eap_server/eap_server.c
index bd919e570c82..84ecafc7ca3e 100644
--- a/src/eap_server/eap_server.c
+++ b/src/eap_server/eap_server.c
@@ -96,7 +96,8 @@ static struct wpabuf * eap_sm_buildInitiateReauthStart(struct eap_sm *sm,
 		plen += 2 + domain_len;
 	}
 
-	msg = eap_msg_alloc(EAP_VENDOR_IETF, EAP_ERP_TYPE_REAUTH_START, plen,
+	msg = eap_msg_alloc(EAP_VENDOR_IETF,
+			    (EapType) EAP_ERP_TYPE_REAUTH_START, plen,
 			    EAP_CODE_INITIATE, id);
 	if (msg == NULL)
 		return NULL;
@@ -714,8 +715,8 @@ static void erp_send_finish_reauth(struct eap_sm *sm,
 	plen = 1 + 2 + 2 + os_strlen(nai);
 	if (hash_len)
 		plen += 1 + hash_len;
-	msg = eap_msg_alloc(EAP_VENDOR_IETF, EAP_ERP_TYPE_REAUTH, plen,
-			    EAP_CODE_FINISH, id);
+	msg = eap_msg_alloc(EAP_VENDOR_IETF, (EapType) EAP_ERP_TYPE_REAUTH,
+			    plen, EAP_CODE_FINISH, id);
 	if (msg == NULL)
 		return;
 	wpabuf_put_u8(msg, flags);
@@ -745,7 +746,7 @@ static void erp_send_finish_reauth(struct eap_sm *sm,
 	wpabuf_free(sm->lastReqData);
 	sm->lastReqData = NULL;
 
-	if (flags & 0x80) {
+	if ((flags & 0x80) || !erp) {
 		sm->eap_if.eapFail = TRUE;
 		wpa_msg(sm->msg_ctx, MSG_INFO, WPA_EVENT_EAP_FAILURE
 			MACSTR, MAC2STR(sm->peer_addr));
@@ -799,7 +800,7 @@ SM_STATE(EAP, INITIATE_RECEIVED)
 
 	sm->rxInitiate = FALSE;
 
-	pos = eap_hdr_validate(EAP_VENDOR_IETF, EAP_ERP_TYPE_REAUTH,
+	pos = eap_hdr_validate(EAP_VENDOR_IETF, (EapType) EAP_ERP_TYPE_REAUTH,
 			       sm->eap_if.eapRespData, &len);
 	if (pos == NULL) {
 		wpa_printf(MSG_INFO, "EAP-Initiate: Invalid frame");
@@ -1246,6 +1247,17 @@ SM_STEP(EAP)
 			break;
 		}
 		SM_ENTER(EAP, SEND_REQUEST);
+		if (sm->eap_if.eapNoReq && !sm->eap_if.eapReq) {
+			/*
+			 * This transition is not mentioned in RFC 4137, but it
+			 * is needed to handle cleanly a case where EAP method
+			 * buildReq fails.
+			 */
+			wpa_printf(MSG_DEBUG,
+				   "EAP: Method did not return a request");
+			SM_ENTER(EAP, FAILURE);
+			break;
+		}
 		break;
 	case EAP_METHOD_RESPONSE:
 		/*
@@ -1802,7 +1814,7 @@ static void eap_user_free(struct eap_user *user)
  * This function allocates and initializes an EAP state machine.
  */
 struct eap_sm * eap_server_sm_init(void *eapol_ctx,
-				   struct eapol_callbacks *eapol_cb,
+				   const struct eapol_callbacks *eapol_cb,
 				   struct eap_config *conf)
 {
 	struct eap_sm *sm;
@@ -1853,6 +1865,7 @@ struct eap_sm * eap_server_sm_init(void *eapol_ctx,
 	sm->server_id = conf->server_id;
 	sm->server_id_len = conf->server_id_len;
 	sm->erp = conf->erp;
+	sm->tls_session_lifetime = conf->tls_session_lifetime;
 
 #ifdef CONFIG_TESTING_OPTIONS
 	sm->tls_test_flags = conf->tls_test_flags;
@@ -1979,3 +1992,25 @@ void eap_server_clear_identity(struct eap_sm *sm)
 	os_free(sm->identity);
 	sm->identity = NULL;
 }
+
+
+#ifdef CONFIG_TESTING_OPTIONS
+void eap_server_mschap_rx_callback(struct eap_sm *sm, const char *source,
+				   const u8 *username, size_t username_len,
+				   const u8 *challenge, const u8 *response)
+{
+	char hex_challenge[30], hex_response[90], user[100];
+
+	/* Print out Challenge and Response in format supported by asleap. */
+	if (username)
+		printf_encode(user, sizeof(user), username, username_len);
+	else
+		user[0] = '\0';
+	wpa_snprintf_hex_sep(hex_challenge, sizeof(hex_challenge),
+			     challenge, sizeof(challenge), ':');
+	wpa_snprintf_hex_sep(hex_response, sizeof(hex_response), response, 24,
+			     ':');
+	wpa_printf(MSG_DEBUG, "[%s/user=%s] asleap -C %s -R %s",
+		   source, user, hex_challenge, hex_response);
+}
+#endif /* CONFIG_TESTING_OPTIONS */
diff --git a/src/eap_server/eap_server_eke.c b/src/eap_server/eap_server_eke.c
index 966f511ddddc..ba82be9c3f3a 100644
--- a/src/eap_server/eap_server_eke.c
+++ b/src/eap_server/eap_server_eke.c
@@ -766,6 +766,29 @@ static Boolean eap_eke_isSuccess(struct eap_sm *sm, void *priv)
 }
 
 
+static u8 * eap_eke_get_session_id(struct eap_sm *sm, void *priv, size_t *len)
+{
+	struct eap_eke_data *data = priv;
+	u8 *sid;
+	size_t sid_len;
+
+	if (data->state != SUCCESS)
+		return NULL;
+
+	sid_len = 1 + 2 * data->sess.nonce_len;
+	sid = os_malloc(sid_len);
+	if (sid == NULL)
+		return NULL;
+	sid[0] = EAP_TYPE_EKE;
+	os_memcpy(sid + 1, data->nonce_p, data->sess.nonce_len);
+	os_memcpy(sid + 1 + data->sess.nonce_len, data->nonce_s,
+		  data->sess.nonce_len);
+	*len = sid_len;
+
+	return sid;
+}
+
+
 int eap_server_eke_register(void)
 {
 	struct eap_method *eap;
@@ -785,6 +808,7 @@ int eap_server_eke_register(void)
 	eap->getKey = eap_eke_getKey;
 	eap->isSuccess = eap_eke_isSuccess;
 	eap->get_emsk = eap_eke_get_emsk;
+	eap->getSessionId = eap_eke_get_session_id;
 
 	ret = eap_server_method_register(eap);
 	if (ret)
diff --git a/src/eap_server/eap_server_fast.c b/src/eap_server/eap_server_fast.c
index 6745100d338c..bd9018e78b56 100644
--- a/src/eap_server/eap_server_fast.c
+++ b/src/eap_server/eap_server_fast.c
@@ -428,7 +428,7 @@ static void * eap_fast_init(struct eap_sm *sm)
 	}
 	data->state = START;
 
-	if (eap_server_tls_ssl_init(sm, &data->ssl, 0)) {
+	if (eap_server_tls_ssl_init(sm, &data->ssl, 0, EAP_TYPE_FAST)) {
 		wpa_printf(MSG_INFO, "EAP-FAST: Failed to initialize SSL.");
 		eap_fast_reset(sm, data);
 		return NULL;
diff --git a/src/eap_server/eap_server_mschapv2.c b/src/eap_server/eap_server_mschapv2.c
index 05848d2eaac5..98d74e0d717e 100644
--- a/src/eap_server/eap_server_mschapv2.c
+++ b/src/eap_server/eap_server_mschapv2.c
@@ -360,6 +360,19 @@ static void eap_mschapv2_process_response(struct eap_sm *sm,
 		}
 	}
 
+#ifdef CONFIG_TESTING_OPTIONS
+	{
+		u8 challenge[8];
+
+		if (challenge_hash(peer_challenge, data->auth_challenge,
+				   username, username_len, challenge) == 0) {
+			eap_server_mschap_rx_callback(sm, "EAP-MSCHAPV2",
+						      username, username_len,
+						      challenge, nt_response);
+		}
+	}
+#endif /* CONFIG_TESTING_OPTIONS */
+
 	if (username_len != user_len ||
 	    os_memcmp(username, user, username_len) != 0) {
 		wpa_printf(MSG_DEBUG, "EAP-MSCHAPV2: Mismatch in user names");
diff --git a/src/eap_server/eap_server_peap.c b/src/eap_server/eap_server_peap.c
index faa0fd2f2387..51062b0987e4 100644
--- a/src/eap_server/eap_server_peap.c
+++ b/src/eap_server/eap_server_peap.c
@@ -95,6 +95,37 @@ static void eap_peap_state(struct eap_peap_data *data, int state)
 		   eap_peap_state_txt(data->state),
 		   eap_peap_state_txt(state));
 	data->state = state;
+	if (state == FAILURE || state == FAILURE_REQ)
+		tls_connection_remove_session(data->ssl.conn);
+}
+
+
+static void eap_peap_valid_session(struct eap_sm *sm,
+				   struct eap_peap_data *data)
+{
+	struct wpabuf *buf;
+
+	if (!sm->tls_session_lifetime ||
+	    tls_connection_resumed(sm->ssl_ctx, data->ssl.conn))
+		return;
+
+	buf = wpabuf_alloc(1 + 1 + sm->identity_len);
+	if (!buf)
+		return;
+	wpabuf_put_u8(buf, EAP_TYPE_PEAP);
+	if (sm->identity) {
+		u8 id_len;
+
+		if (sm->identity_len <= 255)
+			id_len = sm->identity_len;
+		else
+			id_len = 255;
+		wpabuf_put_u8(buf, id_len);
+		wpabuf_put_data(buf, sm->identity, id_len);
+	} else {
+		wpabuf_put_u8(buf, 0);
+	}
+	tls_connection_set_success_data(data->ssl.conn, buf);
 }
 
 
@@ -151,7 +182,7 @@ static void * eap_peap_init(struct eap_sm *sm)
 	data->state = START;
 	data->crypto_binding = OPTIONAL_BINDING;
 
-	if (eap_server_tls_ssl_init(sm, &data->ssl, 0)) {
+	if (eap_server_tls_ssl_init(sm, &data->ssl, 0, EAP_TYPE_PEAP)) {
 		wpa_printf(MSG_INFO, "EAP-PEAP: Failed to initialize SSL.");
 		eap_peap_reset(sm, data);
 		return NULL;
@@ -539,15 +570,14 @@ static Boolean eap_peap_check(struct eap_sm *sm, void *priv,
 
 
 static int eap_peap_phase2_init(struct eap_sm *sm, struct eap_peap_data *data,
-				EapType eap_type)
+				int vendor, EapType eap_type)
 {
 	if (data->phase2_priv && data->phase2_method) {
 		data->phase2_method->reset(sm, data->phase2_priv);
 		data->phase2_method = NULL;
 		data->phase2_priv = NULL;
 	}
-	data->phase2_method = eap_server_get_eap_method(EAP_VENDOR_IETF,
-							eap_type);
+	data->phase2_method = eap_server_get_eap_method(vendor, eap_type);
 	if (!data->phase2_method)
 		return -1;
 
@@ -709,10 +739,12 @@ static void eap_peap_process_phase2_tlv(struct eap_sm *sm,
 		if (status == EAP_TLV_RESULT_SUCCESS) {
 			wpa_printf(MSG_INFO, "EAP-PEAP: TLV Result - Success "
 				   "- requested %s", requested);
-			if (data->tlv_request == TLV_REQ_SUCCESS)
+			if (data->tlv_request == TLV_REQ_SUCCESS) {
 				eap_peap_state(data, SUCCESS);
-			else
+				eap_peap_valid_session(sm, data);
+			} else {
 				eap_peap_state(data, FAILURE);
+			}
 			
 		} else if (status == EAP_TLV_RESULT_FAILURE) {
 			wpa_printf(MSG_INFO, "EAP-PEAP: TLV Result - Failure "
@@ -737,7 +769,7 @@ static void eap_peap_process_phase2_soh(struct eap_sm *sm,
 	const u8 *soh_tlv = NULL;
 	size_t soh_tlv_len = 0;
 	int tlv_type, mandatory, tlv_len, vtlv_len;
-	u8 next_type;
+	u32 next_type;
 	u32 vendor_id;
 
 	pos = eap_hdr_validate(EAP_VENDOR_MICROSOFT, 0x21, in_data, &left);
@@ -852,8 +884,9 @@ auth_method:
 	eap_peap_state(data, PHASE2_METHOD);
 	next_type = sm->user->methods[0].method;
 	sm->user_eap_method_index = 1;
-	wpa_printf(MSG_DEBUG, "EAP-PEAP: try EAP type %d", next_type);
-	eap_peap_phase2_init(sm, data, next_type);
+	wpa_printf(MSG_DEBUG, "EAP-PEAP: try EAP vendor %d type %d",
+		   sm->user->methods[0].vendor, next_type);
+	eap_peap_phase2_init(sm, data, sm->user->methods[0].vendor, next_type);
 }
 #endif /* EAP_SERVER_TNC */
 
@@ -862,7 +895,8 @@ static void eap_peap_process_phase2_response(struct eap_sm *sm,
 					     struct eap_peap_data *data,
 					     struct wpabuf *in_data)
 {
-	u8 next_type = EAP_TYPE_NONE;
+	int next_vendor = EAP_VENDOR_IETF;
+	u32 next_type = EAP_TYPE_NONE;
 	const struct eap_hdr *hdr;
 	const u8 *pos;
 	size_t left;
@@ -894,17 +928,23 @@ static void eap_peap_process_phase2_response(struct eap_sm *sm,
 			    "allowed types", pos + 1, left - 1);
 		eap_sm_process_nak(sm, pos + 1, left - 1);
 		if (sm->user && sm->user_eap_method_index < EAP_MAX_METHODS &&
-		    sm->user->methods[sm->user_eap_method_index].method !=
-		    EAP_TYPE_NONE) {
+		    (sm->user->methods[sm->user_eap_method_index].vendor !=
+		     EAP_VENDOR_IETF ||
+		     sm->user->methods[sm->user_eap_method_index].method !=
+		     EAP_TYPE_NONE)) {
+			next_vendor = sm->user->methods[
+				sm->user_eap_method_index].vendor;
 			next_type = sm->user->methods[
 				sm->user_eap_method_index++].method;
-			wpa_printf(MSG_DEBUG, "EAP-PEAP: try EAP type %d",
-				   next_type);
+			wpa_printf(MSG_DEBUG,
+				   "EAP-PEAP: try EAP vendor %d type 0x%x",
+				   next_vendor, next_type);
 		} else {
 			eap_peap_req_failure(sm, data);
+			next_vendor = EAP_VENDOR_IETF;
 			next_type = EAP_TYPE_NONE;
 		}
-		eap_peap_phase2_init(sm, data, next_type);
+		eap_peap_phase2_init(sm, data, next_vendor, next_type);
 		return;
 	}
 
@@ -929,8 +969,9 @@ static void eap_peap_process_phase2_response(struct eap_sm *sm,
 	if (!data->phase2_method->isSuccess(sm, data->phase2_priv)) {
 		wpa_printf(MSG_DEBUG, "EAP-PEAP: Phase2 method failed");
 		eap_peap_req_failure(sm, data);
+		next_vendor = EAP_VENDOR_IETF;
 		next_type = EAP_TYPE_NONE;
-		eap_peap_phase2_init(sm, data, next_type);
+		eap_peap_phase2_init(sm, data, next_vendor, next_type);
 		return;
 	}
 
@@ -942,7 +983,8 @@ static void eap_peap_process_phase2_response(struct eap_sm *sm,
 			wpa_printf(MSG_DEBUG, "EAP-PEAP: Phase2 getKey "
 				   "failed");
 			eap_peap_req_failure(sm, data);
-			eap_peap_phase2_init(sm, data, EAP_TYPE_NONE);
+			eap_peap_phase2_init(sm, data, EAP_VENDOR_IETF,
+					     EAP_TYPE_NONE);
 			return;
 		}
 	}
@@ -957,6 +999,7 @@ static void eap_peap_process_phase2_response(struct eap_sm *sm,
 					  "database",
 					  sm->identity, sm->identity_len);
 			eap_peap_req_failure(sm, data);
+			next_vendor = EAP_VENDOR_IETF;
 			next_type = EAP_TYPE_NONE;
 			break;
 		}
@@ -967,18 +1010,22 @@ static void eap_peap_process_phase2_response(struct eap_sm *sm,
 			eap_peap_state(data, PHASE2_SOH);
 			wpa_printf(MSG_DEBUG, "EAP-PEAP: Try to initialize "
 				   "TNC (NAP SOH)");
+			next_vendor = EAP_VENDOR_IETF;
 			next_type = EAP_TYPE_NONE;
 			break;
 		}
 #endif /* EAP_SERVER_TNC */
 
 		eap_peap_state(data, PHASE2_METHOD);
+		next_vendor = sm->user->methods[0].vendor;
 		next_type = sm->user->methods[0].method;
 		sm->user_eap_method_index = 1;
-		wpa_printf(MSG_DEBUG, "EAP-PEAP: try EAP type %d", next_type);
+		wpa_printf(MSG_DEBUG, "EAP-PEAP: try EAP vendor %d type 0x%x",
+			   next_vendor, next_type);
 		break;
 	case PHASE2_METHOD:
 		eap_peap_req_success(sm, data);
+		next_vendor = EAP_VENDOR_IETF;
 		next_type = EAP_TYPE_NONE;
 		break;
 	case FAILURE:
@@ -989,7 +1036,7 @@ static void eap_peap_process_phase2_response(struct eap_sm *sm,
 		break;
 	}
 
-	eap_peap_phase2_init(sm, data, next_type);
+	eap_peap_phase2_init(sm, data, next_vendor, next_type);
 }
 
 
@@ -1080,6 +1127,7 @@ static void eap_peap_process_phase2(struct eap_sm *sm,
 		wpa_printf(MSG_DEBUG, "EAP-PEAP: Phase 2 Success");
 		if (data->state == SUCCESS_REQ) {
 			eap_peap_state(data, SUCCESS);
+			eap_peap_valid_session(sm, data);
 		}
 		break;
 	case EAP_CODE_FAILURE:
@@ -1133,7 +1181,8 @@ static void eap_peap_process_msg(struct eap_sm *sm, void *priv,
 		break;
 	case PHASE2_START:
 		eap_peap_state(data, PHASE2_ID);
-		eap_peap_phase2_init(sm, data, EAP_TYPE_IDENTITY);
+		eap_peap_phase2_init(sm, data, EAP_VENDOR_IETF,
+				     EAP_TYPE_IDENTITY);
 		break;
 	case PHASE1_ID2:
 	case PHASE2_ID:
@@ -1144,6 +1193,7 @@ static void eap_peap_process_msg(struct eap_sm *sm, void *priv,
 		break;
 	case SUCCESS_REQ:
 		eap_peap_state(data, SUCCESS);
+		eap_peap_valid_session(sm, data);
 		break;
 	case FAILURE_REQ:
 		eap_peap_state(data, FAILURE);
@@ -1160,10 +1210,65 @@ static void eap_peap_process(struct eap_sm *sm, void *priv,
 			     struct wpabuf *respData)
 {
 	struct eap_peap_data *data = priv;
+	const struct wpabuf *buf;
+	const u8 *pos;
+	u8 id_len;
+
 	if (eap_server_tls_process(sm, &data->ssl, respData, data,
 				   EAP_TYPE_PEAP, eap_peap_process_version,
-				   eap_peap_process_msg) < 0)
+				   eap_peap_process_msg) < 0) {
 		eap_peap_state(data, FAILURE);
+		return;
+	}
+
+	if (data->state == SUCCESS ||
+	    !tls_connection_established(sm->ssl_ctx, data->ssl.conn) ||
+	    !tls_connection_resumed(sm->ssl_ctx, data->ssl.conn))
+		return;
+
+	buf = tls_connection_get_success_data(data->ssl.conn);
+	if (!buf || wpabuf_len(buf) < 2) {
+		wpa_printf(MSG_DEBUG,
+			   "EAP-PEAP: No success data in resumed session - reject attempt");
+		eap_peap_state(data, FAILURE);
+		return;
+	}
+
+	pos = wpabuf_head(buf);
+	if (*pos != EAP_TYPE_PEAP) {
+		wpa_printf(MSG_DEBUG,
+			   "EAP-PEAP: Resumed session for another EAP type (%u) - reject attempt",
+			   *pos);
+		eap_peap_state(data, FAILURE);
+		return;
+	}
+
+	pos++;
+	id_len = *pos++;
+	wpa_hexdump_ascii(MSG_DEBUG, "EAP-PEAP: Identity from cached session",
+			  pos, id_len);
+	os_free(sm->identity);
+	sm->identity = os_malloc(id_len ? id_len : 1);
+	if (!sm->identity) {
+		sm->identity_len = 0;
+		eap_peap_state(data, FAILURE);
+		return;
+	}
+
+	os_memcpy(sm->identity, pos, id_len);
+	sm->identity_len = id_len;
+
+	if (eap_user_get(sm, sm->identity, sm->identity_len, 1) != 0) {
+		wpa_hexdump_ascii(MSG_DEBUG, "EAP-PEAP: Phase2 Identity not found in the user database",
+				  sm->identity, sm->identity_len);
+		eap_peap_state(data, FAILURE);
+		return;
+	}
+
+	wpa_printf(MSG_DEBUG,
+		   "EAP-PEAP: Resuming previous session - skip Phase2");
+	eap_peap_state(data, SUCCESS_REQ);
+	tls_connection_set_success_data_resumed(data->ssl.conn);
 }
 
 
diff --git a/src/eap_server/eap_server_pwd.c b/src/eap_server/eap_server_pwd.c
index 943af0d15078..cb83ff7305bd 100644
--- a/src/eap_server/eap_server_pwd.c
+++ b/src/eap_server/eap_server_pwd.c
@@ -10,6 +10,7 @@
 
 #include "common.h"
 #include "crypto/sha256.h"
+#include "crypto/ms_funcs.h"
 #include "eap_server/eap_i.h"
 #include "eap_common/eap_pwd_common.h"
 
@@ -24,6 +25,7 @@ struct eap_pwd_data {
 	size_t id_server_len;
 	u8 *password;
 	size_t password_len;
+	int password_hash;
 	u32 token;
 	u16 group_num;
 	EAP_PWD_group *grp;
@@ -112,6 +114,7 @@ static void * eap_pwd_init(struct eap_sm *sm)
 	}
 	data->password_len = sm->user->password_len;
 	os_memcpy(data->password, sm->user->password, data->password_len);
+	data->password_hash = sm->user->password_hash;
 
 	data->bnctx = BN_CTX_new();
 	if (data->bnctx == NULL) {
@@ -181,7 +184,8 @@ static void eap_pwd_build_id_req(struct eap_sm *sm, struct eap_pwd_data *data,
 	wpabuf_put_u8(data->outbuf, EAP_PWD_DEFAULT_RAND_FUNC);
 	wpabuf_put_u8(data->outbuf, EAP_PWD_DEFAULT_PRF);
 	wpabuf_put_data(data->outbuf, &data->token, sizeof(data->token));
-	wpabuf_put_u8(data->outbuf, EAP_PWD_PREP_NONE);
+	wpabuf_put_u8(data->outbuf, data->password_hash ? EAP_PWD_PREP_MS :
+		      EAP_PWD_PREP_NONE);
 	wpabuf_put_data(data->outbuf, data->id_server, data->id_server_len);
 }
 
@@ -579,6 +583,10 @@ static void eap_pwd_process_id_resp(struct eap_sm *sm,
 				    const u8 *payload, size_t payload_len)
 {
 	struct eap_pwd_id *id;
+	const u8 *password;
+	size_t password_len;
+	u8 pwhashhash[16];
+	int res;
 
 	if (payload_len < sizeof(struct eap_pwd_id)) {
 		wpa_printf(MSG_INFO, "EAP-pwd: Invalid ID response");
@@ -610,11 +618,25 @@ static void eap_pwd_process_id_resp(struct eap_sm *sm,
 			   "group");
 		return;
 	}
-	if (compute_password_element(data->grp, data->group_num,
-				     data->password, data->password_len,
-				     data->id_server, data->id_server_len,
-				     data->id_peer, data->id_peer_len,
-				     (u8 *) &data->token)) {
+
+	if (data->password_hash) {
+		res = hash_nt_password_hash(data->password, pwhashhash);
+		if (res)
+			return;
+		password = pwhashhash;
+		password_len = sizeof(pwhashhash);
+	} else {
+		password = data->password;
+		password_len = data->password_len;
+	}
+
+	res = compute_password_element(data->grp, data->group_num,
+				       password, password_len,
+				       data->id_server, data->id_server_len,
+				       data->id_peer, data->id_peer_len,
+				       (u8 *) &data->token);
+	os_memset(pwhashhash, 0, sizeof(pwhashhash));
+	if (res) {
 		wpa_printf(MSG_INFO, "EAP-PWD (server): unable to compute "
 			   "PWE");
 		return;
@@ -634,9 +656,21 @@ eap_pwd_process_commit_resp(struct eap_sm *sm, struct eap_pwd_data *data,
 	BIGNUM *x = NULL, *y = NULL, *cofactor = NULL;
 	EC_POINT *K = NULL, *point = NULL;
 	int res = 0;
+	size_t prime_len, order_len;
 
 	wpa_printf(MSG_DEBUG, "EAP-pwd: Received commit response");
 
+	prime_len = BN_num_bytes(data->grp->prime);
+	order_len = BN_num_bytes(data->grp->order);
+
+	if (payload_len != 2 * prime_len + order_len) {
+		wpa_printf(MSG_INFO,
+			   "EAP-pwd: Unexpected Commit payload length %u (expected %u)",
+			   (unsigned int) payload_len,
+			   (unsigned int) (2 * prime_len + order_len));
+		goto fin;
+	}
+
 	if (((data->peer_scalar = BN_new()) == NULL) ||
 	    ((data->k = BN_new()) == NULL) ||
 	    ((cofactor = BN_new()) == NULL) ||
@@ -752,6 +786,13 @@ eap_pwd_process_confirm_resp(struct eap_sm *sm, struct eap_pwd_data *data,
 	u8 conf[SHA256_MAC_LEN], *cruft = NULL, *ptr;
 	int offset;
 
+	if (payload_len != SHA256_MAC_LEN) {
+		wpa_printf(MSG_INFO,
+			   "EAP-pwd: Unexpected Confirm payload length %u (expected %u)",
+			   (unsigned int) payload_len, SHA256_MAC_LEN);
+		goto fin;
+	}
+
 	/* build up the ciphersuite: group | random_function | prf */
 	grp = htons(data->group_num);
 	ptr = (u8 *) &cs;
@@ -901,17 +942,28 @@ static void eap_pwd_process(struct eap_sm *sm, void *priv,
 	 * the first fragment has a total length
 	 */
 	if (EAP_PWD_GET_LENGTH_BIT(lm_exch)) {
+		if (len < 2) {
+			wpa_printf(MSG_DEBUG,
+				   "EAP-pwd: Frame too short to contain Total-Length field");
+			return;
+		}
 		tot_len = WPA_GET_BE16(pos);
 		wpa_printf(MSG_DEBUG, "EAP-pwd: Incoming fragments, total "
 			   "length = %d", tot_len);
 		if (tot_len > 15000)
 			return;
+		if (data->inbuf) {
+			wpa_printf(MSG_DEBUG,
+				   "EAP-pwd: Unexpected new fragment start when previous fragment is still in use");
+			return;
+		}
 		data->inbuf = wpabuf_alloc(tot_len);
 		if (data->inbuf == NULL) {
 			wpa_printf(MSG_INFO, "EAP-pwd: Out of memory to "
 				   "buffer fragments!");
 			return;
 		}
+		data->in_frag_pos = 0;
 		pos += sizeof(u16);
 		len -= sizeof(u16);
 	}
diff --git a/src/eap_server/eap_server_tls.c b/src/eap_server/eap_server_tls.c
index 58cfe8ac64a0..bd18a4ba654c 100644
--- a/src/eap_server/eap_server_tls.c
+++ b/src/eap_server/eap_server_tls.c
@@ -48,6 +48,23 @@ static void eap_tls_state(struct eap_tls_data *data, int state)
 		   eap_tls_state_txt(data->state),
 		   eap_tls_state_txt(state));
 	data->state = state;
+	if (state == FAILURE)
+		tls_connection_remove_session(data->ssl.conn);
+}
+
+
+static void eap_tls_valid_session(struct eap_sm *sm, struct eap_tls_data *data)
+{
+	struct wpabuf *buf;
+
+	if (!sm->tls_session_lifetime)
+		return;
+
+	buf = wpabuf_alloc(1);
+	if (!buf)
+		return;
+	wpabuf_put_u8(buf, data->eap_type);
+	tls_connection_set_success_data(data->ssl.conn, buf);
 }
 
 
@@ -60,7 +77,7 @@ static void * eap_tls_init(struct eap_sm *sm)
 		return NULL;
 	data->state = START;
 
-	if (eap_server_tls_ssl_init(sm, &data->ssl, 1)) {
+	if (eap_server_tls_ssl_init(sm, &data->ssl, 1, EAP_TYPE_TLS)) {
 		wpa_printf(MSG_INFO, "EAP-TLS: Failed to initialize SSL.");
 		eap_tls_reset(sm, data);
 		return NULL;
@@ -82,7 +99,7 @@ static void * eap_unauth_tls_init(struct eap_sm *sm)
 		return NULL;
 	data->state = START;
 
-	if (eap_server_tls_ssl_init(sm, &data->ssl, 0)) {
+	if (eap_server_tls_ssl_init(sm, &data->ssl, 0, EAP_UNAUTH_TLS_TYPE)) {
 		wpa_printf(MSG_INFO, "EAP-TLS: Failed to initialize SSL.");
 		eap_tls_reset(sm, data);
 		return NULL;
@@ -104,7 +121,8 @@ static void * eap_wfa_unauth_tls_init(struct eap_sm *sm)
 		return NULL;
 	data->state = START;
 
-	if (eap_server_tls_ssl_init(sm, &data->ssl, 0)) {
+	if (eap_server_tls_ssl_init(sm, &data->ssl, 0,
+				    EAP_WFA_UNAUTH_TLS_TYPE)) {
 		wpa_printf(MSG_INFO, "EAP-TLS: Failed to initialize SSL.");
 		eap_tls_reset(sm, data);
 		return NULL;
@@ -183,6 +201,7 @@ check_established:
 		 * fragments waiting to be sent out. */
 		wpa_printf(MSG_DEBUG, "EAP-TLS: Done");
 		eap_tls_state(data, SUCCESS);
+		eap_tls_valid_session(sm, data);
 	}
 
 	return res;
@@ -234,10 +253,41 @@ static void eap_tls_process(struct eap_sm *sm, void *priv,
 			    struct wpabuf *respData)
 {
 	struct eap_tls_data *data = priv;
+	const struct wpabuf *buf;
+	const u8 *pos;
+
 	if (eap_server_tls_process(sm, &data->ssl, respData, data,
 				   data->eap_type, NULL, eap_tls_process_msg) <
-	    0)
+	    0) {
 		eap_tls_state(data, FAILURE);
+		return;
+	}
+
+	if (!tls_connection_established(sm->ssl_ctx, data->ssl.conn) ||
+	    !tls_connection_resumed(sm->ssl_ctx, data->ssl.conn))
+		return;
+
+	buf = tls_connection_get_success_data(data->ssl.conn);
+	if (!buf || wpabuf_len(buf) < 1) {
+		wpa_printf(MSG_DEBUG,
+			   "EAP-TLS: No success data in resumed session - reject attempt");
+		eap_tls_state(data, FAILURE);
+		return;
+	}
+
+	pos = wpabuf_head(buf);
+	if (*pos != data->eap_type) {
+		wpa_printf(MSG_DEBUG,
+			   "EAP-TLS: Resumed session for another EAP type (%u) - reject attempt",
+			   *pos);
+		eap_tls_state(data, FAILURE);
+		return;
+	}
+
+	wpa_printf(MSG_DEBUG,
+		   "EAP-TLS: Resuming previous session");
+	eap_tls_state(data, SUCCESS);
+	tls_connection_set_success_data_resumed(data->ssl.conn);
 }
 
 
diff --git a/src/eap_server/eap_server_tls_common.c b/src/eap_server/eap_server_tls_common.c
index 56916c45ac69..05677b70e887 100644
--- a/src/eap_server/eap_server_tls_common.c
+++ b/src/eap_server/eap_server_tls_common.c
@@ -44,8 +44,11 @@ static void eap_server_tls_log_cb(void *ctx, const char *msg)
 
 
 int eap_server_tls_ssl_init(struct eap_sm *sm, struct eap_ssl_data *data,
-			    int verify_peer)
+			    int verify_peer, int eap_type)
 {
+	u8 session_ctx[8];
+	unsigned int flags = 0;
+
 	if (sm->ssl_ctx == NULL) {
 		wpa_printf(MSG_ERROR, "TLS context not initialized - cannot use TLS-based EAP method");
 		return -1;
@@ -68,7 +71,13 @@ int eap_server_tls_ssl_init(struct eap_sm *sm, struct eap_ssl_data *data,
 #endif /* CONFIG_TESTING_OPTIONS */
 #endif /* CONFIG_TLS_INTERNAL */
 
-	if (tls_connection_set_verify(sm->ssl_ctx, data->conn, verify_peer)) {
+	if (eap_type != EAP_TYPE_FAST)
+		flags |= TLS_CONN_DISABLE_SESSION_TICKET;
+	os_memcpy(session_ctx, "hostapd", 7);
+	session_ctx[7] = (u8) eap_type;
+	if (tls_connection_set_verify(sm->ssl_ctx, data->conn, verify_peer,
+				      flags, session_ctx,
+				      sizeof(session_ctx))) {
 		wpa_printf(MSG_INFO, "SSL: Failed to configure verification "
 			   "of TLS peer certificate");
 		tls_connection_deinit(sm->ssl_ctx, data->conn);
@@ -100,43 +109,19 @@ void eap_server_tls_ssl_deinit(struct eap_sm *sm, struct eap_ssl_data *data)
 u8 * eap_server_tls_derive_key(struct eap_sm *sm, struct eap_ssl_data *data,
 			       char *label, size_t len)
 {
-	struct tls_keys keys;
-	u8 *rnd = NULL, *out;
+	u8 *out;
 
 	out = os_malloc(len);
 	if (out == NULL)
 		return NULL;
 
-	if (tls_connection_prf(sm->ssl_ctx, data->conn, label, 0, out, len) ==
-	    0)
-		return out;
+	if (tls_connection_prf(sm->ssl_ctx, data->conn, label, 0, 0,
+			       out, len)) {
+		os_free(out);
+		return NULL;
+	}
 
-	if (tls_connection_get_keys(sm->ssl_ctx, data->conn, &keys))
-		goto fail;
-
-	if (keys.client_random == NULL || keys.server_random == NULL ||
-	    keys.master_key == NULL)
-		goto fail;
-
-	rnd = os_malloc(keys.client_random_len + keys.server_random_len);
-	if (rnd == NULL)
-		goto fail;
-	os_memcpy(rnd, keys.client_random, keys.client_random_len);
-	os_memcpy(rnd + keys.client_random_len, keys.server_random,
-		  keys.server_random_len);
-
-	if (tls_prf_sha1_md5(keys.master_key, keys.master_key_len,
-			     label, rnd, keys.client_random_len +
-			     keys.server_random_len, out, len))
-		goto fail;
-
-	os_free(rnd);
 	return out;
-
-fail:
-	os_free(out);
-	os_free(rnd);
-	return NULL;
 }
 
 
@@ -157,10 +142,10 @@ u8 * eap_server_tls_derive_session_id(struct eap_sm *sm,
 				      struct eap_ssl_data *data, u8 eap_type,
 				      size_t *len)
 {
-	struct tls_keys keys;
+	struct tls_random keys;
 	u8 *out;
 
-	if (tls_connection_get_keys(sm->ssl_ctx, data->conn, &keys))
+	if (tls_connection_get_random(sm->ssl_ctx, data->conn, &keys))
 		return NULL;
 
 	if (keys.client_random == NULL || keys.server_random == NULL)
diff --git a/src/eap_server/eap_server_ttls.c b/src/eap_server/eap_server_ttls.c
index 12a31b07a63b..53ffa1ec6785 100644
--- a/src/eap_server/eap_server_ttls.c
+++ b/src/eap_server/eap_server_ttls.c
@@ -71,6 +71,36 @@ static void eap_ttls_state(struct eap_ttls_data *data, int state)
 		   eap_ttls_state_txt(data->state),
 		   eap_ttls_state_txt(state));
 	data->state = state;
+	if (state == FAILURE)
+		tls_connection_remove_session(data->ssl.conn);
+}
+
+
+static void eap_ttls_valid_session(struct eap_sm *sm,
+				   struct eap_ttls_data *data)
+{
+	struct wpabuf *buf;
+
+	if (!sm->tls_session_lifetime)
+		return;
+
+	buf = wpabuf_alloc(1 + 1 + sm->identity_len);
+	if (!buf)
+		return;
+	wpabuf_put_u8(buf, EAP_TYPE_TTLS);
+	if (sm->identity) {
+		u8 id_len;
+
+		if (sm->identity_len <= 255)
+			id_len = sm->identity_len;
+		else
+			id_len = 255;
+		wpabuf_put_u8(buf, id_len);
+		wpabuf_put_data(buf, sm->identity, id_len);
+	} else {
+		wpabuf_put_u8(buf, 0);
+	}
+	tls_connection_set_success_data(data->ssl.conn, buf);
 }
 
 
@@ -317,7 +347,7 @@ static void * eap_ttls_init(struct eap_sm *sm)
 	data->ttls_version = EAP_TTLS_VERSION;
 	data->state = START;
 
-	if (eap_server_tls_ssl_init(sm, &data->ssl, 0)) {
+	if (eap_server_tls_ssl_init(sm, &data->ssl, 0, EAP_TYPE_TTLS)) {
 		wpa_printf(MSG_INFO, "EAP-TTLS: Failed to initialize SSL.");
 		eap_ttls_reset(sm, data);
 		return NULL;
@@ -518,6 +548,7 @@ static void eap_ttls_process_phase2_pap(struct eap_sm *sm,
 
 	wpa_printf(MSG_DEBUG, "EAP-TTLS/PAP: Correct user password");
 	eap_ttls_state(data, SUCCESS);
+	eap_ttls_valid_session(sm, data);
 }
 
 
@@ -576,6 +607,7 @@ static void eap_ttls_process_phase2_chap(struct eap_sm *sm,
 	    0) {
 		wpa_printf(MSG_DEBUG, "EAP-TTLS/CHAP: Correct user password");
 		eap_ttls_state(data, SUCCESS);
+		eap_ttls_valid_session(sm, data);
 	} else {
 		wpa_printf(MSG_DEBUG, "EAP-TTLS/CHAP: Invalid user password");
 		eap_ttls_state(data, FAILURE);
@@ -618,6 +650,12 @@ static void eap_ttls_process_phase2_mschap(struct eap_sm *sm,
 		return;
 	}
 
+#ifdef CONFIG_TESTING_OPTIONS
+	eap_server_mschap_rx_callback(sm, "TTLS-MSCHAP",
+				      sm->identity, sm->identity_len,
+				      challenge, response + 2 + 24);
+#endif /* CONFIG_TESTING_OPTIONS */
+
 	if (os_memcmp_const(challenge, chal, EAP_TTLS_MSCHAP_CHALLENGE_LEN)
 	    != 0 ||
 	    response[0] != chal[EAP_TTLS_MSCHAP_CHALLENGE_LEN]) {
@@ -637,6 +675,7 @@ static void eap_ttls_process_phase2_mschap(struct eap_sm *sm,
 	if (os_memcmp_const(nt_response, response + 2 + 24, 24) == 0) {
 		wpa_printf(MSG_DEBUG, "EAP-TTLS/MSCHAP: Correct response");
 		eap_ttls_state(data, SUCCESS);
+		eap_ttls_valid_session(sm, data);
 	} else {
 		wpa_printf(MSG_DEBUG, "EAP-TTLS/MSCHAP: Invalid NT-Response");
 		wpa_hexdump(MSG_MSGDUMP, "EAP-TTLS/MSCHAP: Received",
@@ -740,6 +779,18 @@ static void eap_ttls_process_phase2_mschapv2(struct eap_sm *sm,
 	}
 
 	rx_resp = response + 2 + EAP_TTLS_MSCHAPV2_CHALLENGE_LEN + 8;
+#ifdef CONFIG_TESTING_OPTIONS
+	{
+		u8 challenge2[8];
+
+		if (challenge_hash(peer_challenge, auth_challenge,
+				   username, username_len, challenge2) == 0) {
+			eap_server_mschap_rx_callback(sm, "TTLS-MSCHAPV2",
+						      username, username_len,
+						      challenge2, rx_resp);
+		}
+	}
+#endif /* CONFIG_TESTING_OPTIONS */
 	if (os_memcmp_const(nt_response, rx_resp, 24) == 0) {
 		wpa_printf(MSG_DEBUG, "EAP-TTLS/MSCHAPV2: Correct "
 			   "NT-Response");
@@ -888,6 +939,7 @@ static void eap_ttls_process_phase2_eap_response(struct eap_sm *sm,
 		break;
 	case PHASE2_METHOD:
 		eap_ttls_state(data, SUCCESS);
+		eap_ttls_valid_session(sm, data);
 		break;
 	case FAILURE:
 		break;
@@ -1111,6 +1163,7 @@ static void eap_ttls_process_msg(struct eap_sm *sm, void *priv,
 			wpa_printf(MSG_DEBUG, "EAP-TTLS/MSCHAPV2: Peer "
 				   "acknowledged response");
 			eap_ttls_state(data, SUCCESS);
+			eap_ttls_valid_session(sm, data);
 		} else if (!data->mschapv2_resp_ok) {
 			wpa_printf(MSG_DEBUG, "EAP-TTLS/MSCHAPV2: Peer "
 				   "acknowledged error");
@@ -1137,10 +1190,64 @@ static void eap_ttls_process(struct eap_sm *sm, void *priv,
 			     struct wpabuf *respData)
 {
 	struct eap_ttls_data *data = priv;
+	const struct wpabuf *buf;
+	const u8 *pos;
+	u8 id_len;
+
 	if (eap_server_tls_process(sm, &data->ssl, respData, data,
 				   EAP_TYPE_TTLS, eap_ttls_process_version,
-				   eap_ttls_process_msg) < 0)
+				   eap_ttls_process_msg) < 0) {
 		eap_ttls_state(data, FAILURE);
+		return;
+	}
+
+	if (!tls_connection_established(sm->ssl_ctx, data->ssl.conn) ||
+	    !tls_connection_resumed(sm->ssl_ctx, data->ssl.conn))
+		return;
+
+	buf = tls_connection_get_success_data(data->ssl.conn);
+	if (!buf || wpabuf_len(buf) < 1) {
+		wpa_printf(MSG_DEBUG,
+			   "EAP-TTLS: No success data in resumed session - reject attempt");
+		eap_ttls_state(data, FAILURE);
+		return;
+	}
+
+	pos = wpabuf_head(buf);
+	if (*pos != EAP_TYPE_TTLS) {
+		wpa_printf(MSG_DEBUG,
+			   "EAP-TTLS: Resumed session for another EAP type (%u) - reject attempt",
+			   *pos);
+		eap_ttls_state(data, FAILURE);
+		return;
+	}
+
+	pos++;
+	id_len = *pos++;
+	wpa_hexdump_ascii(MSG_DEBUG, "EAP-TTLS: Identity from cached session",
+			  pos, id_len);
+	os_free(sm->identity);
+	sm->identity = os_malloc(id_len ? id_len : 1);
+	if (!sm->identity) {
+		sm->identity_len = 0;
+		eap_ttls_state(data, FAILURE);
+		return;
+	}
+
+	os_memcpy(sm->identity, pos, id_len);
+	sm->identity_len = id_len;
+
+	if (eap_user_get(sm, sm->identity, sm->identity_len, 1) != 0) {
+		wpa_hexdump_ascii(MSG_DEBUG, "EAP-TTLS: Phase2 Identity not found in the user database",
+				  sm->identity, sm->identity_len);
+		eap_ttls_state(data, FAILURE);
+		return;
+	}
+
+	wpa_printf(MSG_DEBUG,
+		   "EAP-TTLS: Resuming previous session - skip Phase2");
+	eap_ttls_state(data, SUCCESS);
+	tls_connection_set_success_data_resumed(data->ssl.conn);
 }
 
 
diff --git a/src/eap_server/eap_tls_common.h b/src/eap_server/eap_tls_common.h
index ddf90b859ee4..dc943eb207d7 100644
--- a/src/eap_server/eap_tls_common.h
+++ b/src/eap_server/eap_tls_common.h
@@ -70,7 +70,7 @@ struct eap_ssl_data {
 struct wpabuf * eap_tls_msg_alloc(EapType type, size_t payload_len,
 				  u8 code, u8 identifier);
 int eap_server_tls_ssl_init(struct eap_sm *sm, struct eap_ssl_data *data,
-			    int verify_peer);
+			    int verify_peer, int eap_type);
 void eap_server_tls_ssl_deinit(struct eap_sm *sm, struct eap_ssl_data *data);
 u8 * eap_server_tls_derive_key(struct eap_sm *sm, struct eap_ssl_data *data,
 			       char *label, size_t len);
diff --git a/src/eapol_auth/Makefile b/src/eapol_auth/Makefile
index adfd3dfd5b9b..7b927a127731 100644
--- a/src/eapol_auth/Makefile
+++ b/src/eapol_auth/Makefile
@@ -1,8 +1,16 @@
-all:
-	@echo Nothing to be made.
+all: libeapol_auth.a
 
 clean:
-	rm -f *~ *.o *.d *.gcno *.gcda *.gcov
+	rm -f *~ *.o *.d *.gcno *.gcda *.gcov libeapol_auth.a
 
 install:
 	@echo Nothing to be made.
+
+include ../lib.rules
+
+LIB_OBJS = eapol_auth_sm.o eapol_auth_dump.o
+
+libeapol_auth.a: $(LIB_OBJS)
+	$(AR) crT $@ $?
+
+-include $(OBJS:%.o=%.d)
diff --git a/src/eapol_auth/eapol_auth_sm.c b/src/eapol_auth/eapol_auth_sm.c
index 0df6eb56416b..ff33d286223f 100644
--- a/src/eapol_auth/eapol_auth_sm.c
+++ b/src/eapol_auth/eapol_auth_sm.c
@@ -1,6 +1,6 @@
 /*
  * IEEE 802.1X-2004 Authenticator - EAPOL state machine
- * Copyright (c) 2002-2014, Jouni Malinen <j@w1.fi>
+ * Copyright (c) 2002-2015, Jouni Malinen <j@w1.fi>
  *
  * This software may be distributed under the terms of the BSD license.
  * See README for more details.
@@ -22,7 +22,7 @@
 #define STATE_MACHINE_DEBUG_PREFIX "IEEE 802.1X"
 #define STATE_MACHINE_ADDR sm->addr
 
-static struct eapol_callbacks eapol_cb;
+static const struct eapol_callbacks eapol_cb;
 
 /* EAPOL state machines are described in IEEE Std 802.1X-2004, Chap. 8.2 */
 
@@ -198,6 +198,18 @@ SM_STATE(AUTH_PAE, INITIALIZE)
 {
 	SM_ENTRY_MA(AUTH_PAE, INITIALIZE, auth_pae);
 	sm->portMode = Auto;
+
+	/*
+	 * Clearing keyRun here is not specified in IEEE Std 802.1X-2004, but
+	 * it looks like this would be logical thing to do here since the
+	 * EAPOL-Key exchange is not possible in this state. It is possible to
+	 * get here on disconnection event without advancing to the
+	 * AUTHENTICATING state to clear keyRun before the IEEE 802.11 RSN
+	 * authenticator state machine runs and that may advance from
+	 * AUTHENTICATION2 to INITPMK if keyRun = TRUE has been left from the
+	 * last association. This can be avoided by clearing keyRun here.
+	 */
+	sm->keyRun = FALSE;
 }
 
 
@@ -835,6 +847,7 @@ eapol_auth_alloc(struct eapol_authenticator *eapol, const u8 *addr,
 	eap_conf.server_id = eapol->conf.server_id;
 	eap_conf.server_id_len = eapol->conf.server_id_len;
 	eap_conf.erp = eapol->conf.erp;
+	eap_conf.tls_session_lifetime = eapol->conf.tls_session_lifetime;
 	sm->eap = eap_server_sm_init(sm, &eapol_cb, &eap_conf);
 	if (sm->eap == NULL) {
 		eapol_auth_free(sm);
@@ -1056,7 +1069,7 @@ static int eapol_sm_erp_add_key(void *ctx, struct eap_server_erp_key *erp)
 }
 
 
-static struct eapol_callbacks eapol_cb =
+static const struct eapol_callbacks eapol_cb =
 {
 	eapol_sm_get_eap_user,
 	eapol_sm_get_eap_req_id_text,
@@ -1080,6 +1093,87 @@ int eapol_auth_eap_pending_cb(struct eapol_state_machine *sm, void *ctx)
 }
 
 
+void eapol_auth_reauthenticate(struct eapol_state_machine *sm)
+{
+	wpa_printf(MSG_DEBUG, "EAPOL: External reauthentication trigger for "
+		   MACSTR, MAC2STR(sm->addr));
+	sm->reAuthenticate = TRUE;
+	eapol_auth_step(sm);
+}
+
+
+int eapol_auth_set_conf(struct eapol_state_machine *sm, const char *param,
+			const char *value)
+{
+	wpa_printf(MSG_DEBUG, "EAPOL: External configuration operation for "
+		   MACSTR " - param=%s value=%s",
+		   MAC2STR(sm->addr), param, value);
+
+	if (os_strcasecmp(param, "AdminControlledDirections") == 0) {
+		if (os_strcmp(value, "Both") == 0)
+			sm->adminControlledDirections = Both;
+		else if (os_strcmp(value, "In") == 0)
+			sm->adminControlledDirections = In;
+		else
+			return -1;
+		eapol_auth_step(sm);
+		return 0;
+	}
+
+	if (os_strcasecmp(param, "AdminControlledPortControl") == 0) {
+		if (os_strcmp(value, "ForceAuthorized") == 0)
+			sm->portControl = ForceAuthorized;
+		else if (os_strcmp(value, "ForceUnauthorized") == 0)
+			sm->portControl = ForceUnauthorized;
+		else if (os_strcmp(value, "Auto") == 0)
+			sm->portControl = Auto;
+		else
+			return -1;
+		eapol_auth_step(sm);
+		return 0;
+	}
+
+	if (os_strcasecmp(param, "quietPeriod") == 0) {
+		sm->quietPeriod = atoi(value);
+		return 0;
+	}
+
+	if (os_strcasecmp(param, "serverTimeout") == 0) {
+		sm->serverTimeout = atoi(value);
+		return 0;
+	}
+
+	if (os_strcasecmp(param, "reAuthPeriod") == 0) {
+		sm->reAuthPeriod = atoi(value);
+		return 0;
+	}
+
+	if (os_strcasecmp(param, "reAuthEnabled") == 0) {
+		if (os_strcmp(value, "TRUE") == 0)
+			sm->reAuthEnabled = TRUE;
+		else if (os_strcmp(value, "FALSE") == 0)
+			sm->reAuthEnabled = FALSE;
+		else
+			return -1;
+		eapol_auth_step(sm);
+		return 0;
+	}
+
+	if (os_strcasecmp(param, "KeyTransmissionEnabled") == 0) {
+		if (os_strcmp(value, "TRUE") == 0)
+			sm->keyTxEnabled = TRUE;
+		else if (os_strcmp(value, "FALSE") == 0)
+			sm->keyTxEnabled = FALSE;
+		else
+			return -1;
+		eapol_auth_step(sm);
+		return 0;
+	}
+
+	return -1;
+}
+
+
 static int eapol_auth_conf_clone(struct eapol_auth_config *dst,
 				 struct eapol_auth_config *src)
 {
@@ -1148,6 +1242,7 @@ static int eapol_auth_conf_clone(struct eapol_auth_config *dst,
 	}
 	dst->erp_send_reauth_start = src->erp_send_reauth_start;
 	dst->erp = src->erp;
+	dst->tls_session_lifetime = src->tls_session_lifetime;
 
 	return 0;
 
diff --git a/src/eapol_auth/eapol_auth_sm.h b/src/eapol_auth/eapol_auth_sm.h
index ebed19adefc7..e1974e4354da 100644
--- a/src/eapol_auth/eapol_auth_sm.h
+++ b/src/eapol_auth/eapol_auth_sm.h
@@ -1,6 +1,6 @@
 /*
  * IEEE 802.1X-2004 Authenticator - EAPOL state machine
- * Copyright (c) 2002-2009, Jouni Malinen <j@w1.fi>
+ * Copyright (c) 2002-2015, Jouni Malinen <j@w1.fi>
  *
  * This software may be distributed under the terms of the BSD license.
  * See README for more details.
@@ -27,6 +27,7 @@ struct eapol_auth_config {
 	int erp_send_reauth_start;
 	char *erp_domain; /* a copy of this will be allocated */
 	int erp; /* Whether ERP is enabled on authentication server */
+	unsigned int tls_session_lifetime;
 	u8 *pac_opaque_encr_key;
 	u8 *eap_fast_a_id;
 	size_t eap_fast_a_id_len;
@@ -94,5 +95,8 @@ void eapol_auth_step(struct eapol_state_machine *sm);
 int eapol_auth_dump_state(struct eapol_state_machine *sm, char *buf,
 			  size_t buflen);
 int eapol_auth_eap_pending_cb(struct eapol_state_machine *sm, void *ctx);
+void eapol_auth_reauthenticate(struct eapol_state_machine *sm);
+int eapol_auth_set_conf(struct eapol_state_machine *sm, const char *param,
+			const char *value);
 
 #endif /* EAPOL_AUTH_SM_H */
diff --git a/src/eapol_supp/Makefile b/src/eapol_supp/Makefile
index adfd3dfd5b9b..80db9d48689e 100644
--- a/src/eapol_supp/Makefile
+++ b/src/eapol_supp/Makefile
@@ -1,8 +1,18 @@
-all:
-	@echo Nothing to be made.
+all: libeapol_supp.a
 
 clean:
-	rm -f *~ *.o *.d *.gcno *.gcda *.gcov
+	rm -f *~ *.o *.d *.gcno *.gcda *.gcov libeapol_supp.a
 
 install:
 	@echo Nothing to be made.
+
+include ../lib.rules
+
+CFLAGS += -DIEEE8021X_EAPOL
+
+LIB_OBJS = eapol_supp_sm.o
+
+libeapol_supp.a: $(LIB_OBJS)
+	$(AR) crT $@ $?
+
+-include $(OBJS:%.o=%.d)
diff --git a/src/eapol_supp/eapol_supp_sm.c b/src/eapol_supp/eapol_supp_sm.c
index 9cc234a82b09..09cf4f6b9222 100644
--- a/src/eapol_supp/eapol_supp_sm.c
+++ b/src/eapol_supp/eapol_supp_sm.c
@@ -244,7 +244,8 @@ SM_STATE(SUPP_PAE, DISCONNECTED)
 
 SM_STATE(SUPP_PAE, CONNECTING)
 {
-	int send_start = sm->SUPP_PAE_state == SUPP_PAE_CONNECTING;
+	int send_start = sm->SUPP_PAE_state == SUPP_PAE_CONNECTING ||
+		sm->SUPP_PAE_state == SUPP_PAE_HELD;
 	SM_ENTRY(SUPP_PAE, CONNECTING);
 
 	if (sm->eapTriggerStart)
@@ -653,7 +654,9 @@ static void eapol_sm_processKey(struct eapol_sm *sm)
 	struct ieee802_1x_eapol_key *key;
 	struct eap_key_data keydata;
 	u8 orig_key_sign[IEEE8021X_KEY_SIGN_LEN], datakey[32];
+#ifndef CONFIG_NO_RC4
 	u8 ekey[IEEE8021X_KEY_IV_LEN + IEEE8021X_ENCR_KEY_LEN];
+#endif /* CONFIG_NO_RC4 */
 	int key_len, res, sign_key_len, encr_key_len;
 	u16 rx_key_length;
 	size_t plen;
@@ -747,6 +750,13 @@ static void eapol_sm_processKey(struct eapol_sm *sm)
 		return;
 	}
 	if (key_len == rx_key_length) {
+#ifdef CONFIG_NO_RC4
+		if (encr_key_len) {
+			/* otherwise unused */
+		}
+		wpa_printf(MSG_ERROR, "EAPOL: RC4 not supported in the build");
+		return;
+#else /* CONFIG_NO_RC4 */
 		os_memcpy(ekey, key->key_iv, IEEE8021X_KEY_IV_LEN);
 		os_memcpy(ekey + IEEE8021X_KEY_IV_LEN, keydata.encr_key,
 			  encr_key_len);
@@ -755,6 +765,7 @@ static void eapol_sm_processKey(struct eapol_sm *sm)
 			 datakey, key_len);
 		wpa_hexdump_key(MSG_DEBUG, "EAPOL: Decrypted(RC4) key",
 				datakey, key_len);
+#endif /* CONFIG_NO_RC4 */
 	} else if (key_len == 0) {
 		/*
 		 * IEEE 802.1X-2004 specifies that least significant Key Length
@@ -1997,7 +2008,7 @@ static void eapol_sm_set_anon_id(void *ctx, const u8 *id, size_t len)
 }
 
 
-static struct eapol_callbacks eapol_cb =
+static const struct eapol_callbacks eapol_cb =
 {
 	eapol_sm_get_config,
 	eapol_sm_get_bool,
diff --git a/src/fst/Makefile b/src/fst/Makefile
new file mode 100644
index 000000000000..9c41962fd7e1
--- /dev/null
+++ b/src/fst/Makefile
@@ -0,0 +1,8 @@
+all:
+	@echo Nothing to be made.
+
+clean:
+	rm -f *~ *.o *.d
+
+install:
+	@echo Nothing to be made.
diff --git a/src/fst/fst.c b/src/fst/fst.c
new file mode 100644
index 000000000000..2880870213e6
--- /dev/null
+++ b/src/fst/fst.c
@@ -0,0 +1,225 @@
+/*
+ * FST module implementation
+ * Copyright (c) 2014, Qualcomm Atheros, Inc.
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#include "utils/includes.h"
+
+#include "utils/common.h"
+#include "utils/eloop.h"
+#include "fst/fst.h"
+#include "fst/fst_internal.h"
+#include "fst/fst_defs.h"
+#include "fst/fst_ctrl_iface.h"
+
+struct dl_list fst_global_ctrls_list;
+
+
+static void fst_ctrl_iface_notify_peer_state_change(struct fst_iface *iface,
+						    Boolean connected,
+						    const u8 *peer_addr)
+{
+	union fst_event_extra extra;
+
+	extra.peer_state.connected = connected;
+	os_strlcpy(extra.peer_state.ifname, fst_iface_get_name(iface),
+		   sizeof(extra.peer_state.ifname));
+	os_memcpy(extra.peer_state.addr, peer_addr, ETH_ALEN);
+
+	foreach_fst_ctrl_call(on_event, EVENT_PEER_STATE_CHANGED,
+			      iface, NULL, &extra);
+}
+
+
+struct fst_iface * fst_attach(const char *ifname, const u8 *own_addr,
+			      const struct fst_wpa_obj *iface_obj,
+			      const struct fst_iface_cfg *cfg)
+{
+	struct fst_group *g;
+	struct fst_group *group = NULL;
+	struct fst_iface *iface = NULL;
+	Boolean new_group = FALSE;
+
+	WPA_ASSERT(ifname != NULL);
+	WPA_ASSERT(iface_obj != NULL);
+	WPA_ASSERT(cfg != NULL);
+
+	foreach_fst_group(g) {
+		if (os_strcmp(cfg->group_id, fst_group_get_id(g)) == 0) {
+			group = g;
+			break;
+		}
+	}
+
+	if (!group) {
+		group = fst_group_create(cfg->group_id);
+		if (!group) {
+			fst_printf(MSG_ERROR, "%s: FST group cannot be created",
+				   cfg->group_id);
+			return NULL;
+		}
+		new_group = TRUE;
+	}
+
+	iface = fst_iface_create(group, ifname, own_addr, iface_obj, cfg);
+	if (!iface) {
+		fst_printf_group(group, MSG_ERROR, "cannot create iface for %s",
+				 ifname);
+		if (new_group)
+			fst_group_delete(group);
+		return NULL;
+	}
+
+	fst_group_attach_iface(group, iface);
+	fst_group_update_ie(group);
+
+	foreach_fst_ctrl_call(on_iface_added, iface);
+
+	fst_printf_iface(iface, MSG_DEBUG,
+			 "iface attached to group %s (prio=%d, llt=%d)",
+			 cfg->group_id, cfg->priority, cfg->llt);
+
+	return iface;
+}
+
+
+void fst_detach(struct fst_iface *iface)
+{
+	struct fst_group *group = fst_iface_get_group(iface);
+
+	fst_printf_iface(iface, MSG_DEBUG, "iface detached from group %s",
+			 fst_group_get_id(group));
+	fst_session_global_on_iface_detached(iface);
+	foreach_fst_ctrl_call(on_iface_removed, iface);
+	fst_group_detach_iface(group, iface);
+	fst_iface_delete(iface);
+	fst_group_update_ie(group);
+	fst_group_delete_if_empty(group);
+}
+
+
+int fst_global_init(void)
+{
+	dl_list_init(&fst_global_groups_list);
+	dl_list_init(&fst_global_ctrls_list);
+	fst_session_global_init();
+	return 0;
+}
+
+
+void fst_global_deinit(void)
+{
+	struct fst_group *group;
+	struct fst_ctrl_handle *h;
+
+	fst_session_global_deinit();
+	while ((group = fst_first_group()) != NULL)
+		fst_group_delete(group);
+	while ((h = dl_list_first(&fst_global_ctrls_list,
+				  struct fst_ctrl_handle,
+				  global_ctrls_lentry)))
+		fst_global_del_ctrl(h);
+}
+
+
+struct fst_ctrl_handle * fst_global_add_ctrl(const struct fst_ctrl *ctrl)
+{
+	struct fst_ctrl_handle *h;
+
+	if (!ctrl)
+		return NULL;
+
+	h = os_zalloc(sizeof(*h));
+	if (!h)
+		return NULL;
+
+	if (ctrl->init && ctrl->init()) {
+		os_free(h);
+		return NULL;
+	}
+
+	h->ctrl = *ctrl;
+	dl_list_add_tail(&fst_global_ctrls_list, &h->global_ctrls_lentry);
+
+	return h;
+}
+
+
+void fst_global_del_ctrl(struct fst_ctrl_handle *h)
+{
+	dl_list_del(&h->global_ctrls_lentry);
+	if (h->ctrl.deinit)
+		h->ctrl.deinit();
+	os_free(h);
+}
+
+
+void fst_rx_action(struct fst_iface *iface, const struct ieee80211_mgmt *mgmt,
+		   size_t len)
+{
+	if (fst_iface_is_connected(iface, mgmt->sa))
+		fst_session_on_action_rx(iface, mgmt, len);
+	else
+		wpa_printf(MSG_DEBUG,
+			   "FST: Ignore FST Action frame - no FST connection with "
+			   MACSTR, MAC2STR(mgmt->sa));
+}
+
+
+void fst_notify_peer_connected(struct fst_iface *iface, const u8 *addr)
+{
+	if (is_zero_ether_addr(addr))
+		return;
+
+#ifndef HOSTAPD
+	fst_group_update_ie(fst_iface_get_group(iface));
+#endif /* HOSTAPD */
+
+	fst_printf_iface(iface, MSG_DEBUG, MACSTR " became connected",
+			 MAC2STR(addr));
+
+	fst_ctrl_iface_notify_peer_state_change(iface, TRUE, addr);
+}
+
+
+void fst_notify_peer_disconnected(struct fst_iface *iface, const u8 *addr)
+{
+	if (is_zero_ether_addr(addr))
+		return;
+
+#ifndef HOSTAPD
+	fst_group_update_ie(fst_iface_get_group(iface));
+#endif /* HOSTAPD */
+
+	fst_printf_iface(iface, MSG_DEBUG, MACSTR " became disconnected",
+			 MAC2STR(addr));
+
+	fst_ctrl_iface_notify_peer_state_change(iface, FALSE, addr);
+}
+
+
+Boolean fst_are_ifaces_aggregated(struct fst_iface *iface1,
+				  struct fst_iface *iface2)
+{
+	return fst_iface_get_group(iface1) == fst_iface_get_group(iface2);
+}
+
+
+enum mb_band_id fst_hw_mode_to_band(enum hostapd_hw_mode mode)
+{
+	switch (mode) {
+	case HOSTAPD_MODE_IEEE80211B:
+	case HOSTAPD_MODE_IEEE80211G:
+		return MB_BAND_ID_WIFI_2_4GHZ;
+	case HOSTAPD_MODE_IEEE80211A:
+		return MB_BAND_ID_WIFI_5GHZ;
+	case HOSTAPD_MODE_IEEE80211AD:
+		return MB_BAND_ID_WIFI_60GHZ;
+	default:
+		WPA_ASSERT(0);
+		return MB_BAND_ID_WIFI_2_4GHZ;
+	}
+}
diff --git a/src/fst/fst.h b/src/fst/fst.h
new file mode 100644
index 000000000000..0c0e435b974b
--- /dev/null
+++ b/src/fst/fst.h
@@ -0,0 +1,296 @@
+/*
+ * FST module - interface definitions
+ * Copyright (c) 2014, Qualcomm Atheros, Inc.
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#ifndef FST_H
+#define FST_H
+
+#ifdef CONFIG_FST
+
+#include "common/defs.h"
+#include "fst/fst_ctrl_iface.h"
+
+/* FST module hostap integration API */
+
+#define US_IN_MS           1000
+#define LLT_UNIT_US        32 /* See 10.32.2.2  Transitioning between states */
+
+#define FST_LLT_MS_TO_VAL(m) (((u32) (m)) * US_IN_MS / LLT_UNIT_US)
+#define FST_LLT_VAL_TO_MS(v) (((u32) (v)) * LLT_UNIT_US / US_IN_MS)
+
+#define FST_MAX_LLT_MS       FST_LLT_VAL_TO_MS(-1)
+#define FST_MAX_PRIO_VALUE   ((u8) -1)
+#define FST_MAX_GROUP_ID_LEN IFNAMSIZ
+
+#define FST_DEFAULT_LLT_CFG_VALUE 50
+
+struct hostapd_hw_modes;
+struct ieee80211_mgmt;
+struct fst_iface;
+struct fst_group;
+struct fst_session;
+struct fst_get_peer_ctx;
+struct fst_ctrl_handle;
+
+struct fst_wpa_obj {
+	void *ctx;
+
+	/**
+	 * get_bssid - Get BSSID of the interface
+	 * @ctx: User context %ctx
+	 * Returns: BSSID for success, %NULL for failure.
+	 *
+	 * NOTE: For AP it returns the own BSSID, while for STA - the BSSID of
+	 * the associated AP.
+	 */
+	const u8 * (*get_bssid)(void *ctx);
+
+	/**
+	 * get_channel_info - Get current channel info
+	 * @ctx: User context %ctx
+	 * @hw_mode: OUT, current HW mode
+	 * @channel: OUT, current channel
+	 */
+	void (*get_channel_info)(void *ctx, enum hostapd_hw_mode *hw_mode,
+				 u8 *channel);
+
+	/**
+	 * get_hw_modes - Get hardware modes
+	 * @ctx: User context %ctx
+	 * @modes: OUT, pointer on array of hw modes
+	 *
+	 * Returns: Number of hw modes available.
+	 */
+	int (*get_hw_modes)(void *ctx, struct hostapd_hw_modes **modes);
+
+	/**
+	 * set_ies - Set interface's MB IE
+	 * @ctx: User context %ctx
+	 * @fst_ies: MB IE buffer (owned by FST module)
+	 */
+	void (*set_ies)(void *ctx, const struct wpabuf *fst_ies);
+
+	/**
+	 * send_action - Send FST Action frame via the interface
+	 * @ctx: User context %ctx
+	 * @addr: Address of the destination STA
+	 * @data: Action frame buffer
+	 * Returns: 0 for success, negative error code for failure.
+	 */
+	int (*send_action)(void *ctx, const u8 *addr, struct wpabuf *data);
+
+	/**
+	 * get_mb_ie - Get last MB IE received from STA
+	 * @ctx: User context %ctx
+	 * @addr: Address of the STA
+	 * Returns: MB IE buffer, %NULL if no MB IE received from the STA
+	 */
+	const struct wpabuf * (*get_mb_ie)(void *ctx, const u8 *addr);
+
+	/**
+	 * update_mb_ie - Update last MB IE received from STA
+	 * @ctx: User context %ctx
+	 * @addr: Address of the STA
+	 * @buf: Buffer that contains the MB IEs data
+	 * @size: Size of data in %buf
+	 */
+	void (*update_mb_ie)(void *ctx, const u8 *addr,
+			     const u8 *buf, size_t size);
+
+	/**
+	 * get_peer_first - Get MAC address of the 1st connected STA
+	 * @ctx: User context %ctx
+	 * @get_ctx: Context to be used for %get_peer_next call
+	 * @mb_only: %TRUE if only multi-band capable peer should be reported
+	 * Returns: Address of the 1st connected STA, %NULL if no STAs connected
+	 */
+	const u8 * (*get_peer_first)(void *ctx,
+				     struct fst_get_peer_ctx **get_ctx,
+				     Boolean mb_only);
+	/**
+	 * get_peer_next - Get MAC address of the next connected STA
+	 * @ctx: User context %ctx
+	 * @get_ctx: Context received from %get_peer_first or previous
+	 *           %get_peer_next call
+	 * @mb_only: %TRUE if only multi-band capable peer should be reported
+	 * Returns: Address of the next connected STA, %NULL if no more STAs
+	 *          connected
+	 */
+	const u8 * (*get_peer_next)(void *ctx,
+				    struct fst_get_peer_ctx **get_ctx,
+				    Boolean mb_only);
+};
+
+/**
+ * fst_global_init - Global FST module initiator
+ * Returns: 0 for success, negative error code for failure.
+ * Note: The purpose of this function is to allocate and initiate global
+ *       FST module data structures (linked lists, static data etc.)
+ *       This function should be called prior to the 1st %fst_attach call.
+ */
+int fst_global_init(void);
+
+/**
+ * fst_global_deinit - Global FST module de-initiator
+ * Note: The purpose of this function is to deallocate and de-initiate global
+ *       FST module data structures (linked lists, static data etc.)
+ */
+void fst_global_deinit(void);
+
+/**
+ * struct fst_ctrl - Notification interface for FST module
+ */
+struct fst_ctrl {
+	/**
+	 * init - Initialize the notification interface
+	 * Returns: 0 for success, negative error code for failure.
+	 */
+	int (*init)(void);
+
+	/**
+	 * deinit - Deinitialize the notification interface
+	 */
+	void (*deinit)(void);
+
+	/**
+	 * on_group_created - Notify about FST group creation
+	 * Returns: 0 for success, negative error code for failure.
+	 */
+	int (*on_group_created)(struct fst_group *g);
+
+	/**
+	 * on_group_deleted - Notify about FST group deletion
+	 */
+	void (*on_group_deleted)(struct fst_group *g);
+
+	/**
+	 * on_iface_added - Notify about interface addition
+	 * Returns: 0 for success, negative error code for failure.
+	 */
+	int (*on_iface_added)(struct fst_iface *i);
+
+	/**
+	 * on_iface_removed - Notify about interface removal
+	 */
+	void (*on_iface_removed)(struct fst_iface *i);
+
+	/**
+	 * on_session_added - Notify about FST session addition
+	 * Returns: 0 for success, negative error code for failure.
+	 */
+	int (*on_session_added)(struct fst_session *s);
+
+	/**
+	 * on_session_removed - Notify about FST session removal
+	 */
+	void (*on_session_removed)(struct fst_session *s);
+
+	/**
+	 * on_event - Notify about FST event
+	 * @event_type: Event type
+	 * @i: Interface object that relates to the event or NULL
+	 * @g: Group object that relates to the event or NULL
+	 * @extra - Event specific data (see fst_ctrl_iface.h for more info)
+	 */
+	void (*on_event)(enum fst_event_type event_type, struct fst_iface *i,
+			 struct fst_session *s,
+			 const union fst_event_extra *extra);
+};
+
+struct fst_ctrl_handle * fst_global_add_ctrl(const struct fst_ctrl *ctrl);
+void fst_global_del_ctrl(struct fst_ctrl_handle *h);
+
+/**
+ * NOTE: These values have to be read from configuration file
+ */
+struct fst_iface_cfg {
+	char group_id[FST_MAX_GROUP_ID_LEN + 1];
+	u8 priority;
+	u32 llt;
+};
+
+/**
+ * fst_attach - Attach interface to an FST group according to configuration read
+ * @ifname: Interface name
+ * @own_addr: Own interface MAC address
+ * @iface_obj: Callbacks to be used by FST module to communicate with
+ *             hostapd/wpa_supplicant
+ * @cfg: FST-related interface configuration read from the configuration file
+ * Returns: FST interface object for success, %NULL for failure.
+ */
+struct fst_iface * fst_attach(const char *ifname,
+			      const u8 *own_addr,
+			      const struct fst_wpa_obj *iface_obj,
+			      const struct fst_iface_cfg *cfg);
+
+/**
+ * fst_detach - Detach an interface
+ * @iface: FST interface object
+ */
+void fst_detach(struct fst_iface *iface);
+
+/* FST module inputs */
+/**
+ * fst_rx_action - FST Action frames handler
+ * @iface: FST interface object
+ * @mgmt: Action frame arrived
+ * @len: Action frame length
+ */
+void fst_rx_action(struct fst_iface *iface, const struct ieee80211_mgmt *mgmt,
+		   size_t len);
+
+/**
+ * fst_notify_peer_connected - FST STA connect handler
+ * @iface: FST interface object
+ * @addr: Address of the connected STA
+ */
+void fst_notify_peer_connected(struct fst_iface *iface, const u8 *addr);
+
+/**
+ * fst_notify_peer_disconnected - FST STA disconnect handler
+ * @iface: FST interface object
+ * @addr: Address of the disconnected STA
+ */
+void fst_notify_peer_disconnected(struct fst_iface *iface, const u8 *addr);
+
+/* FST module auxiliary routines */
+
+/**
+ * fst_are_ifaces_aggregated - Determines whether 2 interfaces belong to the
+ *                             same FST group
+ * @iface1: 1st FST interface object
+ * @iface1: 2nd FST interface object
+ *
+ * Returns: %TRUE if the interfaces belong to the same FST group,
+ *          %FALSE otherwise
+ */
+Boolean fst_are_ifaces_aggregated(struct fst_iface *iface1,
+				  struct fst_iface *iface2);
+
+#else /* CONFIG_FST */
+
+static inline int fst_global_init(void)
+{
+	return 0;
+}
+
+static inline int fst_global_start(void)
+{
+	return 0;
+}
+
+static inline void fst_global_stop(void)
+{
+}
+
+static inline void fst_global_deinit(void)
+{
+}
+
+#endif /* CONFIG_FST */
+
+#endif /* FST_H */
diff --git a/src/fst/fst_ctrl_aux.c b/src/fst/fst_ctrl_aux.c
new file mode 100644
index 000000000000..dc7b2a7d7202
--- /dev/null
+++ b/src/fst/fst_ctrl_aux.c
@@ -0,0 +1,69 @@
+/*
+ * FST module implementation
+ * Copyright (c) 2014, Qualcomm Atheros, Inc.
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#include "utils/includes.h"
+#include "utils/common.h"
+#include "common/defs.h"
+#include "fst_ctrl_defs.h"
+#include "fst_ctrl_aux.h"
+
+
+static const char *session_event_names[] = {
+	[EVENT_FST_ESTABLISHED] FST_PVAL_EVT_TYPE_ESTABLISHED,
+	[EVENT_FST_SETUP] FST_PVAL_EVT_TYPE_SETUP,
+	[EVENT_FST_SESSION_STATE_CHANGED] FST_PVAL_EVT_TYPE_SESSION_STATE,
+};
+
+static const char *reason_names[] = {
+	[REASON_TEARDOWN] FST_CS_PVAL_REASON_TEARDOWN,
+	[REASON_SETUP] FST_CS_PVAL_REASON_SETUP,
+	[REASON_SWITCH] FST_CS_PVAL_REASON_SWITCH,
+	[REASON_STT] FST_CS_PVAL_REASON_STT,
+	[REASON_REJECT] FST_CS_PVAL_REASON_REJECT,
+	[REASON_ERROR_PARAMS] FST_CS_PVAL_REASON_ERROR_PARAMS,
+	[REASON_RESET] FST_CS_PVAL_REASON_RESET,
+	[REASON_DETACH_IFACE] FST_CS_PVAL_REASON_DETACH_IFACE,
+};
+
+static const char *session_state_names[] = {
+	[FST_SESSION_STATE_INITIAL] FST_CS_PVAL_STATE_INITIAL,
+	[FST_SESSION_STATE_SETUP_COMPLETION] FST_CS_PVAL_STATE_SETUP_COMPLETION,
+	[FST_SESSION_STATE_TRANSITION_DONE] FST_CS_PVAL_STATE_TRANSITION_DONE,
+	[FST_SESSION_STATE_TRANSITION_CONFIRMED]
+	FST_CS_PVAL_STATE_TRANSITION_CONFIRMED,
+};
+
+
+/* helpers */
+const char * fst_get_str_name(unsigned index, const char *names[],
+			      size_t names_size)
+{
+	if (index >= names_size || !names[index])
+		return FST_NAME_UNKNOWN;
+	return names[index];
+}
+
+
+const char * fst_session_event_type_name(enum fst_event_type event)
+{
+	return fst_get_str_name(event, session_event_names,
+				ARRAY_SIZE(session_event_names));
+}
+
+
+const char * fst_reason_name(enum fst_reason reason)
+{
+	return fst_get_str_name(reason, reason_names, ARRAY_SIZE(reason_names));
+}
+
+
+const char * fst_session_state_name(enum fst_session_state state)
+{
+	return fst_get_str_name(state, session_state_names,
+				ARRAY_SIZE(session_state_names));
+}
diff --git a/src/fst/fst_ctrl_aux.h b/src/fst/fst_ctrl_aux.h
new file mode 100644
index 000000000000..e2133f5062bd
--- /dev/null
+++ b/src/fst/fst_ctrl_aux.h
@@ -0,0 +1,91 @@
+/*
+ * FST module - miscellaneous definitions
+ * Copyright (c) 2014, Qualcomm Atheros, Inc.
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#ifndef FST_CTRL_AUX_H
+#define FST_CTRL_AUX_H
+
+#include "common/defs.h"
+
+/* FST module control interface API */
+#define FST_INVALID_SESSION_ID ((u32) -1)
+#define FST_MAX_GROUP_ID_SIZE   32
+#define FST_MAX_INTERFACE_SIZE  32
+
+enum fst_session_state {
+	FST_SESSION_STATE_INITIAL,
+	FST_SESSION_STATE_SETUP_COMPLETION,
+	FST_SESSION_STATE_TRANSITION_DONE,
+	FST_SESSION_STATE_TRANSITION_CONFIRMED,
+	FST_SESSION_STATE_LAST
+};
+
+enum fst_event_type {
+	EVENT_FST_IFACE_STATE_CHANGED,  /* An interface has been either attached
+					 * to or detached from an FST group */
+	EVENT_FST_ESTABLISHED,          /* FST Session has been established */
+	EVENT_FST_SETUP,                /* FST Session request received */
+	EVENT_FST_SESSION_STATE_CHANGED,/* FST Session state has been changed */
+	EVENT_PEER_STATE_CHANGED        /* FST related generic event occurred,
+					 * see struct fst_hostap_event_data for
+					 *  more info */
+};
+
+enum fst_initiator {
+	FST_INITIATOR_UNDEFINED,
+	FST_INITIATOR_LOCAL,
+	FST_INITIATOR_REMOTE,
+};
+
+union fst_event_extra {
+	struct fst_event_extra_iface_state {
+		Boolean attached;
+		char ifname[FST_MAX_INTERFACE_SIZE];
+		char group_id[FST_MAX_GROUP_ID_SIZE];
+	} iface_state; /* for EVENT_FST_IFACE_STATE_CHANGED */
+	struct fst_event_extra_peer_state {
+		Boolean connected;
+		char ifname[FST_MAX_INTERFACE_SIZE];
+		u8 addr[ETH_ALEN];
+	} peer_state; /* for EVENT_PEER_STATE_CHANGED */
+	struct fst_event_extra_session_state {
+		enum fst_session_state old_state;
+		enum fst_session_state new_state;
+		union fst_session_state_switch_extra {
+			struct {
+				enum fst_reason {
+					REASON_TEARDOWN,
+					REASON_SETUP,
+					REASON_SWITCH,
+					REASON_STT,
+					REASON_REJECT,
+					REASON_ERROR_PARAMS,
+					REASON_RESET,
+					REASON_DETACH_IFACE,
+				} reason;
+				u8 reject_code; /* REASON_REJECT */
+				/* REASON_SWITCH,
+				 * REASON_TEARDOWN,
+				 * REASON_REJECT
+				 */
+				enum fst_initiator initiator;
+			} to_initial;
+		} extra;
+	} session_state; /* for EVENT_FST_SESSION_STATE_CHANGED */
+};
+
+/* helpers - prints enum in string form */
+#define FST_NAME_UNKNOWN "UNKNOWN"
+
+const char * fst_get_str_name(unsigned index, const char *names[],
+			      size_t names_size);
+
+const char * fst_session_event_type_name(enum fst_event_type);
+const char * fst_reason_name(enum fst_reason reason);
+const char * fst_session_state_name(enum fst_session_state state);
+
+#endif /* FST_CTRL_AUX_H */
diff --git a/src/fst/fst_ctrl_defs.h b/src/fst/fst_ctrl_defs.h
new file mode 100644
index 000000000000..67353890ffcf
--- /dev/null
+++ b/src/fst/fst_ctrl_defs.h
@@ -0,0 +1,109 @@
+/*
+ * FST module - shared Control interface definitions
+ * Copyright (c) 2014, Qualcomm Atheros, Inc.
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#ifndef FST_CTRL_DEFS_H
+#define FST_CTRL_DEFS_H
+
+/* Undefined value */
+#define FST_CTRL_PVAL_NONE                     "NONE"
+
+/* FST-ATTACH parameters */
+#define FST_ATTACH_CMD_PNAME_LLT      "llt"      /* pval = desired LLT */
+#define FST_ATTACH_CMD_PNAME_PRIORITY "priority" /* pval = desired priority */
+
+/* FST-MANAGER parameters */
+/* FST Session states */
+#define FST_CS_PVAL_STATE_INITIAL              "INITIAL"
+#define FST_CS_PVAL_STATE_SETUP_COMPLETION     "SETUP_COMPLETION"
+#define FST_CS_PVAL_STATE_TRANSITION_DONE      "TRANSITION_DONE"
+#define FST_CS_PVAL_STATE_TRANSITION_CONFIRMED "TRANSITION_CONFIRMED"
+
+/* FST Session reset reasons */
+#define FST_CS_PVAL_REASON_TEARDOWN     "REASON_TEARDOWN"
+#define FST_CS_PVAL_REASON_SETUP        "REASON_SETUP"
+#define FST_CS_PVAL_REASON_SWITCH       "REASON_SWITCH"
+#define FST_CS_PVAL_REASON_STT          "REASON_STT"
+#define FST_CS_PVAL_REASON_REJECT       "REASON_REJECT"
+#define FST_CS_PVAL_REASON_ERROR_PARAMS "REASON_ERROR_PARAMS"
+#define FST_CS_PVAL_REASON_RESET        "REASON_RESET"
+#define FST_CS_PVAL_REASON_DETACH_IFACE "REASON_DETACH_IFACE"
+
+/* FST Session responses */
+#define FST_CS_PVAL_RESPONSE_ACCEPT "ACCEPT"
+#define FST_CS_PVAL_RESPONSE_REJECT "REJECT"
+
+/* FST Session action initiator */
+#define FST_CS_PVAL_INITIATOR_LOCAL  "LOCAL"
+#define FST_CS_PVAL_INITIATOR_REMOTE "REMOTE"
+
+/* FST-CLI subcommands and parameter names */
+#define FST_CMD_LIST_GROUPS      "list_groups"
+#define FST_CMD_LIST_IFACES      "list_ifaces"
+#define FST_CMD_IFACE_PEERS      "iface_peers"
+#define FST_CMD_GET_PEER_MBIES   "get_peer_mbies"
+#define FST_CMD_LIST_SESSIONS    "list_sessions"
+#define FST_CMD_SESSION_ADD      "session_add"
+#define FST_CMD_SESSION_REMOVE   "session_remove"
+#define FST_CMD_SESSION_GET      "session_get"
+#define FST_CSG_PNAME_OLD_PEER_ADDR  "old_peer_addr" /* pval = address string */
+#define FST_CSG_PNAME_NEW_PEER_ADDR  "new_peer_addr" /* pval = address string */
+#define FST_CSG_PNAME_OLD_IFNAME "old_ifname" /* pval = ifname */
+#define FST_CSG_PNAME_NEW_IFNAME "new_ifname" /* pval = ifname */
+#define FST_CSG_PNAME_LLT        "llt"        /* pval = numeric llt value */
+#define FST_CSG_PNAME_STATE      "state"      /* pval = FST_CS_PVAL_STATE_... */
+#define FST_CMD_SESSION_SET      "session_set"
+#define FST_CSS_PNAME_OLD_PEER_ADDR  FST_CSG_PNAME_OLD_PEER_ADDR
+#define FST_CSS_PNAME_NEW_PEER_ADDR  FST_CSG_PNAME_NEW_PEER_ADDR
+#define FST_CSS_PNAME_OLD_IFNAME     FST_CSG_PNAME_OLD_IFNAME
+#define FST_CSS_PNAME_NEW_IFNAME     FST_CSG_PNAME_NEW_IFNAME
+#define FST_CSS_PNAME_LLT            FST_CSG_PNAME_LLT
+#define FST_CMD_SESSION_INITIATE "session_initiate"
+#define FST_CMD_SESSION_RESPOND  "session_respond"
+#define FST_CMD_SESSION_TRANSFER "session_transfer"
+#define FST_CMD_SESSION_TEARDOWN "session_teardown"
+
+#ifdef CONFIG_FST_TEST
+#define FST_CTR_PVAL_BAD_NEW_BAND        "bad_new_band"
+
+#define FST_CMD_TEST_REQUEST    "test_request"
+#define FST_CTR_IS_SUPPORTED        "is_supported"
+#define FST_CTR_SEND_SETUP_REQUEST  "send_setup_request"
+#define FST_CTR_SEND_SETUP_RESPONSE "send_setup_response"
+#define FST_CTR_SEND_ACK_REQUEST    "send_ack_request"
+#define FST_CTR_SEND_ACK_RESPONSE   "send_ack_response"
+#define FST_CTR_SEND_TEAR_DOWN      "send_tear_down"
+#define FST_CTR_GET_FSTS_ID         "get_fsts_id"
+#define FST_CTR_GET_LOCAL_MBIES     "get_local_mbies"
+#endif /* CONFIG_FST_TEST */
+
+/* Events */
+#define FST_CTRL_EVENT_IFACE "FST-EVENT-IFACE"
+#define FST_CEI_PNAME_IFNAME       "ifname"
+#define FST_CEI_PNAME_GROUP        "group"
+#define FST_CEI_PNAME_ATTACHED     "attached"
+#define FST_CEI_PNAME_DETACHED     "detached"
+#define FST_CTRL_EVENT_PEER "FST-EVENT-PEER"
+#define FST_CEP_PNAME_IFNAME       "ifname"
+#define FST_CEP_PNAME_ADDR         "peer_addr"
+#define FST_CEP_PNAME_CONNECTED    "connected"
+#define FST_CEP_PNAME_DISCONNECTED "disconnected"
+#define FST_CTRL_EVENT_SESSION "FST-EVENT-SESSION"
+#define FST_CES_PNAME_SESSION_ID "session_id"
+#define FST_CES_PNAME_EVT_TYPE   "event_type"
+#define FST_PVAL_EVT_TYPE_SESSION_STATE "EVENT_FST_SESSION_STATE"
+/* old_state/new_state: pval = FST_CS_PVAL_STATE_... */
+#define FST_CES_PNAME_OLD_STATE   "old_state"
+#define FST_CES_PNAME_NEW_STATE   "new_state"
+#define FST_CES_PNAME_REASON      "reason" /* pval = FST_CS_PVAL_REASON_... */
+#define FST_CES_PNAME_REJECT_CODE "reject_code" /* pval = u8 code */
+/* pval = FST_CS_PVAL_INITIATOR_... */
+#define FST_CES_PNAME_INITIATOR   "initiator"
+#define FST_PVAL_EVT_TYPE_ESTABLISHED "EVENT_FST_ESTABLISHED"
+#define FST_PVAL_EVT_TYPE_SETUP "EVENT_FST_SETUP"
+
+#endif /* FST_CTRL_DEFS_H */
diff --git a/src/fst/fst_ctrl_iface.c b/src/fst/fst_ctrl_iface.c
new file mode 100644
index 000000000000..d0907188a389
--- /dev/null
+++ b/src/fst/fst_ctrl_iface.c
@@ -0,0 +1,948 @@
+/*
+ * FST module - Control Interface implementation
+ * Copyright (c) 2014, Qualcomm Atheros, Inc.
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#include "utils/includes.h"
+#include "utils/common.h"
+#include "common/defs.h"
+#include "list.h"
+#include "fst/fst.h"
+#include "fst/fst_internal.h"
+#include "fst_ctrl_defs.h"
+#include "fst_ctrl_iface.h"
+
+
+static struct fst_group * get_fst_group_by_id(const char *id)
+{
+	struct fst_group *g;
+
+	foreach_fst_group(g) {
+		const char *group_id = fst_group_get_id(g);
+
+		if (os_strncmp(group_id, id, os_strlen(group_id)) == 0)
+			return g;
+	}
+
+	return NULL;
+}
+
+
+/* notifications */
+static Boolean format_session_state_extra(const union fst_event_extra *extra,
+					  char *buffer, size_t size)
+{
+	int len;
+	char reject_str[32] = FST_CTRL_PVAL_NONE;
+	const char *initiator = FST_CTRL_PVAL_NONE;
+	const struct fst_event_extra_session_state *ss;
+
+	ss = &extra->session_state;
+	if (ss->new_state != FST_SESSION_STATE_INITIAL)
+		return TRUE;
+
+	switch (ss->extra.to_initial.reason) {
+	case REASON_REJECT:
+		if (ss->extra.to_initial.reject_code != WLAN_STATUS_SUCCESS)
+			os_snprintf(reject_str, sizeof(reject_str), "%u",
+				    ss->extra.to_initial.reject_code);
+		/* no break */
+	case REASON_TEARDOWN:
+	case REASON_SWITCH:
+		switch (ss->extra.to_initial.initiator) {
+		case FST_INITIATOR_LOCAL:
+			initiator = FST_CS_PVAL_INITIATOR_LOCAL;
+			break;
+		case FST_INITIATOR_REMOTE:
+			initiator = FST_CS_PVAL_INITIATOR_REMOTE;
+			break;
+		default:
+			break;
+		}
+		break;
+	default:
+		break;
+	}
+
+	len = os_snprintf(buffer, size,
+			  FST_CES_PNAME_REASON "=%s "
+			  FST_CES_PNAME_REJECT_CODE "=%s "
+			  FST_CES_PNAME_INITIATOR "=%s",
+			  fst_reason_name(ss->extra.to_initial.reason),
+			  reject_str, initiator);
+
+	return !os_snprintf_error(size, len);
+}
+
+
+static void fst_ctrl_iface_notify(struct fst_iface *f, u32 session_id,
+				  enum fst_event_type event_type,
+				  const union fst_event_extra *extra)
+{
+	struct fst_group *g;
+	char extra_str[128] = "";
+	const struct fst_event_extra_session_state *ss;
+	const struct fst_event_extra_iface_state *is;
+	const struct fst_event_extra_peer_state *ps;
+
+	/*
+	 * FST can use any of interface objects as it only sends messages
+	 * on global Control Interface, so we just pick the 1st one.
+	 */
+
+	if (!f) {
+		foreach_fst_group(g) {
+			f = fst_group_first_iface(g);
+			if (f)
+				break;
+		}
+		if (!f)
+			return;
+	}
+
+	WPA_ASSERT(f->iface_obj.ctx);
+
+	switch (event_type) {
+	case EVENT_FST_IFACE_STATE_CHANGED:
+		if (!extra)
+			return;
+		is = &extra->iface_state;
+		wpa_msg_global_only(f->iface_obj.ctx, MSG_INFO,
+				    FST_CTRL_EVENT_IFACE " %s "
+				    FST_CEI_PNAME_IFNAME "=%s "
+				    FST_CEI_PNAME_GROUP "=%s",
+				    is->attached ? FST_CEI_PNAME_ATTACHED :
+				    FST_CEI_PNAME_DETACHED,
+				    is->ifname, is->group_id);
+		break;
+	case EVENT_PEER_STATE_CHANGED:
+		if (!extra)
+			return;
+		ps = &extra->peer_state;
+		wpa_msg_global_only(fst_iface_get_wpa_obj_ctx(f), MSG_INFO,
+				    FST_CTRL_EVENT_PEER " %s "
+				    FST_CEP_PNAME_IFNAME "=%s "
+				    FST_CEP_PNAME_ADDR "=" MACSTR,
+				    ps->connected ? FST_CEP_PNAME_CONNECTED :
+				    FST_CEP_PNAME_DISCONNECTED,
+				    ps->ifname, MAC2STR(ps->addr));
+		break;
+	case EVENT_FST_SESSION_STATE_CHANGED:
+		if (!extra)
+			return;
+		if (!format_session_state_extra(extra, extra_str,
+						sizeof(extra_str))) {
+			fst_printf(MSG_ERROR,
+				   "CTRL: Cannot format STATE_CHANGE extra");
+			extra_str[0] = 0;
+		}
+		ss = &extra->session_state;
+		wpa_msg_global_only(fst_iface_get_wpa_obj_ctx(f), MSG_INFO,
+				    FST_CTRL_EVENT_SESSION " "
+				    FST_CES_PNAME_SESSION_ID "=%u "
+				    FST_CES_PNAME_EVT_TYPE "=%s "
+				    FST_CES_PNAME_OLD_STATE "=%s "
+				    FST_CES_PNAME_NEW_STATE "=%s %s",
+				    session_id,
+				    fst_session_event_type_name(event_type),
+				    fst_session_state_name(ss->old_state),
+				    fst_session_state_name(ss->new_state),
+				    extra_str);
+		break;
+	case EVENT_FST_ESTABLISHED:
+	case EVENT_FST_SETUP:
+		wpa_msg_global_only(fst_iface_get_wpa_obj_ctx(f), MSG_INFO,
+				    FST_CTRL_EVENT_SESSION " "
+				    FST_CES_PNAME_SESSION_ID "=%u "
+				    FST_CES_PNAME_EVT_TYPE "=%s",
+				    session_id,
+				    fst_session_event_type_name(event_type));
+		break;
+	}
+}
+
+
+/* command processors */
+
+/* fst session_get */
+static int session_get(const char *session_id, char *buf, size_t buflen)
+{
+	struct fst_session *s;
+	struct fst_iface *new_iface, *old_iface;
+	const u8 *old_peer_addr, *new_peer_addr;
+	u32 id;
+
+	id = strtoul(session_id, NULL, 0);
+
+	s = fst_session_get_by_id(id);
+	if (!s) {
+		fst_printf(MSG_WARNING, "CTRL: Cannot find session %u", id);
+		return os_snprintf(buf, buflen, "FAIL\n");
+	}
+
+	old_peer_addr = fst_session_get_peer_addr(s, TRUE);
+	new_peer_addr = fst_session_get_peer_addr(s, FALSE);
+	new_iface = fst_session_get_iface(s, FALSE);
+	old_iface = fst_session_get_iface(s, TRUE);
+
+	return os_snprintf(buf, buflen,
+			   FST_CSG_PNAME_OLD_PEER_ADDR "=" MACSTR "\n"
+			   FST_CSG_PNAME_NEW_PEER_ADDR "=" MACSTR "\n"
+			   FST_CSG_PNAME_NEW_IFNAME "=%s\n"
+			   FST_CSG_PNAME_OLD_IFNAME "=%s\n"
+			   FST_CSG_PNAME_LLT "=%u\n"
+			   FST_CSG_PNAME_STATE "=%s\n",
+			   MAC2STR(old_peer_addr),
+			   MAC2STR(new_peer_addr),
+			   new_iface ? fst_iface_get_name(new_iface) :
+			   FST_CTRL_PVAL_NONE,
+			   old_iface ? fst_iface_get_name(old_iface) :
+			   FST_CTRL_PVAL_NONE,
+			   fst_session_get_llt(s),
+			   fst_session_state_name(fst_session_get_state(s)));
+}
+
+
+/* fst session_set */
+static int session_set(const char *session_id, char *buf, size_t buflen)
+{
+	struct fst_session *s;
+	char *p, *q;
+	u32 id;
+	int ret;
+
+	id = strtoul(session_id, &p, 0);
+
+	s = fst_session_get_by_id(id);
+	if (!s) {
+		fst_printf(MSG_WARNING, "CTRL: Cannot find session %u", id);
+		return os_snprintf(buf, buflen, "FAIL\n");
+	}
+
+	if (*p != ' ' || !(q = os_strchr(p + 1, '=')))
+		return os_snprintf(buf, buflen, "FAIL\n");
+	p++;
+
+	if (os_strncasecmp(p, FST_CSS_PNAME_OLD_IFNAME, q - p) == 0) {
+		ret = fst_session_set_str_ifname(s, q + 1, TRUE);
+	} else if (os_strncasecmp(p, FST_CSS_PNAME_NEW_IFNAME, q - p) == 0) {
+		ret = fst_session_set_str_ifname(s, q + 1, FALSE);
+	} else if (os_strncasecmp(p, FST_CSS_PNAME_OLD_PEER_ADDR, q - p) == 0) {
+		ret = fst_session_set_str_peer_addr(s, q + 1, TRUE);
+	} else if (os_strncasecmp(p, FST_CSS_PNAME_NEW_PEER_ADDR, q - p) == 0) {
+		ret = fst_session_set_str_peer_addr(s, q + 1, FALSE);
+	} else if (os_strncasecmp(p, FST_CSS_PNAME_LLT, q - p) == 0) {
+		ret = fst_session_set_str_llt(s, q + 1);
+	} else {
+		fst_printf(MSG_ERROR, "CTRL: Unknown parameter: %s", p);
+		return os_snprintf(buf, buflen, "FAIL\n");
+	}
+
+	return os_snprintf(buf, buflen, "%s\n", ret ? "FAIL" : "OK");
+}
+
+
+/* fst session_add/remove */
+static int session_add(const char *group_id, char *buf, size_t buflen)
+{
+	struct fst_group *g;
+	struct fst_session *s;
+
+	g = get_fst_group_by_id(group_id);
+	if (!g) {
+		fst_printf(MSG_WARNING, "CTRL: Cannot find group '%s'",
+			   group_id);
+		return os_snprintf(buf, buflen, "FAIL\n");
+	}
+
+	s = fst_session_create(g);
+	if (!s) {
+		fst_printf(MSG_ERROR,
+			   "CTRL: Cannot create session for group '%s'",
+			   group_id);
+		return os_snprintf(buf, buflen, "FAIL\n");
+	}
+
+	return os_snprintf(buf, buflen, "%u\n", fst_session_get_id(s));
+}
+
+
+static int session_remove(const char *session_id, char *buf, size_t buflen)
+{
+	struct fst_session *s;
+	struct fst_group *g;
+	u32 id;
+
+	id = strtoul(session_id, NULL, 0);
+
+	s = fst_session_get_by_id(id);
+	if (!s) {
+		fst_printf(MSG_WARNING, "CTRL: Cannot find session %u", id);
+		return os_snprintf(buf, buflen, "FAIL\n");
+	}
+
+	g = fst_session_get_group(s);
+	fst_session_reset(s);
+	fst_session_delete(s);
+	fst_group_delete_if_empty(g);
+
+	return os_snprintf(buf, buflen, "OK\n");
+}
+
+
+/* fst session_initiate */
+static int session_initiate(const char *session_id, char *buf, size_t buflen)
+{
+	struct fst_session *s;
+	u32 id;
+
+	id = strtoul(session_id, NULL, 0);
+
+	s = fst_session_get_by_id(id);
+	if (!s) {
+		fst_printf(MSG_WARNING, "CTRL: Cannot find session %u", id);
+		return os_snprintf(buf, buflen, "FAIL\n");
+	}
+
+	if (fst_session_initiate_setup(s)) {
+		fst_printf(MSG_WARNING, "CTRL: Cannot initiate session %u", id);
+		return os_snprintf(buf, buflen, "FAIL\n");
+	}
+
+	return os_snprintf(buf, buflen, "OK\n");
+}
+
+
+/* fst session_respond */
+static int session_respond(const char *session_id, char *buf, size_t buflen)
+{
+	struct fst_session *s;
+	char *p;
+	u32 id;
+	u8 status_code;
+
+	id = strtoul(session_id, &p, 0);
+
+	s = fst_session_get_by_id(id);
+	if (!s) {
+		fst_printf(MSG_WARNING, "CTRL: Cannot find session %u", id);
+		return os_snprintf(buf, buflen, "FAIL\n");
+	}
+
+	if (*p != ' ')
+		return os_snprintf(buf, buflen, "FAIL\n");
+	p++;
+
+	if (!os_strcasecmp(p, FST_CS_PVAL_RESPONSE_ACCEPT)) {
+		status_code = WLAN_STATUS_SUCCESS;
+	} else if (!os_strcasecmp(p, FST_CS_PVAL_RESPONSE_REJECT)) {
+		status_code = WLAN_STATUS_PENDING_ADMITTING_FST_SESSION;
+	} else {
+		fst_printf(MSG_WARNING,
+			   "CTRL: session %u: unknown response status: %s",
+			   id, p);
+		return os_snprintf(buf, buflen, "FAIL\n");
+	}
+
+	if (fst_session_respond(s, status_code)) {
+		fst_printf(MSG_WARNING, "CTRL: Cannot respond to session %u",
+			   id);
+		return os_snprintf(buf, buflen, "FAIL\n");
+	}
+
+	fst_printf(MSG_INFO, "CTRL: session %u responded", id);
+
+	return os_snprintf(buf, buflen, "OK\n");
+}
+
+
+/* fst session_transfer */
+static int session_transfer(const char *session_id, char *buf, size_t buflen)
+{
+	struct fst_session *s;
+	u32 id;
+
+	id = strtoul(session_id, NULL, 0);
+
+	s = fst_session_get_by_id(id);
+	if (!s) {
+		fst_printf(MSG_WARNING, "CTRL: Cannot find session %u", id);
+		return os_snprintf(buf, buflen, "FAIL\n");
+	}
+
+	if (fst_session_initiate_switch(s)) {
+		fst_printf(MSG_WARNING,
+			   "CTRL: Cannot initiate ST for session %u", id);
+		return os_snprintf(buf, buflen, "FAIL\n");
+	}
+
+	return os_snprintf(buf, buflen, "OK\n");
+}
+
+
+/* fst session_teardown */
+static int session_teardown(const char *session_id, char *buf, size_t buflen)
+{
+	struct fst_session *s;
+	u32 id;
+
+	id = strtoul(session_id, NULL, 0);
+
+	s = fst_session_get_by_id(id);
+	if (!s) {
+		fst_printf(MSG_WARNING, "CTRL: Cannot find session %u", id);
+		return os_snprintf(buf, buflen, "FAIL\n");
+	}
+
+	if (fst_session_tear_down_setup(s)) {
+		fst_printf(MSG_WARNING, "CTRL: Cannot tear down session %u",
+			   id);
+		return os_snprintf(buf, buflen, "FAIL\n");
+	}
+
+	return os_snprintf(buf, buflen, "OK\n");
+}
+
+
+#ifdef CONFIG_FST_TEST
+/* fst test_request */
+static int test_request(const char *request, char *buf, size_t buflen)
+{
+	const char *p = request;
+	int ret;
+
+	if (!os_strncasecmp(p, FST_CTR_SEND_SETUP_REQUEST,
+			    os_strlen(FST_CTR_SEND_SETUP_REQUEST))) {
+		ret = fst_test_req_send_fst_request(
+			p + os_strlen(FST_CTR_SEND_SETUP_REQUEST));
+	} else if (!os_strncasecmp(p, FST_CTR_SEND_SETUP_RESPONSE,
+				   os_strlen(FST_CTR_SEND_SETUP_RESPONSE))) {
+		ret = fst_test_req_send_fst_response(
+			p + os_strlen(FST_CTR_SEND_SETUP_RESPONSE));
+	} else if (!os_strncasecmp(p, FST_CTR_SEND_ACK_REQUEST,
+				   os_strlen(FST_CTR_SEND_ACK_REQUEST))) {
+		ret = fst_test_req_send_ack_request(
+			p + os_strlen(FST_CTR_SEND_ACK_REQUEST));
+	} else if (!os_strncasecmp(p, FST_CTR_SEND_ACK_RESPONSE,
+				   os_strlen(FST_CTR_SEND_ACK_RESPONSE))) {
+		ret = fst_test_req_send_ack_response(
+			p + os_strlen(FST_CTR_SEND_ACK_RESPONSE));
+	} else if (!os_strncasecmp(p, FST_CTR_SEND_TEAR_DOWN,
+				   os_strlen(FST_CTR_SEND_TEAR_DOWN))) {
+		ret = fst_test_req_send_tear_down(
+			p + os_strlen(FST_CTR_SEND_TEAR_DOWN));
+	} else if (!os_strncasecmp(p, FST_CTR_GET_FSTS_ID,
+				   os_strlen(FST_CTR_GET_FSTS_ID))) {
+		u32 fsts_id = fst_test_req_get_fsts_id(
+			p + os_strlen(FST_CTR_GET_FSTS_ID));
+		if (fsts_id != FST_FSTS_ID_NOT_FOUND)
+			return os_snprintf(buf, buflen, "%u\n", fsts_id);
+		return os_snprintf(buf, buflen, "FAIL\n");
+	} else if (!os_strncasecmp(p, FST_CTR_GET_LOCAL_MBIES,
+				   os_strlen(FST_CTR_GET_LOCAL_MBIES))) {
+		return fst_test_req_get_local_mbies(
+			p + os_strlen(FST_CTR_GET_LOCAL_MBIES), buf, buflen);
+	} else if (!os_strncasecmp(p, FST_CTR_IS_SUPPORTED,
+				   os_strlen(FST_CTR_IS_SUPPORTED))) {
+		ret = 0;
+	} else {
+		fst_printf(MSG_ERROR, "CTRL: Unknown parameter: %s", p);
+		return os_snprintf(buf, buflen, "FAIL\n");
+	}
+
+	return os_snprintf(buf, buflen, "%s\n", ret ? "FAIL" : "OK");
+}
+#endif /* CONFIG_FST_TEST */
+
+
+/* fst list_sessions */
+struct list_sessions_cb_ctx {
+	char *buf;
+	size_t buflen;
+	size_t reply_len;
+};
+
+
+static void list_session_enum_cb(struct fst_group *g, struct fst_session *s,
+				 void *ctx)
+{
+	struct list_sessions_cb_ctx *c = ctx;
+	int ret;
+
+	ret = os_snprintf(c->buf, c->buflen, " %u", fst_session_get_id(s));
+
+	c->buf += ret;
+	c->buflen -= ret;
+	c->reply_len += ret;
+}
+
+
+static int list_sessions(const char *group_id, char *buf, size_t buflen)
+{
+	struct list_sessions_cb_ctx ctx;
+	struct fst_group *g;
+
+	g = get_fst_group_by_id(group_id);
+	if (!g) {
+		fst_printf(MSG_WARNING, "CTRL: Cannot find group '%s'",
+			   group_id);
+		return os_snprintf(buf, buflen, "FAIL\n");
+	}
+
+	ctx.buf = buf;
+	ctx.buflen = buflen;
+	ctx.reply_len = 0;
+
+	fst_session_enum(g, list_session_enum_cb, &ctx);
+
+	ctx.reply_len += os_snprintf(buf + ctx.reply_len, ctx.buflen, "\n");
+
+	return ctx.reply_len;
+}
+
+
+/* fst iface_peers */
+static int iface_peers(const char *group_id, char *buf, size_t buflen)
+{
+	const char *ifname;
+	struct fst_group *g;
+	struct fst_iface *f;
+	struct fst_get_peer_ctx *ctx;
+	const u8 *addr;
+	unsigned found = 0;
+	int ret = 0;
+
+	g = get_fst_group_by_id(group_id);
+	if (!g) {
+		fst_printf(MSG_WARNING, "CTRL: Cannot find group '%s'",
+			   group_id);
+		return os_snprintf(buf, buflen, "FAIL\n");
+	}
+
+	ifname = os_strchr(group_id, ' ');
+	if (!ifname)
+		return os_snprintf(buf, buflen, "FAIL\n");
+	ifname++;
+
+	foreach_fst_group_iface(g, f) {
+		const char *in = fst_iface_get_name(f);
+
+		if (os_strncmp(ifname, in, os_strlen(in)) == 0) {
+			found = 1;
+			break;
+		}
+	}
+
+	if (!found)
+		return os_snprintf(buf, buflen, "FAIL\n");
+
+	addr = fst_iface_get_peer_first(f, &ctx, FALSE);
+	for (; addr != NULL; addr = fst_iface_get_peer_next(f, &ctx, FALSE)) {
+		int res;
+
+		res = os_snprintf(buf + ret, buflen - ret, MACSTR "\n",
+				  MAC2STR(addr));
+		if (os_snprintf_error(buflen - ret, res))
+			break;
+		ret += res;
+	}
+
+	return ret;
+}
+
+
+static int get_peer_mbies(const char *params, char *buf, size_t buflen)
+{
+	char *endp;
+	char ifname[FST_MAX_INTERFACE_SIZE];
+	u8 peer_addr[ETH_ALEN];
+	struct fst_group *g;
+	struct fst_iface *iface = NULL;
+	const struct wpabuf *mbies;
+
+	if (fst_read_next_text_param(params, ifname, sizeof(ifname), &endp) ||
+	    !*ifname)
+		goto problem;
+
+	while (isspace(*endp))
+		endp++;
+	if (fst_read_peer_addr(endp, peer_addr))
+		goto problem;
+
+	foreach_fst_group(g) {
+		iface = fst_group_get_iface_by_name(g, ifname);
+		if (iface)
+			break;
+	}
+	if (!iface)
+		goto problem;
+
+	mbies = fst_iface_get_peer_mb_ie(iface, peer_addr);
+	if (!mbies)
+		goto problem;
+
+	return wpa_snprintf_hex(buf, buflen, wpabuf_head(mbies),
+				wpabuf_len(mbies));
+
+problem:
+	return os_snprintf(buf, buflen, "FAIL\n");
+}
+
+
+/* fst list_ifaces */
+static int list_ifaces(const char *group_id, char *buf, size_t buflen)
+{
+	struct fst_group *g;
+	struct fst_iface *f;
+	int ret = 0;
+
+	g = get_fst_group_by_id(group_id);
+	if (!g) {
+		fst_printf(MSG_WARNING, "CTRL: Cannot find group '%s'",
+			   group_id);
+		return os_snprintf(buf, buflen, "FAIL\n");
+	}
+
+	foreach_fst_group_iface(g, f) {
+		int res;
+		const u8 *iface_addr = fst_iface_get_addr(f);
+
+		res = os_snprintf(buf + ret, buflen - ret,
+				  "%s|" MACSTR "|%u|%u\n",
+				  fst_iface_get_name(f),
+				  MAC2STR(iface_addr),
+				  fst_iface_get_priority(f),
+				  fst_iface_get_llt(f));
+		if (os_snprintf_error(buflen - ret, res))
+			break;
+		ret += res;
+	}
+
+	return ret;
+}
+
+
+/* fst list_groups */
+static int list_groups(const char *cmd, char *buf, size_t buflen)
+{
+	struct fst_group *g;
+	int ret = 0;
+
+	foreach_fst_group(g) {
+		int res;
+
+		res = os_snprintf(buf + ret, buflen - ret, "%s\n",
+				  fst_group_get_id(g));
+		if (os_snprintf_error(buflen - ret, res))
+			break;
+		ret += res;
+	}
+
+	return ret;
+}
+
+
+static const char * band_freq(enum mb_band_id band)
+{
+	static const char *band_names[] = {
+		[MB_BAND_ID_WIFI_2_4GHZ] "2.4GHZ",
+		[MB_BAND_ID_WIFI_5GHZ] "5GHZ",
+		[MB_BAND_ID_WIFI_60GHZ] "60GHZ",
+	};
+
+	return fst_get_str_name(band, band_names, ARRAY_SIZE(band_names));
+}
+
+
+static int print_band(unsigned num, struct fst_iface *iface, const u8 *addr,
+		      char *buf, size_t buflen)
+{
+	const struct wpabuf *wpabuf;
+	enum hostapd_hw_mode hw_mode;
+	u8 channel;
+	int ret = 0;
+
+	fst_iface_get_channel_info(iface, &hw_mode, &channel);
+
+	ret += os_snprintf(buf + ret, buflen - ret, "band%u_frequency=%s\n",
+			   num, band_freq(fst_hw_mode_to_band(hw_mode)));
+	ret += os_snprintf(buf + ret, buflen - ret, "band%u_iface=%s\n",
+			   num, fst_iface_get_name(iface));
+	wpabuf = fst_iface_get_peer_mb_ie(iface, addr);
+	if (wpabuf) {
+		ret += os_snprintf(buf + ret, buflen - ret, "band%u_mb_ies=",
+				   num);
+		ret += wpa_snprintf_hex(buf + ret, buflen - ret,
+					wpabuf_head(wpabuf),
+					wpabuf_len(wpabuf));
+		ret += os_snprintf(buf + ret, buflen - ret, "\n");
+	}
+	ret += os_snprintf(buf + ret, buflen - ret, "band%u_fst_group_id=%s\n",
+			   num, fst_iface_get_group_id(iface));
+	ret += os_snprintf(buf + ret, buflen - ret, "band%u_fst_priority=%u\n",
+			   num, fst_iface_get_priority(iface));
+	ret += os_snprintf(buf + ret, buflen - ret, "band%u_fst_llt=%u\n",
+			   num, fst_iface_get_llt(iface));
+
+	return ret;
+}
+
+
+static void fst_ctrl_iface_on_iface_state_changed(struct fst_iface *i,
+						  Boolean attached)
+{
+	union fst_event_extra extra;
+
+	os_memset(&extra, 0, sizeof(extra));
+	extra.iface_state.attached = attached;
+	os_strlcpy(extra.iface_state.ifname, fst_iface_get_name(i),
+		   sizeof(extra.iface_state.ifname));
+	os_strlcpy(extra.iface_state.group_id, fst_iface_get_group_id(i),
+		   sizeof(extra.iface_state.group_id));
+
+	fst_ctrl_iface_notify(i, FST_INVALID_SESSION_ID,
+			      EVENT_FST_IFACE_STATE_CHANGED, &extra);
+}
+
+
+static int fst_ctrl_iface_on_iface_added(struct fst_iface *i)
+{
+	fst_ctrl_iface_on_iface_state_changed(i, TRUE);
+	return 0;
+}
+
+
+static void fst_ctrl_iface_on_iface_removed(struct fst_iface *i)
+{
+	fst_ctrl_iface_on_iface_state_changed(i, FALSE);
+}
+
+
+static void fst_ctrl_iface_on_event(enum fst_event_type event_type,
+				    struct fst_iface *i, struct fst_session *s,
+				    const union fst_event_extra *extra)
+{
+	u32 session_id = s ? fst_session_get_id(s) : FST_INVALID_SESSION_ID;
+
+	fst_ctrl_iface_notify(i, session_id, event_type, extra);
+}
+
+
+static const struct fst_ctrl ctrl_cli = {
+	.on_iface_added = fst_ctrl_iface_on_iface_added,
+	.on_iface_removed =  fst_ctrl_iface_on_iface_removed,
+	.on_event = fst_ctrl_iface_on_event,
+};
+
+const struct fst_ctrl *fst_ctrl_cli = &ctrl_cli;
+
+
+int fst_ctrl_iface_mb_info(const u8 *addr, char *buf, size_t buflen)
+{
+	struct fst_group *g;
+	struct fst_iface *f;
+	unsigned num = 0;
+	int ret = 0;
+
+	foreach_fst_group(g) {
+		foreach_fst_group_iface(g, f) {
+			if (fst_iface_is_connected(f, addr)) {
+				ret += print_band(num++, f, addr,
+						  buf + ret, buflen - ret);
+			}
+		}
+	}
+
+	return ret;
+}
+
+
+/* fst ctrl processor */
+int fst_ctrl_iface_receive(const char *cmd, char *reply, size_t reply_size)
+{
+	static const struct fst_command {
+		const char *name;
+		unsigned has_param;
+		int (*process)(const char *group_id, char *buf, size_t buflen);
+	} commands[] = {
+		{ FST_CMD_LIST_GROUPS, 0, list_groups},
+		{ FST_CMD_LIST_IFACES, 1, list_ifaces},
+		{ FST_CMD_IFACE_PEERS, 1, iface_peers},
+		{ FST_CMD_GET_PEER_MBIES, 1, get_peer_mbies},
+		{ FST_CMD_LIST_SESSIONS, 1, list_sessions},
+		{ FST_CMD_SESSION_ADD, 1, session_add},
+		{ FST_CMD_SESSION_REMOVE, 1, session_remove},
+		{ FST_CMD_SESSION_GET, 1, session_get},
+		{ FST_CMD_SESSION_SET, 1, session_set},
+		{ FST_CMD_SESSION_INITIATE, 1, session_initiate},
+		{ FST_CMD_SESSION_RESPOND, 1, session_respond},
+		{ FST_CMD_SESSION_TRANSFER, 1, session_transfer},
+		{ FST_CMD_SESSION_TEARDOWN, 1, session_teardown},
+#ifdef CONFIG_FST_TEST
+		{ FST_CMD_TEST_REQUEST, 1, test_request },
+#endif /* CONFIG_FST_TEST */
+		{ NULL, 0, NULL }
+	};
+	const struct fst_command *c;
+	const char *p;
+	const char *temp;
+	Boolean non_spaces_found;
+
+	for (c = commands; c->name; c++) {
+		if (os_strncasecmp(cmd, c->name, os_strlen(c->name)) != 0)
+			continue;
+		p = cmd + os_strlen(c->name);
+		if (c->has_param) {
+			if (!isspace(p[0]))
+				return os_snprintf(reply, reply_size, "FAIL\n");
+			p++;
+			temp = p;
+			non_spaces_found = FALSE;
+			while (*temp) {
+				if (!isspace(*temp)) {
+					non_spaces_found = TRUE;
+					break;
+				}
+				temp++;
+			}
+			if (!non_spaces_found)
+				return os_snprintf(reply, reply_size, "FAIL\n");
+		}
+		return c->process(p, reply, reply_size);
+	}
+
+	return os_snprintf(reply, reply_size, "UNKNOWN FST COMMAND\n");
+}
+
+
+int fst_read_next_int_param(const char *params, Boolean *valid, char **endp)
+{
+	int ret = -1;
+	const char *curp;
+
+	*valid = FALSE;
+	*endp = (char *) params;
+	curp = params;
+	if (*curp) {
+		ret = (int) strtol(curp, endp, 0);
+		if (!**endp || isspace(**endp))
+			*valid = TRUE;
+	}
+
+	return ret;
+}
+
+
+int fst_read_next_text_param(const char *params, char *buf, size_t buflen,
+			     char **endp)
+{
+	size_t max_chars_to_copy;
+	char *cur_dest;
+
+	*endp = (char *) params;
+	while (isspace(**endp))
+		(*endp)++;
+	if (!**endp || buflen <= 1)
+		return -EINVAL;
+
+	max_chars_to_copy = buflen - 1;
+	/* We need 1 byte for the terminating zero */
+	cur_dest = buf;
+	while (**endp && !isspace(**endp) && max_chars_to_copy > 0) {
+		*cur_dest = **endp;
+		(*endp)++;
+		cur_dest++;
+		max_chars_to_copy--;
+	}
+	*cur_dest = 0;
+
+	return 0;
+}
+
+
+int fst_read_peer_addr(const char *mac, u8 *peer_addr)
+{
+	if (hwaddr_aton(mac, peer_addr)) {
+		fst_printf(MSG_WARNING, "Bad peer_mac %s: invalid addr string",
+			   mac);
+		return -1;
+	}
+
+	if (is_zero_ether_addr(peer_addr) ||
+	    is_multicast_ether_addr(peer_addr)) {
+		fst_printf(MSG_WARNING, "Bad peer_mac %s: not a unicast addr",
+			   mac);
+		return -1;
+	}
+
+	return 0;
+}
+
+
+int fst_parse_attach_command(const char *cmd, char *ifname, size_t ifname_size,
+			     struct fst_iface_cfg *cfg)
+{
+	char *pos;
+	char *endp;
+	Boolean is_valid;
+	int val;
+
+	if (fst_read_next_text_param(cmd, ifname, ifname_size, &endp) ||
+	    fst_read_next_text_param(endp, cfg->group_id, sizeof(cfg->group_id),
+				     &endp))
+		return -EINVAL;
+
+	cfg->llt = FST_DEFAULT_LLT_CFG_VALUE;
+	cfg->priority = 0;
+	pos = os_strstr(endp, FST_ATTACH_CMD_PNAME_LLT);
+	if (pos) {
+		pos += os_strlen(FST_ATTACH_CMD_PNAME_LLT);
+		if (*pos == '=') {
+			val = fst_read_next_int_param(pos + 1, &is_valid,
+						      &endp);
+			if (is_valid)
+				cfg->llt = val;
+		}
+	}
+	pos = os_strstr(endp, FST_ATTACH_CMD_PNAME_PRIORITY);
+	if (pos) {
+		pos += os_strlen(FST_ATTACH_CMD_PNAME_PRIORITY);
+		if (*pos == '=') {
+			val = fst_read_next_int_param(pos + 1, &is_valid,
+						      &endp);
+			if (is_valid)
+				cfg->priority = (u8) val;
+		}
+	}
+
+	return 0;
+}
+
+
+int fst_parse_detach_command(const char *cmd, char *ifname, size_t ifname_size)
+{
+	char *endp;
+
+	return fst_read_next_text_param(cmd, ifname, ifname_size, &endp);
+}
+
+
+int fst_iface_detach(const char *ifname)
+{
+	struct fst_group *g;
+
+	foreach_fst_group(g) {
+		struct fst_iface *f;
+
+		f = fst_group_get_iface_by_name(g, ifname);
+		if (f) {
+			fst_detach(f);
+			return 0;
+		}
+	}
+
+	return -EINVAL;
+}
diff --git a/src/fst/fst_ctrl_iface.h b/src/fst/fst_ctrl_iface.h
new file mode 100644
index 000000000000..4d0cd9fce4e0
--- /dev/null
+++ b/src/fst/fst_ctrl_iface.h
@@ -0,0 +1,45 @@
+/*
+ * FST module - internal Control interface definitions
+ * Copyright (c) 2014, Qualcomm Atheros, Inc.
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#ifndef FST_CTRL_IFACE_H
+#define FST_CTRL_IFACE_H
+
+#include "fst/fst_ctrl_aux.h"
+
+#ifdef CONFIG_FST
+
+/* receiver */
+int fst_ctrl_iface_mb_info(const u8 *addr, char *buf, size_t buflen);
+
+int fst_ctrl_iface_receive(const char *txtaddr, char *buf, size_t buflen);
+
+extern const struct fst_ctrl *fst_ctrl_cli;
+
+#else /* CONFIG_FST */
+
+static inline int
+fst_ctrl_iface_mb_info(const u8 *addr, char *buf, size_t buflen)
+{
+	return 0;
+}
+
+#endif /* CONFIG_FST */
+
+int fst_read_next_int_param(const char *params, Boolean *valid, char **endp);
+int fst_read_next_text_param(const char *params, char *buf, size_t buflen,
+			     char **endp);
+int fst_read_peer_addr(const char *mac, u8 *peer_addr);
+
+struct fst_iface_cfg;
+
+int fst_parse_attach_command(const char *cmd, char *ifname, size_t ifname_size,
+			     struct fst_iface_cfg *cfg);
+int fst_parse_detach_command(const char *cmd, char *ifname, size_t ifname_size);
+int fst_iface_detach(const char *ifname);
+
+#endif /* CTRL_IFACE_FST_H */
diff --git a/src/fst/fst_defs.h b/src/fst/fst_defs.h
new file mode 100644
index 000000000000..8ddcc61376b2
--- /dev/null
+++ b/src/fst/fst_defs.h
@@ -0,0 +1,87 @@
+/*
+ * FST module - FST related definitions
+ * Copyright (c) 2014, Qualcomm Atheros, Inc.
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#ifndef IEEE_80211_FST_DEFS_H
+#define IEEE_80211_FST_DEFS_H
+
+/* IEEE Std 802.11ad */
+
+#define MB_STA_CHANNEL_ALL 0
+
+enum session_type {
+	SESSION_TYPE_BSS = 0, /*  Infrastructure BSS */
+	SESSION_TYPE_IBSS = 1,
+	SESSION_TYPE_DLS = 2,
+	SESSION_TYPE_TDLS = 3,
+	SESSION_TYPE_PBSS = 4
+};
+
+#define SESSION_CONTROL(session_type, switch_intent) \
+	(((u8) ((session_type) & 0x7)) | ((switch_intent) ? 0x10 : 0x00))
+
+#define GET_SESSION_CONTROL_TYPE(session_control) \
+	((u8) ((session_control) & 0x7))
+
+#define GET_SESSION_CONTROL_SWITCH_INTENT(session_control) \
+	(((session_control) & 0x10) >> 4)
+
+/* 8.4.2.147  Session Transition element */
+struct session_transition_ie {
+	u8 element_id;
+	u8 length;
+	u32 fsts_id;
+	u8 session_control;
+	u8 new_band_id;
+	u8 new_band_setup;
+	u8 new_band_op;
+	u8 old_band_id;
+	u8 old_band_setup;
+	u8 old_band_op;
+} STRUCT_PACKED;
+
+struct fst_setup_req {
+	u8 action;
+	u8 dialog_token;
+	u32 llt;
+	struct session_transition_ie stie;
+	/* Multi-band (optional) */
+	/* Wakeup Schedule (optional) */
+	/* Awake Window (optional) */
+	/* Switching Stream (optional) */
+} STRUCT_PACKED;
+
+struct fst_setup_res {
+	u8 action;
+	u8 dialog_token;
+	u8 status_code;
+	struct session_transition_ie stie;
+	/* Multi-band (optional) */
+	/* Wakeup Schedule (optional) */
+	/* Awake Window (optional) */
+	/* Switching Stream (optional) */
+	/* Timeout Interval (optional) */
+} STRUCT_PACKED;
+
+struct fst_ack_req {
+	u8 action;
+	u8 dialog_token;
+	u32 fsts_id;
+} STRUCT_PACKED;
+
+struct fst_ack_res {
+	u8 action;
+	u8 dialog_token;
+	u32 fsts_id;
+} STRUCT_PACKED;
+
+struct fst_tear_down {
+	u8 action;
+	u32 fsts_id;
+} STRUCT_PACKED;
+
+#endif /* IEEE_80211_FST_DEFS_H */
diff --git a/src/fst/fst_group.c b/src/fst/fst_group.c
new file mode 100644
index 000000000000..f6c7be9435f4
--- /dev/null
+++ b/src/fst/fst_group.c
@@ -0,0 +1,462 @@
+/*
+ * FST module - FST group object implementation
+ * Copyright (c) 2014, Qualcomm Atheros, Inc.
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#include "utils/includes.h"
+#include "utils/common.h"
+#include "common/defs.h"
+#include "common/ieee802_11_defs.h"
+#include "common/ieee802_11_common.h"
+#include "drivers/driver.h"
+#include "fst/fst_internal.h"
+#include "fst/fst_defs.h"
+
+
+struct dl_list fst_global_groups_list;
+
+#ifndef HOSTAPD
+static Boolean fst_has_fst_peer(struct fst_iface *iface, Boolean *has_peer)
+{
+	const u8 *bssid;
+
+	bssid = fst_iface_get_bssid(iface);
+	if (!bssid) {
+		*has_peer = FALSE;
+		return FALSE;
+	}
+
+	*has_peer = TRUE;
+	return fst_iface_get_peer_mb_ie(iface, bssid) != NULL;
+}
+#endif /* HOSTAPD */
+
+
+static void fst_dump_mb_ies(const char *group_id, const char *ifname,
+			    struct wpabuf *mbies)
+{
+	const u8 *p = wpabuf_head(mbies);
+	size_t s = wpabuf_len(mbies);
+
+	while (s >= 2) {
+		const struct multi_band_ie *mbie =
+			(const struct multi_band_ie *) p;
+		WPA_ASSERT(mbie->eid == WLAN_EID_MULTI_BAND);
+		WPA_ASSERT(2 + mbie->len >= sizeof(*mbie));
+
+		fst_printf(MSG_WARNING,
+			   "%s: %s: mb_ctrl=%u band_id=%u op_class=%u chan=%u bssid="
+			   MACSTR
+			   " beacon_int=%u tsf_offs=[%u %u %u %u %u %u %u %u] mb_cc=0x%02x tmout=%u",
+			   group_id, ifname,
+			   mbie->mb_ctrl, mbie->band_id, mbie->op_class,
+			   mbie->chan, MAC2STR(mbie->bssid), mbie->beacon_int,
+			   mbie->tsf_offs[0], mbie->tsf_offs[1],
+			   mbie->tsf_offs[2], mbie->tsf_offs[3],
+			   mbie->tsf_offs[4], mbie->tsf_offs[5],
+			   mbie->tsf_offs[6], mbie->tsf_offs[7],
+			   mbie->mb_connection_capability,
+			   mbie->fst_session_tmout);
+
+		p += 2 + mbie->len;
+		s -= 2 + mbie->len;
+	}
+}
+
+
+static void fst_fill_mb_ie(struct wpabuf *buf, const u8 *bssid,
+			   const u8 *own_addr, enum mb_band_id band, u8 channel)
+{
+	struct multi_band_ie *mbie;
+	size_t len = sizeof(*mbie);
+
+	if (own_addr)
+		len += ETH_ALEN;
+
+	mbie = wpabuf_put(buf, len);
+
+	os_memset(mbie, 0, len);
+
+	mbie->eid = WLAN_EID_MULTI_BAND;
+	mbie->len = len - 2;
+#ifdef HOSTAPD
+	mbie->mb_ctrl = MB_STA_ROLE_AP;
+	mbie->mb_connection_capability = MB_CONNECTION_CAPABILITY_AP;
+#else /* HOSTAPD */
+	mbie->mb_ctrl = MB_STA_ROLE_NON_PCP_NON_AP;
+	mbie->mb_connection_capability = 0;
+#endif /* HOSTAPD */
+	if (bssid)
+		os_memcpy(mbie->bssid, bssid, ETH_ALEN);
+	mbie->band_id = band;
+	mbie->op_class = 0;  /* means all */
+	mbie->chan = channel;
+	mbie->fst_session_tmout = FST_DEFAULT_SESSION_TIMEOUT_TU;
+
+	if (own_addr) {
+		mbie->mb_ctrl |= MB_CTRL_STA_MAC_PRESENT;
+		os_memcpy(&mbie[1], own_addr, ETH_ALEN);
+	}
+}
+
+
+static unsigned fst_fill_iface_mb_ies(struct fst_iface *f, struct wpabuf *buf)
+{
+	const  u8 *bssid;
+
+	bssid = fst_iface_get_bssid(f);
+	if (bssid) {
+		enum hostapd_hw_mode hw_mode;
+		u8 channel;
+
+		if (buf) {
+			fst_iface_get_channel_info(f, &hw_mode, &channel);
+			fst_fill_mb_ie(buf, bssid, fst_iface_get_addr(f),
+				       fst_hw_mode_to_band(hw_mode), channel);
+		}
+		return 1;
+	} else {
+		unsigned bands[MB_BAND_ID_WIFI_60GHZ + 1] = {};
+		struct hostapd_hw_modes *modes;
+		enum mb_band_id b;
+		int num_modes = fst_iface_get_hw_modes(f, &modes);
+		int ret = 0;
+
+		while (num_modes--) {
+			b = fst_hw_mode_to_band(modes->mode);
+			modes++;
+			if (b >= ARRAY_SIZE(bands) || bands[b]++)
+				continue;
+			ret++;
+			if (buf)
+				fst_fill_mb_ie(buf, NULL, fst_iface_get_addr(f),
+					       b, MB_STA_CHANNEL_ALL);
+		}
+		return ret;
+	}
+}
+
+
+static struct wpabuf * fst_group_create_mb_ie(struct fst_group *g,
+					      struct fst_iface *i)
+{
+	struct wpabuf *buf;
+	struct fst_iface *f;
+	unsigned int nof_mbies = 0;
+	unsigned int nof_ifaces_added = 0;
+#ifndef HOSTAPD
+	Boolean has_peer;
+	Boolean has_fst_peer;
+
+	foreach_fst_group_iface(g, f) {
+		has_fst_peer = fst_has_fst_peer(f, &has_peer);
+		if (has_peer && !has_fst_peer)
+			return NULL;
+	}
+#endif /* HOSTAPD */
+
+	foreach_fst_group_iface(g, f) {
+		if (f == i)
+			continue;
+		nof_mbies += fst_fill_iface_mb_ies(f, NULL);
+	}
+
+	buf = wpabuf_alloc(nof_mbies *
+			   (sizeof(struct multi_band_ie) + ETH_ALEN));
+	if (!buf) {
+		fst_printf_iface(i, MSG_ERROR,
+				 "cannot allocate mem for %u MB IEs",
+				 nof_mbies);
+		return NULL;
+	}
+
+	/* The list is sorted in descending order by priorities, so MB IEs will
+	 * be arranged in the same order, as required by spec (see corresponding
+	 * comment in.fst_attach().
+	 */
+	foreach_fst_group_iface(g, f) {
+		if (f == i)
+			continue;
+
+		fst_fill_iface_mb_ies(f, buf);
+		++nof_ifaces_added;
+
+		fst_printf_iface(i, MSG_DEBUG, "added to MB IE");
+	}
+
+	if (!nof_ifaces_added) {
+		wpabuf_free(buf);
+		buf = NULL;
+		fst_printf_iface(i, MSG_INFO,
+				 "cannot add MB IE: no backup ifaces");
+	} else {
+		fst_dump_mb_ies(fst_group_get_id(g), fst_iface_get_name(i),
+				buf);
+	}
+
+	return buf;
+}
+
+
+static const u8 * fst_mbie_get_peer_addr(const struct multi_band_ie *mbie)
+{
+	const u8 *peer_addr = NULL;
+
+	switch (MB_CTRL_ROLE(mbie->mb_ctrl)) {
+	case MB_STA_ROLE_AP:
+		peer_addr = mbie->bssid;
+		break;
+	case MB_STA_ROLE_NON_PCP_NON_AP:
+		if (mbie->mb_ctrl & MB_CTRL_STA_MAC_PRESENT &&
+		    (size_t) 2 + mbie->len >= sizeof(*mbie) + ETH_ALEN)
+			peer_addr = (const u8 *) &mbie[1];
+		break;
+	default:
+		break;
+	}
+
+	return peer_addr;
+}
+
+
+static struct fst_iface *
+fst_group_get_new_iface_by_mbie_and_band_id(struct fst_group *g,
+					    const u8 *mb_ies_buff,
+					    size_t mb_ies_size,
+					    u8 band_id,
+					    u8 *iface_peer_addr)
+{
+	while (mb_ies_size >= 2) {
+		const struct multi_band_ie *mbie =
+			(const struct multi_band_ie *) mb_ies_buff;
+
+		if (mbie->eid != WLAN_EID_MULTI_BAND ||
+		    (size_t) 2 + mbie->len < sizeof(*mbie))
+			break;
+
+		if (mbie->band_id == band_id) {
+			struct fst_iface *iface;
+
+			foreach_fst_group_iface(g, iface) {
+				const u8 *peer_addr =
+					fst_mbie_get_peer_addr(mbie);
+
+				if (peer_addr &&
+				    fst_iface_is_connected(iface, peer_addr) &&
+				    band_id == fst_iface_get_band_id(iface)) {
+					os_memcpy(iface_peer_addr, peer_addr,
+						  ETH_ALEN);
+					return iface;
+				}
+			}
+			break;
+		}
+
+		mb_ies_buff += 2 + mbie->len;
+		mb_ies_size -= 2 + mbie->len;
+	}
+
+	return NULL;
+}
+
+
+struct fst_iface * fst_group_get_iface_by_name(struct fst_group *g,
+					       const char *ifname)
+{
+	struct fst_iface *f;
+
+	foreach_fst_group_iface(g, f) {
+		const char *in = fst_iface_get_name(f);
+
+		if (os_strncmp(in, ifname, os_strlen(in)) == 0)
+			return f;
+	}
+
+	return NULL;
+}
+
+
+u8 fst_group_assign_dialog_token(struct fst_group *g)
+{
+	g->dialog_token++;
+	if (g->dialog_token == 0)
+		g->dialog_token++;
+	return g->dialog_token;
+}
+
+
+u32 fst_group_assign_fsts_id(struct fst_group *g)
+{
+	g->fsts_id++;
+	return g->fsts_id;
+}
+
+
+static Boolean
+fst_group_does_iface_appear_in_other_mbies(struct fst_group *g,
+					   struct fst_iface *iface,
+					   struct fst_iface *other,
+					   u8 *peer_addr)
+{
+	struct fst_get_peer_ctx *ctx;
+	const u8 *addr;
+	const u8 *iface_addr;
+	enum mb_band_id  iface_band_id;
+
+	WPA_ASSERT(g == fst_iface_get_group(iface));
+	WPA_ASSERT(g == fst_iface_get_group(other));
+
+	iface_addr = fst_iface_get_addr(iface);
+	iface_band_id = fst_iface_get_band_id(iface);
+
+	addr = fst_iface_get_peer_first(other, &ctx, TRUE);
+	for (; addr; addr = fst_iface_get_peer_next(other, &ctx, TRUE)) {
+		const struct wpabuf *mbies;
+		u8 other_iface_peer_addr[ETH_ALEN];
+		struct fst_iface *other_new_iface;
+
+		mbies = fst_iface_get_peer_mb_ie(other, addr);
+		if (!mbies)
+			continue;
+
+		other_new_iface = fst_group_get_new_iface_by_mbie_and_band_id(
+			g, wpabuf_head(mbies), wpabuf_len(mbies),
+			iface_band_id, other_iface_peer_addr);
+		if (other_new_iface == iface &&
+		    os_memcmp(iface_addr, other_iface_peer_addr,
+			      ETH_ALEN) != 0) {
+			os_memcpy(peer_addr, addr, ETH_ALEN);
+			return TRUE;
+		}
+	}
+
+	return FALSE;
+}
+
+
+struct fst_iface *
+fst_group_find_new_iface_by_stie(struct fst_group *g,
+				 struct fst_iface *iface,
+				 const u8 *peer_addr,
+				 const struct session_transition_ie *stie,
+				 u8 *iface_peer_addr)
+{
+	struct fst_iface *i;
+
+	foreach_fst_group_iface(g, i) {
+		if (i == iface ||
+		    stie->new_band_id != fst_iface_get_band_id(i))
+			continue;
+		if (fst_group_does_iface_appear_in_other_mbies(g, iface, i,
+			iface_peer_addr))
+			return i;
+		break;
+	}
+	return NULL;
+}
+
+
+struct fst_iface *
+fst_group_get_new_iface_by_stie_and_mbie(
+	struct fst_group *g, const u8 *mb_ies_buff, size_t mb_ies_size,
+	const struct session_transition_ie *stie, u8 *iface_peer_addr)
+{
+	return fst_group_get_new_iface_by_mbie_and_band_id(
+		g, mb_ies_buff, mb_ies_size, stie->new_band_id,
+		iface_peer_addr);
+}
+
+
+struct fst_group * fst_group_create(const char *group_id)
+{
+	struct fst_group *g;
+
+	g = os_zalloc(sizeof(*g));
+	if (g == NULL) {
+		fst_printf(MSG_ERROR, "%s: Cannot alloc group", group_id);
+		return NULL;
+	}
+
+	dl_list_init(&g->ifaces);
+	os_strlcpy(g->group_id, group_id, sizeof(g->group_id));
+
+	dl_list_add_tail(&fst_global_groups_list, &g->global_groups_lentry);
+	fst_printf_group(g, MSG_DEBUG, "instance created");
+
+	foreach_fst_ctrl_call(on_group_created, g);
+
+	return g;
+}
+
+
+void fst_group_attach_iface(struct fst_group *g, struct fst_iface *i)
+{
+	struct dl_list *list = &g->ifaces;
+	struct fst_iface *f;
+
+	/*
+	 * Add new interface to the list.
+	 * The list is sorted in descending order by priority to allow
+	 * multiple MB IEs creation according to the spec (see 10.32 Multi-band
+	 * operation, 10.32.1 General), as they should be ordered according to
+	 * priorities.
+	 */
+	foreach_fst_group_iface(g, f) {
+		if (fst_iface_get_priority(f) < fst_iface_get_priority(i))
+			break;
+		list = &f->group_lentry;
+	}
+	dl_list_add(list, &i->group_lentry);
+}
+
+
+void fst_group_detach_iface(struct fst_group *g, struct fst_iface *i)
+{
+	dl_list_del(&i->group_lentry);
+}
+
+
+void fst_group_delete(struct fst_group *group)
+{
+	struct fst_session *s;
+
+	dl_list_del(&group->global_groups_lentry);
+	WPA_ASSERT(dl_list_empty(&group->ifaces));
+	foreach_fst_ctrl_call(on_group_deleted, group);
+	fst_printf_group(group, MSG_DEBUG, "instance deleted");
+	while ((s = fst_session_global_get_first_by_group(group)) != NULL)
+		fst_session_delete(s);
+	os_free(group);
+}
+
+
+Boolean fst_group_delete_if_empty(struct fst_group *group)
+{
+	Boolean is_empty = !fst_group_has_ifaces(group) &&
+		!fst_session_global_get_first_by_group(group);
+
+	if (is_empty)
+		fst_group_delete(group);
+
+	return is_empty;
+}
+
+
+void fst_group_update_ie(struct fst_group *g)
+{
+	struct fst_iface *i;
+
+	foreach_fst_group_iface(g, i) {
+		struct wpabuf *mbie = fst_group_create_mb_ie(g, i);
+
+		if (!mbie)
+			fst_printf_iface(i, MSG_WARNING, "cannot create MB IE");
+
+		fst_iface_attach_mbie(i, mbie);
+		fst_iface_set_ies(i, mbie);
+		fst_printf_iface(i, MSG_DEBUG, "multi-band IE set to %p", mbie);
+	}
+}
diff --git a/src/fst/fst_group.h b/src/fst/fst_group.h
new file mode 100644
index 000000000000..3a87c0bc91c9
--- /dev/null
+++ b/src/fst/fst_group.h
@@ -0,0 +1,75 @@
+/*
+ * FST module - FST group object definitions
+ * Copyright (c) 2014, Qualcomm Atheros, Inc.
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#ifndef FST_GROUP_H
+#define FST_GROUP_H
+
+struct fst_group {
+	char group_id[IFNAMSIZ + 1];
+	struct dl_list ifaces;
+	u8 dialog_token;
+	u32 fsts_id;
+	struct dl_list global_groups_lentry;
+};
+
+struct session_transition_ie;
+
+#define foreach_fst_group_iface(g, i) \
+	dl_list_for_each((i), &(g)->ifaces, struct fst_iface, group_lentry)
+
+struct fst_group * fst_group_create(const char *group_id);
+void fst_group_attach_iface(struct fst_group *g, struct fst_iface *i);
+void fst_group_detach_iface(struct fst_group *g, struct fst_iface *i);
+void fst_group_delete(struct fst_group *g);
+
+void fst_group_update_ie(struct fst_group *g);
+
+static inline Boolean fst_group_has_ifaces(struct fst_group *g)
+{
+	return !dl_list_empty(&g->ifaces);
+}
+
+static inline struct fst_iface * fst_group_first_iface(struct fst_group *g)
+{
+	return dl_list_first(&g->ifaces, struct fst_iface, group_lentry);
+}
+
+static inline const char * fst_group_get_id(struct fst_group *g)
+{
+	return g->group_id;
+}
+
+Boolean fst_group_delete_if_empty(struct fst_group *group);
+struct fst_iface * fst_group_get_iface_by_name(struct fst_group *g,
+					       const char *ifname);
+struct fst_iface *
+fst_group_find_new_iface_by_stie(struct fst_group *g,
+				 struct fst_iface *iface,
+				 const u8 *peer_addr,
+				 const struct session_transition_ie *stie,
+				 u8 *iface_peer_addr);
+struct fst_iface *
+fst_group_get_new_iface_by_stie_and_mbie(
+	struct fst_group *g, const u8 *mb_ies_buff, size_t mb_ies_size,
+	const struct session_transition_ie *stie, u8 *iface_peer_addr);
+u8  fst_group_assign_dialog_token(struct fst_group *g);
+u32 fst_group_assign_fsts_id(struct fst_group *g);
+
+extern struct dl_list fst_global_groups_list;
+
+#define foreach_fst_group(g) \
+	dl_list_for_each((g), &fst_global_groups_list, \
+			 struct fst_group, global_groups_lentry)
+
+static inline struct fst_group * fst_first_group(void)
+{
+	return dl_list_first(&fst_global_groups_list, struct fst_group,
+			     global_groups_lentry);
+}
+
+#endif /* FST_GROUP_H */
diff --git a/src/fst/fst_iface.c b/src/fst/fst_iface.c
new file mode 100644
index 000000000000..5a92d2c33e42
--- /dev/null
+++ b/src/fst/fst_iface.c
@@ -0,0 +1,79 @@
+/*
+ * FST module - FST interface object implementation
+ * Copyright (c) 2014, Qualcomm Atheros, Inc.
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#include "utils/includes.h"
+#include "utils/common.h"
+#include "fst/fst_internal.h"
+#include "fst/fst_defs.h"
+
+
+struct fst_iface * fst_iface_create(struct fst_group *g, const char *ifname,
+				    const u8 *own_addr,
+				    const struct fst_wpa_obj *iface_obj,
+				    const struct fst_iface_cfg *cfg)
+{
+	struct fst_iface *i;
+
+	i = os_zalloc(sizeof(*i));
+	if (!i) {
+		fst_printf_group(g, MSG_ERROR, "cannot allocate iface for %s",
+				ifname);
+		return NULL;
+	}
+
+	i->cfg = *cfg;
+	i->iface_obj = *iface_obj;
+	i->group = g;
+	os_strlcpy(i->ifname, ifname, sizeof(i->ifname));
+	os_memcpy(i->own_addr, own_addr, sizeof(i->own_addr));
+
+	if (!i->cfg.llt) {
+		fst_printf_iface(i, MSG_WARNING, "Zero llt adjusted");
+		i->cfg.llt = FST_DEFAULT_LLT_CFG_VALUE;
+	}
+
+	return i;
+}
+
+
+void fst_iface_delete(struct fst_iface *i)
+{
+	fst_iface_set_ies(i, NULL);
+	wpabuf_free(i->mb_ie);
+	os_free(i);
+}
+
+
+Boolean fst_iface_is_connected(struct fst_iface *iface, const u8 *addr)
+{
+	struct fst_get_peer_ctx *ctx;
+	const u8 *a = fst_iface_get_peer_first(iface, &ctx, TRUE);
+
+	for (; a != NULL; a = fst_iface_get_peer_next(iface, &ctx, TRUE))
+		if (os_memcmp(addr, a, ETH_ALEN) == 0)
+			return TRUE;
+
+	return FALSE;
+}
+
+
+void fst_iface_attach_mbie(struct fst_iface *i, struct wpabuf *mbie)
+{
+	wpabuf_free(i->mb_ie);
+	i->mb_ie = mbie;
+}
+
+
+enum mb_band_id fst_iface_get_band_id(struct fst_iface *i)
+{
+	enum hostapd_hw_mode hw_mode;
+	u8 channel;
+
+	fst_iface_get_channel_info(i, &hw_mode, &channel);
+	return fst_hw_mode_to_band(hw_mode);
+}
diff --git a/src/fst/fst_iface.h b/src/fst/fst_iface.h
new file mode 100644
index 000000000000..4670d894f7cb
--- /dev/null
+++ b/src/fst/fst_iface.h
@@ -0,0 +1,135 @@
+/*
+ * FST module - FST interface object definitions
+ * Copyright (c) 2014, Qualcomm Atheros, Inc.
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+
+#ifndef FST_IFACE_H
+#define FST_IFACE_H
+
+#include "utils/includes.h"
+#include "utils/common.h"
+#include "list.h"
+#include "fst.h"
+
+struct fst_iface {
+	struct fst_group *group;
+	struct fst_wpa_obj iface_obj;
+	u8 own_addr[ETH_ALEN];
+	struct wpabuf *mb_ie;
+	char ifname[IFNAMSIZ + 1];
+	struct fst_iface_cfg cfg;
+	struct dl_list group_lentry;
+};
+
+struct fst_iface * fst_iface_create(struct fst_group *g, const char *ifname,
+				    const u8 *own_addr,
+				    const struct fst_wpa_obj *iface_obj,
+				    const struct fst_iface_cfg *cfg);
+void fst_iface_delete(struct fst_iface *i);
+
+static inline struct fst_group * fst_iface_get_group(struct fst_iface *i)
+{
+	return i->group;
+}
+
+static inline const char * fst_iface_get_name(struct fst_iface *i)
+{
+	return i->ifname;
+}
+
+static inline const u8 * fst_iface_get_addr(struct fst_iface *i)
+{
+	return i->own_addr;
+}
+
+static inline const char * fst_iface_get_group_id(struct fst_iface *i)
+{
+	return i->cfg.group_id;
+}
+
+static inline u8 fst_iface_get_priority(struct fst_iface *i)
+{
+	return i->cfg.priority;
+}
+
+static inline u32 fst_iface_get_llt(struct fst_iface *i)
+{
+	return i->cfg.llt;
+}
+
+static inline const struct wpabuf * fst_iface_get_mbie(struct fst_iface *i)
+{
+	return i->mb_ie;
+}
+
+static inline const u8 * fst_iface_get_bssid(struct fst_iface *i)
+{
+	return i->iface_obj.get_bssid(i->iface_obj.ctx);
+}
+
+static inline void fst_iface_get_channel_info(struct fst_iface *i,
+					      enum hostapd_hw_mode *hw_mode,
+					      u8 *channel)
+{
+	i->iface_obj.get_channel_info(i->iface_obj.ctx, hw_mode, channel);
+}
+
+static inline int fst_iface_get_hw_modes(struct fst_iface *i,
+					 struct hostapd_hw_modes **modes)
+{
+	return i->iface_obj.get_hw_modes(i->iface_obj.ctx, modes);
+}
+
+static inline void fst_iface_set_ies(struct fst_iface *i,
+				     const struct wpabuf *fst_ies)
+{
+	i->iface_obj.set_ies(i->iface_obj.ctx, fst_ies);
+}
+
+static inline int fst_iface_send_action(struct fst_iface *i,
+					const u8 *addr, struct wpabuf *data)
+{
+	return i->iface_obj.send_action(i->iface_obj.ctx, addr, data);
+}
+
+static inline const struct wpabuf *
+fst_iface_get_peer_mb_ie(struct fst_iface *i, const u8 *addr)
+{
+	return i->iface_obj.get_mb_ie(i->iface_obj.ctx, addr);
+}
+
+static inline void fst_iface_update_mb_ie(struct fst_iface *i,
+					  const u8 *addr,
+					  const u8 *buf, size_t size)
+{
+	return i->iface_obj.update_mb_ie(i->iface_obj.ctx, addr, buf, size);
+}
+
+static inline const u8 * fst_iface_get_peer_first(struct fst_iface *i,
+						  struct fst_get_peer_ctx **ctx,
+						  Boolean mb_only)
+{
+	return i->iface_obj.get_peer_first(i->iface_obj.ctx, ctx, mb_only);
+}
+
+static inline const u8 * fst_iface_get_peer_next(struct fst_iface *i,
+						 struct fst_get_peer_ctx **ctx,
+						 Boolean mb_only)
+{
+	return i->iface_obj.get_peer_next(i->iface_obj.ctx, ctx, mb_only);
+}
+
+Boolean fst_iface_is_connected(struct fst_iface *iface, const u8 *addr);
+void fst_iface_attach_mbie(struct fst_iface *i, struct wpabuf *mbie);
+enum mb_band_id fst_iface_get_band_id(struct fst_iface *i);
+
+static inline void * fst_iface_get_wpa_obj_ctx(struct fst_iface *i)
+{
+	return i->iface_obj.ctx;
+}
+
+#endif /* FST_IFACE_H */
diff --git a/src/fst/fst_internal.h b/src/fst/fst_internal.h
new file mode 100644
index 000000000000..9fe32b85409b
--- /dev/null
+++ b/src/fst/fst_internal.h
@@ -0,0 +1,49 @@
+/*
+ * FST module - auxiliary definitions
+ * Copyright (c) 2014, Qualcomm Atheros, Inc.
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#ifndef FST_INTERNAL_H
+#define FST_INTERNAL_H
+
+#include "utils/includes.h"
+#include "utils/common.h"
+#include "common/defs.h"
+#include "common/ieee802_11_defs.h"
+#include "fst/fst_iface.h"
+#include "fst/fst_group.h"
+#include "fst/fst_session.h"
+
+#define fst_printf(level, format, ...) \
+	wpa_printf((level), "FST: " format, ##__VA_ARGS__)
+
+#define fst_printf_group(group, level, format, ...) \
+	wpa_printf((level), "FST: %s: " format, \
+		   fst_group_get_id(group), ##__VA_ARGS__)
+
+#define fst_printf_iface(iface, level, format, ...) \
+	fst_printf_group(fst_iface_get_group(iface), (level), "%s: " format, \
+			 fst_iface_get_name(iface), ##__VA_ARGS__)
+
+enum mb_band_id fst_hw_mode_to_band(enum hostapd_hw_mode mode);
+
+struct fst_ctrl_handle {
+	struct fst_ctrl ctrl;
+	struct dl_list global_ctrls_lentry;
+};
+
+extern struct dl_list fst_global_ctrls_list;
+
+#define foreach_fst_ctrl_call(clb, ...) \
+	do { \
+		struct fst_ctrl_handle *__fst_ctrl_h; \
+		dl_list_for_each(__fst_ctrl_h, &fst_global_ctrls_list, \
+			struct fst_ctrl_handle, global_ctrls_lentry) \
+			if (__fst_ctrl_h->ctrl.clb) \
+				__fst_ctrl_h->ctrl.clb(__VA_ARGS__);\
+	} while (0)
+
+#endif /* FST_INTERNAL_H */
diff --git a/src/fst/fst_session.c b/src/fst/fst_session.c
new file mode 100644
index 000000000000..55fa69495e99
--- /dev/null
+++ b/src/fst/fst_session.c
@@ -0,0 +1,1620 @@
+/*
+ * FST module - FST Session implementation
+ * Copyright (c) 2014, Qualcomm Atheros, Inc.
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#include "utils/includes.h"
+
+#include "utils/common.h"
+#include "utils/eloop.h"
+#include "common/defs.h"
+#include "fst/fst_internal.h"
+#include "fst/fst_defs.h"
+#include "fst/fst_ctrl_iface.h"
+#ifdef CONFIG_FST_TEST
+#include "fst/fst_ctrl_defs.h"
+#endif /* CONFIG_FST_TEST */
+
+#define US_80211_TU 1024
+
+#define US_TO_TU(m) ((m) * / US_80211_TU)
+#define TU_TO_US(m) ((m) * US_80211_TU)
+
+#define FST_LLT_SWITCH_IMMEDIATELY 0
+
+#define fst_printf_session(s, level, format, ...) \
+	fst_printf((level), "%u (0x%08x): [" MACSTR "," MACSTR "] :" format, \
+		   (s)->id, (s)->data.fsts_id, \
+		   MAC2STR((s)->data.old_peer_addr), \
+		   MAC2STR((s)->data.new_peer_addr), \
+		   ##__VA_ARGS__)
+
+#define fst_printf_siface(s, iface, level, format, ...) \
+	fst_printf_session((s), (level), "%s: " format, \
+			   fst_iface_get_name(iface), ##__VA_ARGS__)
+
+#define fst_printf_sframe(s, is_old, level, format, ...) \
+	fst_printf_siface((s), \
+		(is_old) ? (s)->data.old_iface : (s)->data.new_iface, \
+		(level), format, ##__VA_ARGS__)
+
+#define FST_LLT_MS_DEFAULT 50
+#define FST_ACTION_MAX_SUPPORTED   FST_ACTION_ON_CHANNEL_TUNNEL
+
+const char * const fst_action_names[] = {
+	[FST_ACTION_SETUP_REQUEST]     = "Setup Request",
+	[FST_ACTION_SETUP_RESPONSE]    = "Setup Response",
+	[FST_ACTION_TEAR_DOWN]         = "Tear Down",
+	[FST_ACTION_ACK_REQUEST]       = "Ack Request",
+	[FST_ACTION_ACK_RESPONSE]      = "Ack Response",
+	[FST_ACTION_ON_CHANNEL_TUNNEL] = "On Channel Tunnel",
+};
+
+struct fst_session {
+	struct {
+		/* Session configuration that can be zeroed on reset */
+		u8 old_peer_addr[ETH_ALEN];
+		u8 new_peer_addr[ETH_ALEN];
+		struct fst_iface *new_iface;
+		struct fst_iface *old_iface;
+		u32 llt_ms;
+		u8 pending_setup_req_dlgt;
+		u32 fsts_id; /* FSTS ID, see spec, 8.4.2.147
+			      * Session Transition element */
+	} data;
+	/* Session object internal fields which won't be zeroed on reset */
+	struct dl_list global_sessions_lentry;
+	u32 id; /* Session object ID used to identify
+		 * specific session object */
+	struct fst_group *group;
+	enum fst_session_state state;
+	Boolean stt_armed;
+};
+
+static struct dl_list global_sessions_list;
+static u32 global_session_id = 0;
+
+#define foreach_fst_session(s) \
+	dl_list_for_each((s), &global_sessions_list, \
+			 struct fst_session, global_sessions_lentry)
+
+#define foreach_fst_session_safe(s, temp) \
+	dl_list_for_each_safe((s), (temp), &global_sessions_list, \
+			      struct fst_session, global_sessions_lentry)
+
+
+static void fst_session_global_inc_id(void)
+{
+	global_session_id++;
+	if (global_session_id == FST_INVALID_SESSION_ID)
+		global_session_id++;
+}
+
+
+int fst_session_global_init(void)
+{
+	dl_list_init(&global_sessions_list);
+	return 0;
+}
+
+
+void fst_session_global_deinit(void)
+{
+	WPA_ASSERT(dl_list_empty(&global_sessions_list));
+}
+
+
+static inline void fst_session_notify_ctrl(struct fst_session *s,
+					   enum fst_event_type event_type,
+					   union fst_event_extra *extra)
+{
+	foreach_fst_ctrl_call(on_event, event_type, NULL, s, extra);
+}
+
+
+static void fst_session_set_state(struct fst_session *s,
+				  enum fst_session_state state,
+				  union fst_session_state_switch_extra *extra)
+{
+	if (s->state != state) {
+		union fst_event_extra evext = {
+			.session_state = {
+				.old_state = s->state,
+				.new_state = state,
+			},
+		};
+
+		if (extra)
+			evext.session_state.extra = *extra;
+		fst_session_notify_ctrl(s, EVENT_FST_SESSION_STATE_CHANGED,
+					&evext);
+		fst_printf_session(s, MSG_INFO, "State: %s => %s",
+				   fst_session_state_name(s->state),
+				   fst_session_state_name(state));
+		s->state = state;
+	}
+}
+
+
+static u32 fst_find_free_session_id(void)
+{
+	u32 i, id = FST_INVALID_SESSION_ID;
+	struct fst_session *s;
+
+	for (i = 0; i < (u32) -1; i++) {
+		Boolean in_use = FALSE;
+
+		foreach_fst_session(s) {
+			if (s->id == global_session_id) {
+				fst_session_global_inc_id();
+				in_use = TRUE;
+				break;
+			}
+		}
+		if (!in_use) {
+			id = global_session_id;
+			fst_session_global_inc_id();
+			break;
+		}
+	}
+
+	return id;
+}
+
+
+static void fst_session_timeout_handler(void *eloop_data, void *user_ctx)
+{
+	struct fst_session *s = user_ctx;
+	union fst_session_state_switch_extra extra = {
+		.to_initial = {
+			.reason = REASON_STT,
+		},
+	};
+
+	fst_printf_session(s, MSG_WARNING, "Session State Timeout");
+	fst_session_set_state(s, FST_SESSION_STATE_INITIAL, &extra);
+}
+
+
+static void fst_session_stt_arm(struct fst_session *s)
+{
+	eloop_register_timeout(0, TU_TO_US(FST_DEFAULT_SESSION_TIMEOUT_TU),
+			       fst_session_timeout_handler, NULL, s);
+	s->stt_armed = TRUE;
+}
+
+
+static void fst_session_stt_disarm(struct fst_session *s)
+{
+	if (s->stt_armed) {
+		eloop_cancel_timeout(fst_session_timeout_handler, NULL, s);
+		s->stt_armed = FALSE;
+	}
+}
+
+
+static Boolean fst_session_is_in_transition(struct fst_session *s)
+{
+	/* See spec, 10.32.2.2  Transitioning between states */
+	return s->stt_armed;
+}
+
+
+static int fst_session_is_in_progress(struct fst_session *s)
+{
+	return s->state != FST_SESSION_STATE_INITIAL;
+}
+
+
+static int fst_session_is_ready_pending(struct fst_session *s)
+{
+	return s->state == FST_SESSION_STATE_SETUP_COMPLETION &&
+		fst_session_is_in_transition(s);
+}
+
+
+static int fst_session_is_ready(struct fst_session *s)
+{
+	return s->state == FST_SESSION_STATE_SETUP_COMPLETION &&
+		!fst_session_is_in_transition(s);
+}
+
+
+static int fst_session_is_switch_requested(struct fst_session *s)
+{
+	return s->state == FST_SESSION_STATE_TRANSITION_DONE &&
+		fst_session_is_in_transition(s);
+}
+
+
+static struct fst_session *
+fst_find_session_in_progress(const u8 *peer_addr, struct fst_group *g)
+{
+	struct fst_session *s;
+
+	foreach_fst_session(s) {
+		if (s->group == g &&
+		    (os_memcmp(s->data.old_peer_addr, peer_addr,
+			       ETH_ALEN) == 0 ||
+		     os_memcmp(s->data.new_peer_addr, peer_addr,
+			       ETH_ALEN) == 0) &&
+		    fst_session_is_in_progress(s))
+			return s;
+	}
+
+	return NULL;
+}
+
+
+static void fst_session_reset_ex(struct fst_session *s, enum fst_reason reason)
+{
+	union fst_session_state_switch_extra evext = {
+		.to_initial = {
+			.reason = reason,
+		},
+	};
+
+	if (s->state == FST_SESSION_STATE_SETUP_COMPLETION ||
+	    s->state == FST_SESSION_STATE_TRANSITION_DONE)
+		fst_session_tear_down_setup(s);
+	fst_session_stt_disarm(s);
+	os_memset(&s->data, 0, sizeof(s->data));
+	fst_session_set_state(s, FST_SESSION_STATE_INITIAL, &evext);
+}
+
+
+static int fst_session_send_action(struct fst_session *s, Boolean old_iface,
+				   const void *payload, size_t size,
+				   const struct wpabuf *extra_buf)
+{
+	size_t len;
+	int res;
+	struct wpabuf *buf;
+	u8 action;
+	struct fst_iface *iface =
+		old_iface ? s->data.old_iface : s->data.new_iface;
+
+	WPA_ASSERT(payload != NULL);
+	WPA_ASSERT(size != 0);
+
+	action = *(const u8 *) payload;
+
+	WPA_ASSERT(action <= FST_ACTION_MAX_SUPPORTED);
+
+	if (!iface) {
+		fst_printf_session(s, MSG_ERROR,
+				   "no %s interface for FST Action '%s' sending",
+				   old_iface ? "old" : "new",
+				   fst_action_names[action]);
+		return -1;
+	}
+
+	len = sizeof(u8) /* category */ + size;
+	if (extra_buf)
+		len += wpabuf_size(extra_buf);
+
+	buf = wpabuf_alloc(len);
+	if (!buf) {
+		fst_printf_session(s, MSG_ERROR,
+				   "cannot allocate buffer of %zu bytes for FST Action '%s' sending",
+				   len, fst_action_names[action]);
+		return -1;
+	}
+
+	wpabuf_put_u8(buf, WLAN_ACTION_FST);
+	wpabuf_put_data(buf, payload, size);
+	if (extra_buf)
+		wpabuf_put_buf(buf, extra_buf);
+
+	res = fst_iface_send_action(iface,
+				    old_iface ? s->data.old_peer_addr :
+				    s->data.new_peer_addr, buf);
+	if (res < 0)
+		fst_printf_siface(s, iface, MSG_ERROR,
+				  "failed to send FST Action '%s'",
+				  fst_action_names[action]);
+	else
+		fst_printf_siface(s, iface, MSG_DEBUG, "FST Action '%s' sent",
+				  fst_action_names[action]);
+	wpabuf_free(buf);
+
+	return res;
+}
+
+
+static int fst_session_send_tear_down(struct fst_session *s)
+{
+	struct fst_tear_down td;
+	int res;
+
+	if (!fst_session_is_in_progress(s)) {
+		fst_printf_session(s, MSG_ERROR, "No FST setup to tear down");
+		return -1;
+	}
+
+	WPA_ASSERT(s->data.old_iface != NULL);
+	WPA_ASSERT(s->data.new_iface != NULL);
+
+	os_memset(&td, 0, sizeof(td));
+
+	td.action = FST_ACTION_TEAR_DOWN;
+	td.fsts_id = host_to_le32(s->data.fsts_id);
+
+	res = fst_session_send_action(s, TRUE, &td, sizeof(td), NULL);
+	if (!res)
+		fst_printf_sframe(s, TRUE, MSG_INFO, "FST TearDown sent");
+	else
+		fst_printf_sframe(s, TRUE, MSG_ERROR,
+				  "failed to send FST TearDown");
+
+	return res;
+}
+
+
+static void fst_session_handle_setup_request(struct fst_iface *iface,
+					     const struct ieee80211_mgmt *mgmt,
+					     size_t frame_len)
+{
+	struct fst_session *s;
+	const struct fst_setup_req *req;
+	struct fst_iface *new_iface = NULL;
+	struct fst_group *g;
+	u8 new_iface_peer_addr[ETH_ALEN];
+	const struct wpabuf *peer_mbies;
+	size_t plen;
+
+	if (frame_len < IEEE80211_HDRLEN + 1 + sizeof(*req))  {
+		fst_printf_iface(iface, MSG_WARNING,
+				 "FST Request dropped: too short (%zu < %zu)",
+				 frame_len,
+				 IEEE80211_HDRLEN + 1 + sizeof(*req));
+		return;
+	}
+	plen = frame_len - IEEE80211_HDRLEN - 1;
+	req = (const struct fst_setup_req *)
+		(((const u8 *) mgmt) + IEEE80211_HDRLEN + 1);
+	if (req->stie.element_id != WLAN_EID_SESSION_TRANSITION ||
+	    req->stie.length < 11) {
+		fst_printf_iface(iface, MSG_WARNING,
+				 "FST Request dropped: invalid STIE");
+		return;
+	}
+
+	if (req->stie.new_band_id == req->stie.old_band_id) {
+		fst_printf_iface(iface, MSG_WARNING,
+				 "FST Request dropped: new and old band IDs are the same");
+		return;
+	}
+
+	g = fst_iface_get_group(iface);
+
+	if (plen > sizeof(*req)) {
+		fst_iface_update_mb_ie(iface, mgmt->sa, (const u8 *) (req + 1),
+				       plen - sizeof(*req));
+		fst_printf_iface(iface, MSG_INFO,
+				 "FST Request: MB IEs updated for " MACSTR,
+				 MAC2STR(mgmt->sa));
+	}
+
+	peer_mbies = fst_iface_get_peer_mb_ie(iface, mgmt->sa);
+	if (peer_mbies) {
+		new_iface = fst_group_get_new_iface_by_stie_and_mbie(
+			g, wpabuf_head(peer_mbies), wpabuf_len(peer_mbies),
+			&req->stie, new_iface_peer_addr);
+		if (new_iface)
+			fst_printf_iface(iface, MSG_INFO,
+					 "FST Request: new iface (%s:" MACSTR
+					 ") found by MB IEs",
+					 fst_iface_get_name(new_iface),
+					 MAC2STR(new_iface_peer_addr));
+	}
+
+	if (!new_iface) {
+		new_iface = fst_group_find_new_iface_by_stie(
+			g, iface, mgmt->sa, &req->stie,
+			new_iface_peer_addr);
+		if (new_iface)
+			fst_printf_iface(iface, MSG_INFO,
+					 "FST Request: new iface (%s:" MACSTR
+					 ") found by others",
+					 fst_iface_get_name(new_iface),
+					 MAC2STR(new_iface_peer_addr));
+	}
+
+	if (!new_iface) {
+		fst_printf_iface(iface, MSG_WARNING,
+				 "FST Request dropped: new iface not found");
+		return;
+	}
+
+	s = fst_find_session_in_progress(mgmt->sa, g);
+	if (s) {
+		union fst_session_state_switch_extra evext = {
+			.to_initial = {
+				.reason = REASON_SETUP,
+			},
+		};
+
+		/*
+		 * 10.32.2.2  Transitioning between states:
+		 * Upon receipt of an FST Setup Request frame, the responder
+		 * shall respond with an FST Setup Response frame unless it has
+		 * a pending FST Setup Request frame addressed to the initiator
+		 * and the responder has a numerically larger MAC address than
+		 * the initiator’s MAC address, in which case, the responder
+		 * shall delete the received FST Setup Request.
+		 */
+		if (os_memcmp(mgmt->da, mgmt->sa, ETH_ALEN) > 0) {
+			fst_printf_session(s, MSG_WARNING,
+					   "FST Request dropped due to MAC comparison (our MAC is "
+					   MACSTR ")",
+					   MAC2STR(mgmt->da));
+			return;
+		}
+
+		if (!fst_session_is_ready_pending(s)) {
+			fst_printf_session(s, MSG_WARNING,
+					   "FST Request from " MACSTR
+					   " dropped due to inappropriate state %s",
+					   MAC2STR(mgmt->da),
+					   fst_session_state_name(s->state));
+			return;
+		}
+
+
+		/*
+		 * If FST Setup Request arrived with the same FSTS ID as one we
+		 * initialized before, it means the other side either didn't
+		 * receive our FST Request or skipped it for some reason (for
+		 * example, due to numerical MAC comparison).
+		 *
+		 * In this case, there's no need to tear down the session.
+		 * Moreover, as FSTS ID is the same, the other side will
+		 * associate this tear down with the session it initiated that
+		 * will break the sync.
+		 */
+		if (le_to_host32(req->stie.fsts_id) != s->data.fsts_id)
+			fst_session_send_tear_down(s);
+		else
+			fst_printf_session(s, MSG_WARNING,
+					   "Skipping TearDown as the FST request has the same FSTS ID as initiated");
+		fst_session_set_state(s, FST_SESSION_STATE_INITIAL, &evext);
+		fst_session_stt_disarm(s);
+		fst_printf_session(s, MSG_WARNING, "reset due to FST request");
+	}
+
+	s = fst_session_create(g);
+	if (!s) {
+		fst_printf(MSG_WARNING,
+			   "FST Request dropped: cannot create session for %s and %s",
+			   fst_iface_get_name(iface),
+			   fst_iface_get_name(new_iface));
+		return;
+	}
+
+	fst_session_set_iface(s, iface, TRUE);
+	fst_session_set_peer_addr(s, mgmt->sa, TRUE);
+	fst_session_set_iface(s, new_iface, FALSE);
+	fst_session_set_peer_addr(s, new_iface_peer_addr, FALSE);
+	fst_session_set_llt(s, FST_LLT_VAL_TO_MS(le_to_host32(req->llt)));
+	s->data.pending_setup_req_dlgt = req->dialog_token;
+	s->data.fsts_id = le_to_host32(req->stie.fsts_id);
+
+	fst_session_stt_arm(s);
+
+	fst_session_notify_ctrl(s, EVENT_FST_SETUP, NULL);
+
+	fst_session_set_state(s, FST_SESSION_STATE_SETUP_COMPLETION, NULL);
+}
+
+
+static void fst_session_handle_setup_response(struct fst_session *s,
+					      struct fst_iface *iface,
+					      const struct ieee80211_mgmt *mgmt,
+					      size_t frame_len)
+{
+	const struct fst_setup_res *res;
+	size_t plen = frame_len - IEEE80211_HDRLEN - 1;
+	enum hostapd_hw_mode hw_mode;
+	u8 channel;
+	union fst_session_state_switch_extra evext = {
+		.to_initial = {0},
+	};
+
+	if (iface != s->data.old_iface) {
+		fst_printf_session(s, MSG_WARNING,
+				   "FST Response dropped: %s is not the old iface",
+				   fst_iface_get_name(iface));
+		return;
+	}
+
+	if (!fst_session_is_ready_pending(s)) {
+		fst_printf_session(s, MSG_WARNING,
+				   "FST Response dropped due to wrong state: %s",
+				   fst_session_state_name(s->state));
+		return;
+	}
+
+	if (plen < sizeof(*res)) {
+		fst_printf_session(s, MSG_WARNING,
+				   "Too short FST Response dropped");
+		return;
+	}
+	res = (const struct fst_setup_res *)
+		(((const u8 *) mgmt) + IEEE80211_HDRLEN + 1);
+	if (res->stie.element_id != WLAN_EID_SESSION_TRANSITION ||
+	    res->stie.length < 11) {
+		fst_printf_iface(iface, MSG_WARNING,
+				 "FST Response dropped: invalid STIE");
+		return;
+	}
+
+	if (res->dialog_token != s->data.pending_setup_req_dlgt)  {
+		fst_printf_session(s, MSG_WARNING,
+				   "FST Response dropped due to wrong dialog token (%u != %u)",
+				   s->data.pending_setup_req_dlgt,
+				   res->dialog_token);
+		return;
+	}
+
+	if (res->status_code == WLAN_STATUS_SUCCESS &&
+	    le_to_host32(res->stie.fsts_id) != s->data.fsts_id) {
+		fst_printf_session(s, MSG_WARNING,
+				   "FST Response dropped due to wrong FST Session ID (%u)",
+				   le_to_host32(res->stie.fsts_id));
+		return;
+	}
+
+	fst_session_stt_disarm(s);
+
+	if (res->status_code != WLAN_STATUS_SUCCESS) {
+		/*
+		 * 10.32.2.2  Transitioning between states
+		 * The initiator shall set the STT to the value of the
+		 * FSTSessionTimeOut field at ... and at each ACK frame sent in
+		 * response to a received FST Setup Response with the Status
+		 * Code field equal to PENDING_ADMITTING_FST_SESSION or
+		 * PENDING_GAP_IN_BA_WINDOW.
+		 */
+		evext.to_initial.reason = REASON_REJECT;
+		evext.to_initial.reject_code = res->status_code;
+		evext.to_initial.initiator = FST_INITIATOR_REMOTE;
+		fst_session_set_state(s, FST_SESSION_STATE_INITIAL, &evext);
+		fst_printf_session(s, MSG_WARNING,
+				   "FST Setup rejected by remote side with status %u",
+				   res->status_code);
+		return;
+	}
+
+	fst_iface_get_channel_info(s->data.new_iface, &hw_mode, &channel);
+
+	if (fst_hw_mode_to_band(hw_mode) != res->stie.new_band_id) {
+		evext.to_initial.reason = REASON_ERROR_PARAMS;
+		fst_session_set_state(s, FST_SESSION_STATE_INITIAL, &evext);
+		fst_printf_session(s, MSG_WARNING,
+				   "invalid FST Setup parameters");
+		fst_session_tear_down_setup(s);
+		return;
+	}
+
+	fst_printf_session(s, MSG_INFO,
+			   "%s: FST Setup established for %s (llt=%u)",
+			   fst_iface_get_name(s->data.old_iface),
+			   fst_iface_get_name(s->data.new_iface),
+			   s->data.llt_ms);
+
+	fst_session_notify_ctrl(s, EVENT_FST_ESTABLISHED, NULL);
+
+	if (s->data.llt_ms == FST_LLT_SWITCH_IMMEDIATELY)
+		fst_session_initiate_switch(s);
+}
+
+
+static void fst_session_handle_tear_down(struct fst_session *s,
+					 struct fst_iface *iface,
+					 const struct ieee80211_mgmt *mgmt,
+					 size_t frame_len)
+{
+	const struct fst_tear_down *td;
+	size_t plen = frame_len - IEEE80211_HDRLEN - 1;
+	union fst_session_state_switch_extra evext = {
+		.to_initial = {
+			.reason = REASON_TEARDOWN,
+			.initiator = FST_INITIATOR_REMOTE,
+		},
+	};
+
+	if (plen < sizeof(*td)) {
+		fst_printf_session(s, MSG_WARNING,
+				   "Too short FST Tear Down dropped");
+		return;
+	}
+	td = (const struct fst_tear_down *)
+		(((const u8 *) mgmt) + IEEE80211_HDRLEN + 1);
+
+	if (le_to_host32(td->fsts_id) != s->data.fsts_id) {
+		fst_printf_siface(s, iface, MSG_WARNING,
+				  "tear down for wrong FST Setup ID (%u)",
+				  le_to_host32(td->fsts_id));
+		return;
+	}
+
+	fst_session_stt_disarm(s);
+
+	fst_session_set_state(s, FST_SESSION_STATE_INITIAL, &evext);
+}
+
+
+static void fst_session_handle_ack_request(struct fst_session *s,
+					   struct fst_iface *iface,
+					   const struct ieee80211_mgmt *mgmt,
+					   size_t frame_len)
+{
+	const struct fst_ack_req *req;
+	size_t plen = frame_len - IEEE80211_HDRLEN - 1;
+	struct fst_ack_res res;
+	union fst_session_state_switch_extra evext = {
+		.to_initial = {
+			.reason = REASON_SWITCH,
+			.initiator = FST_INITIATOR_REMOTE,
+		},
+	};
+
+	if (!fst_session_is_ready(s) && !fst_session_is_switch_requested(s)) {
+		fst_printf_siface(s, iface, MSG_ERROR,
+				  "cannot initiate switch due to wrong session state (%s)",
+				  fst_session_state_name(s->state));
+		return;
+	}
+
+	WPA_ASSERT(s->data.new_iface != NULL);
+
+	if (iface != s->data.new_iface) {
+		fst_printf_siface(s, iface, MSG_ERROR,
+				  "Ack received on wrong interface");
+		return;
+	}
+
+	if (plen < sizeof(*req)) {
+		fst_printf_session(s, MSG_WARNING,
+				   "Too short FST Ack Request dropped");
+		return;
+	}
+	req = (const struct fst_ack_req *)
+		(((const u8 *) mgmt) + IEEE80211_HDRLEN + 1);
+
+	if (le_to_host32(req->fsts_id) != s->data.fsts_id) {
+		fst_printf_siface(s, iface, MSG_WARNING,
+				  "Ack for wrong FST Setup ID (%u)",
+				  le_to_host32(req->fsts_id));
+		return;
+	}
+
+	os_memset(&res, 0, sizeof(res));
+
+	res.action = FST_ACTION_ACK_RESPONSE;
+	res.dialog_token = req->dialog_token;
+	res.fsts_id = req->fsts_id;
+
+	if (!fst_session_send_action(s, FALSE, &res, sizeof(res), NULL)) {
+		fst_printf_sframe(s, FALSE, MSG_INFO, "FST Ack Response sent");
+		fst_session_stt_disarm(s);
+		fst_session_set_state(s, FST_SESSION_STATE_TRANSITION_DONE,
+				      NULL);
+		fst_session_set_state(s, FST_SESSION_STATE_TRANSITION_CONFIRMED,
+				      NULL);
+		fst_session_set_state(s, FST_SESSION_STATE_INITIAL, &evext);
+	}
+}
+
+
+static void
+fst_session_handle_ack_response(struct fst_session *s,
+				struct fst_iface *iface,
+				const struct ieee80211_mgmt *mgmt,
+				size_t frame_len)
+{
+	const struct fst_ack_res *res;
+	size_t plen = frame_len - IEEE80211_HDRLEN - 1;
+	union fst_session_state_switch_extra evext = {
+		.to_initial = {
+			.reason = REASON_SWITCH,
+			.initiator = FST_INITIATOR_LOCAL,
+		},
+	};
+
+	if (!fst_session_is_switch_requested(s)) {
+		fst_printf_siface(s, iface, MSG_ERROR,
+				  "Ack Response in inappropriate session state (%s)",
+				  fst_session_state_name(s->state));
+		return;
+	}
+
+	WPA_ASSERT(s->data.new_iface != NULL);
+
+	if (iface != s->data.new_iface) {
+		fst_printf_siface(s, iface, MSG_ERROR,
+				  "Ack response received on wrong interface");
+		return;
+	}
+
+	if (plen < sizeof(*res)) {
+		fst_printf_session(s, MSG_WARNING,
+				   "Too short FST Ack Response dropped");
+		return;
+	}
+	res = (const struct fst_ack_res *)
+		(((const u8 *) mgmt) + IEEE80211_HDRLEN + 1);
+
+	if (le_to_host32(res->fsts_id) != s->data.fsts_id) {
+		fst_printf_siface(s, iface, MSG_ERROR,
+				  "Ack response for wrong FST Setup ID (%u)",
+				  le_to_host32(res->fsts_id));
+		return;
+	}
+
+	fst_session_set_state(s, FST_SESSION_STATE_TRANSITION_CONFIRMED, NULL);
+	fst_session_set_state(s, FST_SESSION_STATE_INITIAL, &evext);
+
+	fst_session_stt_disarm(s);
+}
+
+
+struct fst_session * fst_session_create(struct fst_group *g)
+{
+	struct fst_session *s;
+	u32 id;
+
+	WPA_ASSERT(!is_zero_ether_addr(own_addr));
+
+	id = fst_find_free_session_id();
+	if (id == FST_INVALID_SESSION_ID) {
+		fst_printf(MSG_ERROR, "Cannot assign new session ID");
+		return NULL;
+	}
+
+	s = os_zalloc(sizeof(*s));
+	if (!s) {
+		fst_printf(MSG_ERROR, "Cannot allocate new session object");
+		return NULL;
+	}
+
+	s->id = id;
+	s->group = g;
+	s->state = FST_SESSION_STATE_INITIAL;
+
+	s->data.llt_ms = FST_LLT_MS_DEFAULT;
+
+	fst_printf(MSG_INFO, "Session %u created", s->id);
+
+	dl_list_add_tail(&global_sessions_list, &s->global_sessions_lentry);
+
+	foreach_fst_ctrl_call(on_session_added, s);
+
+	return s;
+}
+
+
+void fst_session_set_iface(struct fst_session *s, struct fst_iface *iface,
+			   Boolean is_old)
+{
+	if (is_old)
+		s->data.old_iface = iface;
+	else
+		s->data.new_iface = iface;
+
+}
+
+
+void fst_session_set_llt(struct fst_session *s, u32 llt)
+{
+	s->data.llt_ms = llt;
+}
+
+
+void fst_session_set_peer_addr(struct fst_session *s, const u8 *addr,
+			       Boolean is_old)
+{
+	u8 *a = is_old ? s->data.old_peer_addr : s->data.new_peer_addr;
+
+	os_memcpy(a, addr, ETH_ALEN);
+}
+
+
+int fst_session_initiate_setup(struct fst_session *s)
+{
+	struct fst_setup_req req;
+	int res;
+	u32 fsts_id;
+	u8 dialog_token;
+	struct fst_session *_s;
+
+	if (fst_session_is_in_progress(s)) {
+		fst_printf_session(s, MSG_ERROR, "Session in progress");
+		return -EINVAL;
+	}
+
+	if (is_zero_ether_addr(s->data.old_peer_addr)) {
+		fst_printf_session(s, MSG_ERROR, "No old peer MAC address");
+		return -EINVAL;
+	}
+
+	if (is_zero_ether_addr(s->data.new_peer_addr)) {
+		fst_printf_session(s, MSG_ERROR, "No new peer MAC address");
+		return -EINVAL;
+	}
+
+	if (!s->data.old_iface) {
+		fst_printf_session(s, MSG_ERROR, "No old interface defined");
+		return -EINVAL;
+	}
+
+	if (!s->data.new_iface) {
+		fst_printf_session(s, MSG_ERROR, "No new interface defined");
+		return -EINVAL;
+	}
+
+	if (s->data.new_iface == s->data.old_iface) {
+		fst_printf_session(s, MSG_ERROR,
+				   "Same interface set as old and new");
+		return -EINVAL;
+	}
+
+	if (!fst_iface_is_connected(s->data.old_iface, s->data.old_peer_addr)) {
+		fst_printf_session(s, MSG_ERROR,
+				   "The preset old peer address is not connected");
+		return -EINVAL;
+	}
+
+	if (!fst_iface_is_connected(s->data.new_iface, s->data.new_peer_addr)) {
+		fst_printf_session(s, MSG_ERROR,
+				   "The preset new peer address is not connected");
+		return -EINVAL;
+	}
+
+	_s = fst_find_session_in_progress(s->data.old_peer_addr, s->group);
+	if (_s) {
+		fst_printf_session(s, MSG_ERROR,
+				   "There is another session in progress (old): %u",
+				   _s->id);
+		return -EINVAL;
+	}
+
+	_s = fst_find_session_in_progress(s->data.new_peer_addr, s->group);
+	if (_s) {
+		fst_printf_session(s, MSG_ERROR,
+				   "There is another session in progress (new): %u",
+				   _s->id);
+		return -EINVAL;
+	}
+
+	dialog_token = fst_group_assign_dialog_token(s->group);
+	fsts_id = fst_group_assign_fsts_id(s->group);
+
+	os_memset(&req, 0, sizeof(req));
+
+	fst_printf_siface(s, s->data.old_iface, MSG_INFO,
+		"initiating FST setup for %s (llt=%u ms)",
+		fst_iface_get_name(s->data.new_iface), s->data.llt_ms);
+
+	req.action = FST_ACTION_SETUP_REQUEST;
+	req.dialog_token = dialog_token;
+	req.llt = host_to_le32(FST_LLT_MS_TO_VAL(s->data.llt_ms));
+	/* 8.4.2.147 Session Transition element */
+	req.stie.element_id = WLAN_EID_SESSION_TRANSITION;
+	req.stie.length = sizeof(req.stie) - 2;
+	req.stie.fsts_id = host_to_le32(fsts_id);
+	req.stie.session_control = SESSION_CONTROL(SESSION_TYPE_BSS, 0);
+
+	req.stie.new_band_id = fst_iface_get_band_id(s->data.new_iface);
+	req.stie.new_band_op = 1;
+	req.stie.new_band_setup = 0;
+
+	req.stie.old_band_id = fst_iface_get_band_id(s->data.old_iface);
+	req.stie.old_band_op = 1;
+	req.stie.old_band_setup = 0;
+
+	res = fst_session_send_action(s, TRUE, &req, sizeof(req),
+				      fst_iface_get_mbie(s->data.old_iface));
+	if (!res) {
+		s->data.fsts_id = fsts_id;
+		s->data.pending_setup_req_dlgt = dialog_token;
+		fst_printf_sframe(s, TRUE, MSG_INFO, "FST Setup Request sent");
+		fst_session_set_state(s, FST_SESSION_STATE_SETUP_COMPLETION,
+				      NULL);
+
+		fst_session_stt_arm(s);
+	}
+
+	return res;
+}
+
+
+int fst_session_respond(struct fst_session *s, u8 status_code)
+{
+	struct fst_setup_res res;
+	enum hostapd_hw_mode hw_mode;
+	u8 channel;
+
+	if (!fst_session_is_ready_pending(s)) {
+		fst_printf_session(s, MSG_ERROR, "incorrect state: %s",
+				   fst_session_state_name(s->state));
+		return -EINVAL;
+	}
+
+	if (is_zero_ether_addr(s->data.old_peer_addr)) {
+		fst_printf_session(s, MSG_ERROR, "No peer MAC address");
+		return -EINVAL;
+	}
+
+	if (!s->data.old_iface) {
+		fst_printf_session(s, MSG_ERROR, "No old interface defined");
+		return -EINVAL;
+	}
+
+	if (!s->data.new_iface) {
+		fst_printf_session(s, MSG_ERROR, "No new interface defined");
+		return -EINVAL;
+	}
+
+	if (s->data.new_iface == s->data.old_iface) {
+		fst_printf_session(s, MSG_ERROR,
+				   "Same interface set as old and new");
+		return -EINVAL;
+	}
+
+	if (!fst_iface_is_connected(s->data.old_iface, s->data.old_peer_addr)) {
+		fst_printf_session(s, MSG_ERROR,
+				   "The preset peer address is not in the peer list");
+		return -EINVAL;
+	}
+
+	fst_session_stt_disarm(s);
+
+	os_memset(&res, 0, sizeof(res));
+
+	res.action = FST_ACTION_SETUP_RESPONSE;
+	res.dialog_token = s->data.pending_setup_req_dlgt;
+	res.status_code = status_code;
+
+	res.stie.element_id = WLAN_EID_SESSION_TRANSITION;
+	res.stie.length = sizeof(res.stie) - 2;
+
+	if (status_code == WLAN_STATUS_SUCCESS) {
+		res.stie.fsts_id = s->data.fsts_id;
+		res.stie.session_control = SESSION_CONTROL(SESSION_TYPE_BSS, 0);
+
+		fst_iface_get_channel_info(s->data.new_iface, &hw_mode,
+					   &channel);
+		res.stie.new_band_id = fst_hw_mode_to_band(hw_mode);
+		res.stie.new_band_op = 1;
+		res.stie.new_band_setup = 0;
+
+		fst_iface_get_channel_info(s->data.old_iface, &hw_mode,
+					   &channel);
+		res.stie.old_band_id = fst_hw_mode_to_band(hw_mode);
+		res.stie.old_band_op = 1;
+		res.stie.old_band_setup = 0;
+
+		fst_printf_session(s, MSG_INFO,
+				   "%s: FST Setup Request accepted for %s (llt=%u)",
+				   fst_iface_get_name(s->data.old_iface),
+				   fst_iface_get_name(s->data.new_iface),
+				   s->data.llt_ms);
+	} else {
+		fst_printf_session(s, MSG_WARNING,
+				   "%s: FST Setup Request rejected with code %d",
+				   fst_iface_get_name(s->data.old_iface),
+				   status_code);
+	}
+
+	if (fst_session_send_action(s, TRUE, &res, sizeof(res),
+				    fst_iface_get_mbie(s->data.old_iface))) {
+		fst_printf_sframe(s, TRUE, MSG_ERROR,
+				  "cannot send FST Setup Response with code %d",
+				  status_code);
+		return -EINVAL;
+	}
+
+	fst_printf_sframe(s, TRUE, MSG_INFO, "FST Setup Response sent");
+
+	if (status_code != WLAN_STATUS_SUCCESS) {
+		union fst_session_state_switch_extra evext = {
+			.to_initial = {
+				.reason = REASON_REJECT,
+				.reject_code = status_code,
+				.initiator = FST_INITIATOR_LOCAL,
+			},
+		};
+		fst_session_set_state(s, FST_SESSION_STATE_INITIAL, &evext);
+	}
+
+	return 0;
+}
+
+
+int fst_session_initiate_switch(struct fst_session *s)
+{
+	struct fst_ack_req req;
+	int res;
+	u8 dialog_token;
+
+	if (!fst_session_is_ready(s)) {
+		fst_printf_session(s, MSG_ERROR,
+				   "cannot initiate switch due to wrong setup state (%d)",
+				   s->state);
+		return -1;
+	}
+
+	dialog_token = fst_group_assign_dialog_token(s->group);
+
+	WPA_ASSERT(s->data.new_iface != NULL);
+	WPA_ASSERT(s->data.old_iface != NULL);
+
+	fst_printf_session(s, MSG_INFO, "initiating FST switch: %s => %s",
+			   fst_iface_get_name(s->data.old_iface),
+			   fst_iface_get_name(s->data.new_iface));
+
+	os_memset(&req, 0, sizeof(req));
+
+	req.action = FST_ACTION_ACK_REQUEST;
+	req.dialog_token = dialog_token;
+	req.fsts_id = host_to_le32(s->data.fsts_id);
+
+	res = fst_session_send_action(s, FALSE, &req, sizeof(req), NULL);
+	if (!res) {
+		fst_printf_sframe(s, FALSE, MSG_INFO, "FST Ack Request sent");
+		fst_session_set_state(s, FST_SESSION_STATE_TRANSITION_DONE,
+				      NULL);
+		fst_session_stt_arm(s);
+	} else {
+		fst_printf_sframe(s, FALSE, MSG_ERROR,
+				  "Cannot send FST Ack Request");
+	}
+
+	return res;
+}
+
+
+void fst_session_handle_action(struct fst_session *s,
+			       struct fst_iface *iface,
+			       const struct ieee80211_mgmt *mgmt,
+			       size_t frame_len)
+{
+	switch (mgmt->u.action.u.fst_action.action) {
+	case FST_ACTION_SETUP_REQUEST:
+		WPA_ASSERT(0);
+		break;
+	case FST_ACTION_SETUP_RESPONSE:
+		fst_session_handle_setup_response(s, iface, mgmt, frame_len);
+		break;
+	case FST_ACTION_TEAR_DOWN:
+		fst_session_handle_tear_down(s, iface, mgmt, frame_len);
+		break;
+	case FST_ACTION_ACK_REQUEST:
+		fst_session_handle_ack_request(s, iface, mgmt, frame_len);
+		break;
+	case FST_ACTION_ACK_RESPONSE:
+		fst_session_handle_ack_response(s, iface, mgmt, frame_len);
+		break;
+	case FST_ACTION_ON_CHANNEL_TUNNEL:
+	default:
+		fst_printf_sframe(s, FALSE, MSG_ERROR,
+				  "Unsupported FST Action frame");
+		break;
+	}
+}
+
+
+int fst_session_tear_down_setup(struct fst_session *s)
+{
+	int res;
+	union fst_session_state_switch_extra evext = {
+		.to_initial = {
+			.reason = REASON_TEARDOWN,
+			.initiator = FST_INITIATOR_LOCAL,
+		},
+	};
+
+	res = fst_session_send_tear_down(s);
+
+	fst_session_set_state(s, FST_SESSION_STATE_INITIAL, &evext);
+
+	return res;
+}
+
+
+void fst_session_reset(struct fst_session *s)
+{
+	fst_session_reset_ex(s, REASON_RESET);
+}
+
+
+void fst_session_delete(struct fst_session *s)
+{
+	fst_printf(MSG_INFO, "Session %u deleted", s->id);
+	dl_list_del(&s->global_sessions_lentry);
+	foreach_fst_ctrl_call(on_session_removed, s);
+	os_free(s);
+}
+
+
+struct fst_group * fst_session_get_group(struct fst_session *s)
+{
+	return s->group;
+}
+
+
+struct fst_iface * fst_session_get_iface(struct fst_session *s, Boolean is_old)
+{
+	return is_old ? s->data.old_iface : s->data.new_iface;
+}
+
+
+u32 fst_session_get_id(struct fst_session *s)
+{
+	return s->id;
+}
+
+
+const u8 * fst_session_get_peer_addr(struct fst_session *s, Boolean is_old)
+{
+	return is_old ? s->data.old_peer_addr : s->data.new_peer_addr;
+}
+
+
+u32 fst_session_get_llt(struct fst_session *s)
+{
+	return s->data.llt_ms;
+}
+
+
+enum fst_session_state fst_session_get_state(struct fst_session *s)
+{
+	return s->state;
+}
+
+
+struct fst_session * fst_session_get_by_id(u32 id)
+{
+	struct fst_session *s;
+
+	foreach_fst_session(s) {
+		if (id == s->id)
+			return s;
+	}
+
+	return NULL;
+}
+
+
+void fst_session_enum(struct fst_group *g, fst_session_enum_clb clb, void *ctx)
+{
+	struct fst_session *s;
+
+	foreach_fst_session(s) {
+		if (!g || s->group == g)
+			clb(s->group, s, ctx);
+	}
+}
+
+
+void fst_session_on_action_rx(struct fst_iface *iface,
+			      const struct ieee80211_mgmt *mgmt,
+			      size_t len)
+{
+	struct fst_session *s;
+
+	if (len < IEEE80211_HDRLEN + 2 ||
+	    mgmt->u.action.category != WLAN_ACTION_FST) {
+		fst_printf_iface(iface, MSG_ERROR,
+				 "invalid Action frame received");
+		return;
+	}
+
+	if (mgmt->u.action.u.fst_action.action <= FST_ACTION_MAX_SUPPORTED) {
+		fst_printf_iface(iface, MSG_DEBUG,
+				 "FST Action '%s' received!",
+				 fst_action_names[mgmt->u.action.u.fst_action.action]);
+	} else {
+		fst_printf_iface(iface, MSG_WARNING,
+				 "unknown FST Action (%u) received!",
+				 mgmt->u.action.u.fst_action.action);
+		return;
+	}
+
+	if (mgmt->u.action.u.fst_action.action == FST_ACTION_SETUP_REQUEST) {
+		fst_session_handle_setup_request(iface, mgmt, len);
+		return;
+	}
+
+	s = fst_find_session_in_progress(mgmt->sa, fst_iface_get_group(iface));
+	if (s) {
+		fst_session_handle_action(s, iface, mgmt, len);
+	} else {
+		fst_printf_iface(iface, MSG_WARNING,
+				 "FST Action '%s' dropped: no session in progress found",
+				 fst_action_names[mgmt->u.action.u.fst_action.action]);
+	}
+}
+
+
+int fst_session_set_str_ifname(struct fst_session *s, const char *ifname,
+			       Boolean is_old)
+{
+	struct fst_group *g = fst_session_get_group(s);
+	struct fst_iface *i;
+
+	i = fst_group_get_iface_by_name(g, ifname);
+	if (!i) {
+		fst_printf_session(s, MSG_WARNING,
+				   "Cannot set iface %s: no such iface within group '%s'",
+				   ifname, fst_group_get_id(g));
+		return -1;
+	}
+
+	fst_session_set_iface(s, i, is_old);
+
+	return 0;
+}
+
+
+int fst_session_set_str_peer_addr(struct fst_session *s, const char *mac,
+				  Boolean is_old)
+{
+	u8 peer_addr[ETH_ALEN];
+	int res = fst_read_peer_addr(mac, peer_addr);
+
+	if (res)
+		return res;
+
+	fst_session_set_peer_addr(s, peer_addr, is_old);
+
+	return 0;
+}
+
+
+int fst_session_set_str_llt(struct fst_session *s, const char *llt_str)
+{
+	char *endp;
+	long int llt = strtol(llt_str, &endp, 0);
+
+	if (*endp || llt < 0 || (unsigned long int) llt > FST_MAX_LLT_MS) {
+		fst_printf_session(s, MSG_WARNING,
+				   "Cannot set llt %s: Invalid llt value (1..%u expected)",
+				   llt_str, FST_MAX_LLT_MS);
+		return -1;
+	}
+	fst_session_set_llt(s, (u32) llt);
+
+	return 0;
+}
+
+
+void fst_session_global_on_iface_detached(struct fst_iface *iface)
+{
+	struct fst_session *s;
+
+	foreach_fst_session(s) {
+		if (fst_session_is_in_progress(s) &&
+		    (s->data.new_iface == iface ||
+		     s->data.old_iface == iface))
+			fst_session_reset_ex(s, REASON_DETACH_IFACE);
+	}
+}
+
+
+struct fst_session * fst_session_global_get_first_by_group(struct fst_group *g)
+{
+	struct fst_session *s;
+
+	foreach_fst_session(s) {
+		if (s->group == g)
+			return s;
+	}
+
+	return NULL;
+}
+
+
+#ifdef CONFIG_FST_TEST
+
+static int get_group_fill_session(struct fst_group **g, struct fst_session *s)
+{
+	const u8 *old_addr, *new_addr;
+	struct fst_get_peer_ctx *ctx;
+
+	os_memset(s, 0, sizeof(*s));
+	foreach_fst_group(*g) {
+		s->data.new_iface = fst_group_first_iface(*g);
+		if (s->data.new_iface)
+			break;
+	}
+	if (!s->data.new_iface)
+		return -EINVAL;
+
+	s->data.old_iface = dl_list_entry(s->data.new_iface->group_lentry.next,
+					  struct fst_iface, group_lentry);
+	if (!s->data.old_iface)
+		return -EINVAL;
+
+	old_addr = fst_iface_get_peer_first(s->data.old_iface, &ctx, TRUE);
+	if (!old_addr)
+		return -EINVAL;
+
+	new_addr = fst_iface_get_peer_first(s->data.new_iface, &ctx, TRUE);
+	if (!new_addr)
+		return -EINVAL;
+
+	os_memcpy(s->data.old_peer_addr, old_addr, ETH_ALEN);
+	os_memcpy(s->data.new_peer_addr, new_addr, ETH_ALEN);
+
+	return 0;
+}
+
+
+#define FST_MAX_COMMAND_WORD_NAME_LENGTH 16
+
+int fst_test_req_send_fst_request(const char *params)
+{
+	int fsts_id;
+	Boolean is_valid;
+	char *endp;
+	struct fst_setup_req req;
+	struct fst_session s;
+	struct fst_group *g;
+	enum hostapd_hw_mode hw_mode;
+	u8 channel;
+	char additional_param[FST_MAX_COMMAND_WORD_NAME_LENGTH];
+
+	if (params[0] != ' ')
+		return -EINVAL;
+	params++;
+	fsts_id = fst_read_next_int_param(params, &is_valid, &endp);
+	if (!is_valid)
+		return -EINVAL;
+
+	if (get_group_fill_session(&g, &s))
+		return -EINVAL;
+
+	req.action = FST_ACTION_SETUP_REQUEST;
+	req.dialog_token = g->dialog_token;
+	req.llt = host_to_le32(FST_LLT_MS_DEFAULT);
+	/* 8.4.2.147 Session Transition element */
+	req.stie.element_id = WLAN_EID_SESSION_TRANSITION;
+	req.stie.length = sizeof(req.stie) - 2;
+	req.stie.fsts_id = host_to_le32(fsts_id);
+	req.stie.session_control = SESSION_CONTROL(SESSION_TYPE_BSS, 0);
+
+	fst_iface_get_channel_info(s.data.new_iface, &hw_mode, &channel);
+	req.stie.new_band_id = fst_hw_mode_to_band(hw_mode);
+	req.stie.new_band_op = 1;
+	req.stie.new_band_setup = 0;
+
+	fst_iface_get_channel_info(s.data.old_iface, &hw_mode, &channel);
+	req.stie.old_band_id = fst_hw_mode_to_band(hw_mode);
+	req.stie.old_band_op = 1;
+	req.stie.old_band_setup = 0;
+
+	if (!fst_read_next_text_param(endp, additional_param,
+				       sizeof(additional_param), &endp)) {
+		if (!os_strcasecmp(additional_param, FST_CTR_PVAL_BAD_NEW_BAND))
+			req.stie.new_band_id = req.stie.old_band_id;
+	}
+
+	return fst_session_send_action(&s, TRUE, &req, sizeof(req),
+				       s.data.old_iface->mb_ie);
+}
+
+
+int fst_test_req_send_fst_response(const char *params)
+{
+	int fsts_id;
+	Boolean is_valid;
+	char *endp;
+	struct fst_setup_res res;
+	struct fst_session s;
+	struct fst_group *g;
+	enum hostapd_hw_mode hw_mode;
+	u8 status_code;
+	u8 channel;
+	char response[FST_MAX_COMMAND_WORD_NAME_LENGTH];
+	struct fst_session *_s;
+
+	if (params[0] != ' ')
+		return -EINVAL;
+	params++;
+	fsts_id = fst_read_next_int_param(params, &is_valid, &endp);
+	if (!is_valid)
+		return -EINVAL;
+
+	if (get_group_fill_session(&g, &s))
+		return -EINVAL;
+
+	status_code = WLAN_STATUS_SUCCESS;
+	if (!fst_read_next_text_param(endp, response, sizeof(response),
+				      &endp)) {
+		if (!os_strcasecmp(response, FST_CS_PVAL_RESPONSE_REJECT))
+			status_code = WLAN_STATUS_PENDING_ADMITTING_FST_SESSION;
+	}
+
+	os_memset(&res, 0, sizeof(res));
+
+	res.action = FST_ACTION_SETUP_RESPONSE;
+	/*
+	 * If some session has just received an FST Setup Request, then
+	 * use the correct dialog token copied from this request.
+	 */
+	_s = fst_find_session_in_progress(fst_session_get_peer_addr(&s, TRUE),
+					  g);
+	res.dialog_token = (_s && fst_session_is_ready_pending(_s)) ?
+		_s->data.pending_setup_req_dlgt : g->dialog_token;
+	res.status_code  = status_code;
+
+	res.stie.element_id = WLAN_EID_SESSION_TRANSITION;
+	res.stie.length = sizeof(res.stie) - 2;
+
+	if (res.status_code == WLAN_STATUS_SUCCESS) {
+		res.stie.fsts_id = fsts_id;
+		res.stie.session_control = SESSION_CONTROL(SESSION_TYPE_BSS, 0);
+
+		fst_iface_get_channel_info(s.data.new_iface, &hw_mode,
+					    &channel);
+		res.stie.new_band_id = fst_hw_mode_to_band(hw_mode);
+		res.stie.new_band_op = 1;
+		res.stie.new_band_setup = 0;
+
+		fst_iface_get_channel_info(s.data.old_iface, &hw_mode,
+					   &channel);
+		res.stie.old_band_id = fst_hw_mode_to_band(hw_mode);
+		res.stie.old_band_op = 1;
+		res.stie.old_band_setup = 0;
+	}
+
+	if (!fst_read_next_text_param(endp, response, sizeof(response),
+				      &endp)) {
+		if (!os_strcasecmp(response, FST_CTR_PVAL_BAD_NEW_BAND))
+			res.stie.new_band_id = res.stie.old_band_id;
+	}
+
+	return fst_session_send_action(&s, TRUE, &res, sizeof(res),
+				       s.data.old_iface->mb_ie);
+}
+
+
+int fst_test_req_send_ack_request(const char *params)
+{
+	int fsts_id;
+	Boolean is_valid;
+	char *endp;
+	struct fst_ack_req req;
+	struct fst_session s;
+	struct fst_group *g;
+
+	if (params[0] != ' ')
+		return -EINVAL;
+	params++;
+	fsts_id = fst_read_next_int_param(params, &is_valid, &endp);
+	if (!is_valid)
+		return -EINVAL;
+
+	if (get_group_fill_session(&g, &s))
+		return -EINVAL;
+
+	os_memset(&req, 0, sizeof(req));
+	req.action = FST_ACTION_ACK_REQUEST;
+	req.dialog_token = g->dialog_token;
+	req.fsts_id = fsts_id;
+
+	return fst_session_send_action(&s, FALSE, &req, sizeof(req), NULL);
+}
+
+
+int fst_test_req_send_ack_response(const char *params)
+{
+	int fsts_id;
+	Boolean is_valid;
+	char *endp;
+	struct fst_ack_res res;
+	struct fst_session s;
+	struct fst_group *g;
+
+	if (params[0] != ' ')
+		return -EINVAL;
+	params++;
+	fsts_id = fst_read_next_int_param(params, &is_valid, &endp);
+	if (!is_valid)
+		return -EINVAL;
+
+	if (get_group_fill_session(&g, &s))
+		return -EINVAL;
+
+	os_memset(&res, 0, sizeof(res));
+	res.action = FST_ACTION_ACK_RESPONSE;
+	res.dialog_token = g->dialog_token;
+	res.fsts_id = fsts_id;
+
+	return fst_session_send_action(&s, FALSE, &res, sizeof(res), NULL);
+}
+
+
+int fst_test_req_send_tear_down(const char *params)
+{
+	int fsts_id;
+	Boolean is_valid;
+	char *endp;
+	struct fst_tear_down td;
+	struct fst_session s;
+	struct fst_group *g;
+
+	if (params[0] != ' ')
+		return -EINVAL;
+	params++;
+	fsts_id = fst_read_next_int_param(params, &is_valid, &endp);
+	if (!is_valid)
+		return -EINVAL;
+
+	if (get_group_fill_session(&g, &s))
+		return -EINVAL;
+
+	os_memset(&td, 0, sizeof(td));
+	td.action = FST_ACTION_TEAR_DOWN;
+	td.fsts_id = fsts_id;
+
+	return fst_session_send_action(&s, TRUE, &td, sizeof(td), NULL);
+}
+
+
+u32 fst_test_req_get_fsts_id(const char *params)
+{
+	int sid;
+	Boolean is_valid;
+	char *endp;
+	struct fst_session *s;
+
+	if (params[0] != ' ')
+		return FST_FSTS_ID_NOT_FOUND;
+	params++;
+	sid = fst_read_next_int_param(params, &is_valid, &endp);
+	if (!is_valid)
+		return FST_FSTS_ID_NOT_FOUND;
+
+	s = fst_session_get_by_id(sid);
+	if (!s)
+		return FST_FSTS_ID_NOT_FOUND;
+
+	return s->data.fsts_id;
+}
+
+
+int fst_test_req_get_local_mbies(const char *request, char *buf, size_t buflen)
+{
+	char *endp;
+	char ifname[FST_MAX_COMMAND_WORD_NAME_LENGTH];
+	struct fst_group *g;
+	struct fst_iface *iface;
+
+	if (request[0] != ' ')
+		return -EINVAL;
+	request++;
+	if (fst_read_next_text_param(request, ifname, sizeof(ifname), &endp) ||
+	    !*ifname)
+		goto problem;
+	g = dl_list_first(&fst_global_groups_list, struct fst_group,
+			  global_groups_lentry);
+	if (!g)
+		goto problem;
+	iface = fst_group_get_iface_by_name(g, ifname);
+	if (!iface || !iface->mb_ie)
+		goto problem;
+	return wpa_snprintf_hex(buf, buflen, wpabuf_head(iface->mb_ie),
+				wpabuf_len(iface->mb_ie));
+
+problem:
+	return os_snprintf(buf, buflen, "FAIL\n");
+}
+
+#endif /* CONFIG_FST_TEST */
diff --git a/src/fst/fst_session.h b/src/fst/fst_session.h
new file mode 100644
index 000000000000..1162de4b367a
--- /dev/null
+++ b/src/fst/fst_session.h
@@ -0,0 +1,80 @@
+/*
+ * FST module - FST Session related definitions
+ * Copyright (c) 2014, Qualcomm Atheros, Inc.
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#ifndef FST_SESSION_H
+#define FST_SESSION_H
+
+#define FST_DEFAULT_SESSION_TIMEOUT_TU 255 /* u8 */
+
+struct fst_iface;
+struct fst_group;
+struct fst_session;
+enum fst_session_state;
+
+int  fst_session_global_init(void);
+void fst_session_global_deinit(void);
+void fst_session_global_on_iface_detached(struct fst_iface *iface);
+struct fst_session *
+fst_session_global_get_first_by_group(struct fst_group *g);
+
+struct fst_session * fst_session_create(struct fst_group *g);
+void fst_session_set_iface(struct fst_session *s, struct fst_iface *iface,
+			   Boolean is_old);
+void fst_session_set_llt(struct fst_session *s, u32 llt);
+void fst_session_set_peer_addr(struct fst_session *s, const u8 *addr,
+			       Boolean is_old);
+int fst_session_initiate_setup(struct fst_session *s);
+int fst_session_respond(struct fst_session *s, u8 status_code);
+int fst_session_initiate_switch(struct fst_session *s);
+void fst_session_handle_action(struct fst_session *s, struct fst_iface *iface,
+			       const struct ieee80211_mgmt *mgmt,
+			       size_t frame_len);
+int fst_session_tear_down_setup(struct fst_session *s);
+void fst_session_reset(struct fst_session *s);
+void fst_session_delete(struct fst_session *s);
+
+struct fst_group * fst_session_get_group(struct fst_session *s);
+struct fst_iface * fst_session_get_iface(struct fst_session *s, Boolean is_old);
+const u8 * fst_session_get_peer_addr(struct fst_session *s, Boolean is_old);
+u32 fst_session_get_id(struct fst_session *s);
+u32 fst_session_get_llt(struct fst_session *s);
+enum fst_session_state fst_session_get_state(struct fst_session *s);
+
+struct fst_session *fst_session_get_by_id(u32 id);
+
+typedef void (*fst_session_enum_clb)(struct fst_group *g, struct fst_session *s,
+				     void *ctx);
+
+void fst_session_enum(struct fst_group *g, fst_session_enum_clb clb, void *ctx);
+
+void fst_session_on_action_rx(struct fst_iface *iface,
+			      const struct ieee80211_mgmt *mgmt, size_t len);
+
+
+int fst_session_set_str_ifname(struct fst_session *s, const char *ifname,
+			       Boolean is_old);
+int fst_session_set_str_peer_addr(struct fst_session *s, const char *mac,
+				  Boolean is_old);
+int fst_session_set_str_llt(struct fst_session *s, const char *llt_str);
+
+#ifdef CONFIG_FST_TEST
+
+#define FST_FSTS_ID_NOT_FOUND ((u32) -1)
+
+int fst_test_req_send_fst_request(const char *params);
+int fst_test_req_send_fst_response(const char *params);
+int fst_test_req_send_ack_request(const char *params);
+int fst_test_req_send_ack_response(const char *params);
+int fst_test_req_send_tear_down(const char *params);
+u32 fst_test_req_get_fsts_id(const char *params);
+int fst_test_req_get_local_mbies(const char *request, char *buf,
+				 size_t buflen);
+
+#endif /* CONFIG_FST_TEST */
+
+#endif /* FST_SESSION_H */
diff --git a/src/l2_packet/Makefile b/src/l2_packet/Makefile
index adfd3dfd5b9b..47925b790c74 100644
--- a/src/l2_packet/Makefile
+++ b/src/l2_packet/Makefile
@@ -1,8 +1,16 @@
-all:
-	@echo Nothing to be made.
+all: libl2_packet.a
 
 clean:
-	rm -f *~ *.o *.d *.gcno *.gcda *.gcov
+	rm -f *~ *.o *.d *.gcno *.gcda *.gcov libl2_packet.a
 
 install:
 	@echo Nothing to be made.
+
+include ../lib.rules
+
+LIB_OBJS = l2_packet_linux.o
+
+libl2_packet.a: $(LIB_OBJS)
+	$(AR) crT $@ $?
+
+-include $(OBJS:%.o=%.d)
diff --git a/src/p2p/Makefile b/src/p2p/Makefile
index adfd3dfd5b9b..5587fcf281d3 100644
--- a/src/p2p/Makefile
+++ b/src/p2p/Makefile
@@ -1,8 +1,29 @@
-all:
-	@echo Nothing to be made.
+all: libp2p.a
 
 clean:
-	rm -f *~ *.o *.d *.gcno *.gcda *.gcov
+	rm -f *~ *.o *.d *.gcno *.gcda *.gcov libp2p.a
 
 install:
 	@echo Nothing to be made.
+
+include ../lib.rules
+
+CFLAGS += -DCONFIG_WIFI_DISPLAY
+CFLAGS += -DCONFIG_WPS_NFC
+
+LIB_OBJS= \
+	p2p_build.o \
+	p2p.o \
+	p2p_dev_disc.o \
+	p2p_go_neg.o \
+	p2p_group.o \
+	p2p_invitation.o \
+	p2p_parse.o \
+	p2p_pd.o \
+	p2p_sd.o \
+	p2p_utils.o
+
+libp2p.a: $(LIB_OBJS)
+	$(AR) crT $@ $?
+
+-include $(OBJS:%.o=%.d)
diff --git a/src/p2p/p2p.c b/src/p2p/p2p.c
index 6adb3dc2049f..767706c01d6b 100644
--- a/src/p2p/p2p.c
+++ b/src/p2p/p2p.c
@@ -48,9 +48,8 @@ static void p2p_scan_timeout(void *eloop_ctx, void *timeout_ctx);
 #define P2P_PEER_EXPIRATION_AGE 60
 #endif /* P2P_PEER_EXPIRATION_AGE */
 
-#define P2P_PEER_EXPIRATION_INTERVAL (P2P_PEER_EXPIRATION_AGE / 2)
 
-static void p2p_expire_peers(struct p2p_data *p2p)
+void p2p_expire_peers(struct p2p_data *p2p)
 {
 	struct p2p_device *dev, *n;
 	struct os_reltime now;
@@ -103,15 +102,6 @@ static void p2p_expire_peers(struct p2p_data *p2p)
 }
 
 
-static void p2p_expiration_timeout(void *eloop_ctx, void *timeout_ctx)
-{
-	struct p2p_data *p2p = eloop_ctx;
-	p2p_expire_peers(p2p);
-	eloop_register_timeout(P2P_PEER_EXPIRATION_INTERVAL, 0,
-			       p2p_expiration_timeout, p2p, NULL);
-}
-
-
 static const char * p2p_state_txt(int state)
 {
 	switch (state) {
@@ -297,7 +287,7 @@ static void p2p_listen_in_find(struct p2p_data *p2p, int dev_disc)
 		return;
 	}
 
-	ies = p2p_build_probe_resp_ies(p2p);
+	ies = p2p_build_probe_resp_ies(p2p, NULL, 0);
 	if (ies == NULL)
 		return;
 
@@ -346,7 +336,7 @@ int p2p_listen(struct p2p_data *p2p, unsigned int timeout)
 		return 0;
 	}
 
-	ies = p2p_build_probe_resp_ies(p2p);
+	ies = p2p_build_probe_resp_ies(p2p, NULL, 0);
 	if (ies == NULL)
 		return -1;
 
@@ -468,7 +458,8 @@ static void p2p_copy_client_info(struct p2p_device *dev,
 
 static int p2p_add_group_clients(struct p2p_data *p2p, const u8 *go_dev_addr,
 				 const u8 *go_interface_addr, int freq,
-				 const u8 *gi, size_t gi_len)
+				 const u8 *gi, size_t gi_len,
+				 struct os_reltime *rx_time)
 {
 	struct p2p_group_info info;
 	size_t c;
@@ -536,10 +527,11 @@ static int p2p_add_group_clients(struct p2p_data *p2p, const u8 *go_dev_addr,
 
 		os_memcpy(dev->interface_addr, cli->p2p_interface_addr,
 			  ETH_ALEN);
-		os_get_reltime(&dev->last_seen);
+		os_memcpy(&dev->last_seen, rx_time, sizeof(struct os_reltime));
 		os_memcpy(dev->member_in_go_dev, go_dev_addr, ETH_ALEN);
 		os_memcpy(dev->member_in_go_iface, go_interface_addr,
 			  ETH_ALEN);
+		dev->flags |= P2P_DEV_LAST_SEEN_AS_GROUP_CLIENT;
 	}
 
 	return 0;
@@ -758,26 +750,35 @@ int p2p_add_device(struct p2p_data *p2p, const u8 *addr, int freq,
 
 	/*
 	 * Update the device entry only if the new peer
-	 * entry is newer than the one previously stored.
+	 * entry is newer than the one previously stored, or if
+	 * the device was previously seen as a P2P Client in a group
+	 * and the new entry isn't older than a threshold.
 	 */
 	if (dev->last_seen.sec > 0 &&
-	    os_reltime_before(rx_time, &dev->last_seen)) {
-		p2p_dbg(p2p, "Do not update peer entry based on old frame (rx_time=%u.%06u last_seen=%u.%06u)",
+	    os_reltime_before(rx_time, &dev->last_seen) &&
+	    (!(dev->flags & P2P_DEV_LAST_SEEN_AS_GROUP_CLIENT) ||
+	     os_reltime_expired(&dev->last_seen, rx_time,
+				P2P_DEV_GROUP_CLIENT_RESP_THRESHOLD))) {
+		p2p_dbg(p2p,
+			"Do not update peer entry based on old frame (rx_time=%u.%06u last_seen=%u.%06u flags=0x%x)",
 			(unsigned int) rx_time->sec,
 			(unsigned int) rx_time->usec,
 			(unsigned int) dev->last_seen.sec,
-			(unsigned int) dev->last_seen.usec);
+			(unsigned int) dev->last_seen.usec,
+			dev->flags);
 		p2p_parse_free(&msg);
 		return -1;
 	}
 
 	os_memcpy(&dev->last_seen, rx_time, sizeof(struct os_reltime));
 
-	dev->flags &= ~(P2P_DEV_PROBE_REQ_ONLY | P2P_DEV_GROUP_CLIENT_ONLY);
+	dev->flags &= ~(P2P_DEV_PROBE_REQ_ONLY | P2P_DEV_GROUP_CLIENT_ONLY |
+			P2P_DEV_LAST_SEEN_AS_GROUP_CLIENT);
 
 	if (os_memcmp(addr, p2p_dev_addr, ETH_ALEN) != 0)
 		os_memcpy(dev->interface_addr, addr, ETH_ALEN);
 	if (msg.ssid &&
+	    msg.ssid[1] <= sizeof(dev->oper_ssid) &&
 	    (msg.ssid[1] != P2P_WILDCARD_SSID_LEN ||
 	     os_memcmp(msg.ssid + 2, P2P_WILDCARD_SSID, P2P_WILDCARD_SSID_LEN)
 	     != 0)) {
@@ -843,7 +844,8 @@ int p2p_add_device(struct p2p_data *p2p, const u8 *addr, int freq,
 
 	if (scan_res) {
 		p2p_add_group_clients(p2p, p2p_dev_addr, addr, freq,
-				      msg.group_info, msg.group_info_len);
+				      msg.group_info, msg.group_info_len,
+				      rx_time);
 	}
 
 	p2p_parse_free(&msg);
@@ -1127,8 +1129,10 @@ static int p2ps_gen_hash(struct p2p_data *p2p, const char *str, u8 *hash)
 
 	adv_array = (u8 *) str_buf;
 	adv_len = os_strlen(str);
+	if (adv_len >= sizeof(str_buf))
+		return 0;
 
-	for (i = 0; str[i] && i < adv_len; i++) {
+	for (i = 0; i < adv_len; i++) {
 		if (str[i] >= 'A' && str[i] <= 'Z')
 			str_buf[i] = str[i] - 'A' + 'a';
 		else
@@ -1182,27 +1186,25 @@ int p2p_find(struct p2p_data *p2p, unsigned int timeout,
 		 * An empty seek string means no hash values, but still an ASP
 		 * search.
 		 */
+		p2p_dbg(p2p, "ASP search");
 		p2p->p2ps_seek_count = 0;
 		p2p->p2ps_seek = 1;
 	} else if (seek && seek_count <= P2P_MAX_QUERY_HASH) {
 		u8 buf[P2PS_HASH_LEN];
-		int i;
+		int i, count = 0;
 
-		p2p->p2ps_seek_count = seek_count;
 		for (i = 0; i < seek_count; i++) {
 			if (!p2ps_gen_hash(p2p, seek[i], buf))
 				continue;
 
-			/* If asking for wildcard, don't do others */
-			if (os_memcmp(buf, p2p->wild_card_hash,
-				      P2PS_HASH_LEN) == 0) {
-				p2p->p2ps_seek_count = 0;
-				break;
-			}
-
-			os_memcpy(&p2p->query_hash[i * P2PS_HASH_LEN], buf,
-				  P2PS_HASH_LEN);
+			p2p_dbg(p2p, "Seek service %s hash " MACSTR,
+				seek[i], MAC2STR(buf));
+			os_memcpy(&p2p->p2ps_seek_hash[count * P2PS_HASH_LEN],
+				  buf, P2PS_HASH_LEN);
+			count++;
 		}
+
+		p2p->p2ps_seek_count = count;
 		p2p->p2ps_seek = 1;
 	} else {
 		p2p->p2ps_seek_count = 0;
@@ -1212,7 +1214,8 @@ int p2p_find(struct p2p_data *p2p, unsigned int timeout,
 	/* Special case to perform wildcard search */
 	if (p2p->p2ps_seek_count == 0 && p2p->p2ps_seek) {
 		p2p->p2ps_seek_count = 1;
-		os_memcpy(&p2p->query_hash, p2p->wild_card_hash, P2PS_HASH_LEN);
+		os_memcpy(&p2p->p2ps_seek_hash, p2p->wild_card_hash,
+			  P2PS_HASH_LEN);
 	}
 
 	p2p->start_after_scan = P2P_AFTER_SCAN_NOTHING;
@@ -1380,7 +1383,7 @@ static int p2p_prepare_channel_pref(struct p2p_data *p2p,
 static void p2p_prepare_channel_best(struct p2p_data *p2p)
 {
 	u8 op_class, op_channel;
-	const int op_classes_5ghz[] = { 124, 115, 0 };
+	const int op_classes_5ghz[] = { 124, 125, 115, 0 };
 	const int op_classes_ht40[] = { 126, 127, 116, 117, 0 };
 	const int op_classes_vht[] = { 128, 0 };
 
@@ -2147,7 +2150,9 @@ int p2p_match_dev_type(struct p2p_data *p2p, struct wpabuf *wps)
 }
 
 
-struct wpabuf * p2p_build_probe_resp_ies(struct p2p_data *p2p)
+struct wpabuf * p2p_build_probe_resp_ies(struct p2p_data *p2p,
+					 const u8 *query_hash,
+					 u8 query_count)
 {
 	struct wpabuf *buf;
 	u8 *len;
@@ -2162,7 +2167,7 @@ struct wpabuf * p2p_build_probe_resp_ies(struct p2p_data *p2p)
 	if (p2p->vendor_elem && p2p->vendor_elem[VENDOR_ELEM_PROBE_RESP_P2P])
 		extra += wpabuf_len(p2p->vendor_elem[VENDOR_ELEM_PROBE_RESP_P2P]);
 
-	if (p2p->query_count)
+	if (query_count)
 		extra += MAX_SVC_ADV_IE_LEN;
 
 	buf = wpabuf_alloc(1000 + extra);
@@ -2199,9 +2204,8 @@ struct wpabuf * p2p_build_probe_resp_ies(struct p2p_data *p2p)
 	p2p_buf_add_device_info(buf, p2p, NULL);
 	p2p_buf_update_ie_hdr(buf, len);
 
-	if (p2p->query_count) {
-		p2p_buf_add_service_instance(buf, p2p, p2p->query_count,
-					     p2p->query_hash,
+	if (query_count) {
+		p2p_buf_add_service_instance(buf, p2p, query_count, query_hash,
 					     p2p->p2ps_adv_list);
 	}
 
@@ -2212,18 +2216,21 @@ struct wpabuf * p2p_build_probe_resp_ies(struct p2p_data *p2p)
 static int p2p_service_find_asp(struct p2p_data *p2p, const u8 *hash)
 {
 	struct p2ps_advertisement *adv_data;
+	int any_wfa;
 
 	p2p_dbg(p2p, "ASP find - ASP list: %p", p2p->p2ps_adv_list);
 
-	/* Wildcard always matches if we have actual services */
-	if (os_memcmp(hash, p2p->wild_card_hash, P2PS_HASH_LEN) == 0)
-		return p2p->p2ps_adv_list != NULL;
+	/* Wildcard org.wi-fi.wfds matches any WFA spec defined service */
+	any_wfa = os_memcmp(hash, p2p->wild_card_hash, P2PS_HASH_LEN) == 0;
 
 	adv_data = p2p->p2ps_adv_list;
 	while (adv_data) {
-		p2p_dbg(p2p, "ASP hash: %x =? %x", hash[0], adv_data->hash[0]);
 		if (os_memcmp(hash, adv_data->hash, P2PS_HASH_LEN) == 0)
-			return 1;
+			return 1; /* exact hash match */
+		if (any_wfa &&
+		    os_strncmp(adv_data->svc_name, P2PS_WILD_HASH_STR,
+			       os_strlen(P2PS_WILD_HASH_STR)) == 0)
+			return 1; /* WFA service match */
 		adv_data = adv_data->next;
 	}
 
@@ -2233,13 +2240,15 @@ static int p2p_service_find_asp(struct p2p_data *p2p, const u8 *hash)
 
 static enum p2p_probe_req_status
 p2p_reply_probe(struct p2p_data *p2p, const u8 *addr, const u8 *dst,
-		const u8 *bssid, const u8 *ie, size_t ie_len)
+		const u8 *bssid, const u8 *ie, size_t ie_len,
+		unsigned int rx_freq)
 {
 	struct ieee802_11_elems elems;
 	struct wpabuf *buf;
 	struct ieee80211_mgmt *resp;
 	struct p2p_message msg;
 	struct wpabuf *ies;
+	u8 channel, op_class;
 
 	if (ieee802_11_parse_elems((u8 *) ie, ie_len, &elems, 0) ==
 	    ParseFailed) {
@@ -2291,53 +2300,42 @@ p2p_reply_probe(struct p2p_data *p2p, const u8 *addr, const u8 *dst,
 		return P2P_PREQ_NOT_P2P;
 	}
 
-	p2p->p2ps_svc_found = 0;
-
 	if (msg.service_hash && msg.service_hash_count) {
 		const u8 *hash = msg.service_hash;
-		u8 *dest = p2p->query_hash;
 		u8 i;
+		int p2ps_svc_found = 0;
+
+		p2p_dbg(p2p, "in_listen=%d drv_in_listen=%d when received P2PS Probe Request at %u MHz; own Listen channel %u, pending listen freq %u MHz",
+			p2p->in_listen, p2p->drv_in_listen, rx_freq,
+			p2p->cfg->channel, p2p->pending_listen_freq);
+
+		if (!p2p->in_listen && !p2p->drv_in_listen &&
+		    p2p->pending_listen_freq && rx_freq &&
+		    rx_freq != p2p->pending_listen_freq) {
+			p2p_dbg(p2p, "Do not reply to Probe Request frame that was received on %u MHz while waiting to start Listen state on %u MHz",
+				rx_freq, p2p->pending_listen_freq);
+			p2p_parse_free(&msg);
+			return P2P_PREQ_NOT_LISTEN;
+		}
 
-		p2p->query_count = 0;
 		for (i = 0; i < msg.service_hash_count; i++) {
 			if (p2p_service_find_asp(p2p, hash)) {
-				p2p->p2ps_svc_found = 1;
-
-				if (!os_memcmp(hash, p2p->wild_card_hash,
-					       P2PS_HASH_LEN)) {
-					/* We found match(es) but wildcard
-					 * will return all */
-					p2p->query_count = 1;
-					os_memcpy(p2p->query_hash, hash,
-						  P2PS_HASH_LEN);
-					break;
-				}
-
-				/* Save each matching hash */
-				if (p2p->query_count < P2P_MAX_QUERY_HASH) {
-					os_memcpy(dest, hash, P2PS_HASH_LEN);
-					dest += P2PS_HASH_LEN;
-					p2p->query_count++;
-				} else {
-					/* We found match(es) but too many to
-					 * return all */
-					p2p->query_count = 0;
-					break;
-				}
+				p2p_dbg(p2p, "Service Hash match found: "
+					MACSTR, MAC2STR(hash));
+				p2ps_svc_found = 1;
+				break;
 			}
 			hash += P2PS_HASH_LEN;
 		}
 
-		p2p_dbg(p2p, "ASP adv found: %d", p2p->p2ps_svc_found);
-
 		/* Probed hash unknown */
-		if (!p2p->p2ps_svc_found) {
+		if (!p2ps_svc_found) {
+			p2p_dbg(p2p, "No Service Hash match found");
 			p2p_parse_free(&msg);
 			return P2P_PREQ_NOT_PROCESSED;
 		}
 	} else {
 		/* This is not a P2PS Probe Request */
-		p2p->query_count = 0;
 		p2p_dbg(p2p, "No P2PS Hash in Probe Request");
 
 		if (!p2p->in_listen || !p2p->drv_in_listen) {
@@ -2366,11 +2364,11 @@ p2p_reply_probe(struct p2p_data *p2p, const u8 *addr, const u8 *dst,
 		p2p_parse_free(&msg);
 		return P2P_PREQ_NOT_PROCESSED;
 	}
-	p2p_parse_free(&msg);
 
 	if (!p2p->cfg->send_probe_resp) {
 		/* Response generated elsewhere */
 		p2p_dbg(p2p, "Probe Resp generated elsewhere - do not generate additional response");
+		p2p_parse_free(&msg);
 		return P2P_PREQ_NOT_PROCESSED;
 	}
 
@@ -2382,7 +2380,9 @@ p2p_reply_probe(struct p2p_data *p2p, const u8 *addr, const u8 *dst,
 	 * really only used for discovery purposes, not to learn exact BSS
 	 * parameters.
 	 */
-	ies = p2p_build_probe_resp_ies(p2p);
+	ies = p2p_build_probe_resp_ies(p2p, msg.service_hash,
+				       msg.service_hash_count);
+	p2p_parse_free(&msg);
 	if (ies == NULL)
 		return P2P_PREQ_NOT_PROCESSED;
 
@@ -2392,8 +2392,8 @@ p2p_reply_probe(struct p2p_data *p2p, const u8 *addr, const u8 *dst,
 		return P2P_PREQ_NOT_PROCESSED;
 	}
 
-	resp = NULL;
-	resp = wpabuf_put(buf, resp->u.probe_resp.variable - (u8 *) resp);
+	resp = wpabuf_put(buf, offsetof(struct ieee80211_mgmt,
+					u.probe_resp.variable));
 
 	resp->frame_control = host_to_le16((WLAN_FC_TYPE_MGMT << 2) |
 					   (WLAN_FC_STYPE_PROBE_RESP << 4));
@@ -2422,32 +2422,50 @@ p2p_reply_probe(struct p2p_data *p2p, const u8 *addr, const u8 *dst,
 	wpabuf_put_u8(buf, 480 / 5);
 	wpabuf_put_u8(buf, 540 / 5);
 
+	if (!rx_freq) {
+		channel = p2p->cfg->channel;
+	} else if (p2p_freq_to_channel(rx_freq, &op_class, &channel)) {
+		wpabuf_free(ies);
+		wpabuf_free(buf);
+		return P2P_PREQ_NOT_PROCESSED;
+	}
+
 	wpabuf_put_u8(buf, WLAN_EID_DS_PARAMS);
 	wpabuf_put_u8(buf, 1);
-	wpabuf_put_u8(buf, p2p->cfg->channel);
+	wpabuf_put_u8(buf, channel);
 
 	wpabuf_put_buf(buf, ies);
 	wpabuf_free(ies);
 
-	p2p->cfg->send_probe_resp(p2p->cfg->cb_ctx, buf);
+	p2p->cfg->send_probe_resp(p2p->cfg->cb_ctx, buf, rx_freq);
 
 	wpabuf_free(buf);
 
-	return P2P_PREQ_NOT_PROCESSED;
+	return P2P_PREQ_PROCESSED;
 }
 
 
 enum p2p_probe_req_status
 p2p_probe_req_rx(struct p2p_data *p2p, const u8 *addr, const u8 *dst,
-		 const u8 *bssid, const u8 *ie, size_t ie_len)
+		 const u8 *bssid, const u8 *ie, size_t ie_len,
+		 unsigned int rx_freq)
 {
 	enum p2p_probe_req_status res;
 
 	p2p_add_dev_from_probe_req(p2p, addr, ie, ie_len);
 
-	res = p2p_reply_probe(p2p, addr, dst, bssid, ie, ie_len);
-	p2p->query_count = 0;
+	res = p2p_reply_probe(p2p, addr, dst, bssid, ie, ie_len, rx_freq);
+	if (res != P2P_PREQ_PROCESSED && res != P2P_PREQ_NOT_PROCESSED)
+		return res;
 
+	/*
+	 * Activate a pending GO Negotiation/Invite flow if a received Probe
+	 * Request frame is from an expected peer. Some devices may share the
+	 * same address for P2P and non-P2P STA running simultaneously. The
+	 * P2P_PREQ_PROCESSED and P2P_PREQ_NOT_PROCESSED p2p_reply_probe()
+	 * return values verified above ensure we are handling a Probe Request
+	 * frame from a P2P peer.
+	 */
 	if ((p2p->state == P2P_CONNECT || p2p->state == P2P_CONNECT_LISTEN) &&
 	    p2p->go_neg_peer &&
 	    os_memcmp(addr, p2p->go_neg_peer->info.p2p_device_addr, ETH_ALEN)
@@ -2457,7 +2475,7 @@ p2p_probe_req_rx(struct p2p_data *p2p, const u8 *addr, const u8 *dst,
 		p2p_dbg(p2p, "Found GO Negotiation peer - try to start GO negotiation from timeout");
 		eloop_cancel_timeout(p2p_go_neg_start, p2p, NULL);
 		eloop_register_timeout(0, 0, p2p_go_neg_start, p2p, NULL);
-		return P2P_PREQ_PROCESSED;
+		return res;
 	}
 
 	if ((p2p->state == P2P_INVITE || p2p->state == P2P_INVITE_LISTEN) &&
@@ -2469,7 +2487,7 @@ p2p_probe_req_rx(struct p2p_data *p2p, const u8 *addr, const u8 *dst,
 		p2p_dbg(p2p, "Found Invite peer - try to start Invite from timeout");
 		eloop_cancel_timeout(p2p_invite_start, p2p, NULL);
 		eloop_register_timeout(0, 0, p2p_invite_start, p2p, NULL);
-		return P2P_PREQ_PROCESSED;
+		return res;
 	}
 
 	return res;
@@ -2484,10 +2502,21 @@ static int p2p_assoc_req_ie_wlan_ap(struct p2p_data *p2p, const u8 *bssid,
 	size_t tmplen;
 	int res;
 	u8 group_capab;
+	struct p2p_message msg;
 
 	if (p2p_ie == NULL)
 		return 0; /* WLAN AP is not a P2P manager */
 
+	os_memset(&msg, 0, sizeof(msg));
+	if (p2p_parse_p2p_ie(p2p_ie, &msg) < 0)
+		return 0;
+
+	p2p_dbg(p2p, "BSS P2P manageability %s",
+		msg.manageability ? "enabled" : "disabled");
+
+	if (!msg.manageability)
+		return 0;
+
 	/*
 	 * (Re)Association Request - P2P IE
 	 * P2P Capability attribute (shall be present)
@@ -2650,13 +2679,14 @@ int p2p_service_del_asp(struct p2p_data *p2p, u32 adv_id)
 
 int p2p_service_add_asp(struct p2p_data *p2p, int auto_accept, u32 adv_id,
 			const char *adv_str, u8 svc_state, u16 config_methods,
-			const char *svc_info)
+			const char *svc_info, const u8 *cpt_priority)
 {
 	struct p2ps_advertisement *adv_data, *tmp, **prev;
 	u8 buf[P2PS_HASH_LEN];
 	size_t adv_data_len, adv_len, info_len = 0;
+	int i;
 
-	if (!p2p || !adv_str || !adv_str[0])
+	if (!p2p || !adv_str || !adv_str[0] || !cpt_priority)
 		return -1;
 
 	if (!(config_methods & p2p->cfg->config_methods)) {
@@ -2685,6 +2715,11 @@ int p2p_service_add_asp(struct p2p_data *p2p, int auto_accept, u32 adv_id,
 	adv_data->auto_accept = (u8) auto_accept;
 	os_memcpy(adv_data->svc_name, adv_str, adv_len);
 
+	for (i = 0; cpt_priority[i] && i < P2PS_FEATURE_CAPAB_CPT_MAX; i++) {
+		adv_data->cpt_priority[i] = cpt_priority[i];
+		adv_data->cpt_mask |= cpt_priority[i];
+	}
+
 	if (svc_info && info_len) {
 		adv_data->svc_info = &adv_data->svc_name[adv_len + 1];
 		os_memcpy(adv_data->svc_info, svc_info, info_len);
@@ -2723,13 +2758,33 @@ int p2p_service_add_asp(struct p2p_data *p2p, int auto_accept, u32 adv_id,
 
 inserted:
 	p2p_dbg(p2p,
-		"Added ASP advertisement adv_id=0x%x config_methods=0x%x svc_state=0x%x adv_str='%s'",
-		adv_id, adv_data->config_methods, svc_state, adv_str);
+		"Added ASP advertisement adv_id=0x%x config_methods=0x%x svc_state=0x%x adv_str='%s' cpt_mask=0x%x",
+		adv_id, adv_data->config_methods, svc_state, adv_str,
+		adv_data->cpt_mask);
 
 	return 0;
 }
 
 
+void p2p_service_flush_asp(struct p2p_data *p2p)
+{
+	struct p2ps_advertisement *adv, *prev;
+
+	if (!p2p)
+		return;
+
+	adv = p2p->p2ps_adv_list;
+	while (adv) {
+		prev = adv;
+		adv = adv->next;
+		os_free(prev);
+	}
+
+	p2p->p2ps_adv_list = NULL;
+	p2p_dbg(p2p, "All ASP advertisements flushed");
+}
+
+
 int p2p_parse_dev_addr_in_p2p_ie(struct wpabuf *p2p_ie, u8 *dev_addr)
 {
 	struct p2p_message msg;
@@ -2861,9 +2916,6 @@ struct p2p_data * p2p_init(const struct p2p_config *cfg)
 
 	dl_list_init(&p2p->devices);
 
-	eloop_register_timeout(P2P_PEER_EXPIRATION_INTERVAL, 0,
-			       p2p_expiration_timeout, p2p, NULL);
-
 	p2p->go_timeout = 100;
 	p2p->client_timeout = 20;
 	p2p->num_p2p_sd_queries = 0;
@@ -2878,8 +2930,6 @@ struct p2p_data * p2p_init(const struct p2p_config *cfg)
 
 void p2p_deinit(struct p2p_data *p2p)
 {
-	struct p2ps_advertisement *adv, *prev;
-
 #ifdef CONFIG_WIFI_DISPLAY
 	wpabuf_free(p2p->wfd_ie_beacon);
 	wpabuf_free(p2p->wfd_ie_probe_req);
@@ -2894,7 +2944,6 @@ void p2p_deinit(struct p2p_data *p2p)
 	wpabuf_free(p2p->wfd_coupled_sink_info);
 #endif /* CONFIG_WIFI_DISPLAY */
 
-	eloop_cancel_timeout(p2p_expiration_timeout, p2p, NULL);
 	eloop_cancel_timeout(p2p_ext_listen_timeout, p2p, NULL);
 	eloop_cancel_timeout(p2p_scan_timeout, p2p, NULL);
 	eloop_cancel_timeout(p2p_go_neg_start, p2p, NULL);
@@ -2908,18 +2957,12 @@ void p2p_deinit(struct p2p_data *p2p)
 	os_free(p2p->cfg->serial_number);
 	os_free(p2p->cfg->pref_chan);
 	os_free(p2p->groups);
-	os_free(p2p->p2ps_prov);
+	p2ps_prov_free(p2p);
 	wpabuf_free(p2p->sd_resp);
 	os_free(p2p->after_scan_tx);
 	p2p_remove_wps_vendor_extensions(p2p);
 	os_free(p2p->no_go_freq.range);
-
-	adv = p2p->p2ps_adv_list;
-	while (adv) {
-		prev = adv;
-		adv = adv->next;
-		os_free(prev);
-	}
+	p2p_service_flush_asp(p2p);
 
 	os_free(p2p);
 }
@@ -2937,6 +2980,7 @@ void p2p_flush(struct p2p_data *p2p)
 	p2p_free_sd_queries(p2p);
 	os_free(p2p->after_scan_tx);
 	p2p->after_scan_tx = NULL;
+	p2p->ssid_set = 0;
 }
 
 
@@ -4120,7 +4164,7 @@ int p2p_get_peer_info_txt(const struct p2p_peer_info *info,
 			  "country=%c%c\n"
 			  "oper_freq=%d\n"
 			  "req_config_methods=0x%x\n"
-			  "flags=%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n"
+			  "flags=%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n"
 			  "status=%d\n"
 			  "invitation_reqs=%u\n",
 			  (int) (now.sec - dev->last_seen.sec),
@@ -4164,6 +4208,8 @@ int p2p_get_peer_info_txt(const struct p2p_peer_info *info,
 			  "[FORCE_FREQ]" : "",
 			  dev->flags & P2P_DEV_PD_FOR_JOIN ?
 			  "[PD_FOR_JOIN]" : "",
+			  dev->flags & P2P_DEV_LAST_SEEN_AS_GROUP_CLIENT ?
+			  "[LAST_SEEN_AS_GROUP_CLIENT]" : "",
 			  dev->status,
 			  dev->invitation_reqs);
 	if (os_snprintf_error(end - pos, res))
@@ -5215,6 +5261,7 @@ int p2p_process_nfc_connection_handover(struct p2p_data *p2p,
 
 	if (!msg.oob_go_neg_channel) {
 		p2p_dbg(p2p, "OOB GO Negotiation Channel attribute not included");
+		p2p_parse_free(&msg);
 		return -1;
 	}
 
@@ -5226,6 +5273,7 @@ int p2p_process_nfc_connection_handover(struct p2p_data *p2p,
 					   msg.oob_go_neg_channel[4]);
 	if (freq < 0) {
 		p2p_dbg(p2p, "Unknown peer OOB GO Neg channel");
+		p2p_parse_free(&msg);
 		return -1;
 	}
 	role = msg.oob_go_neg_channel[5];
@@ -5246,6 +5294,7 @@ int p2p_process_nfc_connection_handover(struct p2p_data *p2p,
 					   p2p->cfg->channel);
 		if (freq < 0) {
 			p2p_dbg(p2p, "Own listen channel not known");
+			p2p_parse_free(&msg);
 			return -1;
 		}
 		p2p_dbg(p2p, "Use own Listen channel as OOB GO Neg channel: %u MHz", freq);
@@ -5334,3 +5383,20 @@ void p2p_go_neg_wait_timeout(void *eloop_ctx, void *timeout_ctx)
 		"Timeout on waiting peer to become ready for GO Negotiation");
 	p2p_go_neg_failed(p2p, -1);
 }
+
+
+void p2p_set_own_pref_freq_list(struct p2p_data *p2p,
+				const unsigned int *pref_freq_list,
+				unsigned int size)
+{
+	unsigned int i;
+
+	if (size > P2P_MAX_PREF_CHANNELS)
+		size = P2P_MAX_PREF_CHANNELS;
+	p2p->num_pref_freq = size;
+	for (i = 0; i < size; i++) {
+		p2p->pref_freq_list[i] = pref_freq_list[i];
+		p2p_dbg(p2p, "Own preferred frequency list[%u]=%u MHz",
+			i, p2p->pref_freq_list[i]);
+	}
+}
diff --git a/src/p2p/p2p.h b/src/p2p/p2p.h
index 2402db6a7eb9..b4060be477b6 100644
--- a/src/p2p/p2p.h
+++ b/src/p2p/p2p.h
@@ -9,7 +9,8 @@
 #ifndef P2P_H
 #define P2P_H
 
-#include "wps/wps_defs.h"
+#include "common/ieee802_11_defs.h"
+#include "wps/wps.h"
 
 /* P2P ASP Setup Capability */
 #define P2PS_SETUP_NONE 0
@@ -20,6 +21,12 @@
 #define P2PS_WILD_HASH_STR "org.wi-fi.wfds"
 #define P2PS_HASH_LEN 6
 #define P2P_MAX_QUERY_HASH 6
+#define P2PS_FEATURE_CAPAB_CPT_MAX 2
+
+/**
+ * P2P_MAX_PREF_CHANNELS - Maximum number of preferred channels
+ */
+#define P2P_MAX_PREF_CHANNELS 100
 
 /**
  * P2P_MAX_REG_CLASSES - Maximum number of regulatory classes
@@ -95,7 +102,7 @@ struct p2p_go_neg_results {
 	/**
 	 * ssid - SSID of the group
 	 */
-	u8 ssid[32];
+	u8 ssid[SSID_MAX_LEN];
 
 	/**
 	 * ssid_len - Length of SSID in octets
@@ -154,6 +161,11 @@ struct p2p_go_neg_results {
 };
 
 struct p2ps_provision {
+	/**
+	 * pd_seeker - P2PS provision discovery seeker role
+	 */
+	unsigned int pd_seeker:1;
+
 	/**
 	 * status - Remote returned provisioning status code
 	 */
@@ -194,6 +206,23 @@ struct p2ps_provision {
 	 */
 	u8 adv_mac[ETH_ALEN];
 
+	/**
+	 * cpt_mask - Supported Coordination Protocol Transport mask
+	 *
+	 * A bitwise mask of supported ASP Coordination Protocol Transports.
+	 * This property is set together and corresponds with cpt_priority.
+	 */
+	u8 cpt_mask;
+
+	/**
+	 * cpt_priority - Coordination Protocol Transport priority list
+	 *
+	 * Priorities of supported ASP Coordination Protocol Transports.
+	 * This property is set together and corresponds with cpt_mask.
+	 * The CPT priority list is 0 terminated.
+	 */
+	u8 cpt_priority[P2PS_FEATURE_CAPAB_CPT_MAX + 1];
+
 	/**
 	 * info - Vendor defined extra Provisioning information
 	 */
@@ -233,6 +262,23 @@ struct p2ps_advertisement {
 	 */
 	u8 hash[P2PS_HASH_LEN];
 
+	/**
+	 * cpt_mask - supported Coordination Protocol Transport mask
+	 *
+	 * A bitwise mask of supported ASP Coordination Protocol Transports.
+	 * This property is set together and corresponds with cpt_priority.
+	 */
+	u8 cpt_mask;
+
+	/**
+	 * cpt_priority - Coordination Protocol Transport priority list
+	 *
+	 * Priorities of supported ASP Coordinatin Protocol Transports.
+	 * This property is set together and corresponds with cpt_mask.
+	 * The CPT priority list is 0 terminated.
+	 */
+	u8 cpt_priority[P2PS_FEATURE_CAPAB_CPT_MAX + 1];
+
 	/**
 	 * svc_name - NULL Terminated UTF-8 Service Name, and svc_info storage
 	 */
@@ -268,27 +314,27 @@ struct p2p_peer_info {
 	/**
 	 * device_name - Device Name (0..32 octets encoded in UTF-8)
 	 */
-	char device_name[33];
+	char device_name[WPS_DEV_NAME_MAX_LEN + 1];
 
 	/**
 	 * manufacturer - Manufacturer (0..64 octets encoded in UTF-8)
 	 */
-	char manufacturer[65];
+	char manufacturer[WPS_MANUFACTURER_MAX_LEN + 1];
 
 	/**
 	 * model_name - Model Name (0..32 octets encoded in UTF-8)
 	 */
-	char model_name[33];
+	char model_name[WPS_MODEL_NAME_MAX_LEN + 1];
 
 	/**
 	 * model_number - Model Number (0..32 octets encoded in UTF-8)
 	 */
-	char model_number[33];
+	char model_number[WPS_MODEL_NUMBER_MAX_LEN + 1];
 
 	/**
 	 * serial_number - Serial Number (0..32 octets encoded in UTF-8)
 	 */
-	char serial_number[33];
+	char serial_number[WPS_SERIAL_NUMBER_MAX_LEN + 1];
 
 	/**
 	 * level - Signal level
@@ -316,7 +362,7 @@ struct p2p_peer_info {
 	 * This list includes from 0 to 16 Secondary Device Types as indicated
 	 * by wps_sec_dev_type_list_len (8 * number of types).
 	 */
-	u8 wps_sec_dev_type_list[128];
+	u8 wps_sec_dev_type_list[WPS_SEC_DEV_TYPE_MAX_LEN];
 
 	/**
 	 * wps_sec_dev_type_list_len - Length of secondary device type list
@@ -495,7 +541,7 @@ struct p2p_config {
 	 * This data will be added to the end of the SSID after the
 	 * DIRECT-<random two octets> prefix.
 	 */
-	u8 ssid_postfix[32 - 9];
+	u8 ssid_postfix[SSID_MAX_LEN - 9];
 
 	/**
 	 * ssid_postfix_len - Length of the ssid_postfix data
@@ -569,12 +615,14 @@ struct p2p_config {
 	 * send_probe_resp - Transmit a Probe Response frame
 	 * @ctx: Callback context from cb_ctx
 	 * @buf: Probe Response frame (including the header and body)
+	 * @freq: Forced frequency (in MHz) to use or 0.
 	 * Returns: 0 on success, -1 on failure
 	 *
 	 * This function is used to reply to Probe Request frames that were
 	 * indicated with a call to p2p_probe_req_rx(). The response is to be
-	 * sent on the same channel or to be dropped if the driver is not
-	 * anymore listening to Probe Request frames.
+	 * sent on the same channel, unless otherwise specified, or to be
+	 * dropped if the driver is not listening to Probe Request frames
+	 * anymore.
 	 *
 	 * Alternatively, the responsibility for building the Probe Response
 	 * frames in Listen state may be in another system component in which
@@ -585,7 +633,8 @@ struct p2p_config {
 	 * Request frames must be indicated by calling p2p_probe_req_rx() even
 	 * if this send_probe_resp() is not used.
 	 */
-	int (*send_probe_resp)(void *ctx, const struct wpabuf *buf);
+	int (*send_probe_resp)(void *ctx, const struct wpabuf *buf,
+			       unsigned int freq);
 
 	/**
 	 * send_action - Transmit an Action frame
@@ -704,6 +753,7 @@ struct p2p_config {
 	 * @ctx: Callback context from cb_ctx
 	 * @src: Source address of the message triggering this notification
 	 * @dev_passwd_id: WPS Device Password ID
+	 * @go_intent: Peer's GO Intent
 	 *
 	 * This callback is used to notify that a P2P Device is requesting
 	 * group owner negotiation with us, but we do not have all the
@@ -712,7 +762,8 @@ struct p2p_config {
 	 * PIN or PBC button press. This information can be provided with a
 	 * call to p2p_connect().
 	 */
-	void (*go_neg_req_rx)(void *ctx, const u8 *src, u16 dev_passwd_id);
+	void (*go_neg_req_rx)(void *ctx, const u8 *src, u16 dev_passwd_id,
+			      u8 go_intent);
 
 	/**
 	 * go_neg_completed - Notification of GO Negotiation results
@@ -949,18 +1000,21 @@ struct p2p_config {
 
 	/**
 	 * Determine if we have a persistent group we share with remote peer
+	 * and allocate interface for this group if needed
 	 * @ctx: Callback context from cb_ctx
 	 * @addr: Peer device address to search for
 	 * @ssid: Persistent group SSID or %NULL if any
 	 * @ssid_len: Length of @ssid
-	 * @go_dev_addr: Buffer for returning intended GO P2P Device Address
+	 * @go_dev_addr: Buffer for returning GO P2P Device Address
 	 * @ret_ssid: Buffer for returning group SSID
 	 * @ret_ssid_len: Buffer for returning length of @ssid
+	 * @intended_iface_addr: Buffer for returning intended iface address
 	 * Returns: 1 if a matching persistent group was found, 0 otherwise
 	 */
 	int (*get_persistent_group)(void *ctx, const u8 *addr, const u8 *ssid,
 				    size_t ssid_len, u8 *go_dev_addr,
-				    u8 *ret_ssid, size_t *ret_ssid_len);
+				    u8 *ret_ssid, size_t *ret_ssid_len,
+				    u8 *intended_iface_addr);
 
 	/**
 	 * Get information about a possible local GO role
@@ -1001,7 +1055,8 @@ struct p2p_config {
 				   u8 conncap, int passwd_id,
 				   const u8 *persist_ssid,
 				   size_t persist_ssid_size, int response_done,
-				   int prov_start, const char *session_info);
+				   int prov_start, const char *session_info,
+				   const u8 *feat_cap, size_t feat_cap_len);
 
 	/**
 	 * prov_disc_resp_cb - Callback for indicating completion of PD Response
@@ -1023,6 +1078,20 @@ struct p2p_config {
 	 * P2PS_SETUP_* bitmap is used as the parameters and return value.
 	 */
 	u8 (*p2ps_group_capability)(void *ctx, u8 incoming, u8 role);
+
+	/**
+	 * get_pref_freq_list - Get preferred frequency list for an interface
+	 * @ctx: Callback context from cb_ctx
+	 * @go: Whether the use if for GO role
+	 * @len: Length of freq_list in entries (both IN and OUT)
+	 * @freq_list: Buffer for returning the preferred frequencies (MHz)
+	 * Returns: 0 on success, -1 on failure
+	 *
+	 * This function can be used to query the preferred frequency list from
+	 * the driver specific to a particular interface type.
+	 */
+	int (*get_pref_freq_list)(void *ctx, int go,
+				  unsigned int *len, unsigned int *freq_list);
 };
 
 
@@ -1460,11 +1529,13 @@ enum p2p_probe_req_status {
  * @bssid: BSSID if available or %NULL
  * @ie: Information elements from the Probe Request frame body
  * @ie_len: Length of ie buffer in octets
+ * @rx_freq: Probe Request frame RX frequency
  * Returns: value indicating the type and status of the probe request
  */
 enum p2p_probe_req_status
 p2p_probe_req_rx(struct p2p_data *p2p, const u8 *addr, const u8 *dst,
-		 const u8 *bssid, const u8 *ie, size_t ie_len);
+		 const u8 *bssid, const u8 *ie, size_t ie_len,
+		 unsigned int rx_freq);
 
 /**
  * p2p_rx_action - Report received Action frame
@@ -1607,7 +1678,7 @@ struct p2p_group_config {
 	/**
 	 * ssid - Group SSID
 	 */
-	u8 ssid[32];
+	u8 ssid[SSID_MAX_LEN];
 
 	/**
 	 * ssid_len - Length of SSID
@@ -2214,7 +2285,7 @@ struct p2p_nfc_params {
 	size_t oob_dev_pw_len;
 	int go_freq;
 	u8 go_dev_addr[ETH_ALEN];
-	u8 go_ssid[32];
+	u8 go_ssid[SSID_MAX_LEN];
 	size_t go_ssid_len;
 };
 
@@ -2240,8 +2311,33 @@ struct p2ps_advertisement *
 p2p_service_p2ps_id(struct p2p_data *p2p, u32 adv_id);
 int p2p_service_add_asp(struct p2p_data *p2p, int auto_accept, u32 adv_id,
 			const char *adv_str, u8 svc_state,
-			u16 config_methods, const char *svc_info);
+			u16 config_methods, const char *svc_info,
+			const u8 *cpt_priority);
 int p2p_service_del_asp(struct p2p_data *p2p, u32 adv_id);
+void p2p_service_flush_asp(struct p2p_data *p2p);
 struct p2ps_advertisement * p2p_get_p2ps_adv_list(struct p2p_data *p2p);
 
+/**
+ * p2p_expire_peers - Periodic cleanup function to expire peers
+ * @p2p: P2P module context from p2p_init()
+ *
+ * This is a cleanup function that the entity calling p2p_init() is
+ * expected to call periodically to clean up expired peer entries.
+ */
+void p2p_expire_peers(struct p2p_data *p2p);
+
+void p2p_set_own_pref_freq_list(struct p2p_data *p2p,
+				const unsigned int *pref_freq_list,
+				unsigned int size);
+
+/**
+ * p2p_group_get_common_freqs - Get the group common frequencies
+ * @group: P2P group context from p2p_group_init()
+ * @common_freqs: On return will hold the group common frequencies
+ * @num: On return will hold the number of group common frequencies
+ * Returns: 0 on success, -1 otherwise
+ */
+int p2p_group_get_common_freqs(struct p2p_group *group, int *common_freqs,
+			       unsigned int *num);
+
 #endif /* P2P_H */
diff --git a/src/p2p/p2p_build.c b/src/p2p/p2p_build.c
index 92c920662edb..793d28ba7bdd 100644
--- a/src/p2p/p2p_build.c
+++ b/src/p2p/p2p_build.c
@@ -10,6 +10,7 @@
 
 #include "common.h"
 #include "common/ieee802_11_defs.h"
+#include "common/qca-vendor.h"
 #include "wps/wps_i.h"
 #include "p2p_i.h"
 
@@ -109,6 +110,44 @@ void p2p_buf_add_operating_channel(struct wpabuf *buf, const char *country,
 }
 
 
+void p2p_buf_add_pref_channel_list(struct wpabuf *buf,
+				   const u32 *preferred_freq_list,
+				   unsigned int size)
+{
+	unsigned int i, count = 0;
+	u8 op_class, op_channel;
+
+	if (!size)
+		return;
+
+	/*
+	 * First, determine the number of P2P supported channels in the
+	 * pref_freq_list returned from driver. This is needed for calculations
+	 * of the vendor IE size.
+	 */
+	for (i = 0; i < size; i++) {
+		if (p2p_freq_to_channel(preferred_freq_list[i], &op_class,
+					&op_channel) == 0)
+			count++;
+	}
+
+	wpabuf_put_u8(buf, WLAN_EID_VENDOR_SPECIFIC);
+	wpabuf_put_u8(buf, 4 + count * sizeof(u16));
+	wpabuf_put_be24(buf, OUI_QCA);
+	wpabuf_put_u8(buf, QCA_VENDOR_ELEM_P2P_PREF_CHAN_LIST);
+	for (i = 0; i < size; i++) {
+		if (p2p_freq_to_channel(preferred_freq_list[i], &op_class,
+					&op_channel) < 0) {
+			wpa_printf(MSG_DEBUG, "Unsupported frequency %u MHz",
+				   preferred_freq_list[i]);
+			continue;
+		}
+		wpabuf_put_u8(buf, op_class);
+		wpabuf_put_u8(buf, op_channel);
+	}
+}
+
+
 void p2p_buf_add_channel_list(struct wpabuf *buf, const char *country,
 			      struct p2p_channels *chan)
 {
@@ -353,10 +392,10 @@ void p2p_buf_add_service_hash(struct wpabuf *buf, struct p2p_data *p2p)
 	/* Service Hash */
 	wpabuf_put_u8(buf, P2P_ATTR_SERVICE_HASH);
 	wpabuf_put_le16(buf, p2p->p2ps_seek_count * P2PS_HASH_LEN);
-	wpabuf_put_data(buf, p2p->query_hash,
+	wpabuf_put_data(buf, p2p->p2ps_seek_hash,
 			p2p->p2ps_seek_count * P2PS_HASH_LEN);
 	wpa_hexdump(MSG_DEBUG, "P2P: * Service Hash",
-		    p2p->query_hash, p2p->p2ps_seek_count * P2PS_HASH_LEN);
+		    p2p->p2ps_seek_hash, p2p->p2ps_seek_count * P2PS_HASH_LEN);
 }
 
 
@@ -404,152 +443,221 @@ void p2p_buf_add_advertisement_id(struct wpabuf *buf, u32 id, const u8 *mac)
 }
 
 
+static int p2ps_wildcard_hash(struct p2p_data *p2p,
+			      const u8 *hash, u8 hash_count)
+{
+	u8 i;
+	const u8 *test = hash;
+
+	for (i = 0; i < hash_count; i++) {
+		if (os_memcmp(test, p2p->wild_card_hash, P2PS_HASH_LEN) == 0)
+			return 1;
+		test += P2PS_HASH_LEN;
+	}
+
+	return 0;
+}
+
+
+static int p2p_wfa_service_adv(struct p2p_data *p2p)
+{
+	struct p2ps_advertisement *adv;
+
+	for (adv = p2p->p2ps_adv_list; adv; adv = adv->next) {
+		if (os_strncmp(adv->svc_name, P2PS_WILD_HASH_STR,
+			       os_strlen(P2PS_WILD_HASH_STR)) == 0)
+			return 1;
+	}
+
+	return 0;
+}
+
+
+static int p2p_buf_add_service_info(struct wpabuf *buf, struct p2p_data *p2p,
+				    u32 adv_id, u16 config_methods,
+				    const char *svc_name, u8 **ie_len, u8 **pos,
+				    size_t *total_len, u8 *attr_len)
+{
+	size_t svc_len;
+	size_t remaining;
+	size_t info_len;
+
+	p2p_dbg(p2p, "Add service info for %s (adv_id=%u)", svc_name, adv_id);
+	svc_len = os_strlen(svc_name);
+	info_len = sizeof(adv_id) + sizeof(config_methods) + sizeof(u8) +
+		svc_len;
+
+	if (info_len + *total_len > MAX_SVC_ADV_LEN) {
+		p2p_dbg(p2p,
+			"Unsufficient buffer, failed to add advertised service info");
+		return -1;
+	}
+
+	if (svc_len > 255) {
+		p2p_dbg(p2p,
+			"Invalid service name length (%u bytes), failed to add advertised service info",
+			(unsigned int) svc_len);
+		return -1;
+	}
+
+	if (*ie_len) {
+		int ie_data_len = (*pos - *ie_len) - 1;
+
+		if (ie_data_len < 0 || ie_data_len > 255) {
+			p2p_dbg(p2p,
+				"Invalid IE length, failed to add advertised service info");
+			return -1;
+		}
+		remaining = 255 - ie_data_len;
+	} else {
+		/*
+		 * Adding new P2P IE header takes 6 extra bytes:
+		 * - 2 byte IE header (1 byte IE id and 1 byte length)
+		 * - 4 bytes of IE_VENDOR_TYPE are reduced from 255 below
+		 */
+		*ie_len = p2p_buf_add_ie_hdr(buf);
+		remaining = 255 - 4;
+	}
+
+	if (remaining < sizeof(u32) + sizeof(u16) + sizeof(u8)) {
+		/*
+		 * Split adv_id, config_methods, and svc_name_len between two
+		 * IEs.
+		 */
+		size_t front = remaining;
+		size_t back = sizeof(u32) + sizeof(u16) + sizeof(u8) - front;
+		u8 holder[sizeof(u32) + sizeof(u16) + sizeof(u8)];
+
+		WPA_PUT_LE32(holder, adv_id);
+		WPA_PUT_BE16(&holder[sizeof(u32)], config_methods);
+		holder[sizeof(u32) + sizeof(u16)] = svc_len;
+
+		if (front)
+			wpabuf_put_data(buf, holder, front);
+
+		p2p_buf_update_ie_hdr(buf, *ie_len);
+		*ie_len = p2p_buf_add_ie_hdr(buf);
+
+		wpabuf_put_data(buf, &holder[front], back);
+		remaining = 255 - 4 - (sizeof(u32) + sizeof(u16) + sizeof(u8)) -
+			back;
+	} else {
+		wpabuf_put_le32(buf, adv_id);
+		wpabuf_put_be16(buf, config_methods);
+		wpabuf_put_u8(buf, svc_len);
+		remaining -= sizeof(adv_id) + sizeof(config_methods) +
+			sizeof(u8);
+	}
+
+	if (remaining < svc_len) {
+		/* split svc_name between two or three IEs */
+		size_t front = remaining;
+		size_t back = svc_len - front;
+
+		if (front)
+			wpabuf_put_data(buf, svc_name, front);
+
+		p2p_buf_update_ie_hdr(buf, *ie_len);
+		*ie_len = p2p_buf_add_ie_hdr(buf);
+
+		/* In rare cases, we must split across 3 attributes */
+		if (back > 255 - 4) {
+			wpabuf_put_data(buf, &svc_name[front], 255 - 4);
+			back -= 255 - 4;
+			front += 255 - 4;
+			p2p_buf_update_ie_hdr(buf, *ie_len);
+			*ie_len = p2p_buf_add_ie_hdr(buf);
+		}
+
+		wpabuf_put_data(buf, &svc_name[front], back);
+		remaining = 255 - 4 - back;
+	} else {
+		wpabuf_put_data(buf, svc_name, svc_len);
+		remaining -= svc_len;
+	}
+
+	p2p_buf_update_ie_hdr(buf, *ie_len);
+
+	/* set *ie_len to NULL if a new IE has to be added on the next call */
+	if (!remaining)
+		*ie_len = NULL;
+
+	/* set *pos to point to the next byte to update */
+	*pos = wpabuf_put(buf, 0);
+
+	*total_len += info_len;
+	WPA_PUT_LE16(attr_len, (u16) *total_len);
+	return 0;
+}
+
+
 void p2p_buf_add_service_instance(struct wpabuf *buf, struct p2p_data *p2p,
 				  u8 hash_count, const u8 *hash,
 				  struct p2ps_advertisement *adv_list)
 {
 	struct p2ps_advertisement *adv;
-	struct wpabuf *tmp_buf;
-	u8 *tag_len = NULL, *ie_len = NULL;
-	size_t svc_len = 0, remaining = 0, total_len = 0;
+	int p2ps_wildcard;
+	size_t total_len;
+	struct wpabuf *tmp_buf = NULL;
+	u8 *pos, *attr_len, *ie_len = NULL;
 
-	if (!adv_list || !hash)
+	if (!adv_list || !hash || !hash_count)
 		return;
 
+	wpa_hexdump(MSG_DEBUG, "P2PS: Probe Request service hash values",
+		    hash, hash_count * P2PS_HASH_LEN);
+	p2ps_wildcard = p2ps_wildcard_hash(p2p, hash, hash_count) &&
+		p2p_wfa_service_adv(p2p);
+
 	/* Allocate temp buffer, allowing for overflow of 1 instance */
 	tmp_buf = wpabuf_alloc(MAX_SVC_ADV_IE_LEN + 256 + P2PS_HASH_LEN);
 	if (!tmp_buf)
 		return;
 
+	/*
+	 * Attribute data can be split into a number of IEs. Start with the
+	 * first IE and the attribute headers here.
+	 */
+	ie_len = p2p_buf_add_ie_hdr(tmp_buf);
+
+	total_len = 0;
+
+	wpabuf_put_u8(tmp_buf, P2P_ATTR_ADVERTISED_SERVICE);
+	attr_len = wpabuf_put(tmp_buf, sizeof(u16));
+	WPA_PUT_LE16(attr_len, (u16) total_len);
+	p2p_buf_update_ie_hdr(tmp_buf, ie_len);
+	pos = wpabuf_put(tmp_buf, 0);
+
+	if (p2ps_wildcard) {
+		/* org.wi-fi.wfds match found */
+		p2p_buf_add_service_info(tmp_buf, p2p, 0, 0, P2PS_WILD_HASH_STR,
+					 &ie_len, &pos, &total_len, attr_len);
+	}
+
+	/* add advertised service info of matching services */
 	for (adv = adv_list; adv && total_len <= MAX_SVC_ADV_LEN;
 	     adv = adv->next) {
-		u8 count = hash_count;
 		const u8 *test = hash;
+		u8 i;
 
-		while (count--) {
-			/* Check for wildcard */
-			if (os_memcmp(test, p2p->wild_card_hash,
-				      P2PS_HASH_LEN) == 0) {
-				total_len = MAX_SVC_ADV_LEN + 1;
-				goto wild_hash;
-			}
-
-			if (os_memcmp(test, adv->hash, P2PS_HASH_LEN) == 0)
-				goto hash_match;
+		for (i = 0; i < hash_count; i++) {
+			/* exact name hash match */
+			if (os_memcmp(test, adv->hash, P2PS_HASH_LEN) == 0 &&
+			    p2p_buf_add_service_info(tmp_buf, p2p,
+						     adv->id,
+						     adv->config_methods,
+						     adv->svc_name,
+						     &ie_len, &pos,
+						     &total_len,
+						     attr_len))
+				break;
 
 			test += P2PS_HASH_LEN;
 		}
-
-		/* No matches found - Skip this Adv Instance */
-		continue;
-
-hash_match:
-		if (!tag_len) {
-			tag_len = p2p_buf_add_ie_hdr(tmp_buf);
-			remaining = 255 - 4;
-			if (!ie_len) {
-				wpabuf_put_u8(tmp_buf,
-					      P2P_ATTR_ADVERTISED_SERVICE);
-				ie_len = wpabuf_put(tmp_buf, sizeof(u16));
-				remaining -= (sizeof(u8) + sizeof(u16));
-			}
-		}
-
-		svc_len = os_strlen(adv->svc_name);
-
-		if (7 + svc_len + total_len > MAX_SVC_ADV_LEN) {
-			/* Can't fit... return wildcard */
-			total_len = MAX_SVC_ADV_LEN + 1;
-			break;
-		}
-
-		if (remaining <= (sizeof(adv->id) +
-				  sizeof(adv->config_methods))) {
-			size_t front = remaining;
-			size_t back = (sizeof(adv->id) +
-				       sizeof(adv->config_methods)) - front;
-			u8 holder[sizeof(adv->id) +
-				  sizeof(adv->config_methods)];
-
-			/* This works even if front or back == 0 */
-			WPA_PUT_LE32(holder, adv->id);
-			WPA_PUT_BE16(&holder[sizeof(adv->id)],
-				     adv->config_methods);
-			wpabuf_put_data(tmp_buf, holder, front);
-			p2p_buf_update_ie_hdr(tmp_buf, tag_len);
-			tag_len = p2p_buf_add_ie_hdr(tmp_buf);
-			wpabuf_put_data(tmp_buf, &holder[front], back);
-			remaining = 255 - (sizeof(adv->id) +
-					   sizeof(adv->config_methods)) - back;
-		} else {
-			wpabuf_put_le32(tmp_buf, adv->id);
-			wpabuf_put_be16(tmp_buf, adv->config_methods);
-			remaining -= (sizeof(adv->id) +
-				      sizeof(adv->config_methods));
-		}
-
-		/* We are guaranteed at least one byte for svc_len */
-		wpabuf_put_u8(tmp_buf, svc_len);
-		remaining -= sizeof(u8);
-
-		if (remaining < svc_len) {
-			size_t front = remaining;
-			size_t back = svc_len - front;
-
-			wpabuf_put_data(tmp_buf, adv->svc_name, front);
-			p2p_buf_update_ie_hdr(tmp_buf, tag_len);
-			tag_len = p2p_buf_add_ie_hdr(tmp_buf);
-
-			/* In rare cases, we must split across 3 attributes */
-			if (back > 255 - 4) {
-				wpabuf_put_data(tmp_buf,
-						&adv->svc_name[front], 255 - 4);
-				back -= 255 - 4;
-				front += 255 - 4;
-				p2p_buf_update_ie_hdr(tmp_buf, tag_len);
-				tag_len = p2p_buf_add_ie_hdr(tmp_buf);
-			}
-
-			wpabuf_put_data(tmp_buf, &adv->svc_name[front], back);
-			remaining = 255 - 4 - back;
-		} else {
-			wpabuf_put_data(tmp_buf, adv->svc_name, svc_len);
-			remaining -= svc_len;
-		}
-
-		/*           adv_id      config_methods     svc_string */
-		total_len += sizeof(u32) + sizeof(u16) + sizeof(u8) + svc_len;
 	}
 
-	if (tag_len)
-		p2p_buf_update_ie_hdr(tmp_buf, tag_len);
-
-	if (ie_len)
-		WPA_PUT_LE16(ie_len, (u16) total_len);
-
-wild_hash:
-	/* If all fit, return matching instances, otherwise the wildcard */
-	if (total_len <= MAX_SVC_ADV_LEN) {
+	if (total_len)
 		wpabuf_put_buf(buf, tmp_buf);
-	} else {
-		char *wild_card = P2PS_WILD_HASH_STR;
-		u8 wild_len;
-
-		/* Insert wildcard instance */
-		tag_len = p2p_buf_add_ie_hdr(buf);
-		wpabuf_put_u8(buf, P2P_ATTR_ADVERTISED_SERVICE);
-		ie_len = wpabuf_put(buf, sizeof(u16));
-
-		wild_len = (u8) os_strlen(wild_card);
-		wpabuf_put_le32(buf, 0);
-		wpabuf_put_be16(buf, 0);
-		wpabuf_put_u8(buf, wild_len);
-		wpabuf_put_data(buf, wild_card, wild_len);
-
-		WPA_PUT_LE16(ie_len, 4 + 2 + 1 + wild_len);
-		p2p_buf_update_ie_hdr(buf, tag_len);
-	}
-
 	wpabuf_free(tmp_buf);
 }
 
diff --git a/src/p2p/p2p_dev_disc.c b/src/p2p/p2p_dev_disc.c
index 86bae1a2c0db..98805fee2409 100644
--- a/src/p2p/p2p_dev_disc.c
+++ b/src/p2p/p2p_dev_disc.c
@@ -314,7 +314,7 @@ void p2p_process_go_disc_req(struct p2p_data *p2p, const u8 *da, const u8 *sa,
 
 	p2p_dbg(p2p, "Received GO Discoverability Request - remain awake for 100 TU");
 
-	ies = p2p_build_probe_resp_ies(p2p);
+	ies = p2p_build_probe_resp_ies(p2p, NULL, 0);
 	if (ies == NULL)
 		return;
 
diff --git a/src/p2p/p2p_go_neg.c b/src/p2p/p2p_go_neg.c
index 98abf9d2293e..83b43563d945 100644
--- a/src/p2p/p2p_go_neg.c
+++ b/src/p2p/p2p_go_neg.c
@@ -185,6 +185,9 @@ static struct wpabuf * p2p_build_go_neg_req(struct p2p_data *p2p,
 				      p2p->op_reg_class, p2p->op_channel);
 	p2p_buf_update_ie_hdr(buf, len);
 
+	p2p_buf_add_pref_channel_list(buf, p2p->pref_freq_list,
+				      p2p->num_pref_freq);
+
 	/* WPS IE with Device Password ID attribute */
 	pw_id = p2p_wps_method_pw_id(peer->wps_method);
 	if (peer->oob_pw_id)
@@ -312,7 +315,7 @@ static struct wpabuf * p2p_build_go_neg_resp(struct p2p_data *p2p,
 			       group_capab);
 	p2p_buf_add_go_intent(buf, (p2p->go_intent << 1) | tie_breaker);
 	p2p_buf_add_config_timeout(buf, p2p->go_timeout, p2p->client_timeout);
-	if (peer && peer->go_state == REMOTE_GO) {
+	if (peer && peer->go_state == REMOTE_GO && !p2p->num_pref_freq) {
 		p2p_dbg(p2p, "Omit Operating Channel attribute");
 	} else {
 		p2p_buf_add_operating_channel(buf, p2p->cfg->country,
@@ -379,7 +382,7 @@ void p2p_reselect_channel(struct p2p_data *p2p,
 	int freq;
 	u8 op_reg_class, op_channel;
 	unsigned int i;
-	const int op_classes_5ghz[] = { 124, 115, 0 };
+	const int op_classes_5ghz[] = { 124, 125, 115, 0 };
 	const int op_classes_ht40[] = { 126, 127, 116, 117, 0 };
 	const int op_classes_vht[] = { 128, 0 };
 
@@ -542,6 +545,195 @@ int p2p_go_select_channel(struct p2p_data *p2p, struct p2p_device *dev,
 }
 
 
+static void p2p_check_pref_chan_no_recv(struct p2p_data *p2p, int go,
+					struct p2p_device *dev,
+					struct p2p_message *msg,
+					unsigned freq_list[], unsigned int size)
+{
+	u8 op_class, op_channel;
+	unsigned int oper_freq = 0, i, j;
+	int found = 0;
+
+	p2p_dbg(p2p,
+		"Peer didn't provide a preferred frequency list, see if any of our preferred channels are supported by peer device");
+
+	/*
+	 * Search for a common channel in our preferred frequency list which is
+	 * also supported by the peer device.
+	 */
+	for (i = 0; i < size && !found; i++) {
+		/*
+		 * Make sure that the common frequency is:
+		 * 1. Supported by peer
+		 * 2. Allowed for P2P use.
+		 */
+		oper_freq = freq_list[i];
+		if (p2p_freq_to_channel(oper_freq, &op_class,
+					&op_channel) < 0) {
+			p2p_dbg(p2p, "Unsupported frequency %u MHz", oper_freq);
+			continue;
+		}
+		if (!p2p_channels_includes(&p2p->cfg->channels,
+					   op_class, op_channel) &&
+		    (go || !p2p_channels_includes(&p2p->cfg->cli_channels,
+						  op_class, op_channel))) {
+			p2p_dbg(p2p,
+				"Freq %u MHz (oper_class %u channel %u) not allowed for P2P",
+				oper_freq, op_class, op_channel);
+			break;
+		}
+		for (j = 0; j < msg->channel_list_len; j++) {
+
+			if (op_channel != msg->channel_list[j])
+				continue;
+
+			p2p->op_reg_class = op_class;
+			p2p->op_channel = op_channel;
+			os_memcpy(&p2p->channels, &p2p->cfg->channels,
+				  sizeof(struct p2p_channels));
+			found = 1;
+			break;
+		}
+	}
+
+	if (found) {
+		p2p_dbg(p2p,
+			"Freq %d MHz is a preferred channel and is also supported by peer, use it as the operating channel",
+			oper_freq);
+	} else {
+		p2p_dbg(p2p,
+			"None of our preferred channels are supported by peer!. Use: %d MHz for oper_channel",
+			dev->oper_freq);
+	}
+}
+
+
+static void p2p_check_pref_chan_recv(struct p2p_data *p2p, int go,
+				     struct p2p_device *dev,
+				     struct p2p_message *msg,
+				     unsigned freq_list[], unsigned int size)
+{
+	u8 op_class, op_channel;
+	unsigned int oper_freq = 0, i, j;
+	int found = 0;
+
+	/*
+	 * Peer device supports a Preferred Frequency List.
+	 * Search for a common channel in the preferred frequency lists
+	 * of both peer and local devices.
+	 */
+	for (i = 0; i < size && !found; i++) {
+		for (j = 2; j < (msg->pref_freq_list_len / 2); j++) {
+			oper_freq = p2p_channel_to_freq(
+				msg->pref_freq_list[2 * j],
+				msg->pref_freq_list[2 * j + 1]);
+			if (freq_list[i] != oper_freq)
+				continue;
+
+			/*
+			 * Make sure that the found frequency is:
+			 * 1. Supported
+			 * 2. Allowed for P2P use.
+			 */
+			if (p2p_freq_to_channel(oper_freq, &op_class,
+						&op_channel) < 0) {
+				p2p_dbg(p2p, "Unsupported frequency %u MHz",
+					oper_freq);
+				continue;
+			}
+
+			if (!p2p_channels_includes(&p2p->cfg->channels,
+						   op_class, op_channel) &&
+			    (go ||
+			     !p2p_channels_includes(&p2p->cfg->cli_channels,
+						    op_class, op_channel))) {
+				p2p_dbg(p2p,
+					"Freq %u MHz (oper_class %u channel %u) not allowed for P2P",
+					oper_freq, op_class, op_channel);
+				break;
+			}
+			p2p->op_reg_class = op_class;
+			p2p->op_channel = op_channel;
+			os_memcpy(&p2p->channels, &p2p->cfg->channels,
+				  sizeof(struct p2p_channels));
+			found = 1;
+			break;
+		}
+	}
+
+	if (found) {
+		p2p_dbg(p2p,
+			"Freq %d MHz is a common preferred channel for both peer and local, use it as operating channel",
+			oper_freq);
+	} else {
+		p2p_dbg(p2p,
+			"No common preferred channels found! Use: %d MHz for oper_channel",
+			dev->oper_freq);
+	}
+}
+
+
+void p2p_check_pref_chan(struct p2p_data *p2p, int go,
+			 struct p2p_device *dev, struct p2p_message *msg)
+{
+	unsigned int freq_list[P2P_MAX_PREF_CHANNELS], size;
+	unsigned int i;
+	u8 op_class, op_channel;
+
+	/*
+	 * Use the preferred channel list from the driver only if there is no
+	 * forced_freq, e.g., P2P_CONNECT freq=..., and no preferred operating
+	 * channel hardcoded in the configuration file.
+	 */
+	if (!p2p->cfg->get_pref_freq_list || p2p->cfg->num_pref_chan ||
+	    (dev->flags & P2P_DEV_FORCE_FREQ) || p2p->cfg->cfg_op_channel)
+		return;
+
+	/* Obtain our preferred frequency list from driver based on P2P role. */
+	size = P2P_MAX_PREF_CHANNELS;
+	if (p2p->cfg->get_pref_freq_list(p2p->cfg->cb_ctx, go, &size,
+					 freq_list))
+		return;
+
+	/*
+	 * Check if peer's preference of operating channel is in
+	 * our preferred channel list.
+	 */
+	for (i = 0; i < size; i++) {
+		if (freq_list[i] == (unsigned int) dev->oper_freq)
+			break;
+	}
+	if (i != size) {
+		/* Peer operating channel preference matches our preference */
+		if (p2p_freq_to_channel(freq_list[i], &op_class, &op_channel) <
+		    0) {
+			p2p_dbg(p2p,
+				"Peer operating channel preference is unsupported frequency %u MHz",
+				freq_list[i]);
+		} else {
+			p2p->op_reg_class = op_class;
+			p2p->op_channel = op_channel;
+			os_memcpy(&p2p->channels, &p2p->cfg->channels,
+				  sizeof(struct p2p_channels));
+			return;
+		}
+	}
+
+	p2p_dbg(p2p,
+		"Peer operating channel preference: %d MHz is not in our preferred channel list",
+		dev->oper_freq);
+
+	/*
+	  Check if peer's preferred channel list is
+	  * _not_ included in the GO Negotiation Request or Invitation Request.
+	  */
+	if (msg->pref_freq_list_len == 0)
+		p2p_check_pref_chan_no_recv(p2p, go, dev, msg, freq_list, size);
+	else
+		p2p_check_pref_chan_recv(p2p, go, dev, msg, freq_list, size);
+}
+
+
 void p2p_process_go_neg_req(struct p2p_data *p2p, const u8 *sa,
 			    const u8 *data, size_t len, int rx_freq)
 {
@@ -668,7 +860,9 @@ void p2p_process_go_neg_req(struct p2p_data *p2p, const u8 *sa,
 			MAC2STR(sa));
 		status = P2P_SC_FAIL_INFO_CURRENTLY_UNAVAILABLE;
 		p2p->cfg->go_neg_req_rx(p2p->cfg->cb_ctx, sa,
-					msg.dev_password_id);
+					msg.dev_password_id,
+					msg.go_intent ? (*msg.go_intent >> 1) :
+					0);
 	} else if (p2p->go_neg_peer && p2p->go_neg_peer != dev) {
 		p2p_dbg(p2p, "Already in Group Formation with another peer");
 		status = P2P_SC_FAIL_UNABLE_TO_ACCOMMODATE;
@@ -797,6 +991,12 @@ void p2p_process_go_neg_req(struct p2p_data *p2p, const u8 *sa,
 		p2p_dbg(p2p, "Peer operating channel preference: %d MHz",
 			dev->oper_freq);
 
+		/*
+		 * Use the driver preferred frequency list extension if
+		 * supported.
+		 */
+		p2p_check_pref_chan(p2p, go, dev, &msg);
+
 		if (msg.config_timeout) {
 			dev->go_timeout = msg.config_timeout[0];
 			dev->client_timeout = msg.config_timeout[1];
@@ -1148,6 +1348,13 @@ void p2p_process_go_neg_resp(struct p2p_data *p2p, const u8 *sa,
 	if (go && p2p_go_select_channel(p2p, dev, &status) < 0)
 		goto fail;
 
+	/*
+	 * Use the driver preferred frequency list extension if local device is
+	 * GO.
+	 */
+	if (go)
+		p2p_check_pref_chan(p2p, go, dev, &msg);
+
 	p2p_set_state(p2p, P2P_GO_NEG);
 	p2p_clear_timeout(p2p);
 
diff --git a/src/p2p/p2p_group.c b/src/p2p/p2p_group.c
index 41ca99faaf48..0d6699346568 100644
--- a/src/p2p/p2p_group.c
+++ b/src/p2p/p2p_group.c
@@ -1071,3 +1071,43 @@ void p2p_loop_on_all_groups(struct p2p_data *p2p,
 			break;
 	}
 }
+
+
+int p2p_group_get_common_freqs(struct p2p_group *group, int *common_freqs,
+			       unsigned int *num)
+
+{
+	struct p2p_channels intersect, res;
+	struct p2p_group_member *m;
+
+	if (!group || !common_freqs || !num)
+		return -1;
+
+	os_memset(&intersect, 0, sizeof(intersect));
+	os_memset(&res, 0, sizeof(res));
+
+	p2p_channels_union(&intersect, &group->p2p->cfg->channels,
+			   &intersect);
+
+	p2p_channels_dump(group->p2p,
+			  "Group common freqs before iterating members",
+			  &intersect);
+
+	for (m = group->members; m; m = m->next) {
+		struct p2p_device *dev;
+
+		dev = p2p_get_device(group->p2p, m->dev_addr);
+		if (!dev)
+			continue;
+
+		p2p_channels_intersect(&intersect, &dev->channels, &res);
+		intersect = res;
+	}
+
+	p2p_channels_dump(group->p2p, "Group common channels", &intersect);
+
+	os_memset(common_freqs, 0, *num * sizeof(int));
+	*num = p2p_channels_to_freqs(&intersect, common_freqs, *num);
+
+	return 0;
+}
diff --git a/src/p2p/p2p_i.h b/src/p2p/p2p_i.h
index 6af19ceda450..0ce4058fe3e6 100644
--- a/src/p2p/p2p_i.h
+++ b/src/p2p/p2p_i.h
@@ -14,6 +14,12 @@
 
 #define P2P_GO_NEG_CNF_MAX_RETRY_COUNT 1
 
+/*
+ * A threshold (in seconds) to prefer a direct Probe Response frame from a P2P
+ * Device over the P2P Client Info received from a GO.
+ */
+#define P2P_DEV_GROUP_CLIENT_RESP_THRESHOLD 1
+
 enum p2p_role_indication;
 
 /*
@@ -47,6 +53,9 @@ struct p2p_device {
 	 * from Beacon/Probe Response), the interface address is stored here.
 	 * p2p_device_addr must still be set in such a case to the unique
 	 * identifier for the P2P Device.
+	 *
+	 * This field is also used during P2PS PD to store the intended GO
+	 * address of the peer.
 	 */
 	u8 interface_addr[ETH_ALEN];
 
@@ -71,7 +80,7 @@ struct p2p_device {
 	char country[3];
 	struct p2p_channels channels;
 	int oper_freq;
-	u8 oper_ssid[32];
+	u8 oper_ssid[SSID_MAX_LEN];
 	size_t oper_ssid_len;
 
 	/**
@@ -107,6 +116,8 @@ struct p2p_device {
 #define P2P_DEV_WAIT_INV_REQ_ACK BIT(19)
 #define P2P_DEV_P2PS_REPORTED BIT(20)
 #define P2P_DEV_PD_PEER_P2PS BIT(21)
+#define P2P_DEV_LAST_SEEN_AS_GROUP_CLIENT BIT(22)
+
 	unsigned int flags;
 
 	int status; /* enum p2p_status_code */
@@ -322,7 +333,7 @@ struct p2p_data {
 	/**
 	 * ssid - Selected SSID for GO Negotiation (if local end will be GO)
 	 */
-	u8 ssid[32];
+	u8 ssid[SSID_MAX_LEN];
 
 	/**
 	 * ssid_len - ssid length in octets
@@ -403,7 +414,7 @@ struct p2p_data {
 	enum p2p_invite_role inv_role;
 	u8 inv_bssid[ETH_ALEN];
 	int inv_bssid_set;
-	u8 inv_ssid[32];
+	u8 inv_ssid[SSID_MAX_LEN];
 	size_t inv_ssid_len;
 	u8 inv_sa[ETH_ALEN];
 	u8 inv_group_bssid[ETH_ALEN];
@@ -506,11 +517,9 @@ struct p2p_data {
 	struct p2ps_advertisement *p2ps_adv_list;
 	struct p2ps_provision *p2ps_prov;
 	u8 wild_card_hash[P2PS_HASH_LEN];
-	u8 query_hash[P2P_MAX_QUERY_HASH * P2PS_HASH_LEN];
-	u8 query_count;
 	u8 p2ps_seek;
+	u8 p2ps_seek_hash[P2P_MAX_QUERY_HASH * P2PS_HASH_LEN];
 	u8 p2ps_seek_count;
-	u8 p2ps_svc_found;
 
 #ifdef CONFIG_WIFI_DISPLAY
 	struct wpabuf *wfd_ie_beacon;
@@ -529,6 +538,9 @@ struct p2p_data {
 	u16 authorized_oob_dev_pw_id;
 
 	struct wpabuf **vendor_elem;
+
+	unsigned int pref_freq_list[P2P_MAX_PREF_CHANNELS];
+	unsigned int num_pref_freq;
 };
 
 /**
@@ -578,7 +590,7 @@ struct p2p_message {
 	const u8 *p2p_device_addr;
 	const u8 *pri_dev_type;
 	u8 num_sec_dev_types;
-	char device_name[33];
+	char device_name[WPS_DEV_NAME_MAX_LEN + 1];
 	u16 config_methods;
 
 	/* WPS IE */
@@ -631,6 +643,9 @@ struct p2p_message {
 	const u8 *persistent_dev;
 	const u8 *persistent_ssid;
 	size_t persistent_ssid_len;
+
+	const u8 *pref_freq_list;
+	size_t pref_freq_list_len;
 };
 
 
@@ -757,6 +772,8 @@ void p2p_buf_add_persistent_group_info(struct wpabuf *buf, const u8 *dev_addr,
 				       const u8 *ssid, size_t ssid_len);
 int p2p_build_wps_ie(struct p2p_data *p2p, struct wpabuf *buf, int pw_id,
 		     int all_attr);
+void p2p_buf_add_pref_channel_list(struct wpabuf *buf,
+				   const u32 *preferred_freq_list, u32 size);
 
 /* p2p_sd.c */
 struct p2p_sd_query * p2p_pending_sd_req(struct p2p_data *p2p,
@@ -786,6 +803,8 @@ int p2p_connect_send(struct p2p_data *p2p, struct p2p_device *dev);
 u16 p2p_wps_method_pw_id(enum p2p_wps_method wps_method);
 void p2p_reselect_channel(struct p2p_data *p2p,
 			  struct p2p_channels *intersection);
+void p2p_check_pref_chan(struct p2p_data *p2p, int go,
+			 struct p2p_device *dev, struct p2p_message *msg);
 
 /* p2p_pd.c */
 void p2p_process_prov_disc_req(struct p2p_data *p2p, const u8 *sa,
@@ -795,6 +814,7 @@ void p2p_process_prov_disc_resp(struct p2p_data *p2p, const u8 *sa,
 int p2p_send_prov_disc_req(struct p2p_data *p2p, struct p2p_device *dev,
 			   int join, int force_freq);
 void p2p_reset_pending_pd(struct p2p_data *p2p);
+void p2ps_prov_free(struct p2p_data *p2p);
 
 /* p2p_invitation.c */
 void p2p_process_invitation_req(struct p2p_data *p2p, const u8 *sa,
@@ -840,7 +860,9 @@ void p2p_go_complete(struct p2p_data *p2p, struct p2p_device *peer);
 int p2p_match_dev_type(struct p2p_data *p2p, struct wpabuf *wps);
 int dev_type_list_match(const u8 *dev_type, const u8 *req_dev_type[],
 			size_t num_req_dev_type);
-struct wpabuf * p2p_build_probe_resp_ies(struct p2p_data *p2p);
+struct wpabuf * p2p_build_probe_resp_ies(struct p2p_data *p2p,
+					 const u8 *query_hash,
+					 u8 query_count);
 void p2p_build_ssid(struct p2p_data *p2p, u8 *ssid, size_t *ssid_len);
 int p2p_send_action(struct p2p_data *p2p, unsigned int freq, const u8 *dst,
 		    const u8 *src, const u8 *bssid, const u8 *buf,
diff --git a/src/p2p/p2p_invitation.c b/src/p2p/p2p_invitation.c
index 558c6dd0c58f..108e5b7f93e4 100644
--- a/src/p2p/p2p_invitation.c
+++ b/src/p2p/p2p_invitation.c
@@ -85,6 +85,9 @@ static struct wpabuf * p2p_build_invitation_req(struct p2p_data *p2p,
 	p2p_buf_add_device_info(buf, p2p, peer);
 	p2p_buf_update_ie_hdr(buf, len);
 
+	p2p_buf_add_pref_channel_list(buf, p2p->pref_freq_list,
+				      p2p->num_pref_freq);
+
 #ifdef CONFIG_WIFI_DISPLAY
 	if (wfd_ie)
 		wpabuf_put_buf(buf, wfd_ie);
@@ -134,6 +137,9 @@ static struct wpabuf * p2p_build_invitation_resp(struct p2p_data *p2p,
 		extra = wpabuf_len(wfd_ie);
 #endif /* CONFIG_WIFI_DISPLAY */
 
+	if (p2p->vendor_elem && p2p->vendor_elem[VENDOR_ELEM_P2P_INV_RESP])
+		extra += wpabuf_len(p2p->vendor_elem[VENDOR_ELEM_P2P_INV_RESP]);
+
 	buf = wpabuf_alloc(1000 + extra);
 	if (buf == NULL)
 		return NULL;
@@ -158,6 +164,9 @@ static struct wpabuf * p2p_build_invitation_resp(struct p2p_data *p2p,
 		wpabuf_put_buf(buf, wfd_ie);
 #endif /* CONFIG_WIFI_DISPLAY */
 
+	if (p2p->vendor_elem && p2p->vendor_elem[VENDOR_ELEM_P2P_INV_RESP])
+		wpabuf_put_buf(buf, p2p->vendor_elem[VENDOR_ELEM_P2P_INV_RESP]);
+
 	return buf;
 }
 
@@ -337,6 +346,12 @@ void p2p_process_invitation_req(struct p2p_data *p2p, const u8 *sa,
 			p2p_reselect_channel(p2p, &intersection);
 		}
 
+		/*
+		 * Use the driver preferred frequency list extension if
+		 * supported.
+		 */
+		p2p_check_pref_chan(p2p, go, dev, &msg);
+
 		op_freq = p2p_channel_to_freq(p2p->op_reg_class,
 					      p2p->op_channel);
 		if (op_freq < 0) {
@@ -387,7 +402,7 @@ fail:
 	} else
 		p2p->inv_group_bssid_ptr = NULL;
 	if (msg.group_id) {
-		if (msg.group_id_len - ETH_ALEN <= 32) {
+		if (msg.group_id_len - ETH_ALEN <= SSID_MAX_LEN) {
 			os_memcpy(p2p->inv_ssid, msg.group_id + ETH_ALEN,
 				  msg.group_id_len - ETH_ALEN);
 			p2p->inv_ssid_len = msg.group_id_len - ETH_ALEN;
@@ -528,6 +543,12 @@ void p2p_process_invitation_resp(struct p2p_data *p2p, const u8 *sa,
 				peer_oper_freq = 0;
 		}
 
+		/*
+		 * Use the driver preferred frequency list extension if
+		 * supported.
+		 */
+		p2p_check_pref_chan(p2p, 0, dev, &msg);
+
 		p2p->cfg->invitation_result(p2p->cfg->cb_ctx, *msg.status,
 					    msg.group_bssid, channels, sa,
 					    freq, peer_oper_freq);
diff --git a/src/p2p/p2p_parse.c b/src/p2p/p2p_parse.c
index fd6a4610d839..bd1e68bd4241 100644
--- a/src/p2p/p2p_parse.c
+++ b/src/p2p/p2p_parse.c
@@ -149,7 +149,8 @@ static int p2p_parse_attribute(u8 id, const u8 *data, u16 len,
 		pos += 2;
 		nlen = WPA_GET_BE16(pos);
 		pos += 2;
-		if (data + len - pos < (int) nlen || nlen > 32) {
+		if (data + len - pos < (int) nlen ||
+		    nlen > WPS_DEV_NAME_MAX_LEN) {
 			wpa_printf(MSG_DEBUG, "P2P: Invalid Device Name "
 				   "length %d (buf len %d)", (int) nlen,
 				   (int) (data + len - pos));
@@ -160,8 +161,7 @@ static int p2p_parse_attribute(u8 id, const u8 *data, u16 len,
 		for (i = 0; i < nlen; i++) {
 			if (msg->device_name[i] == '\0')
 				break;
-			if (msg->device_name[i] > 0 &&
-			    msg->device_name[i] < 32)
+			if (is_ctrl_char(msg->device_name[i]))
 				msg->device_name[i] = '_';
 		}
 		wpa_printf(MSG_DEBUG, "P2P: * Device Info: addr " MACSTR
@@ -203,7 +203,7 @@ static int p2p_parse_attribute(u8 id, const u8 *data, u16 len,
 			   MAC2STR(msg->group_bssid));
 		break;
 	case P2P_ATTR_GROUP_ID:
-		if (len < ETH_ALEN || len > ETH_ALEN + 32) {
+		if (len < ETH_ALEN || len > ETH_ALEN + SSID_MAX_LEN) {
 			wpa_printf(MSG_DEBUG, "P2P: Invalid P2P Group ID "
 				   "attribute length %d", len);
 			return -1;
@@ -371,9 +371,9 @@ static int p2p_parse_attribute(u8 id, const u8 *data, u16 len,
 		break;
 	case P2P_ATTR_PERSISTENT_GROUP:
 	{
-		if (len < ETH_ALEN) {
+		if (len < ETH_ALEN || len > ETH_ALEN + SSID_MAX_LEN) {
 			wpa_printf(MSG_DEBUG,
-				   "P2P: Too short Persistent Group Info (length %u)",
+				   "P2P: Invalid Persistent Group Info (length %u)",
 				   len);
 			return -1;
 		}
@@ -516,7 +516,7 @@ int p2p_parse_ies(const u8 *data, size_t len, struct p2p_message *msg)
 	struct ieee802_11_elems elems;
 
 	ieee802_11_parse_elems(data, len, &elems, 0);
-	if (elems.ds_params && elems.ds_params_len >= 1)
+	if (elems.ds_params)
 		msg->ds_params = elems.ds_params;
 	if (elems.ssid)
 		msg->ssid = elems.ssid - 2;
@@ -548,6 +548,9 @@ int p2p_parse_ies(const u8 *data, size_t len, struct p2p_message *msg)
 	}
 #endif /* CONFIG_WIFI_DISPLAY */
 
+	msg->pref_freq_list = elems.pref_freq_list;
+	msg->pref_freq_list_len = elems.pref_freq_list_len;
+
 	return 0;
 }
 
@@ -674,8 +677,8 @@ int p2p_group_info_parse(const u8 *gi, size_t gi_len,
 		t += 2;
 		if (count > cend - t)
 			return -1; /* invalid Device Name TLV */
-		if (count >= 32)
-			count = 32;
+		if (count >= WPS_DEV_NAME_MAX_LEN)
+			count = WPS_DEV_NAME_MAX_LEN;
 		cli->dev_name = (const char *) t;
 		cli->dev_name_len = count;
 
@@ -703,7 +706,7 @@ static int p2p_group_info_text(const u8 *gi, size_t gi_len, char *buf,
 
 	for (i = 0; i < info.num_clients; i++) {
 		struct p2p_client_info *cli;
-		char name[33];
+		char name[WPS_DEV_NAME_MAX_LEN + 1];
 		char devtype[WPS_DEV_TYPE_BUFSIZE];
 		u8 s;
 		int count;
@@ -742,7 +745,7 @@ static int p2p_group_info_text(const u8 *gi, size_t gi_len, char *buf,
 		name[cli->dev_name_len] = '\0';
 		count = (int) cli->dev_name_len - 1;
 		while (count >= 0) {
-			if (name[count] > 0 && name[count] < 32)
+			if (is_ctrl_char(name[count]))
 				name[count] = '_';
 			count--;
 		}
diff --git a/src/p2p/p2p_pd.c b/src/p2p/p2p_pd.c
index 328b1e029ce5..890094551821 100644
--- a/src/p2p/p2p_pd.c
+++ b/src/p2p/p2p_pd.c
@@ -44,7 +44,7 @@ static void p2ps_add_new_group_info(struct p2p_data *p2p, struct wpabuf *buf)
 {
 	int found;
 	u8 intended_addr[ETH_ALEN];
-	u8 ssid[32];
+	u8 ssid[SSID_MAX_LEN];
 	size_t ssid_len;
 	int group_iface;
 
@@ -57,7 +57,11 @@ static void p2ps_add_new_group_info(struct p2p_data *p2p, struct wpabuf *buf)
 	if (found) {
 		p2p_buf_add_group_id(buf, p2p->cfg->dev_addr,
 				     ssid, ssid_len);
-		p2p_buf_add_intended_addr(buf, intended_addr);
+
+		if (group_iface)
+			p2p_buf_add_intended_addr(buf, p2p->intended_addr);
+		else
+			p2p_buf_add_intended_addr(buf, intended_addr);
 	} else {
 		if (!p2p->ssid_set) {
 			p2p_build_ssid(p2p, p2p->ssid, &p2p->ssid_len);
@@ -82,11 +86,12 @@ static void p2ps_add_pd_req_attrs(struct p2p_data *p2p, struct p2p_device *dev,
 				  struct wpabuf *buf, u16 config_methods)
 {
 	struct p2ps_provision *prov = p2p->p2ps_prov;
-	u8 feat_cap_mask[] = { 1, 0 };
+	struct p2ps_feature_capab fcap = { prov->cpt_mask, 0 };
 	int shared_group = 0;
-	u8 ssid[32];
+	u8 ssid[SSID_MAX_LEN];
 	size_t ssid_len;
 	u8 go_dev_addr[ETH_ALEN];
+	u8 intended_addr[ETH_ALEN];
 
 	/* If we might be explicite group owner, add GO details */
 	if (prov->conncap & (P2PS_SETUP_GROUP_OWNER |
@@ -101,7 +106,7 @@ static void p2ps_add_pd_req_attrs(struct p2p_data *p2p, struct p2p_device *dev,
 	if (p2p->cfg->get_persistent_group) {
 		shared_group = p2p->cfg->get_persistent_group(
 			p2p->cfg->cb_ctx, dev->info.p2p_device_addr, NULL, 0,
-			go_dev_addr, ssid, &ssid_len);
+			go_dev_addr, ssid, &ssid_len, intended_addr);
 	}
 
 	/* Add Operating Channel if conncap includes GO */
@@ -146,12 +151,17 @@ static void p2ps_add_pd_req_attrs(struct p2p_data *p2p, struct p2p_device *dev,
 
 	p2p_buf_add_session_id(buf, prov->session_id, prov->session_mac);
 
-	p2p_buf_add_feature_capability(buf, sizeof(feat_cap_mask),
-				       feat_cap_mask);
+	p2p_buf_add_feature_capability(buf, sizeof(fcap), (const u8 *) &fcap);
 
-	if (shared_group)
+	if (shared_group) {
 		p2p_buf_add_persistent_group_info(buf, go_dev_addr,
 						  ssid, ssid_len);
+		/* Add intended interface address if it is not added yet */
+		if ((prov->conncap == P2PS_SETUP_NONE ||
+		     prov->conncap == P2PS_SETUP_CLIENT) &&
+		    !is_zero_ether_addr(intended_addr))
+			p2p_buf_add_intended_addr(buf, intended_addr);
+	}
 }
 
 
@@ -232,7 +242,9 @@ static struct wpabuf * p2p_build_prov_disc_resp(struct p2p_data *p2p,
 						const u8 *group_id,
 						size_t group_id_len,
 						const u8 *persist_ssid,
-						size_t persist_ssid_len)
+						size_t persist_ssid_len,
+						const u8 *fcap,
+						u16 fcap_len)
 {
 	struct wpabuf *buf;
 	size_t extra = 0;
@@ -270,7 +282,6 @@ static struct wpabuf * p2p_build_prov_disc_resp(struct p2p_data *p2p,
 
 	/* Add P2P IE for P2PS */
 	if (p2p->p2ps_prov && p2p->p2ps_prov->adv_id == adv_id) {
-		u8 feat_cap_mask[] = { 1, 0 };
 		u8 *len = p2p_buf_add_ie_hdr(buf);
 		struct p2ps_provision *prov = p2p->p2ps_prov;
 		u8 group_capab;
@@ -293,18 +304,23 @@ static struct wpabuf * p2p_build_prov_disc_resp(struct p2p_data *p2p,
 		if (persist_ssid && p2p->cfg->get_persistent_group &&
 		    (status == P2P_SC_SUCCESS ||
 		     status == P2P_SC_SUCCESS_DEFERRED)) {
-			u8 ssid[32];
+			u8 ssid[SSID_MAX_LEN];
 			size_t ssid_len;
 			u8 go_dev_addr[ETH_ALEN];
+			u8 intended_addr[ETH_ALEN];
 
 			persist = p2p->cfg->get_persistent_group(
 				p2p->cfg->cb_ctx,
 				dev->info.p2p_device_addr,
 				persist_ssid, persist_ssid_len, go_dev_addr,
-				ssid, &ssid_len);
-			if (persist)
+				ssid, &ssid_len, intended_addr);
+			if (persist) {
 				p2p_buf_add_persistent_group_info(
 					buf, go_dev_addr, ssid, ssid_len);
+				if (!is_zero_ether_addr(intended_addr))
+					p2p_buf_add_intended_addr(
+						buf, intended_addr);
+			}
 		}
 
 		if (!persist && (prov->conncap & P2PS_SETUP_GROUP_OWNER))
@@ -344,8 +360,7 @@ static struct wpabuf * p2p_build_prov_disc_resp(struct p2p_data *p2p,
 		p2p_buf_add_session_id(buf, prov->session_id,
 				       prov->session_mac);
 
-		p2p_buf_add_feature_capability(buf, sizeof(feat_cap_mask),
-					       feat_cap_mask);
+		p2p_buf_add_feature_capability(buf, fcap_len, fcap);
 		p2p_buf_update_ie_hdr(buf, len);
 	} else if (status != P2P_SC_SUCCESS || adv_id) {
 		u8 *len = p2p_buf_add_ie_hdr(buf);
@@ -400,6 +415,18 @@ static int p2ps_setup_p2ps_prov(struct p2p_data *p2p, u32 adv_id,
 }
 
 
+static u8 p2ps_own_preferred_cpt(const u8 *cpt_priority, u8 req_cpt_mask)
+{
+	int i;
+
+	for (i = 0; cpt_priority[i]; i++)
+		if (req_cpt_mask & cpt_priority[i])
+			return cpt_priority[i];
+
+	return 0;
+}
+
+
 void p2p_process_prov_disc_req(struct p2p_data *p2p, const u8 *sa,
 			       const u8 *data, size_t len, int rx_freq)
 {
@@ -415,9 +442,12 @@ void p2p_process_prov_disc_req(struct p2p_data *p2p, const u8 *sa,
 	u32 session_id = 0;
 	u8 session_mac[ETH_ALEN];
 	u8 adv_mac[ETH_ALEN];
-	u8 group_mac[ETH_ALEN];
+	const u8 *group_mac;
 	int passwd_id = DEV_PW_DEFAULT;
 	u16 config_methods;
+	u16 allowed_config_methods = WPS_CONFIG_DISPLAY | WPS_CONFIG_KEYPAD;
+	struct p2ps_feature_capab resp_fcap = { 0, 0 };
+	struct p2ps_feature_capab *req_fcap;
 
 	if (p2p_parse(data, len, &msg))
 		return;
@@ -425,6 +455,7 @@ void p2p_process_prov_disc_req(struct p2p_data *p2p, const u8 *sa,
 	p2p_dbg(p2p, "Received Provision Discovery Request from " MACSTR
 		" with config methods 0x%x (freq=%d)",
 		MAC2STR(sa), msg.wps_config_methods, rx_freq);
+	group_mac = msg.intended_addr;
 
 	dev = p2p_get_device(p2p, sa);
 	if (dev == NULL || (dev->flags & P2P_DEV_PROBE_REQ_ONLY)) {
@@ -441,9 +472,12 @@ void p2p_process_prov_disc_req(struct p2p_data *p2p, const u8 *sa,
 		dev->info.wfd_subelems = wpabuf_dup(msg.wfd_subelems);
 	}
 
-	if (!(msg.wps_config_methods &
-	      (WPS_CONFIG_DISPLAY | WPS_CONFIG_KEYPAD |
-	       WPS_CONFIG_PUSHBUTTON | WPS_CONFIG_P2PS))) {
+	if (msg.adv_id)
+		allowed_config_methods |= WPS_CONFIG_P2PS;
+	else
+		allowed_config_methods |= WPS_CONFIG_PUSHBUTTON;
+
+	if (!(msg.wps_config_methods & allowed_config_methods)) {
 		p2p_dbg(p2p, "Unsupported Config Methods in Provision Discovery Request");
 		goto out;
 	}
@@ -501,14 +535,23 @@ void p2p_process_prov_disc_req(struct p2p_data *p2p, const u8 *sa,
 
 	os_memset(session_mac, 0, ETH_ALEN);
 	os_memset(adv_mac, 0, ETH_ALEN);
-	os_memset(group_mac, 0, ETH_ALEN);
 
+	/* Note 1: A feature capability attribute structure can be changed
+	 * in the future. The assumption is that such modifications are
+	 * backwards compatible, therefore we allow processing of
+	 * msg.feature_cap exceeding the size of the p2ps_feature_capab
+	 * structure.
+	 * Note 2: Vverification of msg.feature_cap_len below has to be changed
+	 * to allow 2 byte feature capability processing if struct
+	 * p2ps_feature_capab is extended to include additional fields and it
+	 * affects the structure size.
+	 */
 	if (msg.adv_id && msg.session_id && msg.session_mac && msg.adv_mac &&
+	    msg.feature_cap && msg.feature_cap_len >= sizeof(*req_fcap) &&
 	    (msg.status || msg.conn_cap)) {
 		u8 remote_conncap;
 
-		if (msg.intended_addr)
-			os_memcpy(group_mac, msg.intended_addr, ETH_ALEN);
+		req_fcap = (struct p2ps_feature_capab *) msg.feature_cap;
 
 		os_memcpy(session_mac, msg.session_mac, ETH_ALEN);
 		os_memcpy(adv_mac, msg.adv_mac, ETH_ALEN);
@@ -533,9 +576,22 @@ void p2p_process_prov_disc_req(struct p2p_data *p2p, const u8 *sa,
 			p2p_dbg(p2p, "Conncap: local:%d remote:%d result:%d",
 				auto_accept, remote_conncap, conncap);
 
-			if (p2ps_adv->config_methods &&
-			    !(msg.wps_config_methods &
-			      p2ps_adv->config_methods)) {
+			resp_fcap.cpt =
+				p2ps_own_preferred_cpt(p2ps_adv->cpt_priority,
+						       req_fcap->cpt);
+
+			p2p_dbg(p2p,
+				"cpt: service:0x%x remote:0x%x result:0x%x",
+				p2ps_adv->cpt_mask, req_fcap->cpt,
+				resp_fcap.cpt);
+
+			if (!resp_fcap.cpt) {
+				p2p_dbg(p2p,
+					"Incompatible P2PS feature capability CPT bitmask");
+				reject = P2P_SC_FAIL_INCOMPATIBLE_PARAMS;
+			} else if (p2ps_adv->config_methods &&
+				   !(msg.wps_config_methods &
+				   p2ps_adv->config_methods)) {
 				p2p_dbg(p2p,
 					"Unsupported config methods in Provision Discovery Request (own=0x%x peer=0x%x)",
 					p2ps_adv->config_methods,
@@ -624,6 +680,15 @@ void p2p_process_prov_disc_req(struct p2p_data *p2p, const u8 *sa,
 					p2p->p2ps_prov->conncap,
 					remote_conncap, conncap);
 
+				resp_fcap.cpt = p2ps_own_preferred_cpt(
+					p2p->p2ps_prov->cpt_priority,
+					req_fcap->cpt);
+
+				p2p_dbg(p2p,
+					"cpt: local:0x%x remote:0x%x result:0x%x",
+					p2p->p2ps_prov->cpt_mask,
+					req_fcap->cpt, resp_fcap.cpt);
+
 				/*
 				 * Ensure that if we asked for PIN originally,
 				 * our method is consistent with original
@@ -634,14 +699,22 @@ void p2p_process_prov_disc_req(struct p2p_data *p2p, const u8 *sa,
 				else if (method & WPS_CONFIG_KEYPAD)
 					method = WPS_CONFIG_DISPLAY;
 
-				/* Reject this "Deferred Accept* if incompatible
-				 * conncap or method */
 				if (!conncap ||
-				    !(msg.wps_config_methods & method))
+				    !(msg.wps_config_methods & method)) {
+					/*
+					 * Reject this "Deferred Accept*
+					 * if incompatible conncap or method
+					 */
 					reject =
 						P2P_SC_FAIL_INCOMPATIBLE_PARAMS;
-				else
+				} else if (!resp_fcap.cpt) {
+					p2p_dbg(p2p,
+						"Incompatible P2PS feature capability CPT bitmask");
+					reject =
+						P2P_SC_FAIL_INCOMPATIBLE_PARAMS;
+				} else {
 					reject = P2P_SC_SUCCESS;
+				}
 
 				p2p->p2ps_prov->status = reject;
 				p2p->p2ps_prov->conncap = conncap;
@@ -659,7 +732,9 @@ out:
 					config_methods, adv_id,
 					msg.group_id, msg.group_id_len,
 					msg.persistent_ssid,
-					msg.persistent_ssid_len);
+					msg.persistent_ssid_len,
+					(const u8 *) &resp_fcap,
+					sizeof(resp_fcap));
 	if (resp == NULL) {
 		p2p_parse_free(&msg);
 		return;
@@ -696,7 +771,7 @@ out:
 					     NULL, adv_id, session_id,
 					     0, 0, msg.persistent_ssid,
 					     msg.persistent_ssid_len,
-					     0, 0, NULL);
+					     0, 0, NULL, NULL, 0);
 	} else if (msg.status && *msg.status == P2P_SC_SUCCESS_DEFERRED &&
 		   p2p->p2ps_prov) {
 		p2p->p2ps_prov->status = reject;
@@ -709,7 +784,7 @@ out:
 						     session_id, conncap, 0,
 						     msg.persistent_ssid,
 						     msg.persistent_ssid_len, 0,
-						     0, NULL);
+						     0, NULL, NULL, 0);
 		else
 			p2p->cfg->p2ps_prov_complete(p2p->cfg->cb_ctx,
 						     *msg.status,
@@ -719,7 +794,9 @@ out:
 						     passwd_id,
 						     msg.persistent_ssid,
 						     msg.persistent_ssid_len, 0,
-						     0, NULL);
+						     0, NULL,
+						     (const u8 *) &resp_fcap,
+						     sizeof(resp_fcap));
 	} else if (msg.status && p2p->p2ps_prov) {
 		p2p->p2ps_prov->status = P2P_SC_SUCCESS;
 		p2p->cfg->p2ps_prov_complete(p2p->cfg->cb_ctx, *msg.status, sa,
@@ -728,7 +805,9 @@ out:
 					     passwd_id,
 					     msg.persistent_ssid,
 					     msg.persistent_ssid_len,
-					     0, 0, NULL);
+					     0, 0, NULL,
+					     (const u8 *) &resp_fcap,
+					     sizeof(resp_fcap));
 	} else if (msg.status) {
 	} else if (auto_accept && reject == P2P_SC_SUCCESS) {
 		p2p->cfg->p2ps_prov_complete(p2p->cfg->cb_ctx, P2P_SC_SUCCESS,
@@ -737,7 +816,9 @@ out:
 					     conncap, passwd_id,
 					     msg.persistent_ssid,
 					     msg.persistent_ssid_len,
-					     0, 0, NULL);
+					     0, 0, NULL,
+					     (const u8 *) &resp_fcap,
+					     sizeof(resp_fcap));
 	} else if (reject == P2P_SC_FAIL_INFO_CURRENTLY_UNAVAILABLE &&
 		   (!msg.session_info || !msg.session_info_len)) {
 		p2p->p2ps_prov->method = msg.wps_config_methods;
@@ -748,7 +829,9 @@ out:
 					     conncap, passwd_id,
 					     msg.persistent_ssid,
 					     msg.persistent_ssid_len,
-					     0, 1, NULL);
+					     0, 1, NULL,
+					     (const u8 *) &resp_fcap,
+					     sizeof(resp_fcap));
 	} else if (reject == P2P_SC_FAIL_INFO_CURRENTLY_UNAVAILABLE) {
 		size_t buf_len = msg.session_info_len;
 		char *buf = os_malloc(2 * buf_len + 1);
@@ -764,14 +847,45 @@ out:
 				adv_mac, session_mac, group_mac, adv_id,
 				session_id, conncap, passwd_id,
 				msg.persistent_ssid, msg.persistent_ssid_len,
-				0, 1, buf);
+				0, 1, buf,
+				(const u8 *) &resp_fcap, sizeof(resp_fcap));
 
 			os_free(buf);
 		}
 	}
 
-	if (reject == P2P_SC_SUCCESS && p2p->cfg->prov_disc_req) {
+	/*
+	 * prov_disc_req callback is used to generate P2P-PROV-DISC-ENTER-PIN,
+	 * P2P-PROV-DISC-SHOW-PIN, and P2P-PROV-DISC-PBC-REQ events.
+	 * Call it either on legacy P2P PD or on P2PS PD only if we need to
+	 * enter/show PIN.
+	 *
+	 * The callback is called in the following cases:
+	 * 1. Legacy P2P PD request, response status SUCCESS
+	 * 2. P2PS advertiser, method: DISPLAY, autoaccept: TRUE,
+	 *    response status: SUCCESS
+	 * 3. P2PS advertiser, method  DISPLAY, autoaccept: FALSE,
+	 *    response status: INFO_CURRENTLY_UNAVAILABLE
+	 * 4. P2PS advertiser, method: KEYPAD, autoaccept==any,
+	 *    response status: INFO_CURRENTLY_UNAVAILABLE
+	 * 5. P2PS follow-on with SUCCESS_DEFERRED,
+	 *    advertiser role: DISPLAY, autoaccept: FALSE,
+	 *    seeker: KEYPAD, response status: SUCCESS
+	 */
+	if (p2p->cfg->prov_disc_req &&
+	    ((reject == P2P_SC_SUCCESS && !msg.adv_id) ||
+	     (!msg.status &&
+	     (reject == P2P_SC_SUCCESS ||
+	      reject == P2P_SC_FAIL_INFO_CURRENTLY_UNAVAILABLE) &&
+	      passwd_id == DEV_PW_USER_SPECIFIED) ||
+	     (!msg.status &&
+	      reject == P2P_SC_FAIL_INFO_CURRENTLY_UNAVAILABLE &&
+	      passwd_id == DEV_PW_REGISTRAR_SPECIFIED) ||
+	     (reject == P2P_SC_SUCCESS &&
+	      msg.status && *msg.status == P2P_SC_SUCCESS_DEFERRED &&
+	       passwd_id == DEV_PW_REGISTRAR_SPECIFIED))) {
 		const u8 *dev_addr = sa;
+
 		if (msg.p2p_device_addr)
 			dev_addr = msg.p2p_device_addr;
 		p2p->cfg->prov_disc_req(p2p->cfg->cb_ctx, sa,
@@ -783,10 +897,133 @@ out:
 					0,
 					msg.group_id, msg.group_id_len);
 	}
+
+	if (dev && reject == P2P_SC_SUCCESS) {
+		switch (config_methods) {
+		case WPS_CONFIG_DISPLAY:
+			dev->wps_prov_info = WPS_CONFIG_KEYPAD;
+			break;
+		case WPS_CONFIG_KEYPAD:
+			dev->wps_prov_info = WPS_CONFIG_DISPLAY;
+			break;
+		case WPS_CONFIG_PUSHBUTTON:
+			dev->wps_prov_info = WPS_CONFIG_PUSHBUTTON;
+			break;
+		case WPS_CONFIG_P2PS:
+			dev->wps_prov_info = WPS_CONFIG_P2PS;
+			break;
+		default:
+			dev->wps_prov_info = 0;
+			break;
+		}
+
+		if (msg.intended_addr)
+			os_memcpy(dev->interface_addr, msg.intended_addr,
+				  ETH_ALEN);
+	}
 	p2p_parse_free(&msg);
 }
 
 
+static int p2p_validate_p2ps_pd_resp(struct p2p_data *p2p,
+				     struct p2p_message *msg)
+{
+	u8 conn_cap_go = 0;
+	u8 conn_cap_cli = 0;
+	u32 session_id;
+	u32 adv_id;
+
+#define P2PS_PD_RESP_CHECK(_val, _attr) \
+	do { \
+		if ((_val) && !msg->_attr) { \
+			p2p_dbg(p2p, "P2PS PD Response missing " #_attr); \
+			return -1; \
+		} \
+	} while (0)
+
+	P2PS_PD_RESP_CHECK(1, status);
+	P2PS_PD_RESP_CHECK(1, adv_id);
+	P2PS_PD_RESP_CHECK(1, adv_mac);
+	P2PS_PD_RESP_CHECK(1, capability);
+	P2PS_PD_RESP_CHECK(1, p2p_device_info);
+	P2PS_PD_RESP_CHECK(1, session_id);
+	P2PS_PD_RESP_CHECK(1, session_mac);
+	P2PS_PD_RESP_CHECK(1, feature_cap);
+
+	session_id = WPA_GET_LE32(msg->session_id);
+	adv_id = WPA_GET_LE32(msg->adv_id);
+
+	if (p2p->p2ps_prov->session_id != session_id) {
+		p2p_dbg(p2p,
+			"Ignore PD Response with unexpected Session ID");
+		return -1;
+	}
+
+	if (os_memcmp(p2p->p2ps_prov->session_mac, msg->session_mac,
+		      ETH_ALEN)) {
+		p2p_dbg(p2p,
+			"Ignore PD Response with unexpected Session MAC");
+		return -1;
+	}
+
+	if (p2p->p2ps_prov->adv_id != adv_id) {
+		p2p_dbg(p2p,
+			"Ignore PD Response with unexpected Advertisement ID");
+		return -1;
+	}
+
+	if (os_memcmp(p2p->p2ps_prov->adv_mac, msg->adv_mac, ETH_ALEN) != 0) {
+		p2p_dbg(p2p,
+			"Ignore PD Response with unexpected Advertisement MAC");
+		return -1;
+	}
+
+	if (msg->listen_channel) {
+		p2p_dbg(p2p,
+			"Ignore malformed PD Response - unexpected Listen Channel");
+		return -1;
+	}
+
+	if (*msg->status == P2P_SC_SUCCESS &&
+	    !(!!msg->conn_cap ^ !!msg->persistent_dev)) {
+		p2p_dbg(p2p,
+			"Ignore malformed PD Response - either conn_cap or persistent group should be present");
+		return -1;
+	}
+
+	if (msg->persistent_dev && *msg->status != P2P_SC_SUCCESS) {
+		p2p_dbg(p2p,
+			"Ignore malformed PD Response - persistent group is present, but the status isn't success");
+		return -1;
+	}
+
+	if (msg->conn_cap) {
+		conn_cap_go = *msg->conn_cap == P2PS_SETUP_GROUP_OWNER;
+		conn_cap_cli = *msg->conn_cap == P2PS_SETUP_CLIENT;
+	}
+
+	P2PS_PD_RESP_CHECK(msg->persistent_dev || conn_cap_go || conn_cap_cli,
+			   channel_list);
+	P2PS_PD_RESP_CHECK(msg->persistent_dev || conn_cap_go || conn_cap_cli,
+			   config_timeout);
+
+	P2PS_PD_RESP_CHECK(conn_cap_go, group_id);
+	P2PS_PD_RESP_CHECK(conn_cap_go, intended_addr);
+	P2PS_PD_RESP_CHECK(conn_cap_go, operating_channel);
+	/*
+	 * TODO: Also validate that operating channel is present if the device
+	 * is a GO in a persistent group. We can't do it here since we don't
+	 * know what is the role of the peer. It should be probably done in
+	 * p2ps_prov_complete callback, but currently operating channel isn't
+	 * passed to it.
+	 */
+
+#undef P2PS_PD_RESP_CHECK
+
+	return 0;
+}
+
+
 void p2p_process_prov_disc_resp(struct p2p_data *p2p, const u8 *sa,
 				const u8 *data, size_t len)
 {
@@ -794,24 +1031,26 @@ void p2p_process_prov_disc_resp(struct p2p_data *p2p, const u8 *sa,
 	struct p2p_device *dev;
 	u16 report_config_methods = 0, req_config_methods;
 	u8 status = P2P_SC_SUCCESS;
-	int success = 0;
 	u32 adv_id = 0;
 	u8 conncap = P2PS_SETUP_NEW;
 	u8 adv_mac[ETH_ALEN];
-	u8 group_mac[ETH_ALEN];
+	const u8 *group_mac;
 	int passwd_id = DEV_PW_DEFAULT;
+	int p2ps_seeker;
 
 	if (p2p_parse(data, len, &msg))
 		return;
 
+	if (p2p->p2ps_prov && p2p_validate_p2ps_pd_resp(p2p, &msg)) {
+		p2p_parse_free(&msg);
+		return;
+	}
+
 	/* Parse the P2PS members present */
 	if (msg.status)
 		status = *msg.status;
 
-	if (msg.intended_addr)
-		os_memcpy(group_mac, msg.intended_addr, ETH_ALEN);
-	else
-		os_memset(group_mac, 0, ETH_ALEN);
+	group_mac = msg.intended_addr;
 
 	if (msg.adv_mac)
 		os_memcpy(adv_mac, msg.adv_mac, ETH_ALEN);
@@ -859,6 +1098,8 @@ void p2p_process_prov_disc_resp(struct p2p_data *p2p, const u8 *sa,
 		p2p->pending_action_state = P2P_NO_PENDING_ACTION;
 	}
 
+	p2ps_seeker = p2p->p2ps_prov && p2p->p2ps_prov->pd_seeker;
+
 	/*
 	 * Use a local copy of the requested config methods since
 	 * p2p_reset_pending_pd() can clear this in the peer entry.
@@ -881,8 +1122,7 @@ void p2p_process_prov_disc_resp(struct p2p_data *p2p, const u8 *sa,
 						 P2P_PROV_DISC_REJECTED,
 						 adv_id, adv_mac, NULL);
 		p2p_parse_free(&msg);
-		os_free(p2p->p2ps_prov);
-		p2p->p2ps_prov = NULL;
+		p2ps_prov_free(p2p);
 		goto out;
 	}
 
@@ -909,7 +1149,6 @@ void p2p_process_prov_disc_resp(struct p2p_data *p2p, const u8 *sa,
 	}
 
 	if ((msg.conn_cap || msg.persistent_dev) &&
-	    msg.adv_id &&
 	    (status == P2P_SC_SUCCESS || status == P2P_SC_SUCCESS_DEFERRED) &&
 	    p2p->p2ps_prov) {
 		if (p2p->cfg->p2ps_prov_complete) {
@@ -918,23 +1157,20 @@ void p2p_process_prov_disc_resp(struct p2p_data *p2p, const u8 *sa,
 				p2p->p2ps_prov->session_mac,
 				group_mac, adv_id, p2p->p2ps_prov->session_id,
 				conncap, passwd_id, msg.persistent_ssid,
-				msg.persistent_ssid_len, 1, 0, NULL);
+				msg.persistent_ssid_len, 1, 0, NULL,
+				msg.feature_cap, msg.feature_cap_len);
 		}
-		os_free(p2p->p2ps_prov);
-		p2p->p2ps_prov = NULL;
-	}
-
-	if (status != P2P_SC_SUCCESS &&
-	    status != P2P_SC_FAIL_INFO_CURRENTLY_UNAVAILABLE &&
-	    status != P2P_SC_SUCCESS_DEFERRED && p2p->p2ps_prov) {
+		p2ps_prov_free(p2p);
+	} else if (status != P2P_SC_SUCCESS &&
+		   status != P2P_SC_FAIL_INFO_CURRENTLY_UNAVAILABLE &&
+		   status != P2P_SC_SUCCESS_DEFERRED && p2p->p2ps_prov) {
 		if (p2p->cfg->p2ps_prov_complete)
 			p2p->cfg->p2ps_prov_complete(
 				p2p->cfg->cb_ctx, status, sa, adv_mac,
 				p2p->p2ps_prov->session_mac,
 				group_mac, adv_id, p2p->p2ps_prov->session_id,
-				0, 0, NULL, 0, 1, 0, NULL);
-		os_free(p2p->p2ps_prov);
-		p2p->p2ps_prov = NULL;
+				0, 0, NULL, 0, 1, 0, NULL, NULL, 0);
+		p2ps_prov_free(p2p);
 	}
 
 	if (status == P2P_SC_FAIL_INFO_CURRENTLY_UNAVAILABLE) {
@@ -950,8 +1186,7 @@ void p2p_process_prov_disc_resp(struct p2p_data *p2p, const u8 *sa,
 
 			if (!deferred_sess_resp) {
 				p2p_parse_free(&msg);
-				os_free(p2p->p2ps_prov);
-				p2p->p2ps_prov = NULL;
+				p2ps_prov_free(p2p);
 				goto out;
 			}
 			utf8_escape((char *) msg.session_info, info_len,
@@ -970,24 +1205,23 @@ void p2p_process_prov_disc_resp(struct p2p_data *p2p, const u8 *sa,
 					p2p->cfg->cb_ctx, sa,
 					P2P_PROV_DISC_INFO_UNAVAILABLE,
 					adv_id, adv_mac, NULL);
-	} else if (msg.wps_config_methods != dev->req_config_methods ||
-		   status != P2P_SC_SUCCESS) {
+	} else if (status != P2P_SC_SUCCESS) {
 		p2p_dbg(p2p, "Peer rejected our Provision Discovery Request");
 		if (p2p->cfg->prov_disc_fail)
 			p2p->cfg->prov_disc_fail(p2p->cfg->cb_ctx, sa,
-						 P2P_PROV_DISC_REJECTED, 0,
-						 NULL, NULL);
+						 P2P_PROV_DISC_REJECTED,
+						 adv_id, adv_mac, NULL);
 		p2p_parse_free(&msg);
-		os_free(p2p->p2ps_prov);
-		p2p->p2ps_prov = NULL;
+		p2ps_prov_free(p2p);
 		goto out;
 	}
 
 	/* Store the provisioning info */
 	dev->wps_prov_info = msg.wps_config_methods;
+	if (msg.intended_addr)
+		os_memcpy(dev->interface_addr, msg.intended_addr, ETH_ALEN);
 
 	p2p_parse_free(&msg);
-	success = 1;
 
 out:
 	dev->req_config_methods = 0;
@@ -999,7 +1233,28 @@ out:
 		p2p_connect_send(p2p, dev);
 		return;
 	}
-	if (success && p2p->cfg->prov_disc_resp)
+
+	/*
+	 * prov_disc_resp callback is used to generate P2P-PROV-DISC-ENTER-PIN,
+	 * P2P-PROV-DISC-SHOW-PIN, and P2P-PROV-DISC-PBC-REQ events.
+	 * Call it only for a legacy P2P PD or for P2PS PD scenarios where
+	 * show/enter PIN events are needed.
+	 *
+	 * The callback is called in the following cases:
+	 * 1. Legacy P2P PD response with a status SUCCESS
+	 * 2. P2PS, advertiser method: DISPLAY, autoaccept: true,
+	 *    response status: SUCCESS, local method KEYPAD
+	 * 3. P2PS, advertiser method: KEYPAD,Seeker side,
+	 *    response status: INFO_CURRENTLY_UNAVAILABLE,
+	 *    local method: DISPLAY
+	 */
+	if (p2p->cfg->prov_disc_resp &&
+	    ((status == P2P_SC_SUCCESS && !adv_id) ||
+	     (p2ps_seeker && status == P2P_SC_SUCCESS &&
+	      passwd_id == DEV_PW_REGISTRAR_SPECIFIED) ||
+	     (p2ps_seeker &&
+	      status == P2P_SC_FAIL_INFO_CURRENTLY_UNAVAILABLE &&
+	      passwd_id == DEV_PW_USER_SPECIFIED)))
 		p2p->cfg->prov_disc_resp(p2p->cfg->cb_ctx, sa,
 					 report_config_methods);
 
@@ -1120,7 +1375,7 @@ int p2p_prov_disc_req(struct p2p_data *p2p, const u8 *peer_addr,
 
 	/* Reset provisioning info */
 	dev->wps_prov_info = 0;
-	os_free(p2p->p2ps_prov);
+	p2ps_prov_free(p2p);
 	p2p->p2ps_prov = p2ps_prov;
 
 	dev->req_config_methods = config_methods;
@@ -1176,3 +1431,10 @@ void p2p_reset_pending_pd(struct p2p_data *p2p)
 	p2p->pd_retries = 0;
 	p2p->pd_force_freq = 0;
 }
+
+
+void p2ps_prov_free(struct p2p_data *p2p)
+{
+	os_free(p2p->p2ps_prov);
+	p2p->p2ps_prov = NULL;
+}
diff --git a/src/p2p/p2p_utils.c b/src/p2p/p2p_utils.c
index f32751d79ca8..2e2aa8ad06f0 100644
--- a/src/p2p/p2p_utils.c
+++ b/src/p2p/p2p_utils.c
@@ -9,6 +9,7 @@
 #include "includes.h"
 
 #include "common.h"
+#include "common/defs.h"
 #include "common/ieee802_11_common.h"
 #include "p2p_i.h"
 
@@ -67,50 +68,11 @@ int p2p_channel_to_freq(int op_class, int channel)
  */
 int p2p_freq_to_channel(unsigned int freq, u8 *op_class, u8 *channel)
 {
-	/* TODO: more operating classes */
-	if (freq >= 2412 && freq <= 2472) {
-		if ((freq - 2407) % 5)
-			return -1;
+	if (ieee80211_freq_to_channel_ext(freq, 0, 0, op_class, channel) ==
+	    NUM_HOSTAPD_MODES)
+		return -1;
 
-		*op_class = 81; /* 2.407 GHz, channels 1..13 */
-		*channel = (freq - 2407) / 5;
-		return 0;
-	}
-
-	if (freq == 2484) {
-		*op_class = 82; /* channel 14 */
-		*channel = 14;
-		return 0;
-	}
-
-	if (freq >= 5180 && freq <= 5240) {
-		if ((freq - 5000) % 5)
-			return -1;
-
-		*op_class = 115; /* 5 GHz, channels 36..48 */
-		*channel = (freq - 5000) / 5;
-		return 0;
-	}
-
-	if (freq >= 5745 && freq <= 5805) {
-		if ((freq - 5000) % 5)
-			return -1;
-
-		*op_class = 124; /* 5 GHz, channels 149..161 */
-		*channel = (freq - 5000) / 5;
-		return 0;
-	}
-
-	if (freq >= 58320 && freq <= 64800) {
-		if ((freq - 58320) % 2160)
-			return -1;
-
-		*op_class = 180; /* 60 GHz, channels 1..4 */
-		*channel = (freq - 56160) / 2160;
-		return 0;
-	}
-
-	return -1;
+	return 0;
 }
 
 
@@ -497,12 +459,22 @@ int p2p_channels_to_freqs(const struct p2p_channels *channels, int *freq_list,
 			break;
 		for (j = 0; j < c->channels; j++) {
 			int freq;
+			unsigned int k;
+
 			if (idx + 1 == max_len)
 				break;
 			freq = p2p_channel_to_freq(c->reg_class,
 						   c->channel[j]);
 			if (freq < 0)
 				continue;
+
+			for (k = 0; k < idx; k++) {
+				if (freq_list[k] == freq)
+					break;
+			}
+
+			if (k < idx)
+				continue;
 			freq_list[idx++] = freq;
 		}
 	}
diff --git a/src/radius/Makefile b/src/radius/Makefile
index b5d063dac10d..3ad4751dfbfe 100644
--- a/src/radius/Makefile
+++ b/src/radius/Makefile
@@ -14,6 +14,7 @@ CFLAGS += -DCONFIG_IPV6
 LIB_OBJS= \
 	radius.o \
 	radius_client.o \
+	radius_das.o \
 	radius_server.o
 
 libradius.a: $(LIB_OBJS)
diff --git a/src/radius/radius.c b/src/radius/radius.c
index 8d878a4bd078..1ebfd11f3b9a 100644
--- a/src/radius/radius.c
+++ b/src/radius/radius.c
@@ -167,7 +167,7 @@ struct radius_attr_type {
 	} data_type;
 };
 
-static struct radius_attr_type radius_attrs[] =
+static const struct radius_attr_type radius_attrs[] =
 {
 	{ RADIUS_ATTR_USER_NAME, "User-Name", RADIUS_ATTR_TEXT },
 	{ RADIUS_ATTR_USER_PASSWORD, "User-Password", RADIUS_ATTR_UNDIST },
@@ -259,7 +259,7 @@ static struct radius_attr_type radius_attrs[] =
 #define RADIUS_ATTRS ARRAY_SIZE(radius_attrs)
 
 
-static struct radius_attr_type *radius_get_attr_type(u8 type)
+static const struct radius_attr_type *radius_get_attr_type(u8 type)
 {
 	size_t i;
 
@@ -274,7 +274,7 @@ static struct radius_attr_type *radius_get_attr_type(u8 type)
 
 static void radius_msg_dump_attr(struct radius_attr_hdr *hdr)
 {
-	struct radius_attr_type *attr;
+	const struct radius_attr_type *attr;
 	int len;
 	unsigned char *pos;
 	char buf[1000];
@@ -1225,7 +1225,7 @@ int radius_msg_add_mppe_keys(struct radius_msg *msg,
 	}
 
 	/* MS-MPPE-Recv-Key */
-	buf = os_malloc(hlen + send_key_len + 16);
+	buf = os_malloc(hlen + recv_key_len + 16);
 	if (buf == NULL) {
 		return 0;
 	}
@@ -1425,7 +1425,7 @@ struct radius_tunnel_attrs {
 /**
  * radius_msg_get_vlanid - Parse RADIUS attributes for VLAN tunnel information
  * @msg: RADIUS message
- * Returns: VLAN ID for the first tunnel configuration of -1 if none is found
+ * Returns: VLAN ID for the first tunnel configuration or 0 if none is found
  */
 int radius_msg_get_vlanid(struct radius_msg *msg)
 {
@@ -1488,7 +1488,7 @@ int radius_msg_get_vlanid(struct radius_msg *msg)
 			return tun->vlanid;
 	}
 
-	return -1;
+	return 0;
 }
 
 
diff --git a/src/radius/radius_das.c b/src/radius/radius_das.c
index 39ceea879caf..b7d991bbd097 100644
--- a/src/radius/radius_das.c
+++ b/src/radius/radius_das.c
@@ -245,7 +245,7 @@ static void radius_das_receive(int sock, void *eloop_ctx, void *sock_ctx)
 				  (u8 *) &val, 4);
 	if (res == 4) {
 		u32 timestamp = ntohl(val);
-		if ((unsigned int) abs(now.sec - timestamp) >
+		if ((unsigned int) abs((int) (now.sec - timestamp)) >
 		    das->time_window) {
 			wpa_printf(MSG_DEBUG, "DAS: Unacceptable "
 				   "Event-Timestamp (%u; local time %u) in "
diff --git a/src/radius/radius_server.c b/src/radius/radius_server.c
index 85a485e91d93..744283c7dc9d 100644
--- a/src/radius/radius_server.c
+++ b/src/radius/radius_server.c
@@ -35,7 +35,7 @@
  */
 #define RADIUS_MAX_MSG_LEN 3000
 
-static struct eapol_callbacks radius_server_eapol_cb;
+static const struct eapol_callbacks radius_server_eapol_cb;
 
 struct radius_client;
 struct radius_server_data;
@@ -265,6 +265,8 @@ struct radius_server_data {
 
 	struct dl_list erp_keys; /* struct eap_server_erp_key */
 
+	unsigned int tls_session_lifetime;
+
 	/**
 	 * wps - Wi-Fi Protected Setup context
 	 *
@@ -688,6 +690,7 @@ radius_server_get_new_session(struct radius_server_data *data,
 	eap_conf.server_id = (const u8 *) data->server_id;
 	eap_conf.server_id_len = os_strlen(data->server_id);
 	eap_conf.erp = data->erp;
+	eap_conf.tls_session_lifetime = data->tls_session_lifetime;
 	radius_server_testing_options(sess, &eap_conf);
 	sess->eap = eap_server_sm_init(sess, &radius_server_eapol_cb,
 				       &eap_conf);
@@ -1711,8 +1714,10 @@ radius_server_init(struct radius_server_conf *conf)
 	data->ipv6 = conf->ipv6;
 	if (conf->pac_opaque_encr_key) {
 		data->pac_opaque_encr_key = os_malloc(16);
-		os_memcpy(data->pac_opaque_encr_key, conf->pac_opaque_encr_key,
-			  16);
+		if (data->pac_opaque_encr_key) {
+			os_memcpy(data->pac_opaque_encr_key,
+				  conf->pac_opaque_encr_key, 16);
+		}
 	}
 	if (conf->eap_fast_a_id) {
 		data->eap_fast_a_id = os_malloc(conf->eap_fast_a_id_len);
@@ -1743,6 +1748,7 @@ radius_server_init(struct radius_server_conf *conf)
 	}
 	data->erp = conf->erp;
 	data->erp_domain = conf->erp_domain;
+	data->tls_session_lifetime = conf->tls_session_lifetime;
 
 	if (conf->subscr_remediation_url) {
 		data->subscr_remediation_url =
@@ -2035,6 +2041,12 @@ static int radius_server_get_eap_user(void *ctx, const u8 *identity,
 		sess->remediation = user->remediation;
 		sess->macacl = user->macacl;
 	}
+
+	if (ret) {
+		RADIUS_DEBUG("%s: User-Name not found from user database",
+			     __func__);
+	}
+
 	return ret;
 }
 
@@ -2095,7 +2107,7 @@ static int radius_server_erp_add_key(void *ctx, struct eap_server_erp_key *erp)
 #endif /* CONFIG_ERP */
 
 
-static struct eapol_callbacks radius_server_eapol_cb =
+static const struct eapol_callbacks radius_server_eapol_cb =
 {
 	.get_eap_user = radius_server_get_eap_user,
 	.get_eap_req_id_text = radius_server_get_eap_req_id_text,
diff --git a/src/radius/radius_server.h b/src/radius/radius_server.h
index ca4e38c12e99..7a25802c8152 100644
--- a/src/radius/radius_server.h
+++ b/src/radius/radius_server.h
@@ -170,6 +170,8 @@ struct radius_server_conf {
 
 	const char *erp_domain;
 
+	unsigned int tls_session_lifetime;
+
 	/**
 	 * wps - Wi-Fi Protected Setup context
 	 *
diff --git a/src/rsn_supp/Makefile b/src/rsn_supp/Makefile
index adfd3dfd5b9b..d5e61fe72dc8 100644
--- a/src/rsn_supp/Makefile
+++ b/src/rsn_supp/Makefile
@@ -1,8 +1,30 @@
-all:
-	@echo Nothing to be made.
+all: librsn_supp.a
 
 clean:
-	rm -f *~ *.o *.d *.gcno *.gcda *.gcov
+	rm -f *~ *.o *.d *.gcno *.gcda *.gcov librsn_supp.a
 
 install:
 	@echo Nothing to be made.
+
+include ../lib.rules
+
+CFLAGS += -DCONFIG_IEEE80211W
+CFLAGS += -DCONFIG_IEEE80211R
+CFLAGS += -DCONFIG_PEERKEY
+CFLAGS += -DCONFIG_TDLS
+CFLAGS += -DCONFIG_WNM
+CFLAGS += -DIEEE8021X_EAPOL
+
+LIB_OBJS= \
+	pmksa_cache.o \
+	wpa_ft.o \
+	peerkey.o \
+	tdls.o \
+	preauth.o \
+	wpa.o \
+	wpa_ie.o
+
+librsn_supp.a: $(LIB_OBJS)
+	$(AR) crT $@ $?
+
+-include $(OBJS:%.o=%.d)
diff --git a/src/rsn_supp/tdls.c b/src/rsn_supp/tdls.c
index c1d7749191d7..722c20a706f9 100644
--- a/src/rsn_supp/tdls.c
+++ b/src/rsn_supp/tdls.c
@@ -12,6 +12,7 @@
 #include "utils/eloop.h"
 #include "utils/os.h"
 #include "common/ieee802_11_defs.h"
+#include "common/ieee802_11_common.h"
 #include "crypto/sha256.h"
 #include "crypto/crypto.h"
 #include "crypto/aes_wrap.h"
@@ -1577,9 +1578,7 @@ static int copy_supp_rates(const struct wpa_eapol_ie_parse *kde,
 static int copy_peer_ht_capab(const struct wpa_eapol_ie_parse *kde,
 			      struct wpa_tdls_peer *peer)
 {
-	if (!kde->ht_capabilities ||
-	    kde->ht_capabilities_len <
-	    sizeof(struct ieee80211_ht_capabilities) ) {
+	if (!kde->ht_capabilities) {
 		wpa_printf(MSG_DEBUG, "TDLS: No supported ht capabilities "
 			   "received");
 		return 0;
@@ -1605,9 +1604,7 @@ static int copy_peer_ht_capab(const struct wpa_eapol_ie_parse *kde,
 static int copy_peer_vht_capab(const struct wpa_eapol_ie_parse *kde,
 			      struct wpa_tdls_peer *peer)
 {
-	if (!kde->vht_capabilities ||
-	    kde->vht_capabilities_len <
-	    sizeof(struct ieee80211_vht_capabilities) ) {
+	if (!kde->vht_capabilities) {
 		wpa_printf(MSG_DEBUG, "TDLS: No supported vht capabilities "
 			   "received");
 		return 0;
@@ -2863,14 +2860,14 @@ void wpa_tdls_disassoc(struct wpa_sm *sm)
 }
 
 
-static int wpa_tdls_prohibited(struct wpa_eapol_ie_parse *elems)
+static int wpa_tdls_prohibited(struct ieee802_11_elems *elems)
 {
 	/* bit 38 - TDLS Prohibited */
 	return !!(elems->ext_capab[2 + 4] & 0x40);
 }
 
 
-static int wpa_tdls_chan_switch_prohibited(struct wpa_eapol_ie_parse *elems)
+static int wpa_tdls_chan_switch_prohibited(struct ieee802_11_elems *elems)
 {
 	/* bit 39 - TDLS Channel Switch Prohibited */
 	return !!(elems->ext_capab[2 + 4] & 0x80);
@@ -2879,12 +2876,13 @@ static int wpa_tdls_chan_switch_prohibited(struct wpa_eapol_ie_parse *elems)
 
 void wpa_tdls_ap_ies(struct wpa_sm *sm, const u8 *ies, size_t len)
 {
-	struct wpa_eapol_ie_parse elems;
+	struct ieee802_11_elems elems;
 
 	sm->tdls_prohibited = 0;
 	sm->tdls_chan_switch_prohibited = 0;
 
-	if (ies == NULL || wpa_supplicant_parse_ies(ies, len, &elems) < 0 ||
+	if (ies == NULL ||
+	    ieee802_11_parse_elems(ies, len, &elems, 0) == ParseFailed ||
 	    elems.ext_capab == NULL || elems.ext_capab_len < 2 + 5)
 		return;
 
@@ -2900,9 +2898,10 @@ void wpa_tdls_ap_ies(struct wpa_sm *sm, const u8 *ies, size_t len)
 
 void wpa_tdls_assoc_resp_ies(struct wpa_sm *sm, const u8 *ies, size_t len)
 {
-	struct wpa_eapol_ie_parse elems;
+	struct ieee802_11_elems elems;
 
-	if (ies == NULL || wpa_supplicant_parse_ies(ies, len, &elems) < 0 ||
+	if (ies == NULL ||
+	    ieee802_11_parse_elems(ies, len, &elems, 0) == ParseFailed ||
 	    elems.ext_capab == NULL || elems.ext_capab_len < 2 + 5)
 		return;
 
diff --git a/src/rsn_supp/wpa.c b/src/rsn_supp/wpa.c
index 8adeef4af1a5..a9f255e37b43 100644
--- a/src/rsn_supp/wpa.c
+++ b/src/rsn_supp/wpa.c
@@ -249,6 +249,17 @@ static int wpa_supplicant_get_pmk(struct wpa_sm *sm,
 					"RSN: the new PMK matches with the "
 					"PMKID");
 				abort_cached = 0;
+			} else if (sa && !sm->cur_pmksa && pmkid) {
+				/*
+				 * It looks like the authentication server
+				 * derived mismatching MSK. This should not
+				 * really happen, but bugs happen.. There is not
+				 * much we can do here without knowing what
+				 * exactly caused the server to misbehave.
+				 */
+				wpa_dbg(sm->ctx->msg_ctx, MSG_INFO,
+					"RSN: PMKID mismatch - authentication server may have derived different MSK?!");
+				return -1;
 			}
 
 			if (!sm->cur_pmksa)
@@ -1281,8 +1292,8 @@ static int wpa_supplicant_process_1_of_2_rsn(struct wpa_sm *sm,
 					      &gd->key_rsc_len, &gd->alg))
 		return -1;
 
-	wpa_hexdump(MSG_DEBUG, "RSN: received GTK in group key handshake",
-		    ie.gtk, ie.gtk_len);
+	wpa_hexdump_key(MSG_DEBUG, "RSN: received GTK in group key handshake",
+			ie.gtk, ie.gtk_len);
 	gd->keyidx = ie.gtk[0] & 0x3;
 	gd->tx = wpa_supplicant_gtk_tx_bit_workaround(sm,
 						      !!(ie.gtk[0] & BIT(2)));
@@ -1333,6 +1344,11 @@ static int wpa_supplicant_process_1_of_2_wpa(struct wpa_sm *sm,
 	gd->keyidx = (key_info & WPA_KEY_INFO_KEY_INDEX_MASK) >>
 		WPA_KEY_INFO_KEY_INDEX_SHIFT;
 	if (ver == WPA_KEY_INFO_TYPE_HMAC_MD5_RC4 && sm->ptk.kek_len == 16) {
+#ifdef CONFIG_NO_RC4
+		wpa_msg(sm->ctx->msg_ctx, MSG_WARNING,
+			"WPA: RC4 not supported in the build");
+		return -1;
+#else /* CONFIG_NO_RC4 */
 		u8 ek[32];
 		if (key_data_len > sizeof(gd->gtk)) {
 			wpa_msg(sm->ctx->msg_ctx, MSG_WARNING,
@@ -1350,6 +1366,7 @@ static int wpa_supplicant_process_1_of_2_wpa(struct wpa_sm *sm,
 			return -1;
 		}
 		os_memset(ek, 0, sizeof(ek));
+#endif /* CONFIG_NO_RC4 */
 	} else if (ver == WPA_KEY_INFO_TYPE_HMAC_SHA1_AES) {
 		if (maxkeylen % 8) {
 			wpa_msg(sm->ctx->msg_ctx, MSG_WARNING,
@@ -1564,6 +1581,11 @@ static int wpa_supplicant_decrypt_key_data(struct wpa_sm *sm,
 	/* Decrypt key data here so that this operation does not need
 	 * to be implemented separately for each message type. */
 	if (ver == WPA_KEY_INFO_TYPE_HMAC_MD5_RC4 && sm->ptk.kek_len == 16) {
+#ifdef CONFIG_NO_RC4
+		wpa_msg(sm->ctx->msg_ctx, MSG_WARNING,
+			"WPA: RC4 not supported in the build");
+		return -1;
+#else /* CONFIG_NO_RC4 */
 		u8 ek[32];
 		os_memcpy(ek, key->key_iv, 16);
 		os_memcpy(ek + 16, sm->ptk.kek, sm->ptk.kek_len);
@@ -1574,6 +1596,7 @@ static int wpa_supplicant_decrypt_key_data(struct wpa_sm *sm,
 			return -1;
 		}
 		os_memset(ek, 0, sizeof(ek));
+#endif /* CONFIG_NO_RC4 */
 	} else if (ver == WPA_KEY_INFO_TYPE_HMAC_SHA1_AES ||
 		   ver == WPA_KEY_INFO_TYPE_AES_128_CMAC ||
 		   sm->key_mgmt == WPA_KEY_MGMT_OSEN ||
diff --git a/src/rsn_supp/wpa_ft.c b/src/rsn_supp/wpa_ft.c
index 06dea0550f1b..205793e7f43a 100644
--- a/src/rsn_supp/wpa_ft.c
+++ b/src/rsn_supp/wpa_ft.c
@@ -168,9 +168,7 @@ static u8 * wpa_ft_gen_req_ies(struct wpa_sm *sm, size_t *len,
 	pos = (u8 *) (rsnie + 1);
 
 	/* Group Suite Selector */
-	if (sm->group_cipher != WPA_CIPHER_CCMP &&
-	    sm->group_cipher != WPA_CIPHER_GCMP &&
-	    sm->group_cipher != WPA_CIPHER_TKIP) {
+	if (!wpa_cipher_valid_group(sm->group_cipher)) {
 		wpa_printf(MSG_WARNING, "FT: Invalid group cipher (%d)",
 			   sm->group_cipher);
 		os_free(buf);
diff --git a/src/rsn_supp/wpa_ie.c b/src/rsn_supp/wpa_ie.c
index cb334df675be..0c37b35c1ee1 100644
--- a/src/rsn_supp/wpa_ie.c
+++ b/src/rsn_supp/wpa_ie.c
@@ -30,6 +30,9 @@ int wpa_parse_wpa_ie(const u8 *wpa_ie, size_t wpa_ie_len,
 {
 	if (wpa_ie_len >= 1 && wpa_ie[0] == WLAN_EID_RSN)
 		return wpa_parse_wpa_ie_rsn(wpa_ie, wpa_ie_len, data);
+	if (wpa_ie_len >= 6 && wpa_ie[0] == WLAN_EID_VENDOR_SPECIFIC &&
+	    wpa_ie[1] >= 4 && WPA_GET_BE32(&wpa_ie[2]) == OSEN_IE_VENDOR_TYPE)
+		return wpa_parse_wpa_ie_rsn(wpa_ie, wpa_ie_len, data);
 	else
 		return wpa_parse_wpa_ie_wpa(wpa_ie, wpa_ie_len, data);
 }
@@ -508,12 +511,14 @@ int wpa_supplicant_parse_ies(const u8 *buf, size_t len,
 			ie->rsn_ie_len = pos[1] + 2;
 			wpa_hexdump(MSG_DEBUG, "WPA: RSN IE in EAPOL-Key",
 				    ie->rsn_ie, ie->rsn_ie_len);
-		} else if (*pos == WLAN_EID_MOBILITY_DOMAIN) {
+		} else if (*pos == WLAN_EID_MOBILITY_DOMAIN &&
+			   pos[1] >= sizeof(struct rsn_mdie)) {
 			ie->mdie = pos;
 			ie->mdie_len = pos[1] + 2;
 			wpa_hexdump(MSG_DEBUG, "WPA: MDIE in EAPOL-Key",
 				    ie->mdie, ie->mdie_len);
-		} else if (*pos == WLAN_EID_FAST_BSS_TRANSITION) {
+		} else if (*pos == WLAN_EID_FAST_BSS_TRANSITION &&
+			   pos[1] >= sizeof(struct rsn_ftie)) {
 			ie->ftie = pos;
 			ie->ftie_len = pos[1] + 2;
 			wpa_hexdump(MSG_DEBUG, "WPA: FTIE in EAPOL-Key",
@@ -548,15 +553,16 @@ int wpa_supplicant_parse_ies(const u8 *buf, size_t len,
 		} else if (*pos == WLAN_EID_EXT_SUPP_RATES) {
 			ie->ext_supp_rates = pos;
 			ie->ext_supp_rates_len = pos[1] + 2;
-		} else if (*pos == WLAN_EID_HT_CAP) {
+		} else if (*pos == WLAN_EID_HT_CAP &&
+			   pos[1] >= sizeof(struct ieee80211_ht_capabilities)) {
 			ie->ht_capabilities = pos + 2;
-			ie->ht_capabilities_len = pos[1];
 		} else if (*pos == WLAN_EID_VHT_AID) {
 			if (pos[1] >= 2)
 				ie->aid = WPA_GET_LE16(pos + 2) & 0x3fff;
-		} else if (*pos == WLAN_EID_VHT_CAP) {
+		} else if (*pos == WLAN_EID_VHT_CAP &&
+			   pos[1] >= sizeof(struct ieee80211_vht_capabilities))
+		{
 			ie->vht_capabilities = pos + 2;
-			ie->vht_capabilities_len = pos[1];
 		} else if (*pos == WLAN_EID_QOS && pos[1] >= 1) {
 			ie->qosinfo = pos[2];
 		} else if (*pos == WLAN_EID_SUPPORTED_CHANNELS) {
diff --git a/src/rsn_supp/wpa_ie.h b/src/rsn_supp/wpa_ie.h
index 0fc42cc492ac..fe95af0abc51 100644
--- a/src/rsn_supp/wpa_ie.h
+++ b/src/rsn_supp/wpa_ie.h
@@ -50,9 +50,7 @@ struct wpa_eapol_ie_parse {
 	const u8 *ext_supp_rates;
 	size_t ext_supp_rates_len;
 	const u8 *ht_capabilities;
-	size_t ht_capabilities_len;
 	const u8 *vht_capabilities;
-	size_t vht_capabilities_len;
 	const u8 *supp_channels;
 	size_t supp_channels_len;
 	const u8 *supp_oper_classes;
diff --git a/src/tls/libtommath.c b/src/tls/libtommath.c
index 3fb8fbed25e7..8bc824f20dcd 100644
--- a/src/tls/libtommath.c
+++ b/src/tls/libtommath.c
@@ -1472,8 +1472,7 @@ static int mp_init_multi(mp_int *mp, ...)
                 cur_arg = va_arg(clean_args, mp_int*);
             }
             va_end(clean_args);
-            res = MP_MEM;
-            break;
+            return MP_MEM;
         }
         n++;
         cur_arg = va_arg(args, mp_int*);
@@ -1631,7 +1630,7 @@ static int mp_div(mp_int * a, mp_int * b, mp_int * c, mp_int * d)
   }
 	
   /* init our temps */
-  if ((res = mp_init_multi(&ta, &tb, &tq, &q, NULL) != MP_OKAY)) {
+  if ((res = mp_init_multi(&ta, &tb, &tq, &q, NULL)) != MP_OKAY) {
      return res;
   }
 
diff --git a/src/tls/tlsv1_client.c b/src/tls/tlsv1_client.c
index facdd659173d..a6f0587e34c5 100644
--- a/src/tls/tlsv1_client.c
+++ b/src/tls/tlsv1_client.c
@@ -714,12 +714,12 @@ int tlsv1_client_hello_ext(struct tlsv1_client *conn, int ext_type,
 
 
 /**
- * tlsv1_client_get_keys - Get master key and random data from TLS connection
+ * tlsv1_client_get_random - Get random data from TLS connection
  * @conn: TLSv1 client connection data from tlsv1_client_init()
- * @keys: Structure of key/random data (filled on success)
+ * @keys: Structure of random data (filled on success)
  * Returns: 0 on success, -1 on failure
  */
-int tlsv1_client_get_keys(struct tlsv1_client *conn, struct tls_keys *keys)
+int tlsv1_client_get_random(struct tlsv1_client *conn, struct tls_random *keys)
 {
 	os_memset(keys, 0, sizeof(*keys));
 	if (conn->state == CLIENT_HELLO)
@@ -731,8 +731,6 @@ int tlsv1_client_get_keys(struct tlsv1_client *conn, struct tls_keys *keys)
 	if (conn->state != SERVER_HELLO) {
 		keys->server_random = conn->server_random;
 		keys->server_random_len = TLS_RANDOM_LEN;
-		keys->master_key = conn->master_secret;
-		keys->master_key_len = TLS_MASTER_SECRET_LEN;
 	}
 
 	return 0;
diff --git a/src/tls/tlsv1_client.h b/src/tls/tlsv1_client.h
index 8ec85f1a9193..a4e25e969937 100644
--- a/src/tls/tlsv1_client.h
+++ b/src/tls/tlsv1_client.h
@@ -36,7 +36,7 @@ int tlsv1_client_shutdown(struct tlsv1_client *conn);
 int tlsv1_client_resumed(struct tlsv1_client *conn);
 int tlsv1_client_hello_ext(struct tlsv1_client *conn, int ext_type,
 			   const u8 *data, size_t data_len);
-int tlsv1_client_get_keys(struct tlsv1_client *conn, struct tls_keys *keys);
+int tlsv1_client_get_random(struct tlsv1_client *conn, struct tls_random *data);
 int tlsv1_client_get_keyblock_size(struct tlsv1_client *conn);
 int tlsv1_client_set_cipher_list(struct tlsv1_client *conn, u8 *ciphers);
 int tlsv1_client_set_cred(struct tlsv1_client *conn,
diff --git a/src/tls/tlsv1_server.c b/src/tls/tlsv1_server.c
index 93ae4888d898..ba47337bcbb1 100644
--- a/src/tls/tlsv1_server.c
+++ b/src/tls/tlsv1_server.c
@@ -610,12 +610,12 @@ int tlsv1_server_resumed(struct tlsv1_server *conn)
 
 
 /**
- * tlsv1_server_get_keys - Get master key and random data from TLS connection
+ * tlsv1_server_get_random - Get random data from TLS connection
  * @conn: TLSv1 server connection data from tlsv1_server_init()
- * @keys: Structure of key/random data (filled on success)
+ * @keys: Structure of random data (filled on success)
  * Returns: 0 on success, -1 on failure
  */
-int tlsv1_server_get_keys(struct tlsv1_server *conn, struct tls_keys *keys)
+int tlsv1_server_get_random(struct tlsv1_server *conn, struct tls_random *keys)
 {
 	os_memset(keys, 0, sizeof(*keys));
 	if (conn->state == CLIENT_HELLO)
@@ -627,8 +627,6 @@ int tlsv1_server_get_keys(struct tlsv1_server *conn, struct tls_keys *keys)
 	if (conn->state != SERVER_HELLO) {
 		keys->server_random = conn->server_random;
 		keys->server_random_len = TLS_RANDOM_LEN;
-		keys->master_key = conn->master_secret;
-		keys->master_key_len = TLS_MASTER_SECRET_LEN;
 	}
 
 	return 0;
diff --git a/src/tls/tlsv1_server.h b/src/tls/tlsv1_server.h
index b2b28d1e1215..10e7699312b0 100644
--- a/src/tls/tlsv1_server.h
+++ b/src/tls/tlsv1_server.h
@@ -32,7 +32,7 @@ int tlsv1_server_get_cipher(struct tlsv1_server *conn, char *buf,
 			    size_t buflen);
 int tlsv1_server_shutdown(struct tlsv1_server *conn);
 int tlsv1_server_resumed(struct tlsv1_server *conn);
-int tlsv1_server_get_keys(struct tlsv1_server *conn, struct tls_keys *keys);
+int tlsv1_server_get_random(struct tlsv1_server *conn, struct tls_random *data);
 int tlsv1_server_get_keyblock_size(struct tlsv1_server *conn);
 int tlsv1_server_set_cipher_list(struct tlsv1_server *conn, u8 *ciphers);
 int tlsv1_server_set_verify(struct tlsv1_server *conn, int verify_peer);
diff --git a/src/tls/x509v3.c b/src/tls/x509v3.c
index 742af328cf8c..b51dfcd44732 100644
--- a/src/tls/x509v3.c
+++ b/src/tls/x509v3.c
@@ -1511,7 +1511,7 @@ struct x509_certificate * x509_certificate_parse(const u8 *buf, size_t len)
 	if (pos + hdr.length < end) {
 		wpa_hexdump(MSG_MSGDUMP, "X509: Ignoring extra data after DER "
 			    "encoded certificate",
-			    pos + hdr.length, end - pos + hdr.length);
+			    pos + hdr.length, end - (pos + hdr.length));
 		end = pos + hdr.length;
 	}
 
diff --git a/src/utils/browser-wpadebug.c b/src/utils/browser-wpadebug.c
index 5fc40fac610e..59ba4d1e02d8 100644
--- a/src/utils/browser-wpadebug.c
+++ b/src/utils/browser-wpadebug.c
@@ -96,7 +96,7 @@ int hs20_web_browser(const char *url)
 
 	if (pid == 0) {
 		/* run the external command in the child process */
-		char *argv[12];
+		char *argv[14];
 
 		argv[0] = "browser-wpadebug";
 		argv[1] = "start";
@@ -109,7 +109,9 @@ int hs20_web_browser(const char *url)
 		argv[8] = "-e";
 		argv[9] = "w1.fi.wpadebug.URL";
 		argv[10] = (void *) url;
-		argv[11] = NULL;
+		argv[11] = "--user";
+		argv[12] = "-3"; /* USER_CURRENT_OR_SELF */
+		argv[13] = NULL;
 
 		execv("/system/bin/am", argv);
 		wpa_printf(MSG_ERROR, "execv: %s", strerror(errno));
diff --git a/src/utils/common.c b/src/utils/common.c
index 5fd795f3f303..660e9fc985d6 100644
--- a/src/utils/common.c
+++ b/src/utils/common.c
@@ -8,6 +8,7 @@
 
 #include "includes.h"
 
+#include "common/ieee802_11_defs.h"
 #include "common.h"
 
 
@@ -277,6 +278,31 @@ int wpa_scnprintf(char *buf, size_t size, const char *fmt, ...)
 	return ret;
 }
 
+
+int wpa_snprintf_hex_sep(char *buf, size_t buf_size, const u8 *data, size_t len,
+			 char sep)
+{
+	size_t i;
+	char *pos = buf, *end = buf + buf_size;
+	int ret;
+
+	if (buf_size == 0)
+		return 0;
+
+	for (i = 0; i < len; i++) {
+		ret = os_snprintf(pos, end - pos, "%02x%c",
+				  data[i], sep);
+		if (os_snprintf_error(end - pos, ret)) {
+			end[-1] = '\0';
+			return pos - buf;
+		}
+		pos += ret;
+	}
+	pos[-1] = '\0';
+	return pos - buf;
+}
+
+
 static inline int _wpa_snprintf_hex(char *buf, size_t buf_size, const u8 *data,
 				    size_t len, int uppercase)
 {
@@ -584,7 +610,7 @@ size_t printf_decode(u8 *buf, size_t maxlen, const char *str)
  */
 const char * wpa_ssid_txt(const u8 *ssid, size_t ssid_len)
 {
-	static char ssid_txt[32 * 4 + 1];
+	static char ssid_txt[SSID_MAX_LEN * 4 + 1];
 
 	if (ssid == NULL) {
 		ssid_txt[0] = '\0';
@@ -946,6 +972,48 @@ int random_mac_addr_keep_oui(u8 *addr)
 }
 
 
+/**
+ * cstr_token - Get next token from const char string
+ * @str: a constant string to tokenize
+ * @delim: a string of delimiters
+ * @last: a pointer to a character following the returned token
+ *      It has to be set to NULL for the first call and passed for any
+ *      futher call.
+ * Returns: a pointer to token position in str or NULL
+ *
+ * This function is similar to str_token, but it can be used with both
+ * char and const char strings. Differences:
+ * - The str buffer remains unmodified
+ * - The returned token is not a NULL terminated string, but a token
+ *   position in str buffer. If a return value is not NULL a size
+ *   of the returned token could be calculated as (last - token).
+ */
+const char * cstr_token(const char *str, const char *delim, const char **last)
+{
+	const char *end, *token = str;
+
+	if (!str || !delim || !last)
+		return NULL;
+
+	if (*last)
+		token = *last;
+
+	while (*token && os_strchr(delim, *token))
+		token++;
+
+	if (!*token)
+		return NULL;
+
+	end = token + 1;
+
+	while (*end && !os_strchr(delim, *end))
+		end++;
+
+	*last = end;
+	return token;
+}
+
+
 /**
  * str_token - Get next token from a string
  * @buf: String to tokenize. Note that the string might be modified.
@@ -956,25 +1024,12 @@ int random_mac_addr_keep_oui(u8 *addr)
  */
 char * str_token(char *str, const char *delim, char **context)
 {
-	char *end, *pos = str;
+	char *token = (char *) cstr_token(str, delim, (const char **) context);
 
-	if (*context)
-		pos = *context;
+	if (token && **context)
+		*(*context)++ = '\0';
 
-	while (*pos && os_strchr(delim, *pos))
-		pos++;
-	if (!*pos)
-		return NULL;
-
-	end = pos + 1;
-	while (*end && !os_strchr(delim, *end))
-		end++;
-
-	if (*end)
-		*end++ = '\0';
-
-	*context = end;
-	return pos;
+	return token;
 }
 
 
@@ -1062,3 +1117,9 @@ size_t utf8_escape(const char *inp, size_t in_size,
 
 	return res_size;
 }
+
+
+int is_ctrl_char(char c)
+{
+	return c > 0 && c < 32;
+}
diff --git a/src/utils/common.h b/src/utils/common.h
index 576e8e7e2d4e..0b9cc3d88209 100644
--- a/src/utils/common.h
+++ b/src/utils/common.h
@@ -53,16 +53,6 @@ static inline unsigned int bswap_32(unsigned int v)
 }
 #endif /* __APPLE__ */
 
-#ifdef CONFIG_TI_COMPILER
-#define __BIG_ENDIAN 4321
-#define __LITTLE_ENDIAN 1234
-#ifdef __big_endian__
-#define __BYTE_ORDER __BIG_ENDIAN
-#else
-#define __BYTE_ORDER __LITTLE_ENDIAN
-#endif
-#endif /* CONFIG_TI_COMPILER */
-
 #ifdef CONFIG_NATIVE_WINDOWS
 #include <winsock.h>
 
@@ -110,22 +100,6 @@ typedef INT8 s8;
 #define WPA_TYPES_DEFINED
 #endif /* __vxworks */
 
-#ifdef CONFIG_TI_COMPILER
-#ifdef _LLONG_AVAILABLE
-typedef unsigned long long u64;
-#else
-/*
- * TODO: 64-bit variable not available. Using long as a workaround to test the
- * build, but this will likely not work for all operations.
- */
-typedef unsigned long u64;
-#endif
-typedef unsigned int u32;
-typedef unsigned short u16;
-typedef unsigned char u8;
-#define WPA_TYPES_DEFINED
-#endif /* CONFIG_TI_COMPILER */
-
 #ifndef WPA_TYPES_DEFINED
 #ifdef CONFIG_USE_INTTYPES_H
 #include <inttypes.h>
@@ -262,7 +236,7 @@ static inline void WPA_PUT_BE24(u8 *a, u32 val)
 
 static inline u32 WPA_GET_BE32(const u8 *a)
 {
-	return (a[0] << 24) | (a[1] << 16) | (a[2] << 8) | a[3];
+	return ((u32) a[0] << 24) | (a[1] << 16) | (a[2] << 8) | a[3];
 }
 
 static inline void WPA_PUT_BE32(u8 *a, u32 val)
@@ -275,7 +249,7 @@ static inline void WPA_PUT_BE32(u8 *a, u32 val)
 
 static inline u32 WPA_GET_LE32(const u8 *a)
 {
-	return (a[3] << 24) | (a[2] << 16) | (a[1] << 8) | a[0];
+	return ((u32) a[3] << 24) | (a[2] << 16) | (a[1] << 8) | a[0];
 }
 
 static inline void WPA_PUT_LE32(u8 *a, u32 val)
@@ -433,7 +407,7 @@ void perror(const char *s);
 #endif
 
 #ifndef BIT
-#define BIT(x) (1 << (x))
+#define BIT(x) (1U << (x))
 #endif
 
 /*
@@ -480,6 +454,8 @@ int hexstr2bin(const char *hex, u8 *buf, size_t len);
 void inc_byte_array(u8 *counter, size_t len);
 void wpa_get_ntp_timestamp(u8 *buf);
 int wpa_scnprintf(char *buf, size_t size, const char *fmt, ...);
+int wpa_snprintf_hex_sep(char *buf, size_t buf_size, const u8 *data, size_t len,
+			 char sep);
 int wpa_snprintf_hex(char *buf, size_t buf_size, const u8 *data, size_t len);
 int wpa_snprintf_hex_uppercase(char *buf, size_t buf_size, const u8 *data,
 			       size_t len);
@@ -516,6 +492,11 @@ static inline int is_broadcast_ether_addr(const u8 *a)
 	return (a[0] & a[1] & a[2] & a[3] & a[4] & a[5]) == 0xff;
 }
 
+static inline int is_multicast_ether_addr(const u8 *a)
+{
+	return a[0] & 0x01;
+}
+
 #define broadcast_ether_addr (const u8 *) "\xff\xff\xff\xff\xff\xff"
 
 #include "wpa_debug.h"
@@ -547,11 +528,13 @@ void bin_clear_free(void *bin, size_t len);
 int random_mac_addr(u8 *addr);
 int random_mac_addr_keep_oui(u8 *addr);
 
+const char * cstr_token(const char *str, const char *delim, const char **last);
 char * str_token(char *str, const char *delim, char **context);
 size_t utf8_escape(const char *inp, size_t in_size,
 		   char *outp, size_t out_size);
 size_t utf8_unescape(const char *inp, size_t in_size,
 		     char *outp, size_t out_size);
+int is_ctrl_char(char c);
 
 
 /*
diff --git a/src/utils/eloop.c b/src/utils/eloop.c
index 4a565ebdbd0a..8647229b8eb5 100644
--- a/src/utils/eloop.c
+++ b/src/utils/eloop.c
@@ -61,11 +61,8 @@ struct eloop_signal {
 struct eloop_sock_table {
 	int count;
 	struct eloop_sock *table;
-#ifdef CONFIG_ELOOP_EPOLL
 	eloop_event_type type;
-#else /* CONFIG_ELOOP_EPOLL */
 	int changed;
-#endif /* CONFIG_ELOOP_EPOLL */
 };
 
 struct eloop_data {
@@ -256,9 +253,7 @@ static int eloop_sock_table_add_sock(struct eloop_sock_table *table,
 	table->table = tmp;
 	eloop.max_sock = new_max_sock;
 	eloop.count++;
-#ifndef CONFIG_ELOOP_EPOLL
 	table->changed = 1;
-#endif /* CONFIG_ELOOP_EPOLL */
 	eloop_trace_sock_add_ref(table);
 
 #ifdef CONFIG_ELOOP_EPOLL
@@ -314,9 +309,7 @@ static void eloop_sock_table_remove_sock(struct eloop_sock_table *table,
 	}
 	table->count--;
 	eloop.count--;
-#ifndef CONFIG_ELOOP_EPOLL
 	table->changed = 1;
-#endif /* CONFIG_ELOOP_EPOLL */
 	eloop_trace_sock_add_ref(table);
 #ifdef CONFIG_ELOOP_EPOLL
 	if (epoll_ctl(eloop.epollfd, EPOLL_CTL_DEL, sock, NULL) < 0) {
@@ -523,6 +516,10 @@ static void eloop_sock_table_dispatch(struct epoll_event *events, int nfds)
 			continue;
 		table->handler(table->sock, table->eloop_data,
 			       table->user_data);
+		if (eloop.readers.changed ||
+		    eloop.writers.changed ||
+		    eloop.exceptions.changed)
+			break;
 	}
 }
 #endif /* CONFIG_ELOOP_EPOLL */
@@ -923,6 +920,20 @@ void eloop_run(void)
 	       (!dl_list_empty(&eloop.timeout) || eloop.readers.count > 0 ||
 		eloop.writers.count > 0 || eloop.exceptions.count > 0)) {
 		struct eloop_timeout *timeout;
+
+		if (eloop.pending_terminate) {
+			/*
+			 * This may happen in some corner cases where a signal
+			 * is received during a blocking operation. We need to
+			 * process the pending signals and exit if requested to
+			 * avoid hitting the SIGALRM limit if the blocking
+			 * operation took more than two seconds.
+			 */
+			eloop_process_pending_signals();
+			if (eloop.terminate)
+				break;
+		}
+
 		timeout = dl_list_first(&eloop.timeout, struct eloop_timeout,
 					list);
 		if (timeout) {
@@ -977,6 +988,11 @@ void eloop_run(void)
 				   , strerror(errno));
 			goto out;
 		}
+
+		eloop.readers.changed = 0;
+		eloop.writers.changed = 0;
+		eloop.exceptions.changed = 0;
+
 		eloop_process_pending_signals();
 
 		/* check if some registered timeouts have occurred */
@@ -998,6 +1014,19 @@ void eloop_run(void)
 		if (res <= 0)
 			continue;
 
+		if (eloop.readers.changed ||
+		    eloop.writers.changed ||
+		    eloop.exceptions.changed) {
+			 /*
+			  * Sockets may have been closed and reopened with the
+			  * same FD in the signal or timeout handlers, so we
+			  * must skip the previous results and check again
+			  * whether any of the currently registered sockets have
+			  * events.
+			  */
+			continue;
+		}
+
 #ifdef CONFIG_ELOOP_POLL
 		eloop_sock_table_dispatch(&eloop.readers, &eloop.writers,
 					  &eloop.exceptions, eloop.pollfds_map,
@@ -1073,7 +1102,7 @@ void eloop_destroy(void)
 
 int eloop_terminated(void)
 {
-	return eloop.terminate;
+	return eloop.terminate || eloop.pending_terminate;
 }
 
 
diff --git a/src/utils/http_curl.c b/src/utils/http_curl.c
index b38cf796ca2a..653eb541ab47 100644
--- a/src/utils/http_curl.c
+++ b/src/utils/http_curl.c
@@ -855,8 +855,10 @@ static int validate_server_cert(struct http_ctx *ctx, X509 *cert)
 	struct http_cert hcert;
 	int ret;
 
-	if (ctx->cert_cb == NULL)
+	if (ctx->cert_cb == NULL) {
+		wpa_printf(MSG_DEBUG, "%s: no cert_cb configured", __func__);
 		return 0;
+	}
 
 	if (0) {
 		BIO *out;
@@ -950,7 +952,8 @@ static int curl_cb_ssl_verify(int preverify_ok, X509_STORE_CTX *x509_ctx)
 	ssl_ctx = ssl->ctx;
 	ctx = SSL_CTX_get_app_data(ssl_ctx);
 
-	wpa_printf(MSG_DEBUG, "curl_cb_ssl_verify");
+	wpa_printf(MSG_DEBUG, "curl_cb_ssl_verify, preverify_ok: %d",
+		   preverify_ok);
 
 	err = X509_STORE_CTX_get_error(x509_ctx);
 	err_str = X509_verify_cert_error_string(err);
@@ -1249,9 +1252,14 @@ static CURL * setup_curl_post(struct http_ctx *ctx, const char *address,
 			      const char *client_key)
 {
 	CURL *curl;
+#ifdef EAP_TLS_OPENSSL
+	const char *extra = " tls=openssl";
+#else /* EAP_TLS_OPENSSL */
+	const char *extra = "";
+#endif /* EAP_TLS_OPENSSL */
 
 	wpa_printf(MSG_DEBUG, "Start HTTP client: address=%s ca_fname=%s "
-		   "username=%s", address, ca_fname, username);
+		   "username=%s%s", address, ca_fname, username, extra);
 
 	curl = curl_easy_init();
 	if (curl == NULL)
diff --git a/src/utils/includes.h b/src/utils/includes.h
index 6c6ec87d0eaf..75513fc8c1ef 100644
--- a/src/utils/includes.h
+++ b/src/utils/includes.h
@@ -17,26 +17,22 @@
 #include "build_config.h"
 
 #include <stdlib.h>
+#include <stddef.h>
 #include <stdio.h>
 #include <stdarg.h>
 #include <string.h>
 #ifndef _WIN32_WCE
-#ifndef CONFIG_TI_COMPILER
 #include <signal.h>
 #include <sys/types.h>
-#endif /* CONFIG_TI_COMPILER */
 #include <errno.h>
 #endif /* _WIN32_WCE */
 #include <ctype.h>
 
-#ifndef CONFIG_TI_COMPILER
 #ifndef _MSC_VER
 #include <unistd.h>
 #endif /* _MSC_VER */
-#endif /* CONFIG_TI_COMPILER */
 
 #ifndef CONFIG_NATIVE_WINDOWS
-#ifndef CONFIG_TI_COMPILER
 #include <sys/socket.h>
 #include <netinet/in.h>
 #include <arpa/inet.h>
@@ -44,7 +40,6 @@
 #include <sys/uio.h>
 #include <sys/time.h>
 #endif /* __vxworks */
-#endif /* CONFIG_TI_COMPILER */
 #endif /* CONFIG_NATIVE_WINDOWS */
 
 #endif /* INCLUDES_H */
diff --git a/src/utils/os.h b/src/utils/os.h
index 77250d6371c9..9e496fb65978 100644
--- a/src/utils/os.h
+++ b/src/utils/os.h
@@ -246,6 +246,13 @@ char * os_readfile(const char *name, size_t *len);
  */
 int os_file_exists(const char *fname);
 
+/**
+ * os_fdatasync - Sync a file's (for a given stream) state with storage device
+ * @stream: the stream to be flushed
+ * Returns: 0 if the operation succeeded or -1 on failure
+ */
+int os_fdatasync(FILE *stream);
+
 /**
  * os_zalloc - Allocate and zero memory
  * @size: Number of bytes to allocate
@@ -646,4 +653,12 @@ int os_exec(const char *program, const char *arg, int wait_completion);
 #define strcpy OS_DO_NOT_USE_strcpy
 #endif /* OS_REJECT_C_LIB_FUNCTIONS */
 
+
+#if defined(WPA_TRACE_BFD) && defined(CONFIG_TESTING_OPTIONS)
+#define TEST_FAIL() testing_test_fail()
+int testing_test_fail(void);
+#else
+#define TEST_FAIL() 0
+#endif
+
 #endif /* OS_H */
diff --git a/src/utils/os_internal.c b/src/utils/os_internal.c
index 77733ad916cd..ed6eb3c6b677 100644
--- a/src/utils/os_internal.c
+++ b/src/utils/os_internal.c
@@ -243,6 +243,12 @@ char * os_readfile(const char *name, size_t *len)
 }
 
 
+int os_fdatasync(FILE *stream)
+{
+	return 0;
+}
+
+
 void * os_zalloc(size_t size)
 {
 	void *n = os_malloc(size);
diff --git a/src/utils/os_none.c b/src/utils/os_none.c
index 83fe025167b6..0c3214d32ce5 100644
--- a/src/utils/os_none.c
+++ b/src/utils/os_none.c
@@ -102,6 +102,12 @@ char * os_readfile(const char *name, size_t *len)
 }
 
 
+int os_fdatasync(FILE *stream)
+{
+	return 0;
+}
+
+
 void * os_zalloc(size_t size)
 {
 	return NULL;
diff --git a/src/utils/os_unix.c b/src/utils/os_unix.c
index e0c1125d5305..ffa2e788b3db 100644
--- a/src/utils/os_unix.c
+++ b/src/utils/os_unix.c
@@ -17,6 +17,12 @@
 #include <private/android_filesystem_config.h>
 #endif /* ANDROID */
 
+#ifdef __MACH__
+#include <CoreServices/CoreServices.h>
+#include <mach/mach.h>
+#include <mach/mach_time.h>
+#endif /* __MACH__ */
+
 #include "os.h"
 #include "common.h"
 
@@ -36,7 +42,7 @@ struct os_alloc_trace {
 	struct dl_list list;
 	size_t len;
 	WPA_TRACE_INFO
-};
+} __attribute__((aligned(16)));
 
 #endif /* WPA_TRACE */
 
@@ -63,6 +69,7 @@ int os_get_time(struct os_time *t)
 
 int os_get_reltime(struct os_reltime *t)
 {
+#ifndef __MACH__
 #if defined(CLOCK_BOOTTIME)
 	static clockid_t clock_id = CLOCK_BOOTTIME;
 #elif defined(CLOCK_MONOTONIC)
@@ -95,6 +102,23 @@ int os_get_reltime(struct os_reltime *t)
 			return -1;
 		}
 	}
+#else /* __MACH__ */
+	uint64_t abstime, nano;
+	static mach_timebase_info_data_t info = { 0, 0 };
+
+	if (!info.denom) {
+		if (mach_timebase_info(&info) != KERN_SUCCESS)
+			return -1;
+	}
+
+	abstime = mach_absolute_time();
+	nano = (abstime * info.numer) / info.denom;
+
+	t->sec = nano / NSEC_PER_SEC;
+	t->usec = (nano - (((uint64_t) t->sec) * NSEC_PER_SEC)) / NSEC_PER_USEC;
+
+	return 0;
+#endif /* __MACH__ */
 }
 
 
@@ -226,6 +250,9 @@ int os_get_random(unsigned char *buf, size_t len)
 	FILE *f;
 	size_t rc;
 
+	if (TEST_FAIL())
+		return -1;
+
 	f = fopen("/dev/urandom", "rb");
 	if (f == NULL) {
 		printf("Could not open /dev/urandom.\n");
@@ -415,6 +442,25 @@ int os_file_exists(const char *fname)
 }
 
 
+int os_fdatasync(FILE *stream)
+{
+	if (!fflush(stream)) {
+#ifndef __MACH__
+		return fdatasync(fileno(stream));
+#else /* __MACH__ */
+#ifdef F_FULLFSYNC
+		/* OS X does not implement fdatasync(). */
+		return fcntl(fileno(stream), F_FULLFSYNC);
+#else /* F_FULLFSYNC */
+#error Neither fdatasync nor F_FULLSYNC are defined
+#endif /* F_FULLFSYNC */
+#endif /* __MACH__ */
+	}
+
+	return -1;
+}
+
+
 #ifndef WPA_TRACE
 void * os_zalloc(size_t size)
 {
@@ -548,6 +594,78 @@ static int testing_fail_alloc(void)
 	return 0;
 }
 
+
+char wpa_trace_test_fail_func[256] = { 0 };
+unsigned int wpa_trace_test_fail_after;
+
+int testing_test_fail(void)
+{
+	const char *func[WPA_TRACE_LEN];
+	size_t i, res, len;
+	char *pos, *next;
+	int match;
+
+	if (!wpa_trace_test_fail_after)
+		return 0;
+
+	res = wpa_trace_calling_func(func, WPA_TRACE_LEN);
+	i = 0;
+	if (i < res && os_strcmp(func[i], __func__) == 0)
+		i++;
+
+	pos = wpa_trace_test_fail_func;
+
+	match = 0;
+	while (i < res) {
+		int allow_skip = 1;
+		int maybe = 0;
+
+		if (*pos == '=') {
+			allow_skip = 0;
+			pos++;
+		} else if (*pos == '?') {
+			maybe = 1;
+			pos++;
+		}
+		next = os_strchr(pos, ';');
+		if (next)
+			len = next - pos;
+		else
+			len = os_strlen(pos);
+		if (os_memcmp(pos, func[i], len) != 0) {
+			if (maybe && next) {
+				pos = next + 1;
+				continue;
+			}
+			if (allow_skip) {
+				i++;
+				continue;
+			}
+			return 0;
+		}
+		if (!next) {
+			match = 1;
+			break;
+		}
+		pos = next + 1;
+		i++;
+	}
+	if (!match)
+		return 0;
+
+	wpa_trace_test_fail_after--;
+	if (wpa_trace_test_fail_after == 0) {
+		wpa_printf(MSG_INFO, "TESTING: fail at %s",
+			   wpa_trace_test_fail_func);
+		for (i = 0; i < res; i++)
+			wpa_printf(MSG_INFO, "backtrace[%d] = %s",
+				   (int) i, func[i]);
+		return 1;
+	}
+
+	return 0;
+}
+
 #else
 
 static inline int testing_fail_alloc(void)
diff --git a/src/utils/os_win32.c b/src/utils/os_win32.c
index 296ea13f153b..dea27b9f2ad8 100644
--- a/src/utils/os_win32.c
+++ b/src/utils/os_win32.c
@@ -216,6 +216,24 @@ char * os_readfile(const char *name, size_t *len)
 }
 
 
+int os_fdatasync(FILE *stream)
+{
+	HANDLE h;
+
+	if (stream == NULL)
+		return -1;
+
+	h = (HANDLE) _get_osfhandle(_fileno(stream));
+	if (h == INVALID_HANDLE_VALUE)
+		return -1;
+
+	if (!FlushFileBuffers(h))
+		return -1;
+
+	return 0;
+}
+
+
 void * os_zalloc(size_t size)
 {
 	return calloc(1, size);
diff --git a/src/utils/radiotap.c b/src/utils/radiotap.c
index f8f815a86be9..c9a502335592 100644
--- a/src/utils/radiotap.c
+++ b/src/utils/radiotap.c
@@ -123,13 +123,13 @@ int ieee80211_radiotap_iterator_init(
 
 	/* find payload start allowing for extended bitmap(s) */
 
-	if (iterator->_bitmap_shifter & (1<<IEEE80211_RADIOTAP_EXT)) {
+	if (iterator->_bitmap_shifter & BIT(IEEE80211_RADIOTAP_EXT)) {
 		if ((unsigned long)iterator->_arg -
 		    (unsigned long)iterator->_rtheader + sizeof(uint32_t) >
 		    (unsigned long)iterator->_max_length)
 			return -EINVAL;
 		while (get_unaligned_le32(iterator->_arg) &
-					(1 << IEEE80211_RADIOTAP_EXT)) {
+		       BIT(IEEE80211_RADIOTAP_EXT)) {
 			iterator->_arg += sizeof(uint32_t);
 
 			/*
diff --git a/src/utils/utils_module_tests.c b/src/utils/utils_module_tests.c
index 4b97dadd786c..41511b9999a6 100644
--- a/src/utils/utils_module_tests.c
+++ b/src/utils/utils_module_tests.c
@@ -9,10 +9,13 @@
 #include "utils/includes.h"
 
 #include "utils/common.h"
+#include "common/ieee802_11_defs.h"
 #include "utils/bitfield.h"
 #include "utils/ext_password.h"
 #include "utils/trace.h"
 #include "utils/base64.h"
+#include "utils/ip_addr.h"
+#include "utils/eloop.h"
 
 
 struct printf_test_data {
@@ -43,6 +46,7 @@ static int printf_encode_decode_tests(void)
 	char buf[100];
 	u8 bin[100];
 	int errors = 0;
+	int array[10];
 
 	wpa_printf(MSG_INFO, "printf encode/decode tests");
 
@@ -91,9 +95,24 @@ static int printf_encode_decode_tests(void)
 	if (printf_decode(bin, 3, "\\xa") != 1 || bin[0] != 10)
 		errors++;
 
+	if (printf_decode(bin, 3, "\\xq") != 1 || bin[0] != 'q')
+		errors++;
+
 	if (printf_decode(bin, 3, "\\a") != 1 || bin[0] != 'a')
 		errors++;
 
+	array[0] = 10;
+	array[1] = 10;
+	array[2] = 5;
+	array[3] = 10;
+	array[4] = 5;
+	array[5] = 0;
+	if (int_array_len(array) != 5)
+		errors++;
+	int_array_sort_unique(array);
+	if (int_array_len(array) != 2)
+		errors++;
+
 	if (errors) {
 		wpa_printf(MSG_ERROR, "%d printf test(s) failed", errors);
 		return -1;
@@ -335,11 +354,14 @@ static int base64_tests(void)
 
 static int common_tests(void)
 {
-	char buf[3];
+	char buf[3], longbuf[100];
 	u8 addr[ETH_ALEN] = { 1, 2, 3, 4, 5, 6 };
 	u8 bin[3];
 	int errors = 0;
 	struct wpa_freq_range_list ranges;
+	size_t len;
+	const char *txt;
+	u8 ssid[255];
 
 	wpa_printf(MSG_INFO, "common tests");
 
@@ -395,6 +417,21 @@ static int common_tests(void)
 	if (utf8_escape("a", 0, buf, sizeof(buf)) != 1 || buf[0] != 'a')
 		errors++;
 
+	os_memset(ssid, 0, sizeof(ssid));
+	txt = wpa_ssid_txt(ssid, sizeof(ssid));
+	len = os_strlen(txt);
+	/* Verify that SSID_MAX_LEN * 4 buffer limit is enforced. */
+	if (len != SSID_MAX_LEN * 4) {
+		wpa_printf(MSG_ERROR,
+			   "Unexpected wpa_ssid_txt() result with too long SSID");
+		errors++;
+	}
+
+	if (wpa_snprintf_hex_sep(longbuf, 0, addr, ETH_ALEN, '-') != 0 ||
+	    wpa_snprintf_hex_sep(longbuf, 5, addr, ETH_ALEN, '-') != 3 ||
+	    os_strcmp(longbuf, "01-0") != 0)
+		errors++;
+
 	if (errors) {
 		wpa_printf(MSG_ERROR, "%d common test(s) failed", errors);
 		return -1;
@@ -404,6 +441,403 @@ static int common_tests(void)
 }
 
 
+static int os_tests(void)
+{
+	int errors = 0;
+	void *ptr;
+	os_time_t t;
+
+	wpa_printf(MSG_INFO, "os tests");
+
+	ptr = os_calloc((size_t) -1, (size_t) -1);
+	if (ptr) {
+		errors++;
+		os_free(ptr);
+	}
+	ptr = os_calloc((size_t) 2, (size_t) -1);
+	if (ptr) {
+		errors++;
+		os_free(ptr);
+	}
+	ptr = os_calloc((size_t) -1, (size_t) 2);
+	if (ptr) {
+		errors++;
+		os_free(ptr);
+	}
+
+	ptr = os_realloc_array(NULL, (size_t) -1, (size_t) -1);
+	if (ptr) {
+		errors++;
+		os_free(ptr);
+	}
+
+	os_sleep(1, 1);
+
+	if (os_mktime(1969, 1, 1, 1, 1, 1, &t) == 0 ||
+	    os_mktime(1971, 0, 1, 1, 1, 1, &t) == 0 ||
+	    os_mktime(1971, 13, 1, 1, 1, 1, &t) == 0 ||
+	    os_mktime(1971, 1, 0, 1, 1, 1, &t) == 0 ||
+	    os_mktime(1971, 1, 32, 1, 1, 1, &t) == 0 ||
+	    os_mktime(1971, 1, 1, -1, 1, 1, &t) == 0 ||
+	    os_mktime(1971, 1, 1, 24, 1, 1, &t) == 0 ||
+	    os_mktime(1971, 1, 1, 1, -1, 1, &t) == 0 ||
+	    os_mktime(1971, 1, 1, 1, 60, 1, &t) == 0 ||
+	    os_mktime(1971, 1, 1, 1, 1, -1, &t) == 0 ||
+	    os_mktime(1971, 1, 1, 1, 1, 61, &t) == 0 ||
+	    os_mktime(1971, 1, 1, 1, 1, 1, &t) != 0 ||
+	    os_mktime(2020, 1, 2, 3, 4, 5, &t) != 0 ||
+	    os_mktime(2015, 12, 31, 23, 59, 59, &t) != 0)
+		errors++;
+
+	if (os_setenv("hwsim_test_env", "test value", 0) != 0 ||
+	    os_setenv("hwsim_test_env", "test value 2", 1) != 0 ||
+	    os_unsetenv("hwsim_test_env") != 0)
+		errors++;
+
+	if (os_file_exists("/this-file-does-not-exists-hwsim") != 0)
+		errors++;
+
+	if (errors) {
+		wpa_printf(MSG_ERROR, "%d os test(s) failed", errors);
+		return -1;
+	}
+
+	return 0;
+}
+
+
+static int wpabuf_tests(void)
+{
+	int errors = 0;
+	void *ptr;
+	struct wpabuf *buf;
+
+	wpa_printf(MSG_INFO, "wpabuf tests");
+
+	ptr = os_malloc(100);
+	if (ptr) {
+		buf = wpabuf_alloc_ext_data(ptr, 100);
+		if (buf) {
+			if (wpabuf_resize(&buf, 100) < 0)
+				errors++;
+			else
+				wpabuf_put(buf, 100);
+			wpabuf_free(buf);
+		} else {
+			errors++;
+			os_free(ptr);
+		}
+	} else {
+		errors++;
+	}
+
+	buf = wpabuf_alloc(100);
+	if (buf) {
+		struct wpabuf *buf2;
+
+		wpabuf_put(buf, 100);
+		if (wpabuf_resize(&buf, 100) < 0)
+			errors++;
+		else
+			wpabuf_put(buf, 100);
+		buf2 = wpabuf_concat(buf, NULL);
+		if (buf2 != buf)
+			errors++;
+		wpabuf_free(buf2);
+	} else {
+		errors++;
+	}
+
+	buf = NULL;
+	buf = wpabuf_zeropad(buf, 10);
+	if (buf != NULL)
+		errors++;
+
+	if (errors) {
+		wpa_printf(MSG_ERROR, "%d wpabuf test(s) failed", errors);
+		return -1;
+	}
+
+	return 0;
+}
+
+
+static int ip_addr_tests(void)
+{
+	int errors = 0;
+	struct hostapd_ip_addr addr;
+	char buf[100];
+
+	wpa_printf(MSG_INFO, "ip_addr tests");
+
+	if (hostapd_parse_ip_addr("1.2.3.4", &addr) != 0 ||
+	    addr.af != AF_INET ||
+	    hostapd_ip_txt(NULL, buf, sizeof(buf)) != NULL ||
+	    hostapd_ip_txt(&addr, buf, 1) != buf || buf[0] != '\0' ||
+	    hostapd_ip_txt(&addr, buf, 0) != NULL ||
+	    hostapd_ip_txt(&addr, buf, sizeof(buf)) != buf)
+		errors++;
+
+	if (hostapd_parse_ip_addr("::", &addr) != 0 ||
+	    addr.af != AF_INET6 ||
+	    hostapd_ip_txt(&addr, buf, 1) != buf || buf[0] != '\0' ||
+	    hostapd_ip_txt(&addr, buf, sizeof(buf)) != buf)
+		errors++;
+
+	if (errors) {
+		wpa_printf(MSG_ERROR, "%d ip_addr test(s) failed", errors);
+		return -1;
+	}
+
+	return 0;
+}
+
+
+struct test_eloop {
+	unsigned int magic;
+	int close_in_timeout;
+	int pipefd1[2];
+	int pipefd2[2];
+};
+
+
+static void eloop_tests_start(int close_in_timeout);
+
+
+static void eloop_test_read_2(int sock, void *eloop_ctx, void *sock_ctx)
+{
+	struct test_eloop *t = eloop_ctx;
+	ssize_t res;
+	char buf[10];
+
+	wpa_printf(MSG_INFO, "%s: sock=%d", __func__, sock);
+
+	if (t->magic != 0x12345678) {
+		wpa_printf(MSG_INFO, "%s: unexpected magic 0x%x",
+			   __func__, t->magic);
+	}
+
+	if (t->pipefd2[0] != sock) {
+		wpa_printf(MSG_INFO, "%s: unexpected sock %d != %d",
+			   __func__, sock, t->pipefd2[0]);
+	}
+
+	res = read(sock, buf, sizeof(buf));
+	wpa_printf(MSG_INFO, "%s: sock=%d --> res=%d",
+		   __func__, sock, (int) res);
+}
+
+
+static void eloop_test_read_2_wrong(int sock, void *eloop_ctx, void *sock_ctx)
+{
+	struct test_eloop *t = eloop_ctx;
+
+	wpa_printf(MSG_INFO, "%s: sock=%d", __func__, sock);
+
+	if (t->magic != 0x12345678) {
+		wpa_printf(MSG_INFO, "%s: unexpected magic 0x%x",
+			   __func__, t->magic);
+	}
+
+	if (t->pipefd2[0] != sock) {
+		wpa_printf(MSG_INFO, "%s: unexpected sock %d != %d",
+			   __func__, sock, t->pipefd2[0]);
+	}
+
+	/*
+	 * This is expected to block due to the original socket with data having
+	 * been closed and no new data having been written to the new socket
+	 * with the same fd. To avoid blocking the process during test, skip the
+	 * read here.
+	 */
+	wpa_printf(MSG_ERROR, "%s: FAIL - should not have called this function",
+		   __func__);
+}
+
+
+static void reopen_pipefd2(struct test_eloop *t)
+{
+	if (t->pipefd2[0] < 0) {
+		wpa_printf(MSG_INFO, "pipefd2 had been closed");
+	} else {
+		int res;
+
+		wpa_printf(MSG_INFO, "close pipefd2");
+		eloop_unregister_read_sock(t->pipefd2[0]);
+		close(t->pipefd2[0]);
+		t->pipefd2[0] = -1;
+		close(t->pipefd2[1]);
+		t->pipefd2[1] = -1;
+
+		res = pipe(t->pipefd2);
+		if (res < 0) {
+			wpa_printf(MSG_INFO, "pipe: %s", strerror(errno));
+			t->pipefd2[0] = -1;
+			t->pipefd2[1] = -1;
+			return;
+		}
+
+		wpa_printf(MSG_INFO,
+			   "re-register pipefd2 with new sockets %d,%d",
+			   t->pipefd2[0], t->pipefd2[1]);
+		eloop_register_read_sock(t->pipefd2[0], eloop_test_read_2_wrong,
+					 t, NULL);
+	}
+}
+
+
+static void eloop_test_read_1(int sock, void *eloop_ctx, void *sock_ctx)
+{
+	struct test_eloop *t = eloop_ctx;
+	ssize_t res;
+	char buf[10];
+
+	wpa_printf(MSG_INFO, "%s: sock=%d", __func__, sock);
+
+	if (t->magic != 0x12345678) {
+		wpa_printf(MSG_INFO, "%s: unexpected magic 0x%x",
+			   __func__, t->magic);
+	}
+
+	if (t->pipefd1[0] != sock) {
+		wpa_printf(MSG_INFO, "%s: unexpected sock %d != %d",
+			   __func__, sock, t->pipefd1[0]);
+	}
+
+	res = read(sock, buf, sizeof(buf));
+	wpa_printf(MSG_INFO, "%s: sock=%d --> res=%d",
+		   __func__, sock, (int) res);
+
+	if (!t->close_in_timeout)
+		reopen_pipefd2(t);
+}
+
+
+static void eloop_test_cb(void *eloop_data, void *user_ctx)
+{
+	struct test_eloop *t = eloop_data;
+
+	wpa_printf(MSG_INFO, "%s", __func__);
+
+	if (t->magic != 0x12345678) {
+		wpa_printf(MSG_INFO, "%s: unexpected magic 0x%x",
+			   __func__, t->magic);
+	}
+
+	if (t->close_in_timeout)
+		reopen_pipefd2(t);
+}
+
+
+static void eloop_test_timeout(void *eloop_data, void *user_ctx)
+{
+	struct test_eloop *t = eloop_data;
+	int next_run = 0;
+
+	wpa_printf(MSG_INFO, "%s", __func__);
+
+	if (t->magic != 0x12345678) {
+		wpa_printf(MSG_INFO, "%s: unexpected magic 0x%x",
+			   __func__, t->magic);
+	}
+
+	if (t->pipefd1[0] >= 0) {
+		wpa_printf(MSG_INFO, "pipefd1 had not been closed");
+		eloop_unregister_read_sock(t->pipefd1[0]);
+		close(t->pipefd1[0]);
+		t->pipefd1[0] = -1;
+		close(t->pipefd1[1]);
+		t->pipefd1[1] = -1;
+	}
+
+	if (t->pipefd2[0] >= 0) {
+		wpa_printf(MSG_INFO, "pipefd2 had not been closed");
+		eloop_unregister_read_sock(t->pipefd2[0]);
+		close(t->pipefd2[0]);
+		t->pipefd2[0] = -1;
+		close(t->pipefd2[1]);
+		t->pipefd2[1] = -1;
+	}
+
+	next_run = t->close_in_timeout;
+	t->magic = 0;
+	wpa_printf(MSG_INFO, "%s - free(%p)", __func__, t);
+	os_free(t);
+
+	if (next_run)
+		eloop_tests_start(0);
+}
+
+
+static void eloop_tests_start(int close_in_timeout)
+{
+	struct test_eloop *t;
+	int res;
+
+	t = os_zalloc(sizeof(*t));
+	if (!t)
+		return;
+	t->magic = 0x12345678;
+	t->close_in_timeout = close_in_timeout;
+
+	wpa_printf(MSG_INFO, "starting eloop tests (%p) (close_in_timeout=%d)",
+		   t, close_in_timeout);
+
+	res = pipe(t->pipefd1);
+	if (res < 0) {
+		wpa_printf(MSG_INFO, "pipe: %s", strerror(errno));
+		os_free(t);
+		return;
+	}
+
+	res = pipe(t->pipefd2);
+	if (res < 0) {
+		wpa_printf(MSG_INFO, "pipe: %s", strerror(errno));
+		close(t->pipefd1[0]);
+		close(t->pipefd1[1]);
+		os_free(t);
+		return;
+	}
+
+	wpa_printf(MSG_INFO, "pipe fds: %d,%d %d,%d",
+		   t->pipefd1[0], t->pipefd1[1],
+		   t->pipefd2[0], t->pipefd2[1]);
+
+	eloop_register_read_sock(t->pipefd1[0], eloop_test_read_1, t, NULL);
+	eloop_register_read_sock(t->pipefd2[0], eloop_test_read_2, t, NULL);
+	eloop_register_timeout(0, 0, eloop_test_cb, t, NULL);
+	eloop_register_timeout(0, 200000, eloop_test_timeout, t, NULL);
+
+	if (write(t->pipefd1[1], "HELLO", 5) < 0)
+		wpa_printf(MSG_INFO, "write: %s", strerror(errno));
+	if (write(t->pipefd2[1], "TEST", 4) < 0)
+		wpa_printf(MSG_INFO, "write: %s", strerror(errno));
+	os_sleep(0, 50000);
+	wpa_printf(MSG_INFO, "waiting for eloop callbacks");
+}
+
+
+static void eloop_tests_run(void *eloop_data, void *user_ctx)
+{
+	eloop_tests_start(1);
+}
+
+
+static int eloop_tests(void)
+{
+	wpa_printf(MSG_INFO, "schedule eloop tests to be run");
+
+	/*
+	 * Cannot return error from these without a significant design change,
+	 * so for now, run the tests from a scheduled timeout and require
+	 * separate verification of the results from the debug log.
+	 */
+	eloop_register_timeout(0, 0, eloop_tests_run, NULL, NULL);
+
+	return 0;
+}
+
+
 int utils_module_tests(void)
 {
 	int ret = 0;
@@ -416,6 +850,10 @@ int utils_module_tests(void)
 	    bitfield_tests() < 0 ||
 	    base64_tests() < 0 ||
 	    common_tests() < 0 ||
+	    os_tests() < 0 ||
+	    wpabuf_tests() < 0 ||
+	    ip_addr_tests() < 0 ||
+	    eloop_tests() < 0 ||
 	    int_array_tests() < 0)
 		ret = -1;
 
diff --git a/src/utils/wpa_debug.c b/src/utils/wpa_debug.c
index 0d1190518536..61c0d5ce68c7 100644
--- a/src/utils/wpa_debug.c
+++ b/src/utils/wpa_debug.c
@@ -307,7 +307,7 @@ static void _wpa_hexdump(int level, const char *title, const u8 *buf,
 				    "%s - hexdump(len=%lu):%s%s",
 				    title, (long unsigned int) len, display,
 				    len > slen ? " ..." : "");
-		os_free(strbuf);
+		bin_clear_free(strbuf, 1 + 3 * slen);
 		return;
 	}
 #else /* CONFIG_ANDROID_LOG */
@@ -339,7 +339,7 @@ static void _wpa_hexdump(int level, const char *title, const u8 *buf,
 
 		syslog(syslog_priority(level), "%s - hexdump(len=%lu):%s",
 		       title, (unsigned long) len, display);
-		os_free(strbuf);
+		bin_clear_free(strbuf, 1 + 3 * len);
 		return;
 	}
 #endif /* CONFIG_DEBUG_SYSLOG */
@@ -635,8 +635,8 @@ void wpa_msg(void *ctx, int level, const char *fmt, ...)
 	va_end(ap);
 	wpa_printf(level, "%s%s", prefix, buf);
 	if (wpa_msg_cb)
-		wpa_msg_cb(ctx, level, 0, buf, len);
-	os_free(buf);
+		wpa_msg_cb(ctx, level, WPA_MSG_PER_INTERFACE, buf, len);
+	bin_clear_free(buf, buflen);
 }
 
 
@@ -663,8 +663,8 @@ void wpa_msg_ctrl(void *ctx, int level, const char *fmt, ...)
 	va_start(ap, fmt);
 	len = vsnprintf(buf, buflen, fmt, ap);
 	va_end(ap);
-	wpa_msg_cb(ctx, level, 0, buf, len);
-	os_free(buf);
+	wpa_msg_cb(ctx, level, WPA_MSG_PER_INTERFACE, buf, len);
+	bin_clear_free(buf, buflen);
 }
 
 
@@ -690,8 +690,8 @@ void wpa_msg_global(void *ctx, int level, const char *fmt, ...)
 	va_end(ap);
 	wpa_printf(level, "%s", buf);
 	if (wpa_msg_cb)
-		wpa_msg_cb(ctx, level, 1, buf, len);
-	os_free(buf);
+		wpa_msg_cb(ctx, level, WPA_MSG_GLOBAL, buf, len);
+	bin_clear_free(buf, buflen);
 }
 
 
@@ -718,8 +718,8 @@ void wpa_msg_global_ctrl(void *ctx, int level, const char *fmt, ...)
 	va_start(ap, fmt);
 	len = vsnprintf(buf, buflen, fmt, ap);
 	va_end(ap);
-	wpa_msg_cb(ctx, level, 1, buf, len);
-	os_free(buf);
+	wpa_msg_cb(ctx, level, WPA_MSG_GLOBAL, buf, len);
+	bin_clear_free(buf, buflen);
 }
 
 
@@ -745,7 +745,34 @@ void wpa_msg_no_global(void *ctx, int level, const char *fmt, ...)
 	va_end(ap);
 	wpa_printf(level, "%s", buf);
 	if (wpa_msg_cb)
-		wpa_msg_cb(ctx, level, 2, buf, len);
+		wpa_msg_cb(ctx, level, WPA_MSG_NO_GLOBAL, buf, len);
+	bin_clear_free(buf, buflen);
+}
+
+
+void wpa_msg_global_only(void *ctx, int level, const char *fmt, ...)
+{
+	va_list ap;
+	char *buf;
+	int buflen;
+	int len;
+
+	va_start(ap, fmt);
+	buflen = vsnprintf(NULL, 0, fmt, ap) + 1;
+	va_end(ap);
+
+	buf = os_malloc(buflen);
+	if (buf == NULL) {
+		wpa_printf(MSG_ERROR, "%s: Failed to allocate message buffer",
+			   __func__);
+		return;
+	}
+	va_start(ap, fmt);
+	len = vsnprintf(buf, buflen, fmt, ap);
+	va_end(ap);
+	wpa_printf(level, "%s", buf);
+	if (wpa_msg_cb)
+		wpa_msg_cb(ctx, level, WPA_MSG_ONLY_GLOBAL, buf, len);
 	os_free(buf);
 }
 
@@ -789,6 +816,45 @@ void hostapd_logger(void *ctx, const u8 *addr, unsigned int module, int level,
 			   MAC2STR(addr), buf);
 	else
 		wpa_printf(MSG_DEBUG, "hostapd_logger: %s", buf);
-	os_free(buf);
+	bin_clear_free(buf, buflen);
 }
 #endif /* CONFIG_NO_HOSTAPD_LOGGER */
+
+
+const char * debug_level_str(int level)
+{
+	switch (level) {
+	case MSG_EXCESSIVE:
+		return "EXCESSIVE";
+	case MSG_MSGDUMP:
+		return "MSGDUMP";
+	case MSG_DEBUG:
+		return "DEBUG";
+	case MSG_INFO:
+		return "INFO";
+	case MSG_WARNING:
+		return "WARNING";
+	case MSG_ERROR:
+		return "ERROR";
+	default:
+		return "?";
+	}
+}
+
+
+int str_to_debug_level(const char *s)
+{
+	if (os_strcasecmp(s, "EXCESSIVE") == 0)
+		return MSG_EXCESSIVE;
+	if (os_strcasecmp(s, "MSGDUMP") == 0)
+		return MSG_MSGDUMP;
+	if (os_strcasecmp(s, "DEBUG") == 0)
+		return MSG_DEBUG;
+	if (os_strcasecmp(s, "INFO") == 0)
+		return MSG_INFO;
+	if (os_strcasecmp(s, "WARNING") == 0)
+		return MSG_WARNING;
+	if (os_strcasecmp(s, "ERROR") == 0)
+		return MSG_ERROR;
+	return -1;
+}
diff --git a/src/utils/wpa_debug.h b/src/utils/wpa_debug.h
index 400bea9e599f..17d8f963802e 100644
--- a/src/utils/wpa_debug.h
+++ b/src/utils/wpa_debug.h
@@ -164,6 +164,7 @@ void wpa_hexdump_ascii_key(int level, const char *title, const void *buf,
 #define wpa_msg_global(args...) do { } while (0)
 #define wpa_msg_global_ctrl(args...) do { } while (0)
 #define wpa_msg_no_global(args...) do { } while (0)
+#define wpa_msg_global_only(args...) do { } while (0)
 #define wpa_msg_register_cb(f) do { } while (0)
 #define wpa_msg_register_ifname_cb(f) do { } while (0)
 #else /* CONFIG_NO_WPA_MSG */
@@ -243,7 +244,28 @@ PRINTF_FORMAT(3, 4);
 void wpa_msg_no_global(void *ctx, int level, const char *fmt, ...)
 PRINTF_FORMAT(3, 4);
 
-typedef void (*wpa_msg_cb_func)(void *ctx, int level, int global,
+/**
+ * wpa_msg_global_only - Conditional printf for ctrl_iface monitors
+ * @ctx: Pointer to context data; this is the ctx variable registered
+ *	with struct wpa_driver_ops::init()
+ * @level: priority level (MSG_*) of the message
+ * @fmt: printf format string, followed by optional arguments
+ *
+ * This function is used to print conditional debugging and error messages.
+ * This function is like wpa_msg_global(), but it sends the output only as a
+ * global event.
+ */
+void wpa_msg_global_only(void *ctx, int level, const char *fmt, ...)
+PRINTF_FORMAT(3, 4);
+
+enum wpa_msg_type {
+	WPA_MSG_PER_INTERFACE,
+	WPA_MSG_GLOBAL,
+	WPA_MSG_NO_GLOBAL,
+	WPA_MSG_ONLY_GLOBAL,
+};
+
+typedef void (*wpa_msg_cb_func)(void *ctx, int level, enum wpa_msg_type type,
 				const char *txt, size_t len);
 
 /**
@@ -342,4 +364,7 @@ static inline void wpa_debug_close_linux_tracing(void)
 #define WPA_ASSERT(a) do { } while (0)
 #endif
 
+const char * debug_level_str(int level);
+int str_to_debug_level(const char *s);
+
 #endif /* WPA_DEBUG_H */
diff --git a/src/utils/wpabuf.c b/src/utils/wpabuf.c
index 7aafa0a5169b..11e7323619de 100644
--- a/src/utils/wpabuf.c
+++ b/src/utils/wpabuf.c
@@ -17,7 +17,7 @@
 
 struct wpabuf_trace {
 	unsigned int magic;
-};
+} __attribute__((aligned(8)));
 
 static struct wpabuf_trace * wpabuf_get_trace(const struct wpabuf *buf)
 {
diff --git a/src/wps/Makefile b/src/wps/Makefile
index adfd3dfd5b9b..4806fe8dadf7 100644
--- a/src/wps/Makefile
+++ b/src/wps/Makefile
@@ -1,8 +1,41 @@
-all:
-	@echo Nothing to be made.
+all: libwps.a
 
 clean:
-	rm -f *~ *.o *.d *.gcno *.gcda *.gcov
+	rm -f *~ *.o *.d *.gcno *.gcda *.gcov libwps.a
 
 install:
 	@echo Nothing to be made.
+
+include ../lib.rules
+
+CFLAGS += -DCONFIG_P2P
+CFLAGS += -DCONFIG_WPS_OOB
+CFLAGS += -DCONFIG_WPS_NFC
+
+LIB_OBJS= \
+	http_client.o \
+	httpread.o \
+	http_server.o \
+	ndef.o \
+	upnp_xml.o \
+	wps_attr_build.o \
+	wps_attr_parse.o \
+	wps_attr_process.o \
+	wps.o \
+	wps_common.o \
+	wps_dev_attr.o \
+	wps_enrollee.o \
+	wps_er.o \
+	wps_er_ssdp.o \
+	wps_module_tests.o \
+	wps_registrar.o \
+	wps_upnp_ap.o \
+	wps_upnp.o \
+	wps_upnp_event.o \
+	wps_upnp_ssdp.o \
+	wps_upnp_web.o
+
+libwps.a: $(LIB_OBJS)
+	$(AR) crT $@ $?
+
+-include $(OBJS:%.o=%.d)
diff --git a/src/wps/http_client.c b/src/wps/http_client.c
index 029001306cbe..cdf3a5128ed3 100644
--- a/src/wps/http_client.c
+++ b/src/wps/http_client.c
@@ -85,15 +85,16 @@ static void http_client_tx_ready(int sock, void *eloop_ctx, void *sock_ctx)
 {
 	struct http_client *c = eloop_ctx;
 	int res;
+	size_t send_len;
 
+	send_len = wpabuf_len(c->req) - c->req_pos;
 	wpa_printf(MSG_DEBUG, "HTTP: Send client request to %s:%d (%lu of %lu "
 		   "bytes remaining)",
 		   inet_ntoa(c->dst.sin_addr), ntohs(c->dst.sin_port),
 		   (unsigned long) wpabuf_len(c->req),
-		   (unsigned long) wpabuf_len(c->req) - c->req_pos);
+		   (unsigned long) send_len);
 
-	res = send(c->sd, wpabuf_head_u8(c->req) + c->req_pos,
-		   wpabuf_len(c->req) - c->req_pos, 0);
+	res = send(c->sd, wpabuf_head_u8(c->req) + c->req_pos, send_len, 0);
 	if (res < 0) {
 		wpa_printf(MSG_DEBUG, "HTTP: Failed to send buffer: %s",
 			   strerror(errno));
@@ -102,12 +103,11 @@ static void http_client_tx_ready(int sock, void *eloop_ctx, void *sock_ctx)
 		return;
 	}
 
-	if ((size_t) res < wpabuf_len(c->req) - c->req_pos) {
+	if ((size_t) res < send_len) {
 		wpa_printf(MSG_DEBUG, "HTTP: Sent %d of %lu bytes; %lu bytes "
 			   "remaining",
 			   res, (unsigned long) wpabuf_len(c->req),
-			   (unsigned long) wpabuf_len(c->req) - c->req_pos -
-			   res);
+			   (unsigned long) send_len - res);
 		c->req_pos += res;
 		return;
 	}
@@ -146,24 +146,20 @@ struct http_client * http_client_addr(struct sockaddr_in *dst,
 	c->cb_ctx = cb_ctx;
 
 	c->sd = socket(AF_INET, SOCK_STREAM, 0);
-	if (c->sd < 0) {
-		http_client_free(c);
-		return NULL;
-	}
+	if (c->sd < 0)
+		goto fail;
 
 	if (fcntl(c->sd, F_SETFL, O_NONBLOCK) != 0) {
 		wpa_printf(MSG_DEBUG, "HTTP: fnctl(O_NONBLOCK) failed: %s",
 			   strerror(errno));
-		http_client_free(c);
-		return NULL;
+		goto fail;
 	}
 
 	if (connect(c->sd, (struct sockaddr *) dst, sizeof(*dst))) {
 		if (errno != EINPROGRESS) {
 			wpa_printf(MSG_DEBUG, "HTTP: Failed to connect: %s",
 				   strerror(errno));
-			http_client_free(c);
-			return NULL;
+			goto fail;
 		}
 
 		/*
@@ -173,20 +169,18 @@ struct http_client * http_client_addr(struct sockaddr_in *dst,
 	}
 
 	if (eloop_register_sock(c->sd, EVENT_TYPE_WRITE, http_client_tx_ready,
-				c, NULL)) {
-		http_client_free(c);
-		return NULL;
-	}
-
-	if (eloop_register_timeout(HTTP_CLIENT_TIMEOUT_SEC, 0,
-				   http_client_timeout, c, NULL)) {
-		http_client_free(c);
-		return NULL;
-	}
+				c, NULL) ||
+	    eloop_register_timeout(HTTP_CLIENT_TIMEOUT_SEC, 0,
+				   http_client_timeout, c, NULL))
+		goto fail;
 
 	c->req = req;
 
 	return c;
+
+fail:
+	http_client_free(c);
+	return NULL;
 }
 
 
diff --git a/src/wps/http_server.c b/src/wps/http_server.c
index ac088c429d60..507abe870780 100644
--- a/src/wps/http_server.c
+++ b/src/wps/http_server.c
@@ -277,11 +277,9 @@ struct http_server * http_server_init(struct in_addr *addr, int port,
 			   "%s", srv->port, strerror(errno));
 		goto fail;
 	}
-	if (listen(srv->fd, 10 /* max backlog */) < 0)
-		goto fail;
-	if (fcntl(srv->fd, F_SETFL, O_NONBLOCK) < 0)
-		goto fail;
-	if (eloop_register_sock(srv->fd, EVENT_TYPE_READ, http_server_cb,
+	if (listen(srv->fd, 10 /* max backlog */) < 0 ||
+	    fcntl(srv->fd, F_SETFL, O_NONBLOCK) < 0 ||
+	    eloop_register_sock(srv->fd, EVENT_TYPE_READ, http_server_cb,
 				srv, NULL))
 		goto fail;
 
diff --git a/src/wps/httpread.c b/src/wps/httpread.c
index 2f08f37275c0..7a2ba50a9f31 100644
--- a/src/wps/httpread.c
+++ b/src/wps/httpread.c
@@ -44,16 +44,6 @@
 #define HTTPREAD_HEADER_MAX_SIZE 4096   /* max allowed for headers */
 #define HTTPREAD_BODYBUF_DELTA 4096     /* increase allocation by this */
 
-#if 0
-/* httpread_debug -- set this global variable > 0 e.g. from debugger
- * to enable debugs (larger numbers for more debugs)
- * Make this a #define of 0 to eliminate the debugging code.
- */
-int httpread_debug = 99;
-#else
-#define httpread_debug 0        /* eliminates even the debugging code */
-#endif
-
 
 /* control instance -- actual definition (opaque to application)
  */
@@ -136,8 +126,7 @@ static void httpread_timeout_handler(void *eloop_data, void *user_ctx);
  */
 void httpread_destroy(struct httpread *h)
 {
-	if (httpread_debug >= 10)
-		wpa_printf(MSG_DEBUG, "ENTER httpread_destroy(%p)", h);
+	wpa_printf(MSG_DEBUG, "httpread_destroy(%p)", h);
 	if (!h)
 		return;
 
@@ -177,6 +166,12 @@ static int httpread_hdr_option_analyze(
 		if (!isdigit(*hbp))
 			return -1;
 		h->content_length = atol(hbp);
+		if (h->content_length < 0 || h->content_length > h->max_bytes) {
+			wpa_printf(MSG_DEBUG,
+				   "httpread: Unacceptable Content-Length %d",
+				   h->content_length);
+			return -1;
+		}
 		h->got_content_length = 1;
 		return 0;
 	}
@@ -283,8 +278,6 @@ static int httpread_hdr_analyze(struct httpread *h)
 			}
 		}
 		*uri = 0;       /* null terminate */
-		while (isgraph(*hbp))
-			hbp++;
 		while (*hbp == ' ' || *hbp == '\t')
 			hbp++;
 		/* get version */
@@ -380,15 +373,16 @@ static void httpread_read_handler(int sd, void *eloop_ctx, void *sock_ctx)
 	char *bbp;      /* pointer into body buffer */
 	char readbuf[HTTPREAD_READBUF_SIZE];  /* temp use to read into */
 
-	if (httpread_debug >= 20)
-		wpa_printf(MSG_DEBUG, "ENTER httpread_read_handler(%p)", h);
-
 	/* read some at a time, then search for the interal
 	 * boundaries between header and data and etc.
 	 */
+	wpa_printf(MSG_DEBUG, "httpread: Trying to read more data(%p)", h);
 	nread = read(h->sd, readbuf, sizeof(readbuf));
-	if (nread < 0)
+	if (nread < 0) {
+		wpa_printf(MSG_DEBUG, "httpread failed: %s", strerror(errno));
 		goto bad;
+	}
+	wpa_hexdump_ascii(MSG_MSGDUMP, "httpread - read", readbuf, nread);
 	if (nread == 0) {
 		/* end of transmission... this may be normal
 		 * or may be an error... in some cases we can't
@@ -411,8 +405,7 @@ static void httpread_read_handler(int sd, void *eloop_ctx, void *sock_ctx)
 		 * although dropped connections can cause false
 		 * end
 		 */
-		if (httpread_debug >= 10)
-			wpa_printf(MSG_DEBUG, "httpread ok eof(%p)", h);
+		wpa_printf(MSG_DEBUG, "httpread ok eof(%p)", h);
 		h->got_body = 1;
 		goto got_file;
 	}
@@ -432,6 +425,8 @@ static void httpread_read_handler(int sd, void *eloop_ctx, void *sock_ctx)
 			if (nread == 0)
 				goto get_more;
 			if (h->hdr_nbytes == HTTPREAD_HEADER_MAX_SIZE) {
+				wpa_printf(MSG_DEBUG,
+					   "httpread: Too long header");
 				goto bad;
 			}
 			*hbp++ = *rbp++;
@@ -453,16 +448,13 @@ static void httpread_read_handler(int sd, void *eloop_ctx, void *sock_ctx)
 			goto bad;
 		}
 		if (h->max_bytes == 0) {
-			if (httpread_debug >= 10)
-				wpa_printf(MSG_DEBUG,
-					   "httpread no body hdr end(%p)", h);
+			wpa_printf(MSG_DEBUG, "httpread no body hdr end(%p)",
+				   h);
 			goto got_file;
 		}
 		if (h->got_content_length && h->content_length == 0) {
-			if (httpread_debug >= 10)
-				wpa_printf(MSG_DEBUG,
-					   "httpread zero content length(%p)",
-					   h);
+			wpa_printf(MSG_DEBUG,
+				   "httpread zero content length(%p)", h);
 			goto got_file;
 		}
 	}
@@ -475,9 +467,7 @@ static void httpread_read_handler(int sd, void *eloop_ctx, void *sock_ctx)
 	    !os_strncasecmp(h->hdr, "HEAD", 4) ||
 	    !os_strncasecmp(h->hdr, "GET", 3)) {
 		if (!h->got_body) {
-			if (httpread_debug >= 10)
-				wpa_printf(MSG_DEBUG,
-					   "httpread NO BODY for sp. type");
+			wpa_printf(MSG_DEBUG, "httpread NO BODY for sp. type");
 		}
 		h->got_body = 1;
 		goto got_file;
@@ -498,8 +488,12 @@ static void httpread_read_handler(int sd, void *eloop_ctx, void *sock_ctx)
 			char *new_body;
 			int new_alloc_nbytes;
 
-			if (h->body_nbytes >= h->max_bytes)
+			if (h->body_nbytes >= h->max_bytes) {
+				wpa_printf(MSG_DEBUG,
+					   "httpread: body_nbytes=%d >= max_bytes=%d",
+					   h->body_nbytes, h->max_bytes);
 				goto bad;
+			}
 			new_alloc_nbytes = h->body_alloc_nbytes +
 				HTTPREAD_BODYBUF_DELTA;
 			/* For content-length case, the first time
@@ -509,9 +503,23 @@ static void httpread_read_handler(int sd, void *eloop_ctx, void *sock_ctx)
 			if (h->got_content_length &&
 			    new_alloc_nbytes < (h->content_length + 1))
 				new_alloc_nbytes = h->content_length + 1;
-			if ((new_body = os_realloc(h->body, new_alloc_nbytes))
-			    == NULL)
+			if (new_alloc_nbytes < h->body_alloc_nbytes ||
+			    new_alloc_nbytes > h->max_bytes +
+			    HTTPREAD_BODYBUF_DELTA) {
+				wpa_printf(MSG_DEBUG,
+					   "httpread: Unacceptable body length %d (body_alloc_nbytes=%u max_bytes=%u)",
+					   new_alloc_nbytes,
+					   h->body_alloc_nbytes,
+					   h->max_bytes);
 				goto bad;
+			}
+			if ((new_body = os_realloc(h->body, new_alloc_nbytes))
+			    == NULL) {
+				wpa_printf(MSG_DEBUG,
+					   "httpread: Failed to reallocate buffer (len=%d)",
+					   new_alloc_nbytes);
+				goto bad;
+			}
 
 			h->body = new_body;
 			h->body_alloc_nbytes = new_alloc_nbytes;
@@ -530,9 +538,19 @@ static void httpread_read_handler(int sd, void *eloop_ctx, void *sock_ctx)
 					/* hdr line consists solely
 					 * of a hex numeral and CFLF
 					 */
-					if (!isxdigit(*cbp))
+					if (!isxdigit(*cbp)) {
+						wpa_printf(MSG_DEBUG,
+							   "httpread: Unexpected chunk header value (not a hex digit)");
 						goto bad;
+					}
 					h->chunk_size = strtoul(cbp, NULL, 16);
+					if (h->chunk_size < 0 ||
+					    h->chunk_size > h->max_bytes) {
+						wpa_printf(MSG_DEBUG,
+							   "httpread: Invalid chunk size %d",
+							   h->chunk_size);
+						goto bad;
+					}
 					/* throw away chunk header
 					 * so we have only real data
 					 */
@@ -542,10 +560,9 @@ static void httpread_read_handler(int sd, void *eloop_ctx, void *sock_ctx)
 						/* end of chunking */
 						/* trailer follows */
 						h->in_trailer = 1;
-						if (httpread_debug >= 20)
-							wpa_printf(
-								MSG_DEBUG,
-								"httpread end chunks(%p)", h);
+						wpa_printf(MSG_DEBUG,
+							   "httpread end chunks(%p)",
+							   h);
 						break;
 					}
 					h->in_chunk_data = 1;
@@ -563,8 +580,11 @@ static void httpread_read_handler(int sd, void *eloop_ctx, void *sock_ctx)
 					 */
 					if (bbp[-1] == '\n' &&
 					    bbp[-2] == '\r') {
-					} else
+					} else {
+						wpa_printf(MSG_DEBUG,
+							   "httpread: Invalid chunk end");
 						goto bad;
+					}
 					h->body_nbytes -= 2;
 					bbp -= 2;
 					h->chunk_start = h->body_nbytes;
@@ -574,10 +594,8 @@ static void httpread_read_handler(int sd, void *eloop_ctx, void *sock_ctx)
 			} else if (h->got_content_length &&
 				   h->body_nbytes >= h->content_length) {
 				h->got_body = 1;
-				if (httpread_debug >= 10)
-					wpa_printf(
-						MSG_DEBUG,
-						"httpread got content(%p)", h);
+				wpa_printf(MSG_DEBUG,
+					   "httpread got content(%p)", h);
 				goto got_file;
 			}
 			if (nread <= 0)
@@ -601,6 +619,11 @@ static void httpread_read_handler(int sd, void *eloop_ctx, void *sock_ctx)
 				ncopy = nread;
 			}
 			/* Note: should never be 0 */
+			if (ncopy < 0) {
+				wpa_printf(MSG_DEBUG,
+					   "httpread: Invalid ncopy=%d", ncopy);
+				goto bad;
+			}
 			if (ncopy > nread)
 				ncopy = nread;
 			os_memcpy(bbp, rbp, ncopy);
@@ -635,10 +658,9 @@ static void httpread_read_handler(int sd, void *eloop_ctx, void *sock_ctx)
 				if (c == '\n') {
 					h->trailer_state = trailer_line_begin;
 					h->in_trailer = 0;
-					if (httpread_debug >= 10)
-						wpa_printf(
-							MSG_DEBUG,
-							"httpread got content(%p)", h);
+					wpa_printf(MSG_DEBUG,
+						   "httpread got content(%p)",
+						   h);
 					h->got_body = 1;
 					goto got_file;
 				}
@@ -666,13 +688,14 @@ bad:
 	return;
 
 get_more:
+	wpa_printf(MSG_DEBUG, "httpread: get more (%p)", h);
 	return;
 
 got_file:
-	if (httpread_debug >= 10)
-		wpa_printf(MSG_DEBUG,
-			   "httpread got file %d bytes type %d",
-			   h->body_nbytes, h->hdr_type);
+	wpa_printf(MSG_DEBUG, "httpread got file %d bytes type %d",
+		   h->body_nbytes, h->hdr_type);
+	wpa_hexdump_ascii(MSG_MSGDUMP, "httpread: body",
+			  h->body, h->body_nbytes);
 	/* Null terminate for convenience of some applications */
 	if (h->body)
 		h->body[h->body_nbytes] = 0; /* null terminate */
diff --git a/src/wps/ndef.c b/src/wps/ndef.c
index d45dfc8efee6..bb3c055486c0 100644
--- a/src/wps/ndef.c
+++ b/src/wps/ndef.c
@@ -29,8 +29,8 @@ struct ndef_record {
 	u32 total_length;
 };
 
-static char wifi_handover_type[] = "application/vnd.wfa.wsc";
-static char p2p_handover_type[] = "application/vnd.wfa.p2p";
+static const char wifi_handover_type[] = "application/vnd.wfa.wsc";
+static const char p2p_handover_type[] = "application/vnd.wfa.p2p";
 
 static int ndef_parse_record(const u8 *data, u32 size,
 			     struct ndef_record *record)
@@ -45,9 +45,14 @@ static int ndef_parse_record(const u8 *data, u32 size,
 			return -1;
 		record->payload_length = *pos++;
 	} else {
+		u32 len;
+
 		if (size < 6)
 			return -1;
-		record->payload_length = ntohl(*(u32 *)pos);
+		len = WPA_GET_BE32(pos);
+		if (len > size - 6 || len > 20000)
+			return -1;
+		record->payload_length = len;
 		pos += sizeof(u32);
 	}
 
@@ -68,7 +73,8 @@ static int ndef_parse_record(const u8 *data, u32 size,
 	pos += record->payload_length;
 
 	record->total_length = pos - data;
-	if (record->total_length > size)
+	if (record->total_length > size ||
+	    record->total_length < record->payload_length)
 		return -1;
 	return 0;
 }
@@ -97,7 +103,7 @@ static struct wpabuf * ndef_parse_records(const struct wpabuf *buf,
 }
 
 
-static struct wpabuf * ndef_build_record(u8 flags, void *type,
+static struct wpabuf * ndef_build_record(u8 flags, const void *type,
 					 u8 type_length, void *id,
 					 u8 id_length,
 					 const struct wpabuf *payload)
diff --git a/src/wps/wps.c b/src/wps/wps.c
index 2c68be8c62ea..fbaf85aabab4 100644
--- a/src/wps/wps.c
+++ b/src/wps/wps.c
@@ -355,16 +355,16 @@ int wps_is_addr_authorized(const struct wpabuf *msg, const u8 *addr,
 int wps_ap_priority_compar(const struct wpabuf *wps_a,
 			   const struct wpabuf *wps_b)
 {
-	struct wps_parse_attr attr_a, attr_b;
+	struct wps_parse_attr attr;
 	int sel_a, sel_b;
 
-	if (wps_a == NULL || wps_parse_msg(wps_a, &attr_a) < 0)
+	if (wps_a == NULL || wps_parse_msg(wps_a, &attr) < 0)
 		return 1;
-	if (wps_b == NULL || wps_parse_msg(wps_b, &attr_b) < 0)
-		return -1;
+	sel_a = attr.selected_registrar && *attr.selected_registrar != 0;
 
-	sel_a = attr_a.selected_registrar && *attr_a.selected_registrar != 0;
-	sel_b = attr_b.selected_registrar && *attr_b.selected_registrar != 0;
+	if (wps_b == NULL || wps_parse_msg(wps_b, &attr) < 0)
+		return -1;
+	sel_b = attr.selected_registrar && *attr.selected_registrar != 0;
 
 	if (sel_a && !sel_b)
 		return -1;
@@ -618,7 +618,8 @@ int wps_attr_text(struct wpabuf *data, char *buf, char *end)
 		if (str == NULL)
 			return pos - buf;
 		for (i = 0; i < attr.dev_name_len; i++) {
-			if (attr.dev_name[i] < 32)
+			if (attr.dev_name[i] == 0 ||
+			    is_ctrl_char(attr.dev_name[i]))
 				str[i] = '_';
 			else
 				str[i] = attr.dev_name[i];
diff --git a/src/wps/wps.h b/src/wps/wps.h
index 0a7f65dfd6cb..2c91d1678c15 100644
--- a/src/wps/wps.h
+++ b/src/wps/wps.h
@@ -9,6 +9,7 @@
 #ifndef WPS_H
 #define WPS_H
 
+#include "common/ieee802_11_defs.h"
 #include "wps_defs.h"
 
 /**
@@ -44,7 +45,7 @@ struct wps_parse_attr;
  * @cred_attr_len: Length of cred_attr in octets
  */
 struct wps_credential {
-	u8 ssid[32];
+	u8 ssid[SSID_MAX_LEN];
 	size_t ssid_len;
 	u16 auth_type;
 	u16 encr_type;
@@ -78,7 +79,7 @@ struct wps_credential {
  * @sec_dev_type: Array of secondary device types
  * @num_sec_dev_type: Number of secondary device types
  * @os_version: OS Version
- * @rf_bands: RF bands (WPS_RF_24GHZ, WPS_RF_50GHZ flags)
+ * @rf_bands: RF bands (WPS_RF_24GHZ, WPS_RF_50GHZ, WPS_RF_60GHZ flags)
  * @p2p: Whether the device is a P2P device
  */
 struct wps_device_data {
@@ -623,7 +624,7 @@ struct wps_context {
 	 * Credentials. In addition, AP uses it when acting as an Enrollee to
 	 * notify Registrar of the current configuration.
 	 */
-	u8 ssid[32];
+	u8 ssid[SSID_MAX_LEN];
 
 	/**
 	 * ssid_len - Length of ssid in octets
diff --git a/src/wps/wps_attr_parse.c b/src/wps/wps_attr_parse.c
index 40bc1ad2d2c5..11a967ba0ef1 100644
--- a/src/wps/wps_attr_parse.c
+++ b/src/wps/wps_attr_parse.c
@@ -447,25 +447,55 @@ static int wps_set_attr(struct wps_parse_attr *attr, u16 type,
 		break;
 	case ATTR_MANUFACTURER:
 		attr->manufacturer = pos;
-		attr->manufacturer_len = len;
+		if (len > WPS_MANUFACTURER_MAX_LEN)
+			attr->manufacturer_len = WPS_MANUFACTURER_MAX_LEN;
+		else
+			attr->manufacturer_len = len;
 		break;
 	case ATTR_MODEL_NAME:
 		attr->model_name = pos;
-		attr->model_name_len = len;
+		if (len > WPS_MODEL_NAME_MAX_LEN)
+			attr->model_name_len = WPS_MODEL_NAME_MAX_LEN;
+		else
+			attr->model_name_len = len;
 		break;
 	case ATTR_MODEL_NUMBER:
 		attr->model_number = pos;
-		attr->model_number_len = len;
+		if (len > WPS_MODEL_NUMBER_MAX_LEN)
+			attr->model_number_len = WPS_MODEL_NUMBER_MAX_LEN;
+		else
+			attr->model_number_len = len;
 		break;
 	case ATTR_SERIAL_NUMBER:
 		attr->serial_number = pos;
-		attr->serial_number_len = len;
+		if (len > WPS_SERIAL_NUMBER_MAX_LEN)
+			attr->serial_number_len = WPS_SERIAL_NUMBER_MAX_LEN;
+		else
+			attr->serial_number_len = len;
 		break;
 	case ATTR_DEV_NAME:
+		if (len > WPS_DEV_NAME_MAX_LEN) {
+			wpa_printf(MSG_DEBUG,
+				   "WPS: Ignore too long Device Name (len=%u)",
+				   len);
+			break;
+		}
 		attr->dev_name = pos;
 		attr->dev_name_len = len;
 		break;
 	case ATTR_PUBLIC_KEY:
+		/*
+		 * The Public Key attribute is supposed to be exactly 192 bytes
+		 * in length. Allow couple of bytes shorter one to try to
+		 * interoperate with implementations that do not use proper
+		 * zero-padding.
+		 */
+		if (len < 190 || len > 192) {
+			wpa_printf(MSG_DEBUG,
+				   "WPS: Ignore Public Key with unexpected length %u",
+				   len);
+			break;
+		}
 		attr->public_key = pos;
 		attr->public_key_len = len;
 		break;
@@ -485,6 +515,11 @@ static int wps_set_attr(struct wps_parse_attr *attr, u16 type,
 		attr->num_cred++;
 		break;
 	case ATTR_SSID:
+		if (len > SSID_MAX_LEN) {
+			wpa_printf(MSG_DEBUG,
+				   "WPS: Ignore too long SSID (len=%u)", len);
+			break;
+		}
 		attr->ssid = pos;
 		attr->ssid_len = len;
 		break;
diff --git a/src/wps/wps_attr_parse.h b/src/wps/wps_attr_parse.h
index 82c4739f61f5..8188fe9173d4 100644
--- a/src/wps/wps_attr_parse.h
+++ b/src/wps/wps_attr_parse.h
@@ -59,43 +59,44 @@ struct wps_parse_attr {
 
 	/* variable length fields */
 	const u8 *manufacturer;
-	size_t manufacturer_len;
 	const u8 *model_name;
-	size_t model_name_len;
 	const u8 *model_number;
-	size_t model_number_len;
 	const u8 *serial_number;
-	size_t serial_number_len;
 	const u8 *dev_name;
-	size_t dev_name_len;
 	const u8 *public_key;
-	size_t public_key_len;
 	const u8 *encr_settings;
-	size_t encr_settings_len;
 	const u8 *ssid; /* <= 32 octets */
-	size_t ssid_len;
 	const u8 *network_key; /* <= 64 octets */
-	size_t network_key_len;
 	const u8 *authorized_macs; /* <= 30 octets */
-	size_t authorized_macs_len;
 	const u8 *sec_dev_type_list; /* <= 128 octets */
-	size_t sec_dev_type_list_len;
 	const u8 *oob_dev_password; /* 38..54 octets */
-	size_t oob_dev_password_len;
+	u16 manufacturer_len;
+	u16 model_name_len;
+	u16 model_number_len;
+	u16 serial_number_len;
+	u16 dev_name_len;
+	u16 public_key_len;
+	u16 encr_settings_len;
+	u16 ssid_len;
+	u16 network_key_len;
+	u16 authorized_macs_len;
+	u16 sec_dev_type_list_len;
+	u16 oob_dev_password_len;
 
 	/* attributes that can occur multiple times */
 #define MAX_CRED_COUNT 10
-	const u8 *cred[MAX_CRED_COUNT];
-	size_t cred_len[MAX_CRED_COUNT];
-	size_t num_cred;
-
 #define MAX_REQ_DEV_TYPE_COUNT 10
-	const u8 *req_dev_type[MAX_REQ_DEV_TYPE_COUNT];
-	size_t num_req_dev_type;
 
+	unsigned int num_cred;
+	unsigned int num_req_dev_type;
+	unsigned int num_vendor_ext;
+
+	u16 cred_len[MAX_CRED_COUNT];
+	u16 vendor_ext_len[MAX_WPS_PARSE_VENDOR_EXT];
+
+	const u8 *cred[MAX_CRED_COUNT];
+	const u8 *req_dev_type[MAX_REQ_DEV_TYPE_COUNT];
 	const u8 *vendor_ext[MAX_WPS_PARSE_VENDOR_EXT];
-	size_t vendor_ext_len[MAX_WPS_PARSE_VENDOR_EXT];
-	size_t num_vendor_ext;
 };
 
 int wps_parse_msg(const struct wpabuf *msg, struct wps_parse_attr *attr);
diff --git a/src/wps/wps_common.c b/src/wps/wps_common.c
index c1ede6a9ea83..88f85fe83f05 100644
--- a/src/wps/wps_common.c
+++ b/src/wps/wps_common.c
@@ -528,7 +528,7 @@ u16 wps_config_methods_str2bin(const char *str)
 {
 	u16 methods = 0;
 
-	if (str == NULL) {
+	if (str == NULL || str[0] == '\0') {
 		/* Default to enabling methods based on build configuration */
 		methods |= WPS_CONFIG_DISPLAY | WPS_CONFIG_KEYPAD;
 		methods |= WPS_CONFIG_VIRT_DISPLAY;
@@ -764,6 +764,8 @@ static int wps_build_ap_freq(struct wpabuf *msg, int freq)
 		rf_band = WPS_RF_24GHZ;
 	else if (mode == HOSTAPD_MODE_IEEE80211A)
 		rf_band = WPS_RF_50GHZ;
+	else if (mode == HOSTAPD_MODE_IEEE80211AD)
+		rf_band = WPS_RF_60GHZ;
 	else
 		return 0; /* Unknown band */
 	ap_channel = channel;
diff --git a/src/wps/wps_defs.h b/src/wps/wps_defs.h
index 25cd14a0b32a..a23b979d2e3c 100644
--- a/src/wps/wps_defs.h
+++ b/src/wps/wps_defs.h
@@ -41,6 +41,11 @@ extern int wps_corrupt_pkhash;
 #define WPS_OOB_DEVICE_PASSWORD_MIN_LEN 16
 #define WPS_OOB_DEVICE_PASSWORD_LEN 32
 #define WPS_OOB_PUBKEY_HASH_LEN 20
+#define WPS_DEV_NAME_MAX_LEN 32
+#define WPS_MANUFACTURER_MAX_LEN 64
+#define WPS_MODEL_NAME_MAX_LEN 32
+#define WPS_MODEL_NUMBER_MAX_LEN 32
+#define WPS_SERIAL_NUMBER_MAX_LEN 32
 
 /* Attribute Types */
 enum wps_attribute {
@@ -232,6 +237,7 @@ enum wps_error_indication {
 /* RF Bands */
 #define WPS_RF_24GHZ 0x01
 #define WPS_RF_50GHZ 0x02
+#define WPS_RF_60GHZ 0x04
 
 /* Config Methods */
 #define WPS_CONFIG_USBA 0x0001
diff --git a/src/wps/wps_enrollee.c b/src/wps/wps_enrollee.c
index 89957b1a818a..9321b721abd7 100644
--- a/src/wps/wps_enrollee.c
+++ b/src/wps/wps_enrollee.c
@@ -759,7 +759,7 @@ static int wps_process_cred_e(struct wps_data *wps, const u8 *cred,
 
 
 static int wps_process_creds(struct wps_data *wps, const u8 *cred[],
-			     size_t cred_len[], size_t num_cred, int wps2)
+			     u16 cred_len[], unsigned int num_cred, int wps2)
 {
 	size_t i;
 	int ok = 0;
@@ -799,6 +799,7 @@ static int wps_process_ap_settings_e(struct wps_data *wps,
 				     struct wpabuf *attrs, int wps2)
 {
 	struct wps_credential cred;
+	int ret = 0;
 
 	if (!wps->wps->ap)
 		return 0;
@@ -877,10 +878,10 @@ static int wps_process_ap_settings_e(struct wps_data *wps,
 	if (wps->wps->cred_cb) {
 		cred.cred_attr = wpabuf_head(attrs);
 		cred.cred_attr_len = wpabuf_len(attrs);
-		wps->wps->cred_cb(wps->wps->cb_ctx, &cred);
+		ret = wps->wps->cred_cb(wps->wps->cb_ctx, &cred);
 	}
 
-	return 0;
+	return ret;
 }
 
 
diff --git a/src/wps/wps_er.c b/src/wps/wps_er.c
index 078ff72781a6..b840acd924a7 100644
--- a/src/wps/wps_er.c
+++ b/src/wps/wps_er.c
@@ -1649,11 +1649,15 @@ static void wps_er_http_put_message_cb(void *ctx, struct http_client *c,
 	case HTTP_CLIENT_OK:
 		wpa_printf(MSG_DEBUG, "WPS ER: PutMessage OK");
 		reply = http_client_get_body(c);
-		if (reply == NULL)
-			break;
-		msg = os_zalloc(wpabuf_len(reply) + 1);
-		if (msg == NULL)
+		if (reply)
+			msg = os_zalloc(wpabuf_len(reply) + 1);
+		if (msg == NULL) {
+			if (ap->wps) {
+				wps_deinit(ap->wps);
+				ap->wps = NULL;
+			}
 			break;
+		}
 		os_memcpy(msg, wpabuf_head(reply), wpabuf_len(reply));
 		break;
 	case HTTP_CLIENT_FAILED:
@@ -1709,21 +1713,30 @@ static void wps_er_ap_put_message(struct wps_er_ap *ap,
 	url = http_client_url_parse(ap->control_url, &dst, &path);
 	if (url == NULL) {
 		wpa_printf(MSG_DEBUG, "WPS ER: Failed to parse controlURL");
-		return;
+		goto fail;
 	}
 
 	buf = wps_er_soap_hdr(msg, "PutMessage", "NewInMessage", path, &dst,
 			      &len_ptr, &body_ptr);
 	os_free(url);
 	if (buf == NULL)
-		return;
+		goto fail;
 
 	wps_er_soap_end(buf, "PutMessage", len_ptr, body_ptr);
 
 	ap->http = http_client_addr(&dst, buf, 10000,
 				    wps_er_http_put_message_cb, ap);
-	if (ap->http == NULL)
+	if (ap->http == NULL) {
 		wpabuf_free(buf);
+		goto fail;
+	}
+	return;
+
+fail:
+	if (ap->wps) {
+		wps_deinit(ap->wps);
+		ap->wps = NULL;
+	}
 }
 
 
diff --git a/src/wps/wps_er_ssdp.c b/src/wps/wps_er_ssdp.c
index e381fecbdc86..280b2b3bde19 100644
--- a/src/wps/wps_er_ssdp.c
+++ b/src/wps/wps_er_ssdp.c
@@ -78,9 +78,7 @@ static void wps_er_ssdp_rx(int sd, void *eloop_ctx, void *sock_ctx)
 			if (os_strstr(start, "ssdp:byebye"))
 				byebye = 1;
 		} else if (os_strncasecmp(start, "CACHE-CONTROL:", 14) == 0) {
-			start += 9;
-			while (*start == ' ')
-				start++;
+			start += 14;
 			pos2 = os_strstr(start, "max-age=");
 			if (pos2 == NULL)
 				continue;
diff --git a/src/wps/wps_module_tests.c b/src/wps/wps_module_tests.c
index 6800e86db911..350630768be4 100644
--- a/src/wps/wps_module_tests.c
+++ b/src/wps/wps_module_tests.c
@@ -17,7 +17,7 @@ struct wps_attr_parse_test {
 	int extra;
 };
 
-struct wps_attr_parse_test wps_attr_parse_test_cases[] = {
+const struct wps_attr_parse_test wps_attr_parse_test_cases[] = {
 	/* Empty message */
 	{ "", 0, 0 },
 	/* Truncated attribute header */
@@ -271,7 +271,7 @@ static int wps_attr_parse_tests(void)
 	for (i = 0; i < ARRAY_SIZE(wps_attr_parse_test_cases); i++) {
 		struct wpabuf *buf;
 		size_t len;
-		struct wps_attr_parse_test *test =
+		const struct wps_attr_parse_test *test =
 			&wps_attr_parse_test_cases[i];
 
 		len = os_strlen(test->data) / 2;
diff --git a/src/wps/wps_registrar.c b/src/wps/wps_registrar.c
index 48b7e1288af0..4ca3a42d4c73 100644
--- a/src/wps/wps_registrar.c
+++ b/src/wps/wps_registrar.c
@@ -2605,13 +2605,16 @@ static enum wps_process_res wps_process_m1(struct wps_data *wps,
 		token = wps_get_nfc_pw_token(
 			&wps->wps->registrar->nfc_pw_tokens, wps->dev_pw_id);
 		if (token && token->peer_pk_hash_known) {
+			size_t len;
+
 			wpa_printf(MSG_DEBUG, "WPS: Found matching NFC "
 				   "Password Token");
 			dl_list_del(&token->list);
 			wps->nfc_pw_token = token;
 
 			addr[0] = attr->public_key;
-			sha256_vector(1, addr, &attr->public_key_len, hash);
+			len = attr->public_key_len;
+			sha256_vector(1, addr, &len, hash);
 			if (os_memcmp_const(hash,
 					    wps->nfc_pw_token->pubkey_hash,
 					    WPS_OOB_PUBKEY_HASH_LEN) != 0) {
@@ -3226,8 +3229,13 @@ static enum wps_process_res wps_process_wsc_done(struct wps_data *wps,
 		os_memset(&cred, 0, sizeof(cred));
 		os_memcpy(cred.ssid, wps->wps->ssid, wps->wps->ssid_len);
 		cred.ssid_len = wps->wps->ssid_len;
-		cred.auth_type = WPS_AUTH_WPAPSK | WPS_AUTH_WPA2PSK;
-		cred.encr_type = WPS_ENCR_TKIP | WPS_ENCR_AES;
+		if (wps->wps->rf_band_cb(wps->wps->cb_ctx) == WPS_RF_60GHZ) {
+			cred.auth_type = WPS_AUTH_WPA2PSK;
+			cred.encr_type = WPS_ENCR_AES;
+		} else {
+			cred.auth_type = WPS_AUTH_WPAPSK | WPS_AUTH_WPA2PSK;
+			cred.encr_type = WPS_ENCR_TKIP | WPS_ENCR_AES;
+		}
 		os_memcpy(cred.key, wps->new_psk, wps->new_psk_len);
 		cred.key_len = wps->new_psk_len;
 
diff --git a/src/wps/wps_upnp.c b/src/wps/wps_upnp.c
index 933d7340ee8e..44318e094252 100644
--- a/src/wps/wps_upnp.c
+++ b/src/wps/wps_upnp.c
@@ -695,6 +695,7 @@ struct subscription * subscription_start(struct upnp_wps_device_sm *sm,
 	struct subscription *s;
 	time_t now = time(NULL);
 	time_t expire = now + UPNP_SUBSCRIBE_SEC;
+	char str[80];
 
 	/* Get rid of expired subscriptions so we have room */
 	subscription_list_age(sm, now);
@@ -743,8 +744,10 @@ struct subscription * subscription_start(struct upnp_wps_device_sm *sm,
 		subscription_destroy(s);
 		return NULL;
 	}
-	wpa_printf(MSG_DEBUG, "WPS UPnP: Subscription %p started with %s",
-		   s, callback_urls);
+	uuid_bin2str(s->uuid, str, sizeof(str));
+	wpa_printf(MSG_DEBUG,
+		   "WPS UPnP: Subscription %p (SID %s) started with %s",
+		   s, str, callback_urls);
 	/* Schedule sending this */
 	event_send_all_later(sm);
 	return s;
diff --git a/src/wps/wps_upnp_ap.c b/src/wps/wps_upnp_ap.c
index 2949f141220b..cca390530a16 100644
--- a/src/wps/wps_upnp_ap.c
+++ b/src/wps/wps_upnp_ap.c
@@ -34,10 +34,8 @@ int upnp_er_set_selected_registrar(struct wps_registrar *reg,
 
 	wpa_hexdump_buf(MSG_MSGDUMP, "WPS: SetSelectedRegistrar attributes",
 			msg);
-	if (wps_validate_upnp_set_selected_registrar(msg) < 0)
-		return -1;
-
-	if (wps_parse_msg(msg, &attr) < 0)
+	if (wps_validate_upnp_set_selected_registrar(msg) < 0 ||
+	    wps_parse_msg(msg, &attr) < 0)
 		return -1;
 
 	s->reg = reg;
diff --git a/src/wps/wps_upnp_event.c b/src/wps/wps_upnp_event.c
index 2c8ed4f13996..94aae7542b2e 100644
--- a/src/wps/wps_upnp_event.c
+++ b/src/wps/wps_upnp_event.c
@@ -276,11 +276,9 @@ static int event_send_start(struct subscription *s)
 	 * Assume we are called ONLY with no current event and ONLY with
 	 * nonempty event queue and ONLY with at least one address to send to.
 	 */
-	if (dl_list_empty(&s->addr_list))
-		return -1;
-	if (s->current_event)
-		return -1;
-	if (dl_list_empty(&s->event_queue))
+	if (dl_list_empty(&s->addr_list) ||
+	    s->current_event ||
+	    dl_list_empty(&s->event_queue))
 		return -1;
 
 	s->current_event = e = event_dequeue(s);
diff --git a/src/wps/wps_upnp_ssdp.c b/src/wps/wps_upnp_ssdp.c
index 26a740d25224..968fc03f92e7 100644
--- a/src/wps/wps_upnp_ssdp.c
+++ b/src/wps/wps_upnp_ssdp.c
@@ -139,7 +139,7 @@ next_advertisement(struct upnp_wps_device_sm *sm,
 	uuid_bin2str(iface->wps->uuid, uuid_string, sizeof(uuid_string));
 	msg = wpabuf_alloc(800); /* more than big enough */
 	if (msg == NULL)
-		goto fail;
+		return NULL;
 	switch (a->type) {
 	case ADVERTISE_UP:
 	case ADVERTISE_DOWN:
@@ -213,10 +213,6 @@ next_advertisement(struct upnp_wps_device_sm *sm,
 		*islast = 1;
 
 	return msg;
-
-fail:
-	wpabuf_free(msg);
-	return NULL;
 }
 
 
@@ -744,11 +740,9 @@ int ssdp_listener_open(void)
 	int sd;
 
 	sd = socket(AF_INET, SOCK_DGRAM, 0);
-	if (sd < 0)
-		goto fail;
-	if (fcntl(sd, F_SETFL, O_NONBLOCK) != 0)
-		goto fail;
-	if (setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)))
+	if (sd < 0 ||
+	    fcntl(sd, F_SETFL, O_NONBLOCK) != 0 ||
+	    setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)))
 		goto fail;
 	os_memset(&addr, 0, sizeof(addr));
 	addr.sin_family = AF_INET;
@@ -760,9 +754,8 @@ int ssdp_listener_open(void)
 	mcast_addr.imr_interface.s_addr = htonl(INADDR_ANY);
 	mcast_addr.imr_multiaddr.s_addr = inet_addr(UPNP_MULTICAST_ADDRESS);
 	if (setsockopt(sd, IPPROTO_IP, IP_ADD_MEMBERSHIP,
-		       (char *) &mcast_addr, sizeof(mcast_addr)))
-		goto fail;
-	if (setsockopt(sd, IPPROTO_IP, IP_MULTICAST_TTL,
+		       (char *) &mcast_addr, sizeof(mcast_addr)) ||
+	    setsockopt(sd, IPPROTO_IP, IP_MULTICAST_TTL,
 		       &ttl, sizeof(ttl)))
 		goto fail;
 
diff --git a/src/wps/wps_upnp_web.c b/src/wps/wps_upnp_web.c
index b1cf571d8acb..d5b0b5b26e9d 100644
--- a/src/wps/wps_upnp_web.c
+++ b/src/wps/wps_upnp_web.c
@@ -1003,6 +1003,8 @@ static void web_connection_parse_subscribe(struct upnp_wps_device_sm *sm,
 				ret = HTTP_INTERNAL_SERVER_ERROR;
 				goto error;
 			}
+			if (len > 0 && callback_urls[len - 1] == '\r')
+				callback_urls[len - 1] = '\0';
 			continue;
 		}
 		/* SID is only for renewal */
@@ -1214,18 +1216,25 @@ static void web_connection_parse_unsubscribe(struct upnp_wps_device_sm *sm,
 	}
 
 	if (got_uuid) {
+		char str[80];
+
+		uuid_bin2str(uuid, str, sizeof(str));
+
 		s = subscription_find(sm, uuid);
 		if (s) {
 			struct subscr_addr *sa;
 			sa = dl_list_first(&s->addr_list, struct subscr_addr,
 					   list);
-			wpa_printf(MSG_DEBUG, "WPS UPnP: Unsubscribing %p %s",
-				   s, (sa && sa->domain_and_port) ?
+			wpa_printf(MSG_DEBUG,
+				   "WPS UPnP: Unsubscribing %p (SID %s) %s",
+				   s, str, (sa && sa->domain_and_port) ?
 				   sa->domain_and_port : "-null-");
 			dl_list_del(&s->list);
 			subscription_destroy(s);
 		} else {
-			wpa_printf(MSG_INFO, "WPS UPnP: Could not find matching subscription to unsubscribe");
+			wpa_printf(MSG_INFO,
+				   "WPS UPnP: Could not find matching subscription to unsubscribe (SID %s)",
+				   str);
 			ret = HTTP_PRECONDITION_FAILED;
 			goto send_msg;
 		}
diff --git a/src/wps/wps_validate.c b/src/wps/wps_validate.c
index 1c6a14bce4bc..267b565e4784 100644
--- a/src/wps/wps_validate.c
+++ b/src/wps/wps_validate.c
@@ -224,6 +224,8 @@ static int wps_validate_rf_bands(const u8 *rf_bands, int mandatory)
 		return 0;
 	}
 	if (*rf_bands != WPS_RF_24GHZ && *rf_bands != WPS_RF_50GHZ &&
+	    *rf_bands != WPS_RF_60GHZ &&
+	    *rf_bands != (WPS_RF_24GHZ | WPS_RF_50GHZ | WPS_RF_60GHZ) &&
 	    *rf_bands != (WPS_RF_24GHZ | WPS_RF_50GHZ)) {
 		wpa_printf(MSG_INFO, "WPS-STRICT: Invalid Rf Bands "
 			   "attribute value 0x%x", *rf_bands);
diff --git a/wpa_supplicant/ChangeLog b/wpa_supplicant/ChangeLog
index 1ac79b4ae5f9..facd90eea30c 100644
--- a/wpa_supplicant/ChangeLog
+++ b/wpa_supplicant/ChangeLog
@@ -1,5 +1,68 @@
 ChangeLog for wpa_supplicant
 
+2015-09-27 - v2.5
+	* fixed P2P validation of SSID element length before copying it
+	  [http://w1.fi/security/2015-1/] (CVE-2015-1863)
+	* fixed WPS UPnP vulnerability with HTTP chunked transfer encoding
+	  [http://w1.fi/security/2015-2/] (CVE-2015-4141)
+	* fixed WMM Action frame parser (AP mode)
+	  [http://w1.fi/security/2015-3/] (CVE-2015-4142)
+	* fixed EAP-pwd peer missing payload length validation
+	  [http://w1.fi/security/2015-4/]
+	  (CVE-2015-4143, CVE-2015-4144, CVE-2015-4145, CVE-2015-4146)
+	* fixed validation of WPS and P2P NFC NDEF record payload length
+	  [http://w1.fi/security/2015-5/]
+	* nl80211:
+	  - added VHT configuration for IBSS
+	  - fixed vendor command handling to check OUI properly
+	  - allow driver-based roaming to change ESS
+	* added AVG_BEACON_RSSI to SIGNAL_POLL output
+	* wpa_cli: added tab completion for number of commands
+	* removed unmaintained and not yet completed SChannel/CryptoAPI support
+	* modified Extended Capabilities element use in Probe Request frames to
+	  include all cases if any of the values are non-zero
+	* added support for dynamically creating/removing a virtual interface
+	  with interface_add/interface_remove
+	* added support for hashed password (NtHash) in EAP-pwd peer
+	* added support for memory-only PSK/passphrase (mem_only_psk=1 and
+	  CTRL-REQ/RSP-PSK_PASSPHRASE)
+	* P2P
+	  - optimize scan frequencies list when re-joining a persistent group
+	  - fixed number of sequences with nl80211 P2P Device interface
+	  - added operating class 125 for P2P use cases (this allows 5 GHz
+	    channels 161 and 169 to be used if they are enabled in the current
+	    regulatory domain)
+	  - number of fixes to P2PS functionality
+	  - do not allow 40 MHz co-ex PRI/SEC switch to force MCC
+	  - extended support for preferred channel listing
+	* D-Bus:
+	  - fixed WPS property of fi.w1.wpa_supplicant1.BSS interface
+	  - fixed PresenceRequest to use group interface
+	  - added new signals: FindStopped, WPS pbc-overlap,
+	    GroupFormationFailure, WPS timeout, InvitationReceived
+	  - added new methods: WPS Cancel, P2P Cancel, Reconnect, RemoveClient
+	  - added manufacturer info
+	* added EAP-EKE peer support for deriving Session-Id
+	* added wps_priority configuration parameter to set the default priority
+	  for all network profiles added by WPS
+	* added support to request a scan with specific SSIDs with the SCAN
+	  command (optional "ssid <hexdump>" arguments)
+	* removed support for WEP40/WEP104 as a group cipher with WPA/WPA2
+	* fixed SAE group selection in an error case
+	* modified SAE routines to be more robust and PWE generation to be
+	  stronger against timing attacks
+	* added support for Brainpool Elliptic Curves with SAE
+	* added support for CCMP-256 and GCMP-256 as group ciphers with FT
+	* fixed BSS selection based on estimated throughput
+	* added option to disable TLSv1.0 with OpenSSL
+	  (phase1="tls_disable_tlsv1_0=1")
+	* added Fast Session Transfer (FST) module
+	* fixed OpenSSL PKCS#12 extra certificate handling
+	* fixed key derivation for Suite B 192-bit AKM (this breaks
+	  compatibility with the earlier version)
+	* added RSN IE to Mesh Peering Open/Confirm frames
+	* number of small fixes
+
 2015-03-15 - v2.4
 	* allow OpenSSL cipher configuration to be set for internal EAP server
 	  (openssl_ciphers parameter)
diff --git a/wpa_supplicant/Makefile b/wpa_supplicant/Makefile
index 0f82af9d76ab..ad9ead90a8a9 100644
--- a/wpa_supplicant/Makefile
+++ b/wpa_supplicant/Makefile
@@ -7,6 +7,7 @@ CFLAGS = -MMD -O2 -Wall -g
 endif
 
 export LIBDIR ?= /usr/local/lib/
+export INCDIR ?= /usr/local/include/
 export BINDIR ?= /usr/local/sbin/
 PKG_CONFIG ?= pkg-config
 
@@ -35,6 +36,9 @@ ALL += systemd/wpa_supplicant-nl80211@.service
 ALL += systemd/wpa_supplicant-wired@.service
 ALL += dbus/fi.epitest.hostap.WPASupplicant.service
 ALL += dbus/fi.w1.wpa_supplicant1.service
+ifdef CONFIG_BUILD_WPA_CLIENT_SO
+ALL += libwpa_client.so
+endif
 
 
 all: verify_config $(ALL) dynamic_eap_methods
@@ -61,6 +65,10 @@ $(DESTDIR)$(BINDIR)/%: %
 
 install: $(addprefix $(DESTDIR)$(BINDIR)/,$(BINALL))
 	$(MAKE) -C ../src install
+ifdef CONFIG_BUILD_WPA_CLIENT_SO
+	install -m 0644 -D libwpa_client.so $(DESTDIR)/$(LIBDIR)/libwpa_client.so
+	install -m 0644 -D ../src/common/wpa_ctrl.h $(DESTDIR)/$(INCDIR)/wpa_ctrl.h
+endif
 
 ifdef CONFIG_FIPS
 CONFIG_NO_RANDOM_POOL=
@@ -106,6 +114,7 @@ OBJS_p += ../src/utils/trace.o
 OBJS_c += ../src/utils/trace.o
 OBJS_priv += ../src/utils/trace.o
 LIBCTRL += ../src/utils/trace.o
+LIBCTRLSO += ../src/utils/trace.c
 LDFLAGS += -rdynamic
 CFLAGS += -funwind-tables
 ifdef CONFIG_WPA_TRACE_BFD
@@ -122,12 +131,15 @@ endif
 OBJS += ../src/utils/$(CONFIG_ELOOP).o
 OBJS_c += ../src/utils/$(CONFIG_ELOOP).o
 
+ifndef CONFIG_OSX
 ifeq ($(CONFIG_ELOOP), eloop)
 # Using glibc < 2.17 requires -lrt for clock_gettime()
+# OS X has an alternate implementation
 LIBS += -lrt
 LIBS_c += -lrt
 LIBS_p += -lrt
 endif
+endif
 
 ifdef CONFIG_ELOOP_POLL
 CFLAGS += -DCONFIG_ELOOP_POLL
@@ -276,6 +288,7 @@ endif
 
 ifdef CONFIG_P2P
 OBJS += p2p_supplicant.o
+OBJS += p2p_supplicant_sd.o
 OBJS += ../src/p2p/p2p.o
 OBJS += ../src/p2p/p2p_utils.o
 OBJS += ../src/p2p/p2p_parse.o
@@ -316,6 +329,10 @@ CFLAGS += -DCONFIG_INTERWORKING
 NEED_GAS=y
 endif
 
+ifdef CONFIG_NO_ROAMING
+CFLAGS += -DCONFIG_NO_ROAMING
+endif
+
 include ../src/drivers/drivers.mak
 ifdef CONFIG_AP
 OBJS_d += $(DRV_BOTH_OBJS)
@@ -378,7 +395,7 @@ endif
 ifdef CONFIG_EAP_UNAUTH_TLS
 # EAP-UNAUTH-TLS
 CFLAGS += -DEAP_UNAUTH_TLS
-ifndef CONFIG_EAP_UNAUTH_TLS
+ifndef CONFIG_EAP_TLS
 OBJS += ../src/eap_peer/eap_tls.o
 OBJS_h += ../src/eap_server/eap_server_tls.o
 TLS_FUNCS=y
@@ -411,9 +428,11 @@ CFLAGS += -DEAP_TTLS
 OBJS += ../src/eap_peer/eap_ttls.o
 OBJS_h += ../src/eap_server/eap_server_ttls.o
 endif
-MS_FUNCS=y
 TLS_FUNCS=y
+ifndef CONFIG_FIPS
+MS_FUNCS=y
 CHAP=y
+endif
 CONFIG_IEEE8021X_EAPOL=y
 endif
 
@@ -646,6 +665,7 @@ CONFIG_IEEE8021X_EAPOL=y
 NEED_DH_GROUPS=y
 NEED_DH_GROUPS_ALL=y
 NEED_SHA256=y
+NEED_AES_CBC=y
 endif
 
 ifdef CONFIG_WPS
@@ -983,9 +1003,12 @@ LIBS += -lssl
 endif
 OBJS += ../src/crypto/crypto_openssl.o
 OBJS_p += ../src/crypto/crypto_openssl.o
+OBJS_priv += ../src/crypto/crypto_openssl.o
 ifdef NEED_FIPS186_2_PRF
 OBJS += ../src/crypto/fips_prf_openssl.o
 endif
+NEED_SHA256=y
+NEED_TLS_PRF_SHA256=y
 LIBS += -lcrypto
 LIBS_p += -lcrypto
 ifdef CONFIG_TLS_ADD_DL
@@ -1001,6 +1024,7 @@ LIBS += -lgnutls -lgpg-error
 endif
 OBJS += ../src/crypto/crypto_gnutls.o
 OBJS_p += ../src/crypto/crypto_gnutls.o
+OBJS_priv += ../src/crypto/crypto_gnutls.o
 ifdef NEED_FIPS186_2_PRF
 OBJS += ../src/crypto/fips_prf_internal.o
 SHA1OBJS += ../src/crypto/sha1-internal.o
@@ -1012,21 +1036,6 @@ CONFIG_INTERNAL_RC4=y
 CONFIG_INTERNAL_DH_GROUP5=y
 endif
 
-ifeq ($(CONFIG_TLS), schannel)
-ifdef TLS_FUNCS
-OBJS += ../src/crypto/tls_schannel.o
-endif
-OBJS += ../src/crypto/crypto_cryptoapi.o
-OBJS_p += ../src/crypto/crypto_cryptoapi.o
-ifdef NEED_FIPS186_2_PRF
-OBJS += ../src/crypto/fips_prf_internal.o
-SHA1OBJS += ../src/crypto/sha1-internal.o
-endif
-CONFIG_INTERNAL_SHA256=y
-CONFIG_INTERNAL_RC4=y
-CONFIG_INTERNAL_DH_GROUP5=y
-endif
-
 ifeq ($(CONFIG_TLS), internal)
 ifndef CONFIG_CRYPTO
 CONFIG_CRYPTO=internal
@@ -1145,6 +1154,20 @@ AESOBJS += ../src/crypto/aes-internal.o ../src/crypto/aes-internal-dec.o
 endif
 
 ifneq ($(CONFIG_TLS), openssl)
+NEED_INTERNAL_AES_WRAP=y
+endif
+ifdef CONFIG_OPENSSL_INTERNAL_AES_WRAP
+# Seems to be needed at least with BoringSSL
+NEED_INTERNAL_AES_WRAP=y
+CFLAGS += -DCONFIG_OPENSSL_INTERNAL_AES_WRAP
+endif
+ifdef CONFIG_FIPS
+# Have to use internal AES key wrap routines to use OpenSSL EVP since the
+# OpenSSL AES_wrap_key()/AES_unwrap_key() API is not available in FIPS mode.
+NEED_INTERNAL_AES_WRAP=y
+endif
+
+ifdef NEED_INTERNAL_AES_WRAP
 AESOBJS += ../src/crypto/aes-unwrap.o
 endif
 ifdef NEED_AES_EAX
@@ -1170,14 +1193,16 @@ AESOBJS += ../src/crypto/aes-siv.o
 endif
 ifdef NEED_AES_WRAP
 NEED_AES_ENC=y
-ifneq ($(CONFIG_TLS), openssl)
+ifdef NEED_INTERNAL_AES_WRAP
 AESOBJS += ../src/crypto/aes-wrap.o
 endif
 endif
 ifdef NEED_AES_CBC
 NEED_AES_ENC=y
+ifneq ($(CONFIG_TLS), openssl)
 AESOBJS += ../src/crypto/aes-cbc.o
 endif
+endif
 ifdef NEED_AES_ENC
 ifdef CONFIG_INTERNAL_AES
 AESOBJS += ../src/crypto/aes-internal-enc.o
@@ -1224,6 +1249,7 @@ MD5OBJS += ../src/crypto/md5-internal.o
 endif
 OBJS += $(MD5OBJS)
 OBJS_p += $(MD5OBJS)
+OBJS_priv += $(MD5OBJS)
 endif
 
 ifdef NEED_MD4
@@ -1239,11 +1265,17 @@ DESOBJS += ../src/crypto/des-internal.o
 endif
 endif
 
+ifdef CONFIG_NO_RC4
+CFLAGS += -DCONFIG_NO_RC4
+endif
+
 ifdef NEED_RC4
 ifdef CONFIG_INTERNAL_RC4
+ifndef CONFIG_NO_RC4
 OBJS += ../src/crypto/rc4.o
 endif
 endif
+endif
 
 SHA256OBJS = # none by default
 ifdef NEED_SHA256
@@ -1265,6 +1297,7 @@ OBJS += $(SHA256OBJS)
 endif
 ifdef NEED_SHA384
 CFLAGS += -DCONFIG_SHA384
+OBJS += ../src/crypto/sha384-prf.o
 endif
 
 ifdef NEED_DH_GROUPS
@@ -1460,6 +1493,7 @@ OBJS += $(SHA1OBJS) $(DESOBJS)
 
 OBJS_p += $(SHA1OBJS)
 OBJS_p += $(SHA256OBJS)
+OBJS_priv += $(SHA1OBJS)
 
 ifdef CONFIG_BGSCAN_SIMPLE
 CFLAGS += -DCONFIG_BGSCAN_SIMPLE
@@ -1586,6 +1620,24 @@ EXTRALIBS += WbemUuid.Lib
 endif
 endif
 
+ifdef CONFIG_FST
+CFLAGS += -DCONFIG_FST
+ifdef CONFIG_FST_TEST
+CFLAGS += -DCONFIG_FST_TEST
+endif
+FST_OBJS += ../src/fst/fst.o
+FST_OBJS += ../src/fst/fst_session.o
+FST_OBJS += ../src/fst/fst_iface.o
+FST_OBJS += ../src/fst/fst_group.o
+FST_OBJS += ../src/fst/fst_ctrl_aux.o
+ifdef CONFIG_CTRL_IFACE
+FST_OBJS += ../src/fst/fst_ctrl_iface.o
+endif
+OBJS += $(FST_OBJS)
+OBJS_t += $(FST_OBJS)
+OBJS_t2 += $(FST_OBJS)
+endif
+
 ifndef LDO
 LDO=$(CC)
 endif
@@ -1640,12 +1692,19 @@ wpa_cli: $(OBJS_c)
 LIBCTRL += ../src/common/wpa_ctrl.o
 LIBCTRL += ../src/utils/os_$(CONFIG_OS).o
 LIBCTRL += ../src/utils/wpa_debug.o
+LIBCTRLSO += ../src/common/wpa_ctrl.c
+LIBCTRLSO += ../src/utils/os_$(CONFIG_OS).c
+LIBCTRLSO += ../src/utils/wpa_debug.c
 
-libwpa_ctrl.a: $(LIBCTRL)
+libwpa_client.a: $(LIBCTRL)
 	$(Q)rm -f $@
 	$(Q)$(AR) crs $@ $?
 	@$(E) "  AR " $@
 
+libwpa_client.so: $(LIBCTRLSO)
+	@$(E) "  CC  $@ ($^)"
+	$(Q)$(CC) $(LDFLAGS) -o $@ $(CFLAGS) -shared -fPIC $^
+
 link_test: $(OBJS) $(OBJS_h) tests/link_test.o
 	$(Q)$(LDO) $(LDFLAGS) -o link_test $(OBJS) $(OBJS_h) tests/link_test.o $(LIBS)
 	@$(E) "  LD " $@
@@ -1728,7 +1787,7 @@ wpa_gui:
 	@echo "wpa_gui has been removed - see wpa_gui-qt4 for replacement"
 
 wpa_gui-qt4/Makefile:
-	qmake -o wpa_gui-qt4/Makefile wpa_gui-qt4/wpa_gui.pro 
+	qmake -o wpa_gui-qt4/Makefile wpa_gui-qt4/wpa_gui.pro
 
 wpa_gui-qt4/lang/wpa_gui_de.qm: wpa_gui-qt4/lang/wpa_gui_de.ts
 	lrelease wpa_gui-qt4/wpa_gui.pro
@@ -1765,6 +1824,7 @@ clean:
 	rm -f nfc_pw_token
 	rm -f lcov.info
 	rm -rf lcov-html
-	rm -f libwpa_ctrl.a
+	rm -f libwpa_client.a
+	rm -f libwpa_client.so
 
 -include $(OBJS:%.o=%.d)
diff --git a/wpa_supplicant/ap.c b/wpa_supplicant/ap.c
index 7ecf7a85c308..7a4f4cf4fbe9 100644
--- a/wpa_supplicant/ap.c
+++ b/wpa_supplicant/ap.c
@@ -142,6 +142,29 @@ void wpa_supplicant_conf_ap_ht(struct wpa_supplicant *wpa_s,
 			}
 		}
 	}
+
+	if (conf->secondary_channel) {
+		struct wpa_supplicant *iface;
+
+		for (iface = wpa_s->global->ifaces; iface; iface = iface->next)
+		{
+			if (iface == wpa_s ||
+			    iface->wpa_state < WPA_AUTHENTICATING ||
+			    (int) iface->assoc_freq != ssid->frequency)
+				continue;
+
+			/*
+			 * Do not allow 40 MHz co-ex PRI/SEC switch to force us
+			 * to change our PRI channel since we have an existing,
+			 * concurrent connection on that channel and doing
+			 * multi-channel concurrency is likely to cause more
+			 * harm than using different PRI/SEC selection in
+			 * environment with multiple BSSes on these two channels
+			 * with mixed 20 MHz or PRI channel selection.
+			 */
+			conf->no_pri_sec_switch = 1;
+		}
+	}
 #endif /* CONFIG_IEEE80211N */
 }
 
@@ -485,8 +508,13 @@ static int ap_probe_req_rx(void *ctx, const u8 *sa, const u8 *da,
 			   int ssi_signal)
 {
 	struct wpa_supplicant *wpa_s = ctx;
+	unsigned int freq = 0;
+
+	if (wpa_s->ap_iface)
+		freq = wpa_s->ap_iface->freq;
+
 	return wpas_p2p_probe_req_rx(wpa_s, sa, da, bssid, ie, ie_len,
-				     ssi_signal);
+				     freq, ssi_signal);
 }
 
 
@@ -1156,6 +1184,7 @@ int ap_switch_channel(struct wpa_supplicant *wpa_s,
 }
 
 
+#ifdef CONFIG_CTRL_IFACE
 int ap_ctrl_iface_chanswitch(struct wpa_supplicant *wpa_s, const char *pos)
 {
 	struct csa_settings settings;
@@ -1166,6 +1195,7 @@ int ap_ctrl_iface_chanswitch(struct wpa_supplicant *wpa_s, const char *pos)
 
 	return ap_switch_channel(wpa_s, &settings);
 }
+#endif /* CONFIG_CTRL_IFACE */
 
 
 void wpas_ap_ch_switch(struct wpa_supplicant *wpa_s, int freq, int ht,
@@ -1175,7 +1205,10 @@ void wpas_ap_ch_switch(struct wpa_supplicant *wpa_s, int freq, int ht,
 		return;
 
 	wpa_s->assoc_freq = freq;
-	hostapd_event_ch_switch(wpa_s->ap_iface->bss[0], freq, ht, offset, width, cf1, cf1);
+	if (wpa_s->current_ssid)
+		wpa_s->current_ssid->frequency = freq;
+	hostapd_event_ch_switch(wpa_s->ap_iface->bss[0], freq, ht,
+				offset, width, cf1, cf2);
 }
 
 
@@ -1265,6 +1298,7 @@ int wpas_ap_wps_add_nfc_pw(struct wpa_supplicant *wpa_s, u16 pw_id,
 #endif /* CONFIG_WPS_NFC */
 
 
+#ifdef CONFIG_CTRL_IFACE
 int wpas_ap_stop_ap(struct wpa_supplicant *wpa_s)
 {
 	struct hostapd_data *hapd;
@@ -1274,6 +1308,7 @@ int wpas_ap_stop_ap(struct wpa_supplicant *wpa_s)
 	hapd = wpa_s->ap_iface->bss[0];
 	return hostapd_ctrl_iface_stop_ap(hapd);
 }
+#endif /* CONFIG_CTRL_IFACE */
 
 
 #ifdef NEED_AP_MLME
@@ -1337,3 +1372,10 @@ void wpas_event_dfs_cac_nop_finished(struct wpa_supplicant *wpa_s,
 				 radar->chan_width, radar->cf1, radar->cf2);
 }
 #endif /* NEED_AP_MLME */
+
+
+void ap_periodic(struct wpa_supplicant *wpa_s)
+{
+	if (wpa_s->ap_iface)
+		hostapd_periodic_iface(wpa_s->ap_iface);
+}
diff --git a/wpa_supplicant/ap.h b/wpa_supplicant/ap.h
index 3f4151d8cb94..594168cf1e03 100644
--- a/wpa_supplicant/ap.h
+++ b/wpa_supplicant/ap.h
@@ -93,4 +93,6 @@ void wpas_event_dfs_cac_aborted(struct wpa_supplicant *wpa_s,
 void wpas_event_dfs_cac_nop_finished(struct wpa_supplicant *wpa_s,
 				     struct dfs_event *radar);
 
+void ap_periodic(struct wpa_supplicant *wpa_s);
+
 #endif /* AP_H */
diff --git a/wpa_supplicant/bss.c b/wpa_supplicant/bss.c
index b4c47e21051d..1051ee3a4c55 100644
--- a/wpa_supplicant/bss.c
+++ b/wpa_supplicant/bss.c
@@ -19,11 +19,6 @@
 #include "bss.h"
 
 
-/**
- * WPA_BSS_EXPIRATION_PERIOD - Period of expiration run in seconds
- */
-#define WPA_BSS_EXPIRATION_PERIOD 10
-
 #define WPA_BSS_FREQ_CHANGED_FLAG	BIT(0)
 #define WPA_BSS_SIGNAL_CHANGED_FLAG	BIT(1)
 #define WPA_BSS_PRIVACY_CHANGED_FLAG	BIT(2)
@@ -311,10 +306,18 @@ static int wpa_bss_known(struct wpa_supplicant *wpa_s, struct wpa_bss *bss)
 
 static int wpa_bss_in_use(struct wpa_supplicant *wpa_s, struct wpa_bss *bss)
 {
-	return bss == wpa_s->current_bss ||
-		(!is_zero_ether_addr(bss->bssid) &&
-		 (os_memcmp(bss->bssid, wpa_s->bssid, ETH_ALEN) == 0 ||
-		  os_memcmp(bss->bssid, wpa_s->pending_bssid, ETH_ALEN) == 0));
+	if (bss == wpa_s->current_bss)
+		return 1;
+
+	if (wpa_s->current_bss &&
+	    (bss->ssid_len != wpa_s->current_bss->ssid_len ||
+	     os_memcmp(bss->ssid, wpa_s->current_bss->ssid,
+		       bss->ssid_len) != 0))
+		return 0; /* SSID has changed */
+
+	return !is_zero_ether_addr(bss->bssid) &&
+		(os_memcmp(bss->bssid, wpa_s->bssid, ETH_ALEN) == 0 ||
+		 os_memcmp(bss->bssid, wpa_s->pending_bssid, ETH_ALEN) == 0);
 }
 
 
@@ -390,15 +393,16 @@ static struct wpa_bss * wpa_bss_add(struct wpa_supplicant *wpa_s,
 	dl_list_add_tail(&wpa_s->bss_id, &bss->list_id);
 	wpa_s->num_bss++;
 	wpa_dbg(wpa_s, MSG_DEBUG, "BSS: Add new id %u BSSID " MACSTR
-		" SSID '%s'",
-		bss->id, MAC2STR(bss->bssid), wpa_ssid_txt(ssid, ssid_len));
+		" SSID '%s' freq %d",
+		bss->id, MAC2STR(bss->bssid), wpa_ssid_txt(ssid, ssid_len),
+		bss->freq);
 	wpas_notify_bss_added(wpa_s, bss->bssid, bss->id);
 	return bss;
 }
 
 
 static int are_ies_equal(const struct wpa_bss *old,
-			 const struct wpa_scan_res *new, u32 ie)
+			 const struct wpa_scan_res *new_res, u32 ie)
 {
 	const u8 *old_ie, *new_ie;
 	struct wpabuf *old_ie_buff = NULL;
@@ -408,19 +412,19 @@ static int are_ies_equal(const struct wpa_bss *old,
 	switch (ie) {
 	case WPA_IE_VENDOR_TYPE:
 		old_ie = wpa_bss_get_vendor_ie(old, ie);
-		new_ie = wpa_scan_get_vendor_ie(new, ie);
+		new_ie = wpa_scan_get_vendor_ie(new_res, ie);
 		is_multi = 0;
 		break;
 	case WPS_IE_VENDOR_TYPE:
 		old_ie_buff = wpa_bss_get_vendor_ie_multi(old, ie);
-		new_ie_buff = wpa_scan_get_vendor_ie_multi(new, ie);
+		new_ie_buff = wpa_scan_get_vendor_ie_multi(new_res, ie);
 		is_multi = 1;
 		break;
 	case WLAN_EID_RSN:
 	case WLAN_EID_SUPP_RATES:
 	case WLAN_EID_EXT_SUPP_RATES:
 		old_ie = wpa_bss_get_ie(old, ie);
-		new_ie = wpa_scan_get_ie(new, ie);
+		new_ie = wpa_scan_get_ie(new_res, ie);
 		is_multi = 0;
 		break;
 	default:
@@ -454,15 +458,15 @@ static int are_ies_equal(const struct wpa_bss *old,
 
 
 static u32 wpa_bss_compare_res(const struct wpa_bss *old,
-			       const struct wpa_scan_res *new)
+			       const struct wpa_scan_res *new_res)
 {
 	u32 changes = 0;
-	int caps_diff = old->caps ^ new->caps;
+	int caps_diff = old->caps ^ new_res->caps;
 
-	if (old->freq != new->freq)
+	if (old->freq != new_res->freq)
 		changes |= WPA_BSS_FREQ_CHANGED_FLAG;
 
-	if (old->level != new->level)
+	if (old->level != new_res->level)
 		changes |= WPA_BSS_SIGNAL_CHANGED_FLAG;
 
 	if (caps_diff & IEEE80211_CAP_PRIVACY)
@@ -471,22 +475,22 @@ static u32 wpa_bss_compare_res(const struct wpa_bss *old,
 	if (caps_diff & IEEE80211_CAP_IBSS)
 		changes |= WPA_BSS_MODE_CHANGED_FLAG;
 
-	if (old->ie_len == new->ie_len &&
-	    os_memcmp(old + 1, new + 1, old->ie_len) == 0)
+	if (old->ie_len == new_res->ie_len &&
+	    os_memcmp(old + 1, new_res + 1, old->ie_len) == 0)
 		return changes;
 	changes |= WPA_BSS_IES_CHANGED_FLAG;
 
-	if (!are_ies_equal(old, new, WPA_IE_VENDOR_TYPE))
+	if (!are_ies_equal(old, new_res, WPA_IE_VENDOR_TYPE))
 		changes |= WPA_BSS_WPAIE_CHANGED_FLAG;
 
-	if (!are_ies_equal(old, new, WLAN_EID_RSN))
+	if (!are_ies_equal(old, new_res, WLAN_EID_RSN))
 		changes |= WPA_BSS_RSNIE_CHANGED_FLAG;
 
-	if (!are_ies_equal(old, new, WPS_IE_VENDOR_TYPE))
+	if (!are_ies_equal(old, new_res, WPS_IE_VENDOR_TYPE))
 		changes |= WPA_BSS_WPS_CHANGED_FLAG;
 
-	if (!are_ies_equal(old, new, WLAN_EID_SUPP_RATES) ||
-	    !are_ies_equal(old, new, WLAN_EID_EXT_SUPP_RATES))
+	if (!are_ies_equal(old, new_res, WLAN_EID_SUPP_RATES) ||
+	    !are_ies_equal(old, new_res, WLAN_EID_EXT_SUPP_RATES))
 		changes |= WPA_BSS_RATES_CHANGED_FLAG;
 
 	return changes;
@@ -534,6 +538,9 @@ wpa_bss_update(struct wpa_supplicant *wpa_s, struct wpa_bss *bss,
 	u32 changes;
 
 	changes = wpa_bss_compare_res(bss, res);
+	if (changes & WPA_BSS_FREQ_CHANGED_FLAG)
+		wpa_printf(MSG_DEBUG, "BSS: " MACSTR " changed freq %d --> %d",
+			   MAC2STR(bss->bssid), bss->freq, res->freq);
 	bss->scan_miss_count = 0;
 	bss->last_update_idx = wpa_s->bss_update_idx;
 	wpa_bss_copy_res(bss, res, fetch_time);
@@ -652,7 +659,7 @@ void wpa_bss_update_scan_res(struct wpa_supplicant *wpa_s,
 			MACSTR, MAC2STR(res->bssid));
 		return;
 	}
-	if (ssid[1] > 32) {
+	if (ssid[1] > SSID_MAX_LEN) {
 		wpa_dbg(wpa_s, MSG_DEBUG, "BSS: Too long SSID IE included for "
 			MACSTR, MAC2STR(res->bssid));
 		return;
@@ -679,7 +686,7 @@ void wpa_bss_update_scan_res(struct wpa_supplicant *wpa_s,
 	 * (to save memory) */
 
 	mesh = wpa_scan_get_ie(res, WLAN_EID_MESH_ID);
-	if (mesh && mesh[1] <= 32)
+	if (mesh && mesh[1] <= SSID_MAX_LEN)
 		ssid = mesh;
 
 	bss = wpa_bss_get(wpa_s, res->bssid, ssid + 2, ssid[1]);
@@ -828,16 +835,6 @@ void wpa_bss_flush_by_age(struct wpa_supplicant *wpa_s, int age)
 }
 
 
-static void wpa_bss_timeout(void *eloop_ctx, void *timeout_ctx)
-{
-	struct wpa_supplicant *wpa_s = eloop_ctx;
-
-	wpa_bss_flush_by_age(wpa_s, wpa_s->conf->bss_expiration_age);
-	eloop_register_timeout(WPA_BSS_EXPIRATION_PERIOD, 0,
-			       wpa_bss_timeout, wpa_s, NULL);
-}
-
-
 /**
  * wpa_bss_init - Initialize BSS table
  * @wpa_s: Pointer to wpa_supplicant data
@@ -850,8 +847,6 @@ int wpa_bss_init(struct wpa_supplicant *wpa_s)
 {
 	dl_list_init(&wpa_s->bss);
 	dl_list_init(&wpa_s->bss_id);
-	eloop_register_timeout(WPA_BSS_EXPIRATION_PERIOD, 0,
-			       wpa_bss_timeout, wpa_s, NULL);
 	return 0;
 }
 
@@ -883,7 +878,6 @@ void wpa_bss_flush(struct wpa_supplicant *wpa_s)
  */
 void wpa_bss_deinit(struct wpa_supplicant *wpa_s)
 {
-	eloop_cancel_timeout(wpa_bss_timeout, wpa_s, NULL);
 	wpa_bss_flush(wpa_s);
 }
 
diff --git a/wpa_supplicant/bss.h b/wpa_supplicant/bss.h
index 634aa3cc06d0..b215380eeb15 100644
--- a/wpa_supplicant/bss.h
+++ b/wpa_supplicant/bss.h
@@ -69,7 +69,7 @@ struct wpa_bss {
 	/** HESSID */
 	u8 hessid[ETH_ALEN];
 	/** SSID */
-	u8 ssid[32];
+	u8 ssid[SSID_MAX_LEN];
 	/** Length of SSID */
 	size_t ssid_len;
 	/** Frequency of the channel in MHz (e.g., 2412 = channel 1) */
diff --git a/wpa_supplicant/config.c b/wpa_supplicant/config.c
index 8e6cd2006b0e..b1adab77bbe0 100644
--- a/wpa_supplicant/config.c
+++ b/wpa_supplicant/config.c
@@ -15,6 +15,7 @@
 #include "rsn_supp/wpa.h"
 #include "eap_peer/eap.h"
 #include "p2p/p2p.h"
+#include "fst/fst.h"
 #include "config.h"
 
 
@@ -967,6 +968,13 @@ static int wpa_config_parse_group(const struct parse_data *data,
 	val = wpa_config_parse_cipher(line, value);
 	if (val == -1)
 		return -1;
+
+	/*
+	 * Backwards compatibility - filter out WEP ciphers that were previously
+	 * allowed.
+	 */
+	val &= ~(WPA_CIPHER_WEP104 | WPA_CIPHER_WEP40);
+
 	if (val & ~WPA_ALLOWED_GROUP_CIPHERS) {
 		wpa_printf(MSG_ERROR, "Line %d: not allowed group cipher "
 			   "(0x%x).", line, val);
@@ -1296,6 +1304,7 @@ static int wpa_config_parse_eap(const struct parse_data *data,
 }
 
 
+#ifndef NO_CONFIG_WRITE
 static char * wpa_config_write_eap(const struct parse_data *data,
 				   struct wpa_ssid *ssid)
 {
@@ -1329,6 +1338,7 @@ static char * wpa_config_write_eap(const struct parse_data *data,
 
 	return buf;
 }
+#endif /* NO_CONFIG_WRITE */
 
 
 static int wpa_config_parse_password(const struct parse_data *data,
@@ -1411,6 +1421,7 @@ static int wpa_config_parse_password(const struct parse_data *data,
 }
 
 
+#ifndef NO_CONFIG_WRITE
 static char * wpa_config_write_password(const struct parse_data *data,
 					struct wpa_ssid *ssid)
 {
@@ -1444,6 +1455,7 @@ static char * wpa_config_write_password(const struct parse_data *data,
 
 	return buf;
 }
+#endif /* NO_CONFIG_WRITE */
 #endif /* IEEE8021X_EAPOL */
 
 
@@ -1810,12 +1822,13 @@ static char * wpa_config_write_mesh_basic_rates(const struct parse_data *data,
  * functions.
  */
 static const struct parse_data ssid_fields[] = {
-	{ STR_RANGE(ssid, 0, MAX_SSID_LEN) },
+	{ STR_RANGE(ssid, 0, SSID_MAX_LEN) },
 	{ INT_RANGE(scan_ssid, 0, 1) },
 	{ FUNC(bssid) },
 	{ FUNC(bssid_blacklist) },
 	{ FUNC(bssid_whitelist) },
 	{ FUNC_KEY(psk) },
+	{ INT(mem_only_psk) },
 	{ FUNC(proto) },
 	{ FUNC(key_mgmt) },
 	{ INT(bg_scan_period) },
@@ -2257,6 +2270,7 @@ void wpa_config_free(struct wpa_config *config)
 	os_free(config->osu_dir);
 	os_free(config->bgscan);
 	os_free(config->wowlan_triggers);
+	os_free(config->fst_group_id);
 	os_free(config);
 }
 
@@ -2516,6 +2530,9 @@ int wpa_config_set_quoted(struct wpa_ssid *ssid, const char *var,
  */
 char ** wpa_config_get_all(struct wpa_ssid *ssid, int get_keys)
 {
+#ifdef NO_CONFIG_WRITE
+	return NULL;
+#else /* NO_CONFIG_WRITE */
 	const struct parse_data *field;
 	char *key, *value;
 	size_t i;
@@ -2561,6 +2578,7 @@ err:
 		os_free(value++);
 	os_free(props);
 	return NULL;
+#endif /* NO_CONFIG_WRITE */
 }
 
 
@@ -2947,7 +2965,7 @@ int wpa_config_set_cred(struct wpa_cred *cred, const char *var,
 	if (os_strcmp(var, "excluded_ssid") == 0) {
 		struct excluded_ssid *e;
 
-		if (len > MAX_SSID_LEN) {
+		if (len > SSID_MAX_LEN) {
 			wpa_printf(MSG_ERROR, "Line %d: invalid "
 				   "excluded_ssid length %d", line, (int) len);
 			os_free(val);
@@ -3499,9 +3517,12 @@ struct wpa_config * wpa_config_alloc_empty(const char *ctrl_interface,
 	config->user_mpm = DEFAULT_USER_MPM;
 	config->max_peer_links = DEFAULT_MAX_PEER_LINKS;
 	config->mesh_max_inactivity = DEFAULT_MESH_MAX_INACTIVITY;
+	config->dot11RSNASAERetransPeriod =
+		DEFAULT_DOT11_RSNA_SAE_RETRANS_PERIOD;
 	config->fast_reauth = DEFAULT_FAST_REAUTH;
 	config->p2p_go_intent = DEFAULT_P2P_GO_INTENT;
 	config->p2p_intra_bss = DEFAULT_P2P_INTRA_BSS;
+	config->p2p_go_freq_change_policy = DEFAULT_P2P_GO_FREQ_MOVE;
 	config->p2p_go_max_inactivity = DEFAULT_P2P_GO_MAX_INACTIVITY;
 	config->p2p_optimize_listen_chan = DEFAULT_P2P_OPTIMIZE_LISTEN_CHAN;
 	config->p2p_go_ctwindow = DEFAULT_P2P_GO_CTWINDOW;
@@ -4052,6 +4073,31 @@ static int wpa_config_get_str(const char *name, struct wpa_config *config,
 }
 
 
+#ifdef CONFIG_P2P
+static int wpa_config_get_ipv4(const char *name, struct wpa_config *config,
+			       long offset, char *buf, size_t buflen,
+			       int pretty_print)
+{
+	void *val = ((u8 *) config) + (long) offset;
+	int res;
+	char addr[INET_ADDRSTRLEN];
+
+	if (!val || !inet_ntop(AF_INET, val, addr, sizeof(addr)))
+		return -1;
+
+	if (pretty_print)
+		res = os_snprintf(buf, buflen, "%s=%s\n", name, addr);
+	else
+		res = os_snprintf(buf, buflen, "%s", addr);
+
+	if (os_snprintf_error(buflen, res))
+		res = -1;
+
+	return res;
+}
+#endif /* CONFIG_P2P */
+
+
 #ifdef OFFSET
 #undef OFFSET
 #endif /* OFFSET */
@@ -4067,7 +4113,8 @@ static int wpa_config_get_str(const char *name, struct wpa_config *config,
 #define STR(f) _STR(f), NULL, NULL
 #define STR_RANGE(f, min, max) _STR(f), (void *) min, (void *) max
 #define BIN(f) #f, wpa_global_config_parse_bin, NULL, OFFSET(f), NULL, NULL
-#define IPV4(f) #f, wpa_global_config_parse_ipv4, NULL, OFFSET(f), NULL, NULL
+#define IPV4(f) #f, wpa_global_config_parse_ipv4, wpa_config_get_ipv4,  \
+	OFFSET(f), NULL, NULL
 
 static const struct global_parse_data global_fields[] = {
 #ifdef CONFIG_CTRL_IFACE
@@ -4086,6 +4133,7 @@ static const struct global_parse_data global_fields[] = {
 	{ INT(user_mpm), 0 },
 	{ INT_RANGE(max_peer_links, 0, 255), 0 },
 	{ INT(mesh_max_inactivity), 0 },
+	{ INT(dot11RSNASAERetransPeriod), 0 },
 #endif /* CONFIG_MESH */
 	{ INT(disable_scan_offload), 0 },
 	{ INT(fast_reauth), 0 },
@@ -4106,7 +4154,8 @@ static const struct global_parse_data global_fields[] = {
 	{ FUNC_NO_VAR(load_dynamic_eap), 0 },
 #ifdef CONFIG_WPS
 	{ FUNC(uuid), CFG_CHANGED_UUID },
-	{ STR_RANGE(device_name, 0, 32), CFG_CHANGED_DEVICE_NAME },
+	{ STR_RANGE(device_name, 0, WPS_DEV_NAME_MAX_LEN),
+	  CFG_CHANGED_DEVICE_NAME },
 	{ STR_RANGE(manufacturer, 0, 64), CFG_CHANGED_WPS_STRING },
 	{ STR_RANGE(model_name, 0, 32), CFG_CHANGED_WPS_STRING },
 	{ STR_RANGE(model_number, 0, 32), CFG_CHANGED_WPS_STRING },
@@ -4119,8 +4168,8 @@ static const struct global_parse_data global_fields[] = {
 #endif /* CONFIG_WPS */
 #ifdef CONFIG_P2P
 	{ FUNC(sec_device_type), CFG_CHANGED_SEC_DEVICE_TYPE },
-	{ INT(p2p_listen_reg_class), 0 },
-	{ INT(p2p_listen_channel), 0 },
+	{ INT(p2p_listen_reg_class), CFG_CHANGED_P2P_LISTEN_CHANNEL },
+	{ INT(p2p_listen_channel), CFG_CHANGED_P2P_LISTEN_CHANNEL },
 	{ INT(p2p_oper_reg_class), CFG_CHANGED_P2P_OPER_CHANNEL },
 	{ INT(p2p_oper_channel), CFG_CHANGED_P2P_OPER_CHANNEL },
 	{ INT_RANGE(p2p_go_intent, 0, 15), 0 },
@@ -4128,6 +4177,7 @@ static const struct global_parse_data global_fields[] = {
 	{ INT_RANGE(persistent_reconnect, 0, 1), 0 },
 	{ INT_RANGE(p2p_intra_bss, 0, 1), CFG_CHANGED_P2P_INTRA_BSS },
 	{ INT(p2p_group_idle), 0 },
+	{ INT_RANGE(p2p_go_freq_change_policy, 0, P2P_GO_FREQ_MOVE_MAX), 0 },
 	{ INT_RANGE(p2p_passphrase_len, 8, 63),
 	  CFG_CHANGED_P2P_PASSPHRASE_LEN },
 	{ FUNC(p2p_pref_chan), CFG_CHANGED_P2P_PREF_CHAN },
@@ -4144,6 +4194,7 @@ static const struct global_parse_data global_fields[] = {
 	{ IPV4(ip_addr_mask), 0 },
 	{ IPV4(ip_addr_start), 0 },
 	{ IPV4(ip_addr_end), 0 },
+	{ INT_RANGE(p2p_cli_probe, 0, 1), 0 },
 #endif /* CONFIG_P2P */
 	{ FUNC(country), CFG_CHANGED_COUNTRY },
 	{ INT(bss_max_count), 0 },
@@ -4189,6 +4240,12 @@ static const struct global_parse_data global_fields[] = {
 	{ INT(key_mgmt_offload), 0},
 	{ INT(passive_scan), 0 },
 	{ INT(reassoc_same_bss_optim), 0 },
+	{ INT(wps_priority), 0},
+#ifdef CONFIG_FST
+	{ STR_RANGE(fst_group_id, 1, FST_MAX_GROUP_ID_LEN), 0 },
+	{ INT_RANGE(fst_priority, 1, FST_MAX_PRIO_VALUE), 0 },
+	{ INT_RANGE(fst_llt, 1, FST_MAX_LLT_MS), 0 },
+#endif /* CONFIG_FST */
 };
 
 #undef FUNC
diff --git a/wpa_supplicant/config.h b/wpa_supplicant/config.h
index 34b754e09c53..627f38b6e005 100644
--- a/wpa_supplicant/config.h
+++ b/wpa_supplicant/config.h
@@ -18,6 +18,11 @@
 #define DEFAULT_USER_MPM 1
 #define DEFAULT_MAX_PEER_LINKS 99
 #define DEFAULT_MESH_MAX_INACTIVITY 300
+/*
+ * The default dot11RSNASAERetransPeriod is defined as 40 ms in the standard,
+ * but use 1000 ms in practice to avoid issues on low power CPUs.
+ */
+#define DEFAULT_DOT11_RSNA_SAE_RETRANS_PERIOD 1000
 #define DEFAULT_FAST_REAUTH 1
 #define DEFAULT_P2P_GO_INTENT 7
 #define DEFAULT_P2P_INTRA_BSS 1
@@ -37,6 +42,7 @@
 
 #include "config_ssid.h"
 #include "wps/wps.h"
+#include "common/ieee802_11_defs.h"
 #include "common/ieee802_11_common.h"
 
 
@@ -241,7 +247,7 @@ struct wpa_cred {
 	char *phase2;
 
 	struct excluded_ssid {
-		u8 ssid[MAX_SSID_LEN];
+		u8 ssid[SSID_MAX_LEN];
 		size_t ssid_len;
 	} *excluded_ssid;
 	size_t num_excluded_ssid;
@@ -400,6 +406,11 @@ struct wpa_config {
 	 * one by one until the driver reports successful association; each
 	 * network block should have explicit security policy (i.e., only one
 	 * option in the lists) for key_mgmt, pairwise, group, proto variables.
+	 *
+	 * Note: ap_scan=2 should not be used with the nl80211 driver interface
+	 * (the current Linux interface). ap_scan=1 is optimized work working
+	 * with nl80211. For finding networks using hidden SSID, scan_ssid=1 in
+	 * the network block can be used with nl80211.
 	 */
 	int ap_scan;
 
@@ -732,6 +743,34 @@ struct wpa_config {
 	 */
 	int p2p_group_idle;
 
+	/**
+	 * p2p_go_freq_change_policy - The GO frequency change policy
+	 *
+	 * This controls the behavior of the GO when there is a change in the
+	 * map of the currently used frequencies in case more than one channel
+	 * is supported.
+	 *
+	 * @P2P_GO_FREQ_MOVE_SCM: Prefer working in a single channel mode if
+	 * possible. In case the GO is the only interface using its frequency
+	 * and there are other station interfaces on other frequencies, the GO
+	 * will migrate to one of these frequencies.
+	 *
+	 * @P2P_GO_FREQ_MOVE_SCM_PEER_SUPPORTS: Same as P2P_GO_FREQ_MOVE_SCM,
+	 * but a transition is possible only in case one of the other used
+	 * frequencies is one of the frequencies in the intersection of the
+	 * frequency list of the local device and the peer device.
+	 *
+	 * @P2P_GO_FREQ_MOVE_STAY: Prefer to stay on the current frequency.
+	 */
+	enum {
+		P2P_GO_FREQ_MOVE_SCM = 0,
+		P2P_GO_FREQ_MOVE_SCM_PEER_SUPPORTS = 1,
+		P2P_GO_FREQ_MOVE_STAY = 2,
+		P2P_GO_FREQ_MOVE_MAX = P2P_GO_FREQ_MOVE_STAY,
+	} p2p_go_freq_change_policy;
+
+#define DEFAULT_P2P_GO_FREQ_MOVE P2P_GO_FREQ_MOVE_STAY
+
 	/**
 	 * p2p_passphrase_len - Passphrase length (8..63) for P2P GO
 	 *
@@ -966,6 +1005,18 @@ struct wpa_config {
 	 */
 	int p2p_no_group_iface;
 
+	/**
+	 * p2p_cli_probe - Enable/disable P2P CLI probe request handling
+	 *
+	 * If this parameter is set to 1, a connected P2P Client will receive
+	 * and handle Probe Request frames. Setting this parameter to 0
+	 * disables this option. Default value: 0.
+	 *
+	 * Note: Setting this property at run time takes effect on the following
+	 * interface state transition to/from the WPA_COMPLETED state.
+	 */
+	int p2p_cli_probe;
+
 	/**
 	 * okc - Whether to enable opportunistic key caching by default
 	 *
@@ -1147,6 +1198,15 @@ struct wpa_config {
 	 */
 	int mesh_max_inactivity;
 
+	/**
+	 * dot11RSNASAERetransPeriod - Timeout to retransmit SAE Auth frame
+	 *
+	 * This timeout value is used in mesh STA to retransmit
+	 * SAE Authentication frame.
+	 * By default: 1000 milliseconds.
+	 */
+	int dot11RSNASAERetransPeriod;
+
 	/**
 	 * passive_scan - Whether to force passive scan for network connection
 	 *
@@ -1163,6 +1223,30 @@ struct wpa_config {
 	 * reassoc_same_bss_optim - Whether to optimize reassoc-to-same-BSS
 	 */
 	int reassoc_same_bss_optim;
+
+	/**
+	 * wps_priority - Priority for the networks added through WPS
+	 *
+	 * This priority value will be set to each network profile that is added
+	 * by executing the WPS protocol.
+	 */
+	int wps_priority;
+
+	/**
+	 * fst_group_id - FST group ID
+	 */
+	char *fst_group_id;
+
+	/**
+	 * fst_priority - priority of the interface within the FST group
+	 */
+	int fst_priority;
+
+	/**
+	 * fst_llt - default FST LLT (Link-Lost Timeout) to be used for the
+	 * interface.
+	 */
+	int fst_llt;
 };
 
 
diff --git a/wpa_supplicant/config_file.c b/wpa_supplicant/config_file.c
index 3d3a6e404fd8..fb438ea43e13 100644
--- a/wpa_supplicant/config_file.c
+++ b/wpa_supplicant/config_file.c
@@ -501,7 +501,12 @@ static void write_bssid(FILE *f, struct wpa_ssid *ssid)
 
 static void write_psk(FILE *f, struct wpa_ssid *ssid)
 {
-	char *value = wpa_config_get(ssid, "psk");
+	char *value;
+
+	if (ssid->mem_only_psk)
+		return;
+
+	value = wpa_config_get(ssid, "psk");
 	if (value == NULL)
 		return;
 	fprintf(f, "\tpsk=%s\n", value);
@@ -673,6 +678,7 @@ static void wpa_config_write_network(FILE *f, struct wpa_ssid *ssid)
 	write_str(f, "bssid_blacklist", ssid);
 	write_str(f, "bssid_whitelist", ssid);
 	write_psk(f, ssid);
+	INT(mem_only_psk);
 	write_proto(f, ssid);
 	write_key_mgmt(f, ssid);
 	INT_DEF(bg_scan_period, DEFAULT_BG_SCAN_PERIOD);
@@ -1010,13 +1016,13 @@ static void wpa_config_write_global(FILE *f, struct wpa_config *config)
 	if (config->driver_param)
 		fprintf(f, "driver_param=%s\n", config->driver_param);
 	if (config->dot11RSNAConfigPMKLifetime)
-		fprintf(f, "dot11RSNAConfigPMKLifetime=%d\n",
+		fprintf(f, "dot11RSNAConfigPMKLifetime=%u\n",
 			config->dot11RSNAConfigPMKLifetime);
 	if (config->dot11RSNAConfigPMKReauthThreshold)
-		fprintf(f, "dot11RSNAConfigPMKReauthThreshold=%d\n",
+		fprintf(f, "dot11RSNAConfigPMKReauthThreshold=%u\n",
 			config->dot11RSNAConfigPMKReauthThreshold);
 	if (config->dot11RSNAConfigSATimeout)
-		fprintf(f, "dot11RSNAConfigSATimeout=%d\n",
+		fprintf(f, "dot11RSNAConfigSATimeout=%u\n",
 			config->dot11RSNAConfigSATimeout);
 	if (config->update_config)
 		fprintf(f, "update_config=%d\n", config->update_config);
@@ -1064,27 +1070,27 @@ static void wpa_config_write_global(FILE *f, struct wpa_config *config)
 #endif /* CONFIG_WPS */
 #ifdef CONFIG_P2P
 	if (config->p2p_listen_reg_class)
-		fprintf(f, "p2p_listen_reg_class=%u\n",
+		fprintf(f, "p2p_listen_reg_class=%d\n",
 			config->p2p_listen_reg_class);
 	if (config->p2p_listen_channel)
-		fprintf(f, "p2p_listen_channel=%u\n",
+		fprintf(f, "p2p_listen_channel=%d\n",
 			config->p2p_listen_channel);
 	if (config->p2p_oper_reg_class)
-		fprintf(f, "p2p_oper_reg_class=%u\n",
+		fprintf(f, "p2p_oper_reg_class=%d\n",
 			config->p2p_oper_reg_class);
 	if (config->p2p_oper_channel)
-		fprintf(f, "p2p_oper_channel=%u\n", config->p2p_oper_channel);
+		fprintf(f, "p2p_oper_channel=%d\n", config->p2p_oper_channel);
 	if (config->p2p_go_intent != DEFAULT_P2P_GO_INTENT)
-		fprintf(f, "p2p_go_intent=%u\n", config->p2p_go_intent);
+		fprintf(f, "p2p_go_intent=%d\n", config->p2p_go_intent);
 	if (config->p2p_ssid_postfix)
 		fprintf(f, "p2p_ssid_postfix=%s\n", config->p2p_ssid_postfix);
 	if (config->persistent_reconnect)
-		fprintf(f, "persistent_reconnect=%u\n",
+		fprintf(f, "persistent_reconnect=%d\n",
 			config->persistent_reconnect);
 	if (config->p2p_intra_bss != DEFAULT_P2P_INTRA_BSS)
-		fprintf(f, "p2p_intra_bss=%u\n", config->p2p_intra_bss);
+		fprintf(f, "p2p_intra_bss=%d\n", config->p2p_intra_bss);
 	if (config->p2p_group_idle)
-		fprintf(f, "p2p_group_idle=%u\n", config->p2p_group_idle);
+		fprintf(f, "p2p_group_idle=%d\n", config->p2p_group_idle);
 	if (config->p2p_passphrase_len)
 		fprintf(f, "p2p_passphrase_len=%u\n",
 			config->p2p_passphrase_len);
@@ -1112,19 +1118,24 @@ static void wpa_config_write_global(FILE *f, struct wpa_config *config)
 		fprintf(f, "p2p_optimize_listen_chan=%d\n",
 			config->p2p_optimize_listen_chan);
 	if (config->p2p_go_ht40)
-		fprintf(f, "p2p_go_ht40=%u\n", config->p2p_go_ht40);
+		fprintf(f, "p2p_go_ht40=%d\n", config->p2p_go_ht40);
 	if (config->p2p_go_vht)
-		fprintf(f, "p2p_go_vht=%u\n", config->p2p_go_vht);
+		fprintf(f, "p2p_go_vht=%d\n", config->p2p_go_vht);
 	if (config->p2p_go_ctwindow != DEFAULT_P2P_GO_CTWINDOW)
-		fprintf(f, "p2p_go_ctwindow=%u\n", config->p2p_go_ctwindow);
+		fprintf(f, "p2p_go_ctwindow=%d\n", config->p2p_go_ctwindow);
 	if (config->p2p_disabled)
-		fprintf(f, "p2p_disabled=%u\n", config->p2p_disabled);
+		fprintf(f, "p2p_disabled=%d\n", config->p2p_disabled);
 	if (config->p2p_no_group_iface)
-		fprintf(f, "p2p_no_group_iface=%u\n",
+		fprintf(f, "p2p_no_group_iface=%d\n",
 			config->p2p_no_group_iface);
 	if (config->p2p_ignore_shared_freq)
-		fprintf(f, "p2p_ignore_shared_freq=%u\n",
+		fprintf(f, "p2p_ignore_shared_freq=%d\n",
 			config->p2p_ignore_shared_freq);
+	if (config->p2p_cli_probe)
+		fprintf(f, "p2p_cli_probe=%d\n", config->p2p_cli_probe);
+	if (config->p2p_go_freq_change_policy != DEFAULT_P2P_GO_FREQ_MOVE)
+		fprintf(f, "p2p_go_freq_change_policy=%u\n",
+			config->p2p_go_freq_change_policy);
 #endif /* CONFIG_P2P */
 	if (config->country[0] && config->country[1]) {
 		fprintf(f, "country=%c%c\n",
@@ -1144,14 +1155,14 @@ static void wpa_config_write_global(FILE *f, struct wpa_config *config)
 	if (config->max_num_sta != DEFAULT_MAX_NUM_STA)
 		fprintf(f, "max_num_sta=%u\n", config->max_num_sta);
 	if (config->disassoc_low_ack)
-		fprintf(f, "disassoc_low_ack=%u\n", config->disassoc_low_ack);
+		fprintf(f, "disassoc_low_ack=%d\n", config->disassoc_low_ack);
 #ifdef CONFIG_HS20
 	if (config->hs20)
 		fprintf(f, "hs20=1\n");
 #endif /* CONFIG_HS20 */
 #ifdef CONFIG_INTERWORKING
 	if (config->interworking)
-		fprintf(f, "interworking=%u\n", config->interworking);
+		fprintf(f, "interworking=%d\n", config->interworking);
 	if (!is_zero_ether_addr(config->hessid))
 		fprintf(f, "hessid=" MACSTR "\n", MAC2STR(config->hessid));
 	if (config->access_network_type != DEFAULT_ACCESS_NETWORK_TYPE)
@@ -1159,7 +1170,7 @@ static void wpa_config_write_global(FILE *f, struct wpa_config *config)
 			config->access_network_type);
 #endif /* CONFIG_INTERWORKING */
 	if (config->pbc_in_m1)
-		fprintf(f, "pbc_in_m1=%u\n", config->pbc_in_m1);
+		fprintf(f, "pbc_in_m1=%d\n", config->pbc_in_m1);
 	if (config->wps_nfc_pw_from_config) {
 		if (config->wps_nfc_dev_pw_id)
 			fprintf(f, "wps_nfc_dev_pw_id=%d\n",
@@ -1218,7 +1229,7 @@ static void wpa_config_write_global(FILE *f, struct wpa_config *config)
 		int i;
 		fprintf(f, "freq_list=");
 		for (i = 0; config->freq_list[i]; i++) {
-			fprintf(f, "%s%u", i > 0 ? " " : "",
+			fprintf(f, "%s%d", i > 0 ? " " : "",
 				config->freq_list[i]);
 		}
 		fprintf(f, "\n");
@@ -1259,7 +1270,7 @@ static void wpa_config_write_global(FILE *f, struct wpa_config *config)
 		fprintf(f, "preassoc_mac_addr=%d\n", config->preassoc_mac_addr);
 
 	if (config->key_mgmt_offload != DEFAULT_KEY_MGMT_OFFLOAD)
-		fprintf(f, "key_mgmt_offload=%u\n", config->key_mgmt_offload);
+		fprintf(f, "key_mgmt_offload=%d\n", config->key_mgmt_offload);
 
 	if (config->user_mpm != DEFAULT_USER_MPM)
 		fprintf(f, "user_mpm=%d\n", config->user_mpm);
@@ -1274,12 +1285,20 @@ static void wpa_config_write_global(FILE *f, struct wpa_config *config)
 		fprintf(f, "mesh_max_inactivity=%d\n",
 			config->mesh_max_inactivity);
 
+	if (config->dot11RSNASAERetransPeriod !=
+	    DEFAULT_DOT11_RSNA_SAE_RETRANS_PERIOD)
+		fprintf(f, "dot11RSNASAERetransPeriod=%d\n",
+			config->dot11RSNASAERetransPeriod);
+
 	if (config->passive_scan)
 		fprintf(f, "passive_scan=%d\n", config->passive_scan);
 
 	if (config->reassoc_same_bss_optim)
 		fprintf(f, "reassoc_same_bss_optim=%d\n",
 			config->reassoc_same_bss_optim);
+
+	if (config->wps_priority)
+		fprintf(f, "wps_priority=%d\n", config->wps_priority);
 }
 
 #endif /* CONFIG_NO_CONFIG_WRITE */
@@ -1342,6 +1361,8 @@ int wpa_config_write(const char *name, struct wpa_config *config)
 	}
 #endif /* CONFIG_NO_CONFIG_BLOBS */
 
+	os_fdatasync(f);
+
 	fclose(f);
 
 	if (tmp_name) {
diff --git a/wpa_supplicant/config_ssid.h b/wpa_supplicant/config_ssid.h
index 7c826cfd983f..7ef326cfbed6 100644
--- a/wpa_supplicant/config_ssid.h
+++ b/wpa_supplicant/config_ssid.h
@@ -13,8 +13,6 @@
 #include "utils/list.h"
 #include "eap_peer/eap_config.h"
 
-#define MAX_SSID_LEN 32
-
 
 #define DEFAULT_EAP_WORKAROUND ((unsigned int) -1)
 #define DEFAULT_EAPOL_FLAGS (EAPOL_FLAG_REQUIRE_KEY_UNICAST | \
@@ -22,8 +20,7 @@
 #define DEFAULT_PROTO (WPA_PROTO_WPA | WPA_PROTO_RSN)
 #define DEFAULT_KEY_MGMT (WPA_KEY_MGMT_PSK | WPA_KEY_MGMT_IEEE8021X)
 #define DEFAULT_PAIRWISE (WPA_CIPHER_CCMP | WPA_CIPHER_TKIP)
-#define DEFAULT_GROUP (WPA_CIPHER_CCMP | WPA_CIPHER_TKIP | \
-		       WPA_CIPHER_WEP104 | WPA_CIPHER_WEP40)
+#define DEFAULT_GROUP (WPA_CIPHER_CCMP | WPA_CIPHER_TKIP)
 #define DEFAULT_FRAGMENT_SIZE 1398
 
 #define DEFAULT_BG_SCAN_PERIOD -1
@@ -180,6 +177,14 @@ struct wpa_ssid {
 	 */
 	char *ext_psk;
 
+	/**
+	 * mem_only_psk - Whether to keep PSK/passphrase only in memory
+	 *
+	 * 0 = allow psk/passphrase to be stored to the configuration file
+	 * 1 = do not store psk/passphrase to the configuration file
+	 */
+	int mem_only_psk;
+
 	/**
 	 * pairwise_cipher - Bitfield of allowed pairwise ciphers, WPA_CIPHER_*
 	 */
@@ -220,7 +225,9 @@ struct wpa_ssid {
 	 *
 	 * scan_ssid can be used to scan for APs using hidden SSIDs.
 	 * Note: Many drivers do not support this. ap_mode=2 can be used with
-	 * such drivers to use hidden SSIDs.
+	 * such drivers to use hidden SSIDs. Note2: Most nl80211-based drivers
+	 * do support scan_ssid=1 and that should be used with them instead of
+	 * ap_scan=2.
 	 */
 	int scan_ssid;
 
diff --git a/wpa_supplicant/ctrl_iface.c b/wpa_supplicant/ctrl_iface.c
index b4aefb65ec25..3b97806d871d 100644
--- a/wpa_supplicant/ctrl_iface.c
+++ b/wpa_supplicant/ctrl_iface.c
@@ -28,6 +28,8 @@
 #include "rsn_supp/pmksa_cache.h"
 #include "l2_packet/l2_packet.h"
 #include "wps/wps.h"
+#include "fst/fst.h"
+#include "fst/fst_ctrl_iface.h"
 #include "config.h"
 #include "wpa_supplicant_i.h"
 #include "driver_i.h"
@@ -153,7 +155,8 @@ static int set_disallow_aps(struct wpa_supplicant *wpa_s, char *val)
 			}
 			ssid = ns;
 
-			if ((end - pos) & 0x01 || end - pos > 2 * 32 ||
+			if ((end - pos) & 0x01 ||
+			    end - pos > 2 * SSID_MAX_LEN ||
 			    hexstr2bin(pos, ssid[ssid_count].ssid,
 				       (end - pos) / 2) < 0) {
 				os_free(ssid);
@@ -283,6 +286,30 @@ static int wpas_ctrl_pno(struct wpa_supplicant *wpa_s, char *cmd)
 }
 
 
+static int wpas_ctrl_set_band(struct wpa_supplicant *wpa_s, char *band)
+{
+	union wpa_event_data event;
+
+	if (os_strcmp(band, "AUTO") == 0)
+		wpa_s->setband = WPA_SETBAND_AUTO;
+	else if (os_strcmp(band, "5G") == 0)
+		wpa_s->setband = WPA_SETBAND_5G;
+	else if (os_strcmp(band, "2G") == 0)
+		wpa_s->setband = WPA_SETBAND_2G;
+	else
+		return -1;
+
+	if (wpa_drv_setband(wpa_s, wpa_s->setband) == 0) {
+		os_memset(&event, 0, sizeof(event));
+		event.channel_list_changed.initiator = REGDOM_SET_BY_USER;
+		event.channel_list_changed.type = REGDOM_TYPE_UNKNOWN;
+		wpa_supplicant_event(wpa_s, EVENT_CHANNEL_LIST_CHANGED, &event);
+	}
+
+	return 0;
+}
+
+
 static int wpa_supplicant_ctrl_iface_set(struct wpa_supplicant *wpa_s,
 					 char *cmd)
 {
@@ -446,14 +473,7 @@ static int wpa_supplicant_ctrl_iface_set(struct wpa_supplicant *wpa_s,
 		ret = wpas_ctrl_set_blob(wpa_s, value);
 #endif /* CONFIG_NO_CONFIG_BLOBS */
 	} else if (os_strcasecmp(cmd, "setband") == 0) {
-		if (os_strcmp(value, "AUTO") == 0)
-			wpa_s->setband = WPA_SETBAND_AUTO;
-		else if (os_strcmp(value, "5G") == 0)
-			wpa_s->setband = WPA_SETBAND_5G;
-		else if (os_strcmp(value, "2G") == 0)
-			wpa_s->setband = WPA_SETBAND_2G;
-		else
-			ret = -1;
+		ret = wpas_ctrl_set_band(wpa_s, value);
 	} else {
 		value[-1] = '=';
 		ret = wpa_config_process_global(wpa_s->conf, cmd, -1);
@@ -759,6 +779,33 @@ static int wpa_supplicant_ctrl_iface_tdls_cancel_chan_switch(
 	return wpa_tdls_disable_chan_switch(wpa_s->wpa, peer);
 }
 
+
+static int wpa_supplicant_ctrl_iface_tdls_link_status(
+	struct wpa_supplicant *wpa_s, const char *addr,
+	char *buf, size_t buflen)
+{
+	u8 peer[ETH_ALEN];
+	const char *tdls_status;
+	int ret;
+
+	if (hwaddr_aton(addr, peer)) {
+		wpa_printf(MSG_DEBUG,
+			   "CTRL_IFACE TDLS_LINK_STATUS: Invalid address '%s'",
+			   addr);
+		return -1;
+	}
+	wpa_printf(MSG_DEBUG, "CTRL_IFACE TDLS_LINK_STATUS " MACSTR,
+		   MAC2STR(peer));
+
+	tdls_status = wpa_tdls_get_link_status(wpa_s->wpa, peer);
+	wpa_printf(MSG_DEBUG, "CTRL_IFACE TDLS_LINK_STATUS: %s", tdls_status);
+	ret = os_snprintf(buf, buflen, "TDLS link status: %s\n", tdls_status);
+	if (os_snprintf_error(buflen, ret))
+		return -1;
+
+	return ret;
+}
+
 #endif /* CONFIG_TDLS */
 
 
@@ -1728,7 +1775,7 @@ static int wpa_supplicant_ctrl_iface_status(struct wpa_supplicant *wpa_s,
 		if (ssid) {
 			u8 *_ssid = ssid->ssid;
 			size_t ssid_len = ssid->ssid_len;
-			u8 ssid_buf[MAX_SSID_LEN];
+			u8 ssid_buf[SSID_MAX_LEN];
 			if (ssid_len == 0) {
 				int _res = wpa_drv_get_ssid(wpa_s, ssid_buf);
 				if (_res < 0)
@@ -2089,45 +2136,6 @@ static int wpa_supplicant_ctrl_iface_blacklist(struct wpa_supplicant *wpa_s,
 }
 
 
-static const char * debug_level_str(int level)
-{
-	switch (level) {
-	case MSG_EXCESSIVE:
-		return "EXCESSIVE";
-	case MSG_MSGDUMP:
-		return "MSGDUMP";
-	case MSG_DEBUG:
-		return "DEBUG";
-	case MSG_INFO:
-		return "INFO";
-	case MSG_WARNING:
-		return "WARNING";
-	case MSG_ERROR:
-		return "ERROR";
-	default:
-		return "?";
-	}
-}
-
-
-static int str_to_debug_level(const char *s)
-{
-	if (os_strcasecmp(s, "EXCESSIVE") == 0)
-		return MSG_EXCESSIVE;
-	if (os_strcasecmp(s, "MSGDUMP") == 0)
-		return MSG_MSGDUMP;
-	if (os_strcasecmp(s, "DEBUG") == 0)
-		return MSG_DEBUG;
-	if (os_strcasecmp(s, "INFO") == 0)
-		return MSG_INFO;
-	if (os_strcasecmp(s, "WARNING") == 0)
-		return MSG_WARNING;
-	if (os_strcasecmp(s, "ERROR") == 0)
-		return MSG_ERROR;
-	return -1;
-}
-
-
 static int wpa_supplicant_ctrl_iface_log_level(struct wpa_supplicant *wpa_s,
 					       char *cmd, char *buf,
 					       size_t buflen)
@@ -2160,7 +2168,7 @@ static int wpa_supplicant_ctrl_iface_log_level(struct wpa_supplicant *wpa_s,
 		}
 	}
 
-	if (cmd && os_strlen(cmd)) {
+	if (os_strlen(cmd)) {
 		int level = str_to_debug_level(cmd);
 		if (level < 0)
 			return -1;
@@ -2366,6 +2374,14 @@ static char * wpa_supplicant_ie_txt(char *pos, char *end, const char *proto,
 	}
 #endif /* CONFIG_SUITEB192 */
 
+	if (data.key_mgmt & WPA_KEY_MGMT_OSEN) {
+		ret = os_snprintf(pos, end - pos, "%sOSEN",
+				  pos == start ? "" : "+");
+		if (os_snprintf_error(end - pos, ret))
+			return pos;
+		pos += ret;
+	}
+
 	pos = wpa_supplicant_cipher_txt(pos, end, data.pairwise_cipher);
 
 	if (data.capabilities & WPA_CAPABILITY_PREAUTH) {
@@ -2433,7 +2449,7 @@ static int wpa_supplicant_ctrl_iface_scan_result(
 {
 	char *pos, *end;
 	int ret;
-	const u8 *ie, *ie2, *p2p, *mesh;
+	const u8 *ie, *ie2, *osen_ie, *p2p, *mesh;
 
 	mesh = wpa_bss_get_ie(bss, WLAN_EID_MESH_ID);
 	p2p = wpa_bss_get_vendor_ie(bss, P2P_IE_VENDOR_TYPE);
@@ -2460,8 +2476,12 @@ static int wpa_supplicant_ctrl_iface_scan_result(
 		pos = wpa_supplicant_ie_txt(pos, end, mesh ? "RSN" : "WPA2",
 					    ie2, 2 + ie2[1]);
 	}
+	osen_ie = wpa_bss_get_vendor_ie(bss, OSEN_IE_VENDOR_TYPE);
+	if (osen_ie)
+		pos = wpa_supplicant_ie_txt(pos, end, "OSEN",
+					    osen_ie, 2 + osen_ie[1]);
 	pos = wpa_supplicant_wps_ie_txt(wpa_s, pos, end, bss);
-	if (!ie && !ie2 && bss->caps & IEEE80211_CAP_PRIVACY) {
+	if (!ie && !ie2 && !osen_ie && (bss->caps & IEEE80211_CAP_PRIVACY)) {
 		ret = os_snprintf(pos, end - pos, "[WEP]");
 		if (os_snprintf_error(end - pos, ret))
 			return -1;
@@ -2525,6 +2545,14 @@ static int wpa_supplicant_ctrl_iface_scan_result(
 		pos += ret;
 	}
 #endif /* CONFIG_HS20 */
+#ifdef CONFIG_FST
+	if (wpa_bss_get_ie(bss, WLAN_EID_MULTI_BAND)) {
+		ret = os_snprintf(pos, end - pos, "[FST]");
+		if (os_snprintf_error(end - pos, ret))
+			return -1;
+		pos += ret;
+	}
+#endif /* CONFIG_FST */
 
 	ret = os_snprintf(pos, end - pos, "\t%s",
 			  wpa_ssid_txt(bss->ssid, bss->ssid_len));
@@ -2716,6 +2744,8 @@ static int wpa_supplicant_ctrl_iface_select_network(
 		}
 	}
 
+	wpa_s->scan_min_time.sec = 0;
+	wpa_s->scan_min_time.usec = 0;
 	wpa_supplicant_select_network(wpa_s, ssid);
 
 	return 0;
@@ -2753,6 +2783,8 @@ static int wpa_supplicant_ctrl_iface_enable_network(
 			return 0;
 		}
 	}
+	wpa_s->scan_min_time.sec = 0;
+	wpa_s->scan_min_time.usec = 0;
 	wpa_supplicant_enable_network(wpa_s, ssid);
 
 	return 0;
@@ -2836,7 +2868,8 @@ static int wpa_supplicant_ctrl_iface_remove_network(
 #endif /* CONFIG_SME */
 			wpa_sm_set_config(wpa_s->wpa, NULL);
 			eapol_sm_notify_config(wpa_s->eapol, NULL, NULL);
-			wpa_s->own_disconnect_req = 1;
+			if (wpa_s->wpa_state >= WPA_AUTHENTICATING)
+				wpa_s->own_disconnect_req = 1;
 			wpa_supplicant_deauthenticate(
 				wpa_s, WLAN_REASON_DEAUTH_LEAVING);
 		}
@@ -2883,7 +2916,8 @@ static int wpa_supplicant_ctrl_iface_remove_network(
 		wpa_sm_set_config(wpa_s->wpa, NULL);
 		eapol_sm_notify_config(wpa_s->eapol, NULL, NULL);
 
-		wpa_s->own_disconnect_req = 1;
+		if (wpa_s->wpa_state >= WPA_AUTHENTICATING)
+			wpa_s->own_disconnect_req = 1;
 		wpa_supplicant_deauthenticate(wpa_s,
 					      WLAN_REASON_DEAUTH_LEAVING);
 	}
@@ -3005,19 +3039,19 @@ static int wpa_supplicant_ctrl_iface_get_network(
 	*name++ = '\0';
 
 	id = atoi(cmd);
-	wpa_printf(MSG_DEBUG, "CTRL_IFACE: GET_NETWORK id=%d name='%s'",
+	wpa_printf(MSG_EXCESSIVE, "CTRL_IFACE: GET_NETWORK id=%d name='%s'",
 		   id, name);
 
 	ssid = wpa_config_get_network(wpa_s->conf, id);
 	if (ssid == NULL) {
-		wpa_printf(MSG_DEBUG, "CTRL_IFACE: Could not find network "
+		wpa_printf(MSG_EXCESSIVE, "CTRL_IFACE: Could not find network "
 			   "id=%d", id);
 		return -1;
 	}
 
 	value = wpa_config_get_no_key(ssid, name);
 	if (value == NULL) {
-		wpa_printf(MSG_DEBUG, "CTRL_IFACE: Failed to get network "
+		wpa_printf(MSG_EXCESSIVE, "CTRL_IFACE: Failed to get network "
 			   "variable '%s'", name);
 		return -1;
 	}
@@ -3035,7 +3069,8 @@ static int wpa_supplicant_ctrl_iface_get_network(
 
 
 static int wpa_supplicant_ctrl_iface_dup_network(
-	struct wpa_supplicant *wpa_s, char *cmd)
+	struct wpa_supplicant *wpa_s, char *cmd,
+	struct wpa_supplicant *dst_wpa_s)
 {
 	struct wpa_ssid *ssid_s, *ssid_d;
 	char *name, *id, *value;
@@ -3054,8 +3089,10 @@ static int wpa_supplicant_ctrl_iface_dup_network(
 
 	id_s = atoi(cmd);
 	id_d = atoi(id);
-	wpa_printf(MSG_DEBUG, "CTRL_IFACE: DUP_NETWORK id=%d -> %d name='%s'",
-		   id_s, id_d, name);
+
+	wpa_printf(MSG_DEBUG,
+		   "CTRL_IFACE: DUP_NETWORK ifname=%s->%s id=%d->%d name='%s'",
+		   wpa_s->ifname, dst_wpa_s->ifname, id_s, id_d, name);
 
 	ssid_s = wpa_config_get_network(wpa_s->conf, id_s);
 	if (ssid_s == NULL) {
@@ -3064,7 +3101,7 @@ static int wpa_supplicant_ctrl_iface_dup_network(
 		return -1;
 	}
 
-	ssid_d = wpa_config_get_network(wpa_s->conf, id_d);
+	ssid_d = wpa_config_get_network(dst_wpa_s->conf, id_d);
 	if (ssid_d == NULL) {
 		wpa_printf(MSG_DEBUG, "CTRL_IFACE: Could not find "
 			   "network id=%d", id_d);
@@ -3078,7 +3115,7 @@ static int wpa_supplicant_ctrl_iface_dup_network(
 		return -1;
 	}
 
-	ret = wpa_supplicant_ctrl_iface_update_network(wpa_s, ssid_d, name,
+	ret = wpa_supplicant_ctrl_iface_update_network(dst_wpa_s, ssid_d, name,
 						       value);
 
 	os_free(value);
@@ -3889,6 +3926,15 @@ static int wpa_supplicant_ctrl_iface_get_capability(
 	}
 #endif /* CONFIG_EPR */
 
+#ifdef CONFIG_FIPS
+	if (os_strcmp(field, "fips") == 0) {
+		res = os_snprintf(buf, buflen, "FIPS");
+		if (os_snprintf_error(buflen, res))
+			return -1;
+		return res;
+	}
+#endif /* CONFIG_FIPS */
+
 	wpa_printf(MSG_DEBUG, "CTRL_IFACE: Unknown GET_CAPABILITY field '%s'",
 		   field);
 
@@ -3937,7 +3983,7 @@ static int print_bss_info(struct wpa_supplicant *wpa_s, struct wpa_bss *bss,
 	size_t i;
 	int ret;
 	char *pos, *end;
-	const u8 *ie, *ie2;
+	const u8 *ie, *ie2, *osen_ie;
 
 	pos = buf;
 	end = buf + buflen;
@@ -4054,8 +4100,13 @@ static int print_bss_info(struct wpa_supplicant *wpa_s, struct wpa_bss *bss,
 		if (ie2)
 			pos = wpa_supplicant_ie_txt(pos, end, "WPA2", ie2,
 						    2 + ie2[1]);
+		osen_ie = wpa_bss_get_vendor_ie(bss, OSEN_IE_VENDOR_TYPE);
+		if (osen_ie)
+			pos = wpa_supplicant_ie_txt(pos, end, "OSEN",
+						    osen_ie, 2 + osen_ie[1]);
 		pos = wpa_supplicant_wps_ie_txt(wpa_s, pos, end, bss);
-		if (!ie && !ie2 && bss->caps & IEEE80211_CAP_PRIVACY) {
+		if (!ie && !ie2 && !osen_ie &&
+		    (bss->caps & IEEE80211_CAP_PRIVACY)) {
 			ret = os_snprintf(pos, end - pos, "[WEP]");
 			if (os_snprintf_error(end - pos, ret))
 				return 0;
@@ -4133,9 +4184,10 @@ static int print_bss_info(struct wpa_supplicant *wpa_s, struct wpa_bss *bss,
 	if (mask & WPA_BSS_MASK_WPS_SCAN) {
 		ie = (const u8 *) (bss + 1);
 		ret = wpas_wps_scan_result_text(ie, bss->ie_len, pos, end);
-		if (ret < 0 || ret >= end - pos)
+		if (ret >= end - pos)
 			return 0;
-		pos += ret;
+		if (ret > 0)
+			pos += ret;
 	}
 #endif /* CONFIG_WPS */
 
@@ -4236,6 +4288,15 @@ static int print_bss_info(struct wpa_supplicant *wpa_s, struct wpa_bss *bss,
 		pos += ret;
 	}
 
+#ifdef CONFIG_FST
+	if (mask & WPA_BSS_MASK_FST) {
+		ret = fst_ctrl_iface_mb_info(bss->bssid, pos, end - pos);
+		if (ret < 0 || ret >= end - pos)
+			return 0;
+		pos += ret;
+	}
+#endif /* CONFIG_FST */
+
 	if (mask & WPA_BSS_MASK_DELIM) {
 		ret = os_snprintf(pos, end - pos, "====\n");
 		if (os_snprintf_error(end - pos, ret))
@@ -4548,24 +4609,6 @@ static int p2p_ctrl_find(struct wpa_supplicant *wpa_s, char *cmd)
 	} else
 		search_delay = wpas_p2p_search_delay(wpa_s);
 
-	/* Must be searched for last, because it adds nul termination */
-	pos = os_strstr(cmd, " seek=");
-	while (pos && seek_count < P2P_MAX_QUERY_HASH + 1) {
-		char *term;
-
-		term = os_strchr(pos + 1, ' ');
-		_seek[seek_count++] = pos + 6;
-		seek = _seek;
-		pos = os_strstr(pos + 6, " seek=");
-
-		if (term)
-			*term = '\0';
-	}
-	if (seek_count > P2P_MAX_QUERY_HASH) {
-		seek[0] = NULL;
-		seek_count = 1;
-	}
-
 	pos = os_strstr(cmd, "freq=");
 	if (pos) {
 		pos += 5;
@@ -4574,11 +4617,75 @@ static int p2p_ctrl_find(struct wpa_supplicant *wpa_s, char *cmd)
 			return -1;
 	}
 
+	/* Must be searched for last, because it adds nul termination */
+	pos = os_strstr(cmd, " seek=");
+	if (pos)
+		pos += 6;
+	while (pos && seek_count < P2P_MAX_QUERY_HASH + 1) {
+		char *term;
+
+		_seek[seek_count++] = pos;
+		seek = _seek;
+		term = os_strchr(pos, ' ');
+		if (!term)
+			break;
+		*term = '\0';
+		pos = os_strstr(term + 1, "seek=");
+		if (pos)
+			pos += 5;
+	}
+	if (seek_count > P2P_MAX_QUERY_HASH) {
+		seek[0] = NULL;
+		seek_count = 1;
+	}
+
 	return wpas_p2p_find(wpa_s, timeout, type, _dev_type != NULL, _dev_type,
 			     _dev_id, search_delay, seek_count, seek, freq);
 }
 
 
+static int p2ps_ctrl_parse_cpt_priority(const char *pos, u8 *cpt)
+{
+	const char *last = NULL;
+	const char *token;
+	long int token_len;
+	unsigned int i;
+
+	/* Expected predefined CPT names delimited by ':' */
+	for (i = 0; (token = cstr_token(pos, ": \t", &last)); i++) {
+		if (i >= P2PS_FEATURE_CAPAB_CPT_MAX) {
+			wpa_printf(MSG_ERROR,
+				   "P2PS: CPT name list is too long, expected up to %d names",
+				   P2PS_FEATURE_CAPAB_CPT_MAX);
+			cpt[0] = 0;
+			return -1;
+		}
+
+		token_len = last - token;
+
+		if (token_len  == 3 &&
+		    os_memcmp(token, "UDP", token_len) == 0) {
+			cpt[i] = P2PS_FEATURE_CAPAB_UDP_TRANSPORT;
+		} else if (token_len == 3 &&
+			   os_memcmp(token, "MAC", token_len) == 0) {
+			cpt[i] = P2PS_FEATURE_CAPAB_MAC_TRANSPORT;
+		} else {
+			wpa_printf(MSG_ERROR,
+				   "P2PS: Unsupported CPT name '%s'", token);
+			cpt[0] = 0;
+			return -1;
+		}
+
+		if (isblank(*last)) {
+			i++;
+			break;
+		}
+	}
+	cpt[i] = 0;
+	return 0;
+}
+
+
 static struct p2ps_provision * p2p_parse_asp_provision_cmd(const char *cmd)
 {
 	struct p2ps_provision *p2ps_prov;
@@ -4587,6 +4694,7 @@ static struct p2ps_provision * p2p_parse_asp_provision_cmd(const char *cmd)
 	char *info = NULL;
 	u8 role = P2PS_SETUP_NONE;
 	long long unsigned val;
+	int i;
 
 	pos = os_strstr(cmd, "info=");
 	if (pos) {
@@ -4645,6 +4753,18 @@ static struct p2ps_provision * p2p_parse_asp_provision_cmd(const char *cmd)
 	if (!pos || hwaddr_aton(pos + 12, p2ps_prov->session_mac))
 		goto invalid_args;
 
+	pos = os_strstr(cmd, "cpt=");
+	if (pos) {
+		if (p2ps_ctrl_parse_cpt_priority(pos + 4,
+						 p2ps_prov->cpt_priority))
+			goto invalid_args;
+	} else {
+		p2ps_prov->cpt_priority[0] = P2PS_FEATURE_CAPAB_UDP_TRANSPORT;
+	}
+
+	for (i = 0; p2ps_prov->cpt_priority[i]; i++)
+		p2ps_prov->cpt_mask |= p2ps_prov->cpt_priority[i];
+
 	/* force conncap with tstCap (no sanity checks) */
 	pos = os_strstr(cmd, "tstCap=");
 	if (pos) {
@@ -4721,6 +4841,8 @@ static int p2p_ctrl_asp_provision(struct wpa_supplicant *wpa_s, char *cmd)
 	if (!p2ps_prov)
 		return -1;
 
+	p2ps_prov->pd_seeker = 1;
+
 	return wpas_p2p_prov_disc(wpa_s, addr, NULL, WPAS_P2P_PD_FOR_ASP,
 				  p2ps_prov);
 }
@@ -5140,6 +5262,8 @@ static int p2p_ctrl_service_add_asp(struct wpa_supplicant *wpa_s,
 	char *adv_str;
 	u32 auto_accept, adv_id, svc_state, config_methods;
 	char *svc_info = NULL;
+	char *cpt_prio_str;
+	u8 cpt_prio[P2PS_FEATURE_CAPAB_CPT_MAX + 1];
 
 	pos = os_strchr(cmd, ' ');
 	if (pos == NULL)
@@ -5212,6 +5336,19 @@ static int p2p_ctrl_service_add_asp(struct wpa_supplicant *wpa_s,
 	if (pos != NULL)
 		*pos++ = '\0';
 
+	cpt_prio_str = (pos && pos[0]) ? os_strstr(pos, "cpt=") : NULL;
+	if (cpt_prio_str) {
+		pos = os_strchr(pos, ' ');
+		if (pos != NULL)
+			*pos++ = '\0';
+
+		if (p2ps_ctrl_parse_cpt_priority(cpt_prio_str + 4, cpt_prio))
+			return -1;
+	} else {
+		cpt_prio[0] = P2PS_FEATURE_CAPAB_UDP_TRANSPORT;
+		cpt_prio[1] = 0;
+	}
+
 	/* Service and Response Information are optional */
 	if (pos && pos[0]) {
 		size_t len;
@@ -5229,7 +5366,7 @@ static int p2p_ctrl_service_add_asp(struct wpa_supplicant *wpa_s,
 
 	return wpas_p2p_service_add_asp(wpa_s, auto_accept, adv_id, adv_str,
 					(u8) svc_state, (u16) config_methods,
-					svc_info);
+					svc_info, cpt_prio);
 }
 
 
@@ -5299,6 +5436,11 @@ static int p2p_ctrl_service_del_asp(struct wpa_supplicant *wpa_s, char *cmd)
 {
 	u32 adv_id;
 
+	if (os_strcmp(cmd, "all") == 0) {
+		wpas_p2p_service_flush_asp(wpa_s);
+		return 0;
+	}
+
 	if (sscanf(cmd, "%x", &adv_id) != 1)
 		return -1;
 
@@ -5449,13 +5591,10 @@ static int p2p_ctrl_invite(struct wpa_supplicant *wpa_s, char *cmd)
 
 
 static int p2p_ctrl_group_add_persistent(struct wpa_supplicant *wpa_s,
-					 char *cmd, int freq, int ht40,
-					 int vht)
+					 int id, int freq, int ht40, int vht)
 {
-	int id;
 	struct wpa_ssid *ssid;
 
-	id = atoi(cmd);
 	ssid = wpa_config_get_network(wpa_s->conf, id);
 	if (ssid == NULL || ssid->disabled != 2) {
 		wpa_printf(MSG_DEBUG, "CTRL_IFACE: Could not find SSID id=%d "
@@ -5465,37 +5604,41 @@ static int p2p_ctrl_group_add_persistent(struct wpa_supplicant *wpa_s,
 	}
 
 	return wpas_p2p_group_add_persistent(wpa_s, ssid, 0, freq, 0, ht40, vht,
-					     NULL, 0);
+					     NULL, 0, 0);
 }
 
 
 static int p2p_ctrl_group_add(struct wpa_supplicant *wpa_s, char *cmd)
 {
-	int freq = 0, ht40, vht;
-	char *pos;
+	int freq = 0, persistent = 0, group_id = -1;
+	int vht = wpa_s->conf->p2p_go_vht;
+	int ht40 = wpa_s->conf->p2p_go_ht40 || vht;
+	char *token, *context = NULL;
 
-	pos = os_strstr(cmd, "freq=");
-	if (pos)
-		freq = atoi(pos + 5);
+	while ((token = str_token(cmd, " ", &context))) {
+		if (sscanf(token, "freq=%d", &freq) == 1 ||
+		    sscanf(token, "persistent=%d", &group_id) == 1) {
+			continue;
+		} else if (os_strcmp(token, "ht40") == 0) {
+			ht40 = 1;
+		} else if (os_strcmp(token, "vht") == 0) {
+			vht = 1;
+			ht40 = 1;
+		} else if (os_strcmp(token, "persistent") == 0) {
+			persistent = 1;
+		} else {
+			wpa_printf(MSG_DEBUG,
+				   "CTRL: Invalid P2P_GROUP_ADD parameter: '%s'",
+				   token);
+			return -1;
+		}
+	}
 
-	vht = (os_strstr(cmd, "vht") != NULL) || wpa_s->conf->p2p_go_vht;
-	ht40 = (os_strstr(cmd, "ht40") != NULL) || wpa_s->conf->p2p_go_ht40 ||
-		vht;
+	if (group_id >= 0)
+		return p2p_ctrl_group_add_persistent(wpa_s, group_id,
+						     freq, ht40, vht);
 
-	if (os_strncmp(cmd, "persistent=", 11) == 0)
-		return p2p_ctrl_group_add_persistent(wpa_s, cmd + 11, freq,
-						     ht40, vht);
-	if (os_strcmp(cmd, "persistent") == 0 ||
-	    os_strncmp(cmd, "persistent ", 11) == 0)
-		return wpas_p2p_group_add(wpa_s, 1, freq, ht40, vht);
-	if (os_strncmp(cmd, "freq=", 5) == 0)
-		return wpas_p2p_group_add(wpa_s, 0, freq, ht40, vht);
-	if (ht40)
-		return wpas_p2p_group_add(wpa_s, 0, freq, ht40, vht);
-
-	wpa_printf(MSG_DEBUG, "CTRL: Invalid P2P_GROUP_ADD parameters '%s'",
-		   cmd);
-	return -1;
+	return wpas_p2p_group_add(wpa_s, persistent, freq, ht40, vht);
 }
 
 
@@ -5625,7 +5768,7 @@ static int p2p_ctrl_disallow_freq(struct wpa_supplicant *wpa_s,
 			   freq->min, freq->max);
 	}
 
-	wpas_p2p_update_channel_list(wpa_s);
+	wpas_p2p_update_channel_list(wpa_s, WPAS_P2P_CHANNEL_UPDATE_DISALLOW);
 	return 0;
 }
 
@@ -5842,6 +5985,7 @@ static void p2p_ctrl_flush(struct wpa_supplicant *wpa_s)
 	os_memset(wpa_s->p2p_auth_invite, 0, ETH_ALEN);
 	wpa_s->force_long_sd = 0;
 	wpas_p2p_stop_find(wpa_s);
+	wpa_s->parent->p2ps_method_config_any = 0;
 	if (wpa_s->global->p2p)
 		p2p_flush(wpa_s->global->p2p);
 }
@@ -6476,6 +6620,61 @@ static int wpa_supplicant_signal_poll(struct wpa_supplicant *wpa_s, char *buf,
 		pos += ret;
 	}
 
+	if (si.avg_beacon_signal) {
+		ret = os_snprintf(pos, end - pos,
+				  "AVG_BEACON_RSSI=%d\n", si.avg_beacon_signal);
+		if (os_snprintf_error(end - pos, ret))
+			return -1;
+		pos += ret;
+	}
+
+	return pos - buf;
+}
+
+
+static int wpas_ctrl_iface_get_pref_freq_list(
+	struct wpa_supplicant *wpa_s, char *cmd, char *buf, size_t buflen)
+{
+	unsigned int freq_list[100], num = 100, i;
+	int ret;
+	enum wpa_driver_if_type iface_type;
+	char *pos, *end;
+
+	pos = buf;
+	end = buf + buflen;
+
+	/* buf: "<interface_type>" */
+	if (os_strcmp(cmd, "STATION") == 0)
+		iface_type = WPA_IF_STATION;
+	else if (os_strcmp(cmd, "AP") == 0)
+		iface_type = WPA_IF_AP_BSS;
+	else if (os_strcmp(cmd, "P2P_GO") == 0)
+		iface_type = WPA_IF_P2P_GO;
+	else if (os_strcmp(cmd, "P2P_CLIENT") == 0)
+		iface_type = WPA_IF_P2P_CLIENT;
+	else if (os_strcmp(cmd, "IBSS") == 0)
+		iface_type = WPA_IF_IBSS;
+	else if (os_strcmp(cmd, "TDLS") == 0)
+		iface_type = WPA_IF_TDLS;
+	else
+		return -1;
+
+	wpa_printf(MSG_DEBUG,
+		   "CTRL_IFACE: GET_PREF_FREQ_LIST iface_type=%d (%s)",
+		   iface_type, buf);
+
+	ret = wpa_drv_get_pref_freq_list(wpa_s, iface_type, &num, freq_list);
+	if (ret)
+		return -1;
+
+	for (i = 0; i < num; i++) {
+		ret = os_snprintf(pos, end - pos, "%s%u",
+				  i > 0 ? "," : "", freq_list[i]);
+		if (os_snprintf_error(end - pos, ret))
+			return -1;
+		pos += ret;
+	}
+
 	return pos - buf;
 }
 
@@ -6602,6 +6801,7 @@ static void wpa_supplicant_ctrl_iface_flush(struct wpa_supplicant *wpa_s)
 	p2p_wpa_s->p2p_disable_ip_addr_req = 0;
 	os_free(p2p_wpa_s->global->p2p_go_avoid_freq.range);
 	p2p_wpa_s->global->p2p_go_avoid_freq.range = NULL;
+	p2p_wpa_s->global->p2p_go_avoid_freq.num = 0;
 	p2p_wpa_s->global->pending_p2ps_group = 0;
 #endif /* CONFIG_P2P */
 
@@ -6689,6 +6889,8 @@ static void wpa_supplicant_ctrl_iface_flush(struct wpa_supplicant *wpa_s)
 			   MAC2STR(wpa_s->bssid),
 			   MAC2STR(wpa_s->pending_bssid));
 	}
+
+	eloop_cancel_timeout(wpas_network_reenabled, wpa_s, NULL);
 }
 
 
@@ -6922,6 +7124,8 @@ static void wpas_ctrl_scan(struct wpa_supplicant *wpa_s, char *params,
 	void (*scan_res_handler)(struct wpa_supplicant *wpa_s,
 				 struct wpa_scan_results *scan_res);
 	int *manual_scan_freqs = NULL;
+	struct wpa_ssid_value *ssid = NULL, *ns;
+	unsigned int ssid_count = 0;
 
 	if (wpa_s->wpa_state == WPA_INTERFACE_DISABLED) {
 		*reply_len = -1;
@@ -6935,6 +7139,15 @@ static void wpas_ctrl_scan(struct wpa_supplicant *wpa_s, char *params,
 		return;
 	}
 
+#ifdef CONFIG_INTERWORKING
+	if (wpa_s->fetch_anqp_in_progress || wpa_s->network_select) {
+		wpa_printf(MSG_DEBUG,
+			   "Interworking select in progress - reject new scan");
+		*reply_len = os_snprintf(reply, reply_size, "FAIL-BUSY\n");
+		return;
+	}
+#endif /* CONFIG_INTERWORKING */
+
 	if (params) {
 		if (os_strncasecmp(params, "TYPE=ONLY", 9) == 0)
 			scan_only = 1;
@@ -6967,6 +7180,60 @@ static void wpas_ctrl_scan(struct wpa_supplicant *wpa_s, char *params,
 			*reply_len = -1;
 			goto done;
 		}
+
+		pos = params;
+		while (pos && *pos != '\0') {
+			if (os_strncmp(pos, "ssid ", 5) == 0) {
+				char *end;
+
+				pos += 5;
+				end = pos;
+				while (*end) {
+					if (*end == '\0' || *end == ' ')
+						break;
+					end++;
+				}
+
+				ns = os_realloc_array(
+					ssid, ssid_count + 1,
+					sizeof(struct wpa_ssid_value));
+				if (ns == NULL) {
+					*reply_len = -1;
+					goto done;
+				}
+				ssid = ns;
+
+				if ((end - pos) & 0x01 ||
+				    end - pos > 2 * SSID_MAX_LEN ||
+				    hexstr2bin(pos, ssid[ssid_count].ssid,
+					       (end - pos) / 2) < 0) {
+					wpa_printf(MSG_DEBUG,
+						   "Invalid SSID value '%s'",
+						   pos);
+					*reply_len = -1;
+					goto done;
+				}
+				ssid[ssid_count].ssid_len = (end - pos) / 2;
+				wpa_hexdump_ascii(MSG_DEBUG, "scan SSID",
+						  ssid[ssid_count].ssid,
+						  ssid[ssid_count].ssid_len);
+				ssid_count++;
+				pos = end;
+			}
+
+			pos = os_strchr(pos, ' ');
+			if (pos)
+				pos++;
+		}
+	}
+
+	wpa_s->num_ssids_from_scan_req = ssid_count;
+	os_free(wpa_s->ssids_from_scan_req);
+	if (ssid_count) {
+		wpa_s->ssids_from_scan_req = ssid;
+		ssid = NULL;
+	} else {
+		wpa_s->ssids_from_scan_req = NULL;
 	}
 
 	if (scan_only)
@@ -7030,6 +7297,7 @@ static void wpas_ctrl_scan(struct wpa_supplicant *wpa_s, char *params,
 
 done:
 	os_free(manual_scan_freqs);
+	os_free(ssid);
 }
 
 
@@ -7231,7 +7499,7 @@ void wpas_data_test_rx(void *ctx, const u8 *src_addr, const u8 *buf, size_t len)
 {
 	struct wpa_supplicant *wpa_s = ctx;
 	const struct ether_header *eth;
-	const struct iphdr *ip;
+	struct iphdr ip;
 	const u8 *pos;
 	unsigned int i;
 
@@ -7239,14 +7507,14 @@ void wpas_data_test_rx(void *ctx, const u8 *src_addr, const u8 *buf, size_t len)
 		return;
 
 	eth = (const struct ether_header *) buf;
-	ip = (const struct iphdr *) (eth + 1);
-	pos = (const u8 *) (ip + 1);
+	os_memcpy(&ip, eth + 1, sizeof(ip));
+	pos = &buf[sizeof(*eth) + sizeof(ip)];
 
-	if (ip->ihl != 5 || ip->version != 4 ||
-	    ntohs(ip->tot_len) != HWSIM_IP_LEN)
+	if (ip.ihl != 5 || ip.version != 4 ||
+	    ntohs(ip.tot_len) != HWSIM_IP_LEN)
 		return;
 
-	for (i = 0; i < HWSIM_IP_LEN - sizeof(*ip); i++) {
+	for (i = 0; i < HWSIM_IP_LEN - sizeof(ip); i++) {
 		if (*pos != (u8) i)
 			return;
 		pos++;
@@ -7293,7 +7561,7 @@ static int wpas_ctrl_iface_data_test_tx(struct wpa_supplicant *wpa_s, char *cmd)
 	int used;
 	long int val;
 	u8 tos;
-	u8 buf[HWSIM_PACKETLEN];
+	u8 buf[2 + HWSIM_PACKETLEN];
 	struct ether_header *eth;
 	struct iphdr *ip;
 	u8 *dpos;
@@ -7321,7 +7589,7 @@ static int wpas_ctrl_iface_data_test_tx(struct wpa_supplicant *wpa_s, char *cmd)
 		return -1;
 	tos = val;
 
-	eth = (struct ether_header *) buf;
+	eth = (struct ether_header *) &buf[2];
 	os_memcpy(eth->ether_dhost, dst, ETH_ALEN);
 	os_memcpy(eth->ether_shost, src, ETH_ALEN);
 	eth->ether_type = htons(ETHERTYPE_IP);
@@ -7333,14 +7601,14 @@ static int wpas_ctrl_iface_data_test_tx(struct wpa_supplicant *wpa_s, char *cmd)
 	ip->tos = tos;
 	ip->tot_len = htons(HWSIM_IP_LEN);
 	ip->protocol = 1;
-	ip->saddr = htonl(192 << 24 | 168 << 16 | 1 << 8 | 1);
-	ip->daddr = htonl(192 << 24 | 168 << 16 | 1 << 8 | 2);
+	ip->saddr = htonl(192U << 24 | 168 << 16 | 1 << 8 | 1);
+	ip->daddr = htonl(192U << 24 | 168 << 16 | 1 << 8 | 2);
 	ip->check = ipv4_hdr_checksum(ip, sizeof(*ip));
 	dpos = (u8 *) (ip + 1);
 	for (i = 0; i < HWSIM_IP_LEN - sizeof(*ip); i++)
 		*dpos++ = i;
 
-	if (l2_packet_send(wpa_s->l2_test, dst, ETHERTYPE_IP, buf,
+	if (l2_packet_send(wpa_s->l2_test, dst, ETHERTYPE_IP, &buf[2],
 			   HWSIM_PACKETLEN) < 0)
 		return -1;
 
@@ -7429,6 +7697,44 @@ static int wpas_ctrl_get_alloc_fail(struct wpa_supplicant *wpa_s,
 #endif /* WPA_TRACE_BFD */
 }
 
+
+static int wpas_ctrl_test_fail(struct wpa_supplicant *wpa_s, char *cmd)
+{
+#ifdef WPA_TRACE_BFD
+	extern char wpa_trace_test_fail_func[256];
+	extern unsigned int wpa_trace_test_fail_after;
+	char *pos;
+
+	wpa_trace_test_fail_after = atoi(cmd);
+	pos = os_strchr(cmd, ':');
+	if (pos) {
+		pos++;
+		os_strlcpy(wpa_trace_test_fail_func, pos,
+			   sizeof(wpa_trace_test_fail_func));
+	} else {
+		wpa_trace_test_fail_after = 0;
+	}
+	return 0;
+#else /* WPA_TRACE_BFD */
+	return -1;
+#endif /* WPA_TRACE_BFD */
+}
+
+
+static int wpas_ctrl_get_fail(struct wpa_supplicant *wpa_s,
+				    char *buf, size_t buflen)
+{
+#ifdef WPA_TRACE_BFD
+	extern char wpa_trace_test_fail_func[256];
+	extern unsigned int wpa_trace_test_fail_after;
+
+	return os_snprintf(buf, buflen, "%u:%s", wpa_trace_test_fail_after,
+			   wpa_trace_test_fail_func);
+#else /* WPA_TRACE_BFD */
+	return -1;
+#endif /* WPA_TRACE_BFD */
+}
+
 #endif /* CONFIG_TESTING_OPTIONS */
 
 
@@ -7665,7 +7971,7 @@ static int wpas_ctrl_iface_send_neigbor_rep(struct wpa_supplicant *wpa_s,
 
 	if (os_strncmp(cmd, " ssid=", 6) == 0) {
 		ssid.ssid_len = os_strlen(cmd + 6);
-		if (ssid.ssid_len > 32)
+		if (ssid.ssid_len > SSID_MAX_LEN)
 			return -1;
 		ssid.ssid = (u8 *) (cmd + 6);
 		ssid_p = &ssid;
@@ -7806,6 +8112,19 @@ static int wpas_ctrl_iface_mac_rand_scan(struct wpa_supplicant *wpa_s,
 }
 
 
+static int wpas_ctrl_cmd_debug_level(const char *cmd)
+{
+	if (os_strcmp(cmd, "PING") == 0 ||
+	    os_strncmp(cmd, "BSS ", 4) == 0 ||
+	    os_strncmp(cmd, "GET_NETWORK ", 12) == 0 ||
+	    os_strncmp(cmd, "STATUS", 6) == 0 ||
+	    os_strncmp(cmd, "STA ", 4) == 0 ||
+	    os_strncmp(cmd, "STA-", 4) == 0)
+		return MSG_EXCESSIVE;
+	return MSG_DEBUG;
+}
+
+
 char * wpa_supplicant_ctrl_iface_process(struct wpa_supplicant *wpa_s,
 					 char *buf, size_t *resp_len)
 {
@@ -7829,9 +8148,7 @@ char * wpa_supplicant_ctrl_iface_process(struct wpa_supplicant *wpa_s,
 		wpa_hexdump_ascii_key(MSG_DEBUG, "RX ctrl_iface",
 				      (const u8 *) buf, os_strlen(buf));
 	} else {
-		int level = MSG_DEBUG;
-		if (os_strcmp(buf, "PING") == 0)
-			level = MSG_EXCESSIVE;
+		int level = wpas_ctrl_cmd_debug_level(buf);
 		wpa_dbg(wpa_s, level, "Control interface command '%s'", buf);
 	}
 
@@ -8066,7 +8383,7 @@ char * wpa_supplicant_ctrl_iface_process(struct wpa_supplicant *wpa_s,
 		if (wpas_p2p_group_remove(wpa_s, buf + 17))
 			reply_len = -1;
 	} else if (os_strcmp(buf, "P2P_GROUP_ADD") == 0) {
-		if (wpas_p2p_group_add(wpa_s, 0, 0, 0, 0))
+		if (p2p_ctrl_group_add(wpa_s, ""))
 			reply_len = -1;
 	} else if (os_strncmp(buf, "P2P_GROUP_ADD ", 14) == 0) {
 		if (p2p_ctrl_group_add(wpa_s, buf + 14))
@@ -8240,6 +8557,7 @@ char * wpa_supplicant_ctrl_iface_process(struct wpa_supplicant *wpa_s,
 		wpa_supplicant_cancel_scan(wpa_s);
 		wpa_supplicant_deauthenticate(wpa_s,
 					      WLAN_REASON_DEAUTH_LEAVING);
+		eloop_cancel_timeout(wpas_network_reenabled, wpa_s, NULL);
 	} else if (os_strcmp(buf, "SCAN") == 0) {
 		wpas_ctrl_scan(wpa_s, NULL, reply, reply_size, &reply_len);
 	} else if (os_strncmp(buf, "SCAN ", 5) == 0) {
@@ -8269,7 +8587,8 @@ char * wpa_supplicant_ctrl_iface_process(struct wpa_supplicant *wpa_s,
 		reply_len = wpa_supplicant_ctrl_iface_get_network(
 			wpa_s, buf + 12, reply, reply_size);
 	} else if (os_strncmp(buf, "DUP_NETWORK ", 12) == 0) {
-		if (wpa_supplicant_ctrl_iface_dup_network(wpa_s, buf + 12))
+		if (wpa_supplicant_ctrl_iface_dup_network(wpa_s, buf + 12,
+							  wpa_s))
 			reply_len = -1;
 	} else if (os_strcmp(buf, "LIST_CREDS") == 0) {
 		reply_len = wpa_supplicant_ctrl_iface_list_creds(
@@ -8372,6 +8691,9 @@ char * wpa_supplicant_ctrl_iface_process(struct wpa_supplicant *wpa_s,
 		if (wpa_supplicant_ctrl_iface_tdls_cancel_chan_switch(wpa_s,
 								      buf + 24))
 			reply_len = -1;
+	} else if (os_strncmp(buf, "TDLS_LINK_STATUS ", 17) == 0) {
+		reply_len = wpa_supplicant_ctrl_iface_tdls_link_status(
+			wpa_s, buf + 17, reply, reply_size);
 #endif /* CONFIG_TDLS */
 	} else if (os_strcmp(buf, "WMM_AC_STATUS") == 0) {
 		reply_len = wpas_wmm_ac_status(wpa_s, reply, reply_size);
@@ -8442,6 +8764,11 @@ char * wpa_supplicant_ctrl_iface_process(struct wpa_supplicant *wpa_s,
 			reply_len = -1;
 	} else if (os_strcmp(buf, "GET_ALLOC_FAIL") == 0) {
 		reply_len = wpas_ctrl_get_alloc_fail(wpa_s, reply, reply_size);
+	} else if (os_strncmp(buf, "TEST_FAIL ", 10) == 0) {
+		if (wpas_ctrl_test_fail(wpa_s, buf + 10) < 0)
+			reply_len = -1;
+	} else if (os_strcmp(buf, "GET_FAIL") == 0) {
+		reply_len = wpas_ctrl_get_fail(wpa_s, reply, reply_size);
 #endif /* CONFIG_TESTING_OPTIONS */
 	} else if (os_strncmp(buf, "VENDOR_ELEM_ADD ", 16) == 0) {
 		if (wpas_ctrl_vendor_elem_add(wpa_s, buf + 16) < 0)
@@ -8460,6 +8787,9 @@ char * wpa_supplicant_ctrl_iface_process(struct wpa_supplicant *wpa_s,
 	} else if (os_strncmp(buf, "MAC_RAND_SCAN ", 14) == 0) {
 		if (wpas_ctrl_iface_mac_rand_scan(wpa_s, buf + 14))
 			reply_len = -1;
+	} else if (os_strncmp(buf, "GET_PREF_FREQ_LIST ", 19) == 0) {
+		reply_len = wpas_ctrl_iface_get_pref_freq_list(
+			wpa_s, buf + 19, reply, reply_size);
 	} else {
 		os_memcpy(reply, "UNKNOWN COMMAND\n", 16);
 		reply_len = 16;
@@ -8479,11 +8809,14 @@ static int wpa_supplicant_global_iface_add(struct wpa_global *global,
 					   char *cmd)
 {
 	struct wpa_interface iface;
-	char *pos;
+	char *pos, *extra;
+	struct wpa_supplicant *wpa_s;
+	unsigned int create_iface = 0;
+	u8 mac_addr[ETH_ALEN];
 
 	/*
 	 * <ifname>TAB<confname>TAB<driver>TAB<ctrl_interface>TAB<driver_param>
-	 * TAB<bridge_ifname>
+	 * TAB<bridge_ifname>[TAB<create>]
 	 */
 	wpa_printf(MSG_DEBUG, "CTRL_IFACE GLOBAL INTERFACE_ADD '%s'", cmd);
 
@@ -8543,12 +8876,54 @@ static int wpa_supplicant_global_iface_add(struct wpa_global *global,
 			iface.bridge_ifname = NULL;
 		if (pos == NULL)
 			break;
+
+		extra = pos;
+		pos = os_strchr(pos, '\t');
+		if (pos)
+			*pos++ = '\0';
+		if (!extra[0])
+			break;
+
+		if (os_strcmp(extra, "create") == 0)
+			create_iface = 1;
+		else {
+			wpa_printf(MSG_DEBUG,
+				   "INTERFACE_ADD unsupported extra parameter: '%s'",
+				   extra);
+			return -1;
+		}
 	} while (0);
 
-	if (wpa_supplicant_get_iface(global, iface.ifname))
-		return -1;
+	if (create_iface) {
+		wpa_printf(MSG_DEBUG, "CTRL_IFACE creating interface '%s'",
+			   iface.ifname);
+		if (!global->ifaces)
+			return -1;
+		if (wpa_drv_if_add(global->ifaces, WPA_IF_STATION, iface.ifname,
+				   NULL, NULL, NULL, mac_addr, NULL) < 0) {
+			wpa_printf(MSG_ERROR,
+				   "CTRL_IFACE interface creation failed");
+			return -1;
+		}
 
-	return wpa_supplicant_add_iface(global, &iface, NULL) ? 0 : -1;
+		wpa_printf(MSG_DEBUG,
+			   "CTRL_IFACE interface '%s' created with MAC addr: "
+			   MACSTR, iface.ifname, MAC2STR(mac_addr));
+	}
+
+	if (wpa_supplicant_get_iface(global, iface.ifname))
+		goto fail;
+
+	wpa_s = wpa_supplicant_add_iface(global, &iface, NULL);
+	if (!wpa_s)
+		goto fail;
+	wpa_s->added_vif = create_iface;
+	return 0;
+
+fail:
+	if (create_iface)
+		wpa_drv_if_remove(global->ifaces, WPA_IF_STATION, iface.ifname);
+	return -1;
 }
 
 
@@ -8556,13 +8931,22 @@ static int wpa_supplicant_global_iface_remove(struct wpa_global *global,
 					      char *cmd)
 {
 	struct wpa_supplicant *wpa_s;
+	int ret;
+	unsigned int delete_iface;
 
 	wpa_printf(MSG_DEBUG, "CTRL_IFACE GLOBAL INTERFACE_REMOVE '%s'", cmd);
 
 	wpa_s = wpa_supplicant_get_iface(global, cmd);
 	if (wpa_s == NULL)
 		return -1;
-	return wpa_supplicant_remove_iface(global, wpa_s, 0);
+	delete_iface = wpa_s->added_vif;
+	ret = wpa_supplicant_remove_iface(global, wpa_s, 0);
+	if (!ret && delete_iface) {
+		wpa_printf(MSG_DEBUG, "CTRL_IFACE deleting the interface '%s'",
+			   cmd);
+		ret = wpa_drv_if_remove(global->ifaces, WPA_IF_STATION, cmd);
+	}
+	return ret;
 }
 
 
@@ -8589,7 +8973,7 @@ static int wpa_supplicant_global_iface_list(struct wpa_global *global,
 	char *pos, *end;
 
 	for (i = 0; wpa_drivers[i]; i++) {
-		struct wpa_driver_ops *drv = wpa_drivers[i];
+		const struct wpa_driver_ops *drv = wpa_drivers[i];
 		if (drv->get_interfaces == NULL)
 			continue;
 		tmp = drv->get_interfaces(global->drv_priv[i]);
@@ -8807,6 +9191,41 @@ static int wpas_global_ctrl_iface_set(struct wpa_global *global, char *cmd)
 }
 
 
+static int wpas_global_ctrl_iface_dup_network(struct wpa_global *global,
+					      char *cmd)
+{
+	struct wpa_supplicant *wpa_s[2]; /* src, dst */
+	char *p;
+	unsigned int i;
+
+	/* cmd: "<src ifname> <dst ifname> <src network id> <dst network id>
+	 * <variable name> */
+
+	for (i = 0; i < ARRAY_SIZE(wpa_s) ; i++) {
+		p = os_strchr(cmd, ' ');
+		if (p == NULL)
+			return -1;
+		*p = '\0';
+
+		wpa_s[i] = global->ifaces;
+		for (; wpa_s[i]; wpa_s[i] = wpa_s[i]->next) {
+			if (os_strcmp(cmd, wpa_s[i]->ifname) == 0)
+				break;
+		}
+
+		if (!wpa_s[i]) {
+			wpa_printf(MSG_DEBUG,
+				   "CTRL_IFACE: Could not find iface=%s", cmd);
+			return -1;
+		}
+
+		cmd = p + 1;
+	}
+
+	return wpa_supplicant_ctrl_iface_dup_network(wpa_s[0], cmd, wpa_s[1]);
+}
+
+
 #ifndef CONFIG_NO_CONFIG_WRITE
 static int wpas_global_ctrl_iface_save_config(struct wpa_global *global)
 {
@@ -8888,6 +9307,59 @@ static int wpas_global_ctrl_iface_status(struct wpa_global *global,
 }
 
 
+#ifdef CONFIG_FST
+
+static int wpas_global_ctrl_iface_fst_attach(struct wpa_global *global,
+					     char *cmd, char *buf,
+					     size_t reply_size)
+{
+	char ifname[IFNAMSIZ + 1];
+	struct fst_iface_cfg cfg;
+	struct wpa_supplicant *wpa_s;
+	struct fst_wpa_obj iface_obj;
+
+	if (!fst_parse_attach_command(cmd, ifname, sizeof(ifname), &cfg)) {
+		wpa_s = wpa_supplicant_get_iface(global, ifname);
+		if (wpa_s) {
+			if (wpa_s->fst) {
+				wpa_printf(MSG_INFO, "FST: Already attached");
+				return -1;
+			}
+			fst_wpa_supplicant_fill_iface_obj(wpa_s, &iface_obj);
+			wpa_s->fst = fst_attach(ifname, wpa_s->own_addr,
+						&iface_obj, &cfg);
+			if (wpa_s->fst)
+				return os_snprintf(buf, reply_size, "OK\n");
+		}
+	}
+
+	return -1;
+}
+
+
+static int wpas_global_ctrl_iface_fst_detach(struct wpa_global *global,
+					     char *cmd, char *buf,
+					     size_t reply_size)
+{
+	char ifname[IFNAMSIZ + 1];
+	struct wpa_supplicant *wpa_s;
+
+	if (!fst_parse_detach_command(cmd, ifname, sizeof(ifname))) {
+		wpa_s = wpa_supplicant_get_iface(global, ifname);
+		if (wpa_s) {
+			if (!fst_iface_detach(ifname)) {
+				wpa_s->fst = NULL;
+				return os_snprintf(buf, reply_size, "OK\n");
+			}
+		}
+	}
+
+	return -1;
+}
+
+#endif /* CONFIG_FST */
+
+
 char * wpa_supplicant_global_ctrl_iface_process(struct wpa_global *global,
 						char *buf, size_t *resp_len)
 {
@@ -8939,6 +9411,18 @@ char * wpa_supplicant_global_ctrl_iface_process(struct wpa_global *global,
 	} else if (os_strcmp(buf, "INTERFACES") == 0) {
 		reply_len = wpa_supplicant_global_iface_interfaces(
 			global, reply, reply_size);
+#ifdef CONFIG_FST
+	} else if (os_strncmp(buf, "FST-ATTACH ", 11) == 0) {
+		reply_len = wpas_global_ctrl_iface_fst_attach(global, buf + 11,
+							      reply,
+							      reply_size);
+	} else if (os_strncmp(buf, "FST-DETACH ", 11) == 0) {
+		reply_len = wpas_global_ctrl_iface_fst_detach(global, buf + 11,
+							      reply,
+							      reply_size);
+	} else if (os_strncmp(buf, "FST-MANAGER ", 12) == 0) {
+		reply_len = fst_ctrl_iface_receive(buf + 12, reply, reply_size);
+#endif /* CONFIG_FST */
 	} else if (os_strcmp(buf, "TERMINATE") == 0) {
 		wpa_supplicant_terminate_proc(global);
 	} else if (os_strcmp(buf, "SUSPEND") == 0) {
@@ -8959,6 +9443,9 @@ char * wpa_supplicant_global_ctrl_iface_process(struct wpa_global *global,
 #endif /* CONFIG_P2P */
 			reply_len = -1;
 		}
+	} else if (os_strncmp(buf, "DUP_NETWORK ", 12) == 0) {
+		if (wpas_global_ctrl_iface_dup_network(global, buf + 12))
+			reply_len = -1;
 #ifndef CONFIG_NO_CONFIG_WRITE
 	} else if (os_strcmp(buf, "SAVE_CONFIG") == 0) {
 		if (wpas_global_ctrl_iface_save_config(global))
diff --git a/wpa_supplicant/ctrl_iface_named_pipe.c b/wpa_supplicant/ctrl_iface_named_pipe.c
index dc02db213a48..54e0e2fac583 100644
--- a/wpa_supplicant/ctrl_iface_named_pipe.c
+++ b/wpa_supplicant/ctrl_iface_named_pipe.c
@@ -423,7 +423,8 @@ static int ctrl_iface_parse(struct ctrl_iface_priv *priv, const char *params)
 }
 
 
-static void wpa_supplicant_ctrl_iface_msg_cb(void *ctx, int level, int global,
+static void wpa_supplicant_ctrl_iface_msg_cb(void *ctx, int level,
+					     enum wpa_msg_type type,
 					     const char *txt, size_t len)
 {
 	struct wpa_supplicant *wpa_s = ctx;
diff --git a/wpa_supplicant/ctrl_iface_udp.c b/wpa_supplicant/ctrl_iface_udp.c
index bf6a3df6c3c5..76f69f2b57bb 100644
--- a/wpa_supplicant/ctrl_iface_udp.c
+++ b/wpa_supplicant/ctrl_iface_udp.c
@@ -322,7 +322,8 @@ static void wpa_supplicant_ctrl_iface_receive(int sock, void *eloop_ctx,
 }
 
 
-static void wpa_supplicant_ctrl_iface_msg_cb(void *ctx, int level, int global,
+static void wpa_supplicant_ctrl_iface_msg_cb(void *ctx, int level,
+					     enum wpa_msg_type type,
 					     const char *txt, size_t len)
 {
 	struct wpa_supplicant *wpa_s = ctx;
diff --git a/wpa_supplicant/ctrl_iface_unix.c b/wpa_supplicant/ctrl_iface_unix.c
index b1ac76668897..11f281473130 100644
--- a/wpa_supplicant/ctrl_iface_unix.c
+++ b/wpa_supplicant/ctrl_iface_unix.c
@@ -13,6 +13,10 @@
 #include <stddef.h>
 #include <unistd.h>
 #include <fcntl.h>
+#ifdef __linux__
+#include <sys/ioctl.h>
+#include <linux/sockios.h>
+#endif /* __linux__ */
 #ifdef ANDROID
 #include <cutils/sockets.h>
 #endif /* ANDROID */
@@ -72,6 +76,32 @@ static int wpas_ctrl_iface_global_reinit(struct wpa_global *global,
 					 struct ctrl_iface_global_priv *priv);
 
 
+static void wpas_ctrl_sock_debug(const char *title, int sock, const char *buf,
+				 size_t len)
+{
+#ifdef __linux__
+	socklen_t optlen;
+	int sndbuf, outq;
+	int level = MSG_MSGDUMP;
+
+	if (len >= 5 && os_strncmp(buf, "PONG\n", 5) == 0)
+		level = MSG_EXCESSIVE;
+
+	optlen = sizeof(sndbuf);
+	sndbuf = 0;
+	if (getsockopt(sock, SOL_SOCKET, SO_SNDBUF, &sndbuf, &optlen) < 0)
+		sndbuf = -1;
+
+	if (ioctl(sock, SIOCOUTQ, &outq) < 0)
+		outq = -1;
+
+	wpa_printf(level,
+		   "CTRL-DEBUG: %s: sock=%d sndbuf=%d outq=%d send_len=%d",
+		   title, sock, sndbuf, outq, (int) len);
+#endif /* __linux__ */
+}
+
+
 static int wpa_supplicant_ctrl_iface_attach(struct dl_list *ctrl_dst,
 					    struct sockaddr_un *from,
 					    socklen_t fromlen, int global)
@@ -197,6 +227,13 @@ static void wpa_supplicant_ctrl_iface_receive(int sock, void *eloop_ctx,
 		reply_buf = wpa_supplicant_ctrl_iface_process(wpa_s, buf,
 							      &reply_len);
 		reply = reply_buf;
+
+		/*
+		 * There could be some password/key material in the command, so
+		 * clear the buffer explicitly now that it is not needed
+		 * anymore.
+		 */
+		os_memset(buf, 0, res);
 	}
 
 	if (!reply && reply_len == 1) {
@@ -208,6 +245,8 @@ static void wpa_supplicant_ctrl_iface_receive(int sock, void *eloop_ctx,
 	}
 
 	if (reply) {
+		wpas_ctrl_sock_debug("ctrl_sock-sendto", sock, reply,
+				     reply_len);
 		if (sendto(sock, reply, reply_len, 0, (struct sockaddr *) &from,
 			   fromlen) < 0) {
 			int _errno = errno;
@@ -295,7 +334,8 @@ static char * wpa_supplicant_ctrl_iface_path(struct wpa_supplicant *wpa_s)
 }
 
 
-static void wpa_supplicant_ctrl_iface_msg_cb(void *ctx, int level, int global,
+static void wpa_supplicant_ctrl_iface_msg_cb(void *ctx, int level,
+					     enum wpa_msg_type type,
 					     const char *txt, size_t len)
 {
 	struct wpa_supplicant *wpa_s = ctx;
@@ -303,19 +343,19 @@ static void wpa_supplicant_ctrl_iface_msg_cb(void *ctx, int level, int global,
 	if (wpa_s == NULL)
 		return;
 
-	if (global != 2 && wpa_s->global->ctrl_iface) {
+	if (type != WPA_MSG_NO_GLOBAL && wpa_s->global->ctrl_iface) {
 		struct ctrl_iface_global_priv *priv = wpa_s->global->ctrl_iface;
 		if (!dl_list_empty(&priv->ctrl_dst)) {
-			wpa_supplicant_ctrl_iface_send(wpa_s, global ? NULL :
-						       wpa_s->ifname,
-						       priv->sock,
-						       &priv->ctrl_dst,
-						       level, txt, len, NULL,
-						       priv);
+			wpa_supplicant_ctrl_iface_send(
+				wpa_s,
+				type != WPA_MSG_PER_INTERFACE ?
+				NULL : wpa_s->ifname,
+				priv->sock, &priv->ctrl_dst, level, txt, len,
+				NULL, priv);
 		}
 	}
 
-	if (wpa_s->ctrl_iface == NULL)
+	if (type == WPA_MSG_ONLY_GLOBAL || wpa_s->ctrl_iface == NULL)
 		return;
 	wpa_supplicant_ctrl_iface_send(wpa_s, NULL, wpa_s->ctrl_iface->sock,
 				       &wpa_s->ctrl_iface->ctrl_dst,
@@ -544,6 +584,53 @@ wpa_supplicant_ctrl_iface_init(struct wpa_supplicant *wpa_s)
 	if (wpa_s->conf->ctrl_interface == NULL)
 		return priv;
 
+#ifdef ANDROID
+	if (wpa_s->global->params.ctrl_interface) {
+		int same = 0;
+
+		if (wpa_s->global->params.ctrl_interface[0] == '/') {
+			if (os_strcmp(wpa_s->global->params.ctrl_interface,
+				      wpa_s->conf->ctrl_interface) == 0)
+				same = 1;
+		} else if (os_strncmp(wpa_s->global->params.ctrl_interface,
+				      "@android:", 9) == 0 ||
+			   os_strncmp(wpa_s->global->params.ctrl_interface,
+				      "@abstract:", 10) == 0) {
+			char *pos;
+
+			/*
+			 * Currently, Android uses @android:wpa_* as the naming
+			 * convention for the global ctrl interface. This logic
+			 * needs to be revisited if the above naming convention
+			 * is modified.
+			 */
+			pos = os_strchr(wpa_s->global->params.ctrl_interface,
+					'_');
+			if (pos &&
+			    os_strcmp(pos + 1,
+				      wpa_s->conf->ctrl_interface) == 0)
+				same = 1;
+		}
+
+		if (same) {
+			/*
+			 * The invalid configuration combination might be
+			 * possible to hit in an Android OTA upgrade case, so
+			 * instead of refusing to start the wpa_supplicant
+			 * process, do not open the per-interface ctrl_iface
+			 * and continue with the global control interface that
+			 * was set from the command line since the Wi-Fi
+			 * framework will use it for operations.
+			 */
+			wpa_printf(MSG_ERROR,
+				   "global ctrl interface %s matches ctrl interface %s - do not open per-interface ctrl interface",
+				   wpa_s->global->params.ctrl_interface,
+				   wpa_s->conf->ctrl_interface);
+			return priv;
+		}
+	}
+#endif /* ANDROID */
+
 	if (wpas_ctrl_iface_open_sock(wpa_s, priv) < 0) {
 		os_free(priv);
 		return NULL;
@@ -708,8 +795,10 @@ static void wpa_supplicant_ctrl_iface_send(struct wpa_supplicant *wpa_s,
 			      offsetof(struct sockaddr_un, sun_path));
 		msg.msg_name = (void *) &dst->addr;
 		msg.msg_namelen = dst->addrlen;
+		wpas_ctrl_sock_debug("ctrl_sock-sendmsg", sock, buf, len);
 		if (sendmsg(sock, &msg, MSG_DONTWAIT) >= 0) {
-			wpa_printf(MSG_DEBUG, "CTRL_IFACE monitor sent successfully to %s",
+			wpa_printf(MSG_MSGDUMP,
+				   "CTRL_IFACE monitor sent successfully to %s",
 				   addr_txt);
 			dst->errors = 0;
 			continue;
@@ -846,6 +935,13 @@ static void wpa_supplicant_global_ctrl_iface_receive(int sock, void *eloop_ctx,
 		reply_buf = wpa_supplicant_global_ctrl_iface_process(
 			global, buf, &reply_len);
 		reply = reply_buf;
+
+		/*
+		 * There could be some password/key material in the command, so
+		 * clear the buffer explicitly now that it is not needed
+		 * anymore.
+		 */
+		os_memset(buf, 0, res);
 	}
 
 	if (!reply && reply_len == 1) {
@@ -857,6 +953,8 @@ static void wpa_supplicant_global_ctrl_iface_receive(int sock, void *eloop_ctx,
 	}
 
 	if (reply) {
+		wpas_ctrl_sock_debug("global_ctrl_sock-sendto",
+				     sock, reply, reply_len);
 		if (sendto(sock, reply, reply_len, 0, (struct sockaddr *) &from,
 			   fromlen) < 0) {
 			wpa_printf(MSG_DEBUG, "ctrl_iface sendto failed: %s",
diff --git a/wpa_supplicant/dbus/dbus_new.c b/wpa_supplicant/dbus/dbus_new.c
index 30ef03a7453b..67d0e2877a47 100644
--- a/wpa_supplicant/dbus/dbus_new.c
+++ b/wpa_supplicant/dbus/dbus_new.c
@@ -137,7 +137,7 @@ static void wpas_dbus_signal_interface(struct wpa_supplicant *wpa_s,
 	iface = wpa_s->global->dbus;
 
 	/* Do nothing if the control interface is not turned on */
-	if (iface == NULL)
+	if (iface == NULL || !wpa_s->dbus_new_path)
 		return;
 
 	msg = dbus_message_new_signal(WPAS_DBUS_NEW_PATH,
@@ -200,7 +200,7 @@ void wpas_dbus_signal_scan_done(struct wpa_supplicant *wpa_s, int success)
 	iface = wpa_s->global->dbus;
 
 	/* Do nothing if the control interface is not turned on */
-	if (iface == NULL)
+	if (iface == NULL || !wpa_s->dbus_new_path)
 		return;
 
 	msg = dbus_message_new_signal(wpa_s->dbus_new_path,
@@ -239,7 +239,7 @@ static void wpas_dbus_signal_bss(struct wpa_supplicant *wpa_s,
 	iface = wpa_s->global->dbus;
 
 	/* Do nothing if the control interface is not turned on */
-	if (iface == NULL)
+	if (iface == NULL || !wpa_s->dbus_new_path)
 		return;
 
 	msg = dbus_message_new_signal(wpa_s->dbus_new_path,
@@ -307,7 +307,7 @@ static void wpas_dbus_signal_blob(struct wpa_supplicant *wpa_s,
 	iface = wpa_s->global->dbus;
 
 	/* Do nothing if the control interface is not turned on */
-	if (iface == NULL)
+	if (iface == NULL || !wpa_s->dbus_new_path)
 		return;
 
 	msg = dbus_message_new_signal(wpa_s->dbus_new_path,
@@ -374,7 +374,7 @@ static void wpas_dbus_signal_network(struct wpa_supplicant *wpa_s,
 	iface = wpa_s->global->dbus;
 
 	/* Do nothing if the control interface is not turned on */
-	if (iface == NULL)
+	if (iface == NULL || !wpa_s->dbus_new_path)
 		return;
 
 	os_snprintf(net_obj_path, WPAS_DBUS_OBJECT_PATH_MAX,
@@ -467,7 +467,7 @@ void wpas_dbus_signal_network_request(struct wpa_supplicant *wpa_s,
 	iface = wpa_s->global->dbus;
 
 	/* Do nothing if the control interface is not turned on */
-	if (iface == NULL)
+	if (iface == NULL || !wpa_s->dbus_new_path)
 		return;
 
 	field = wpa_supplicant_ctrl_req_to_string(rtype, default_txt, &txt);
@@ -511,6 +511,8 @@ void wpas_dbus_signal_network_enabled_changed(struct wpa_supplicant *wpa_s,
 
 	char path[WPAS_DBUS_OBJECT_PATH_MAX];
 
+	if (!wpa_s->dbus_new_path)
+		return;
 	os_snprintf(path, WPAS_DBUS_OBJECT_PATH_MAX,
 		    "%s/" WPAS_DBUS_NEW_NETWORKS_PART "/%d",
 		    wpa_s->dbus_new_path, ssid->id);
@@ -522,6 +524,44 @@ void wpas_dbus_signal_network_enabled_changed(struct wpa_supplicant *wpa_s,
 
 #ifdef CONFIG_WPS
 
+/**
+ * wpas_dbus_signal_wps_event_pbc_overlap - Signals PBC overlap WPS event
+ * @wpa_s: %wpa_supplicant network interface data
+ *
+ * Sends Event dbus signal with name "pbc-overlap" and empty dict as arguments
+ */
+void wpas_dbus_signal_wps_event_pbc_overlap(struct wpa_supplicant *wpa_s)
+{
+
+	DBusMessage *msg;
+	DBusMessageIter iter, dict_iter;
+	struct wpas_dbus_priv *iface;
+	char *key = "pbc-overlap";
+
+	iface = wpa_s->global->dbus;
+
+	/* Do nothing if the control interface is not turned on */
+	if (iface == NULL || !wpa_s->dbus_new_path)
+		return;
+
+	msg = dbus_message_new_signal(wpa_s->dbus_new_path,
+				      WPAS_DBUS_NEW_IFACE_WPS, "Event");
+	if (msg == NULL)
+		return;
+
+	dbus_message_iter_init_append(msg, &iter);
+
+	if (!dbus_message_iter_append_basic(&iter, DBUS_TYPE_STRING, &key) ||
+	    !wpa_dbus_dict_open_write(&iter, &dict_iter) ||
+	    !wpa_dbus_dict_close_write(&iter, &dict_iter))
+		wpa_printf(MSG_ERROR, "dbus: Failed to construct signal");
+	else
+		dbus_connection_send(iface->con, msg, NULL);
+
+	dbus_message_unref(msg);
+}
+
+
 /**
  * wpas_dbus_signal_wps_event_success - Signals Success WPS event
  * @wpa_s: %wpa_supplicant network interface data
@@ -539,7 +579,7 @@ void wpas_dbus_signal_wps_event_success(struct wpa_supplicant *wpa_s)
 	iface = wpa_s->global->dbus;
 
 	/* Do nothing if the control interface is not turned on */
-	if (iface == NULL)
+	if (iface == NULL || !wpa_s->dbus_new_path)
 		return;
 
 	msg = dbus_message_new_signal(wpa_s->dbus_new_path,
@@ -563,6 +603,7 @@ void wpas_dbus_signal_wps_event_success(struct wpa_supplicant *wpa_s)
 /**
  * wpas_dbus_signal_wps_event_fail - Signals Fail WPS event
  * @wpa_s: %wpa_supplicant network interface data
+ * @fail: WPS failure information
  *
  * Sends Event dbus signal with name "fail" and dictionary containing
  * "msg field with fail message number (int32) as arguments
@@ -579,7 +620,7 @@ void wpas_dbus_signal_wps_event_fail(struct wpa_supplicant *wpa_s,
 	iface = wpa_s->global->dbus;
 
 	/* Do nothing if the control interface is not turned on */
-	if (iface == NULL)
+	if (iface == NULL || !wpa_s->dbus_new_path)
 		return;
 
 	msg = dbus_message_new_signal(wpa_s->dbus_new_path,
@@ -592,6 +633,10 @@ void wpas_dbus_signal_wps_event_fail(struct wpa_supplicant *wpa_s,
 	if (!dbus_message_iter_append_basic(&iter, DBUS_TYPE_STRING, &key) ||
 	    !wpa_dbus_dict_open_write(&iter, &dict_iter) ||
 	    !wpa_dbus_dict_append_int32(&dict_iter, "msg", fail->msg) ||
+	    !wpa_dbus_dict_append_int32(&dict_iter, "config_error",
+					fail->config_error) ||
+	    !wpa_dbus_dict_append_int32(&dict_iter, "error_indication",
+					fail->error_indication) ||
 	    !wpa_dbus_dict_close_write(&iter, &dict_iter))
 		wpa_printf(MSG_ERROR, "dbus: Failed to construct signal");
 	else
@@ -604,6 +649,7 @@ void wpas_dbus_signal_wps_event_fail(struct wpa_supplicant *wpa_s,
 /**
  * wpas_dbus_signal_wps_event_m2d - Signals M2D WPS event
  * @wpa_s: %wpa_supplicant network interface data
+ * @m2d: M2D event data information
  *
  * Sends Event dbus signal with name "m2d" and dictionary containing
  * fields of wps_event_m2d structure.
@@ -620,7 +666,7 @@ void wpas_dbus_signal_wps_event_m2d(struct wpa_supplicant *wpa_s,
 	iface = wpa_s->global->dbus;
 
 	/* Do nothing if the control interface is not turned on */
-	if (iface == NULL)
+	if (iface == NULL || !wpa_s->dbus_new_path)
 		return;
 
 	msg = dbus_message_new_signal(wpa_s->dbus_new_path,
@@ -669,6 +715,7 @@ void wpas_dbus_signal_wps_event_m2d(struct wpa_supplicant *wpa_s,
 /**
  * wpas_dbus_signal_wps_cred - Signals new credentials
  * @wpa_s: %wpa_supplicant network interface data
+ * @cred: WPS Credential information
  *
  * Sends signal with credentials in directory argument
  */
@@ -686,7 +733,7 @@ void wpas_dbus_signal_wps_cred(struct wpa_supplicant *wpa_s,
 	iface = wpa_s->global->dbus;
 
 	/* Do nothing if the control interface is not turned on */
-	if (iface == NULL)
+	if (iface == NULL || !wpa_s->dbus_new_path)
 		return;
 
 	msg = dbus_message_new_signal(wpa_s->dbus_new_path,
@@ -760,7 +807,7 @@ void wpas_dbus_signal_certification(struct wpa_supplicant *wpa_s,
 	iface = wpa_s->global->dbus;
 
 	/* Do nothing if the control interface is not turned on */
-	if (iface == NULL)
+	if (iface == NULL || !wpa_s->dbus_new_path)
 		return;
 
 	msg = dbus_message_new_signal(wpa_s->dbus_new_path,
@@ -801,7 +848,7 @@ void wpas_dbus_signal_eap_status(struct wpa_supplicant *wpa_s,
 	iface = wpa_s->global->dbus;
 
 	/* Do nothing if the control interface is not turned on */
-	if (iface == NULL)
+	if (iface == NULL || !wpa_s->dbus_new_path)
 		return;
 
 	msg = dbus_message_new_signal(wpa_s->dbus_new_path,
@@ -844,7 +891,7 @@ static void wpas_dbus_signal_sta(struct wpa_supplicant *wpa_s,
 	iface = wpa_s->global->dbus;
 
 	/* Do nothing if the control interface is not turned on */
-	if (iface == NULL)
+	if (iface == NULL || !wpa_s->dbus_new_path)
 		return;
 
 	msg = dbus_message_new_signal(wpa_s->dbus_new_path,
@@ -916,7 +963,8 @@ void wpas_dbus_signal_p2p_group_removed(struct wpa_supplicant *wpa_s,
 	if (parent->p2p_mgmt)
 		parent = parent->parent;
 
-	if (!wpa_s->dbus_groupobj_path)
+	if (!wpa_s->dbus_groupobj_path || !wpa_s->dbus_new_path ||
+	    !parent->dbus_new_path)
 		return;
 
 	msg = dbus_message_new_signal(parent->dbus_new_path,
@@ -984,6 +1032,8 @@ void wpas_dbus_signal_p2p_provision_discovery(struct wpa_supplicant *wpa_s,
 
 	if (wpa_s->p2p_mgmt)
 		wpa_s = wpa_s->parent;
+	if (!wpa_s->dbus_new_path)
+		return;
 
 	if (request || !status) {
 		if (config_methods & WPS_CONFIG_DISPLAY)
@@ -1057,8 +1107,19 @@ error:
 }
 
 
+/**
+ * wpas_dbus_signal_p2p_go_neg_req - Signal P2P GO Negotiation Request RX
+ * @wpa_s: %wpa_supplicant network interface data
+ * @src: Source address of the message triggering this notification
+ * @dev_passwd_id: WPS Device Password Id
+ * @go_intent: Peer's GO Intent value
+ *
+ * Sends signal to notify that a peer P2P Device is requesting group owner
+ * negotiation with us.
+ */
 void wpas_dbus_signal_p2p_go_neg_req(struct wpa_supplicant *wpa_s,
-				     const u8 *src, u16 dev_passwd_id)
+				     const u8 *src, u16 dev_passwd_id,
+				     u8 go_intent)
 {
 	DBusMessage *msg;
 	DBusMessageIter iter;
@@ -1073,6 +1134,8 @@ void wpas_dbus_signal_p2p_go_neg_req(struct wpa_supplicant *wpa_s,
 
 	if (wpa_s->p2p_mgmt)
 		wpa_s = wpa_s->parent;
+	if (!wpa_s->dbus_new_path)
+		return;
 
 	os_snprintf(peer_obj_path, WPAS_DBUS_OBJECT_PATH_MAX,
 		    "%s/" WPAS_DBUS_NEW_P2P_PEERS_PART "/" COMPACT_MACSTR,
@@ -1090,7 +1153,9 @@ void wpas_dbus_signal_p2p_go_neg_req(struct wpa_supplicant *wpa_s,
 	if (!dbus_message_iter_append_basic(&iter, DBUS_TYPE_OBJECT_PATH,
 					    &path) ||
 	    !dbus_message_iter_append_basic(&iter, DBUS_TYPE_UINT16,
-					    &dev_passwd_id))
+					    &dev_passwd_id) ||
+	    !dbus_message_iter_append_basic(&iter, DBUS_TYPE_BYTE,
+					    &go_intent))
 		wpa_printf(MSG_ERROR, "dbus: Failed to construct signal");
 	else
 		dbus_connection_send(iface->con, msg, NULL);
@@ -1105,7 +1170,8 @@ static int wpas_dbus_get_group_obj_path(struct wpa_supplicant *wpa_s,
 {
 	char group_name[3];
 
-	if (os_memcmp(ssid->ssid, P2P_WILDCARD_SSID, P2P_WILDCARD_SSID_LEN))
+	if (!wpa_s->dbus_new_path ||
+	    os_memcmp(ssid->ssid, P2P_WILDCARD_SSID, P2P_WILDCARD_SSID_LEN))
 		return -1;
 
 	os_memcpy(group_name, ssid->ssid + P2P_WILDCARD_SSID_LEN, 2);
@@ -1209,7 +1275,7 @@ void wpas_dbus_signal_p2p_group_started(struct wpa_supplicant *wpa_s,
 	iface = parent->global->dbus;
 
 	/* Do nothing if the control interface is not turned on */
-	if (iface == NULL)
+	if (iface == NULL || !parent->dbus_new_path || !wpa_s->dbus_new_path)
 		return;
 
 	if (wpa_s->dbus_groupobj_path == NULL)
@@ -1248,10 +1314,9 @@ void wpas_dbus_signal_p2p_group_started(struct wpa_supplicant *wpa_s,
 
 
 /**
- *
- * Method to emit GONegotiation Success or Failure signals based
- * on status.
- * @status: Status of the GO neg request. 0 for success, other for errors.
+ * wpas_dbus_signal_p2p_go_neg_resp - Emit GONegotiation Success/Failure signal
+ * @wpa_s: %wpa_supplicant network interface data
+ * @res: Result of the GO Neg Request
  */
 void wpas_dbus_signal_p2p_go_neg_resp(struct wpa_supplicant *wpa_s,
 				      struct p2p_go_neg_results *res)
@@ -1272,7 +1337,7 @@ void wpas_dbus_signal_p2p_go_neg_resp(struct wpa_supplicant *wpa_s,
 
 	os_memset(freqs, 0, sizeof(freqs));
 	/* Do nothing if the control interface is not turned on */
-	if (iface == NULL)
+	if (iface == NULL || !wpa_s->dbus_new_path)
 		return;
 
 	os_snprintf(peer_obj_path, WPAS_DBUS_OBJECT_PATH_MAX,
@@ -1363,12 +1428,10 @@ err:
 
 
 /**
- *
- * Method to emit Invitation Result signal based on status and
- * bssid
- * @status: Status of the Invite request. 0 for success, other
- * for errors
- * @bssid : Basic Service Set Identifier
+ * wpas_dbus_signal_p2p_invitation_result - Emit InvitationResult signal
+ * @wpa_s: %wpa_supplicant network interface data
+ * @status: Status of invitation process
+ * @bssid: Basic Service Set Identifier
  */
 void wpas_dbus_signal_p2p_invitation_result(struct wpa_supplicant *wpa_s,
 					    int status, const u8 *bssid)
@@ -1386,6 +1449,8 @@ void wpas_dbus_signal_p2p_invitation_result(struct wpa_supplicant *wpa_s,
 
 	if (wpa_s->p2p_mgmt)
 		wpa_s = wpa_s->parent;
+	if (!wpa_s->dbus_new_path)
+		return;
 
 	msg = dbus_message_new_signal(wpa_s->dbus_new_path,
 				      WPAS_DBUS_NEW_IFACE_P2PDEVICE,
@@ -1439,6 +1504,8 @@ void wpas_dbus_signal_p2p_peer_joined(struct wpa_supplicant *wpa_s,
 	parent = wpa_s->parent;
 	if (parent->p2p_mgmt)
 		parent = parent->parent;
+	if (!parent->dbus_new_path)
+		return;
 
 	os_snprintf(peer_obj_path, WPAS_DBUS_OBJECT_PATH_MAX,
 			"%s/" WPAS_DBUS_NEW_P2P_PEERS_PART "/"
@@ -1494,6 +1561,8 @@ void wpas_dbus_signal_p2p_peer_disconnected(struct wpa_supplicant *wpa_s,
 	parent = wpa_s->parent;
 	if (parent->p2p_mgmt)
 		parent = parent->parent;
+	if (!parent->dbus_new_path)
+		return;
 
 	os_snprintf(peer_obj_path, WPAS_DBUS_OBJECT_PATH_MAX,
 			"%s/" WPAS_DBUS_NEW_P2P_PEERS_PART "/"
@@ -1551,6 +1620,8 @@ void wpas_dbus_signal_p2p_sd_request(struct wpa_supplicant *wpa_s,
 
 	if (wpa_s->p2p_mgmt)
 		wpa_s = wpa_s->parent;
+	if (!wpa_s->dbus_new_path)
+		return;
 
 	/* Check if this is a known peer */
 	if (!p2p_peer_known(wpa_s->global->p2p, sa))
@@ -1617,6 +1688,8 @@ void wpas_dbus_signal_p2p_sd_response(struct wpa_supplicant *wpa_s,
 
 	if (wpa_s->p2p_mgmt)
 		wpa_s = wpa_s->parent;
+	if (!wpa_s->dbus_new_path)
+		return;
 
 	/* Check if this is a known peer */
 	if (!p2p_peer_known(wpa_s->global->p2p, sa))
@@ -1678,6 +1751,8 @@ static void wpas_dbus_signal_persistent_group(struct wpa_supplicant *wpa_s,
 
 	if (wpa_s->p2p_mgmt)
 		wpa_s = wpa_s->parent;
+	if (!wpa_s->dbus_new_path)
+		return;
 
 	os_snprintf(pgrp_obj_path, WPAS_DBUS_OBJECT_PATH_MAX,
 		    "%s/" WPAS_DBUS_NEW_PERSISTENT_GROUPS_PART "/%u",
@@ -1740,6 +1815,7 @@ static void wpas_dbus_signal_persistent_group_removed(
 /**
  * wpas_dbus_signal_p2p_wps_failed - Signals WpsFailed event
  * @wpa_s: %wpa_supplicant network interface data
+ * @fail: WPS failure information
  *
  * Sends Event dbus signal with name "fail" and dictionary containing
  * "msg" field with fail message number (int32) as arguments
@@ -1762,6 +1838,8 @@ void wpas_dbus_signal_p2p_wps_failed(struct wpa_supplicant *wpa_s,
 	if (wpa_s->p2p_mgmt)
 		wpa_s = wpa_s->parent;
 
+	if (!wpa_s->dbus_new_path)
+		return;
 	msg = dbus_message_new_signal(wpa_s->dbus_new_path,
 				      WPAS_DBUS_NEW_IFACE_P2PDEVICE,
 				      "WpsFailed");
@@ -1783,6 +1861,98 @@ void wpas_dbus_signal_p2p_wps_failed(struct wpa_supplicant *wpa_s,
 	dbus_message_unref(msg);
 }
 
+
+/**
+ * wpas_dbus_signal_p2p_group_formation_failure - Signals GroupFormationFailure event
+ * @wpa_s: %wpa_supplicant network interface data
+ * @reason: indicates the reason code for group formation failure
+ *
+ * Sends Event dbus signal and string reason code when available.
+ */
+void wpas_dbus_signal_p2p_group_formation_failure(struct wpa_supplicant *wpa_s,
+						  const char *reason)
+{
+	DBusMessage *msg;
+	struct wpas_dbus_priv *iface;
+
+	iface = wpa_s->global->dbus;
+
+	/* Do nothing if the control interface is not turned on */
+	if (iface == NULL)
+		return;
+
+	msg = dbus_message_new_signal(wpa_s->dbus_new_path,
+				      WPAS_DBUS_NEW_IFACE_P2PDEVICE,
+				      "GroupFormationFailure");
+	if (msg == NULL)
+		return;
+
+	if (dbus_message_append_args(msg, DBUS_TYPE_STRING, &reason,
+				     DBUS_TYPE_INVALID))
+		dbus_connection_send(iface->con, msg, NULL);
+	else
+		wpa_printf(MSG_ERROR, "dbus: Failed to construct signal");
+
+	dbus_message_unref(msg);
+}
+
+
+/**
+ * wpas_dbus_signal_p2p_invitation_received - Emit InvitationReceived signal
+ * @wpa_s: %wpa_supplicant network interface data
+ * @sa: Source address of the Invitation Request
+ * @dev_add: GO Device Address
+ * @bssid: P2P Group BSSID or %NULL if not received
+ * @id: Persistent group id or %0 if not persistent group
+ * @op_freq: Operating frequency for the group
+ */
+
+void wpas_dbus_signal_p2p_invitation_received(struct wpa_supplicant *wpa_s,
+					      const u8 *sa, const u8 *dev_addr,
+					      const u8 *bssid, int id,
+					      int op_freq)
+{
+	DBusMessage *msg;
+	DBusMessageIter iter, dict_iter;
+	struct wpas_dbus_priv *iface;
+
+	iface = wpa_s->global->dbus;
+
+	/* Do nothing if the control interface is not turned on */
+	if (iface == NULL)
+		return;
+
+	msg = dbus_message_new_signal(wpa_s->dbus_new_path,
+				      WPAS_DBUS_NEW_IFACE_P2PDEVICE,
+				      "InvitationReceived");
+	if (msg == NULL)
+		return;
+
+	dbus_message_iter_init_append(msg, &iter);
+	if (!wpa_dbus_dict_open_write(&iter, &dict_iter) ||
+	    (sa &&
+	     !wpa_dbus_dict_append_byte_array(&dict_iter, "sa",
+					      (const char *) sa, ETH_ALEN)) ||
+	    (dev_addr &&
+	     !wpa_dbus_dict_append_byte_array(&dict_iter, "go_dev_addr",
+					      (const char *) dev_addr,
+					      ETH_ALEN)) ||
+	    (bssid &&
+	     !wpa_dbus_dict_append_byte_array(&dict_iter, "bssid",
+					      (const char *) bssid,
+					      ETH_ALEN)) ||
+	    (id &&
+	     !wpa_dbus_dict_append_int32(&dict_iter, "persistent_id", id)) ||
+	    !wpa_dbus_dict_append_int32(&dict_iter, "op_freq", op_freq) ||
+	    !wpa_dbus_dict_close_write(&iter, &dict_iter)) {
+		dbus_message_unref(msg);
+		return;
+	}
+
+	dbus_connection_send(iface->con, msg, NULL);
+}
+
+
 #endif /* CONFIG_P2P */
 
 
@@ -1862,6 +2032,9 @@ void wpas_dbus_bss_signal_prop_changed(struct wpa_supplicant *wpa_s,
 	char path[WPAS_DBUS_OBJECT_PATH_MAX];
 	char *prop;
 
+	if (!wpa_s->dbus_new_path)
+		return;
+
 	switch (property) {
 	case WPAS_DBUS_BSS_PROP_SIGNAL:
 		prop = "Signal";
@@ -2177,7 +2350,7 @@ int wpas_dbus_register_network(struct wpa_supplicant *wpa_s,
 #endif /* CONFIG_P2P */
 
 	/* Do nothing if the control interface is not turned on */
-	if (wpa_s == NULL || wpa_s->global == NULL)
+	if (wpa_s == NULL || wpa_s->global == NULL || !wpa_s->dbus_new_path)
 		return 0;
 	ctrl_iface = wpa_s->global->dbus;
 	if (ctrl_iface == NULL)
@@ -2351,7 +2524,7 @@ int wpas_dbus_unregister_bss(struct wpa_supplicant *wpa_s,
 	char bss_obj_path[WPAS_DBUS_OBJECT_PATH_MAX];
 
 	/* Do nothing if the control interface is not turned on */
-	if (wpa_s == NULL || wpa_s->global == NULL)
+	if (wpa_s == NULL || wpa_s->global == NULL || !wpa_s->dbus_new_path)
 		return 0;
 	ctrl_iface = wpa_s->global->dbus;
 	if (ctrl_iface == NULL)
@@ -2394,7 +2567,7 @@ int wpas_dbus_register_bss(struct wpa_supplicant *wpa_s,
 	struct bss_handler_args *arg;
 
 	/* Do nothing if the control interface is not turned on */
-	if (wpa_s == NULL || wpa_s->global == NULL)
+	if (wpa_s == NULL || wpa_s->global == NULL || !wpa_s->dbus_new_path)
 		return 0;
 	ctrl_iface = wpa_s->global->dbus;
 	if (ctrl_iface == NULL)
@@ -2486,6 +2659,12 @@ static const struct wpa_dbus_method_desc wpas_dbus_interface_methods[] = {
 		  END_ARGS
 	  }
 	},
+	{ "Reconnect", WPAS_DBUS_NEW_IFACE_INTERFACE,
+	  (WPADBusMethodHandler) wpas_dbus_handler_reconnect,
+	  {
+		  END_ARGS
+	  }
+	},
 	{ "RemoveNetwork", WPAS_DBUS_NEW_IFACE_INTERFACE,
 	  (WPADBusMethodHandler) wpas_dbus_handler_remove_network,
 	  {
@@ -2558,6 +2737,12 @@ static const struct wpa_dbus_method_desc wpas_dbus_interface_methods[] = {
 		  END_ARGS
 	  }
 	},
+	{ "Cancel", WPAS_DBUS_NEW_IFACE_WPS,
+	  (WPADBusMethodHandler) wpas_dbus_handler_wps_cancel,
+	  {
+		  END_ARGS
+	  }
+	},
 #endif /* CONFIG_WPS */
 #ifdef CONFIG_P2P
 	{ "Find", WPAS_DBUS_NEW_IFACE_P2PDEVICE,
@@ -2617,6 +2802,12 @@ static const struct wpa_dbus_method_desc wpas_dbus_interface_methods[] = {
 		  END_ARGS
 	  }
 	},
+	{ "Cancel", WPAS_DBUS_NEW_IFACE_P2PDEVICE,
+	  (WPADBusMethodHandler) wpas_dbus_handler_p2p_cancel,
+	  {
+		  END_ARGS
+	  }
+	},
 	{ "Invite", WPAS_DBUS_NEW_IFACE_P2PDEVICE,
 	  (WPADBusMethodHandler) wpas_dbus_handler_p2p_invite,
 	  {
@@ -2637,6 +2828,13 @@ static const struct wpa_dbus_method_desc wpas_dbus_interface_methods[] = {
 		  END_ARGS
 	  }
 	},
+	{ "RemoveClient", WPAS_DBUS_NEW_IFACE_P2PDEVICE,
+	  (WPADBusMethodHandler) wpas_dbus_handler_p2p_remove_client,
+	  {
+		  { "args", "a{sv}", ARG_IN },
+		  END_ARGS
+	  }
+	},
 	{ "Flush", WPAS_DBUS_NEW_IFACE_P2PDEVICE,
 	  (WPADBusMethodHandler) wpas_dbus_handler_p2p_flush,
 	  {
@@ -3014,6 +3212,11 @@ static const struct wpa_dbus_signal_desc wpas_dbus_interface_signals[] = {
 		  END_ARGS
 	  }
 	},
+	{ "FindStopped", WPAS_DBUS_NEW_IFACE_P2PDEVICE,
+	  {
+		  END_ARGS
+	  }
+	},
 	{ "ProvisionDiscoveryRequestDisplayPin", WPAS_DBUS_NEW_IFACE_P2PDEVICE,
 	  {
 		  { "peer_object", "o", ARG_OUT },
@@ -3065,6 +3268,12 @@ static const struct wpa_dbus_signal_desc wpas_dbus_interface_signals[] = {
 		  END_ARGS
 	  }
 	},
+	{ "GroupFormationFailure", WPAS_DBUS_NEW_IFACE_P2PDEVICE,
+	  {
+		  { "reason", "s", ARG_OUT },
+		  END_ARGS
+	  }
+	},
 	{ "GONegotiationSuccess", WPAS_DBUS_NEW_IFACE_P2PDEVICE,
 	  {
 		  { "properties", "a{sv}", ARG_OUT },
@@ -3080,7 +3289,8 @@ static const struct wpa_dbus_signal_desc wpas_dbus_interface_signals[] = {
 	{ "GONegotiationRequest", WPAS_DBUS_NEW_IFACE_P2PDEVICE,
 	  {
 		  { "path", "o", ARG_OUT },
-		  { "dev_passwd_id", "i", ARG_OUT },
+		  { "dev_passwd_id", "q", ARG_OUT },
+		  { "device_go_intent", "y", ARG_OUT },
 		  END_ARGS
 	  }
 	},
@@ -3128,6 +3338,12 @@ static const struct wpa_dbus_signal_desc wpas_dbus_interface_signals[] = {
 		  END_ARGS
 	  }
 	},
+	{ "InvitationReceived", WPAS_DBUS_NEW_IFACE_P2PDEVICE,
+	  {
+		  { "properties", "a{sv}", ARG_OUT },
+		  END_ARGS
+	  }
+	},
 #endif /* CONFIG_P2P */
 #ifdef CONFIG_AP
 	{ "ProbeRequest", WPAS_DBUS_NEW_IFACE_INTERFACE,
@@ -3174,6 +3390,11 @@ static const struct wpa_dbus_signal_desc wpas_dbus_interface_signals[] = {
 };
 
 
+/**
+ * wpas_dbus_register_interface - Register an interface with D-Bus
+ * @wpa_s: wpa_supplicant interface structure
+ * Returns: 0 on success, -1 on failure
+ */
 int wpas_dbus_register_interface(struct wpa_supplicant *wpa_s)
 {
 
@@ -3224,6 +3445,11 @@ err:
 }
 
 
+/**
+ * wpas_dbus_unregister_interface - Unregister the interface from D-Bus
+ * @wpa_s: wpa_supplicant interface structure
+ * Returns: 0 on success, -1 on failure
+ */
 int wpas_dbus_unregister_interface(struct wpa_supplicant *wpa_s)
 {
 	struct wpas_dbus_priv *ctrl_iface;
@@ -3265,6 +3491,22 @@ static const struct wpa_dbus_property_desc wpas_dbus_p2p_peer_properties[] = {
 	  wpas_dbus_getter_p2p_peer_device_name,
 	  NULL
 	},
+	{ "Manufacturer", WPAS_DBUS_NEW_IFACE_P2P_PEER, "s",
+	  wpas_dbus_getter_p2p_peer_manufacturer,
+	  NULL
+	},
+	{ "ModelName", WPAS_DBUS_NEW_IFACE_P2P_PEER, "s",
+	  wpas_dbus_getter_p2p_peer_modelname,
+	  NULL
+	},
+	{ "ModelNumber", WPAS_DBUS_NEW_IFACE_P2P_PEER, "s",
+	  wpas_dbus_getter_p2p_peer_modelnumber,
+	  NULL
+	},
+	{ "SerialNumber", WPAS_DBUS_NEW_IFACE_P2P_PEER, "s",
+	  wpas_dbus_getter_p2p_peer_serialnumber,
+	  NULL
+	},
 	{ "PrimaryDeviceType", WPAS_DBUS_NEW_IFACE_P2P_PEER, "ay",
 	  wpas_dbus_getter_p2p_peer_primary_device_type,
 	  NULL
@@ -3345,7 +3587,7 @@ static void wpas_dbus_signal_peer(struct wpa_supplicant *wpa_s,
 	iface = wpa_s->global->dbus;
 
 	/* Do nothing if the control interface is not turned on */
-	if (iface == NULL)
+	if (iface == NULL || !wpa_s->dbus_new_path)
 		return;
 
 	os_snprintf(peer_obj_path, WPAS_DBUS_OBJECT_PATH_MAX,
@@ -3372,7 +3614,7 @@ static void wpas_dbus_signal_peer(struct wpa_supplicant *wpa_s,
 /**
  * wpas_dbus_signal_peer_found - Send a peer found signal
  * @wpa_s: %wpa_supplicant network interface data
- * @dev: peer device object
+ * @dev_addr: Peer P2P Device Address
  *
  * Notify listeners about find a p2p peer device found
  */
@@ -3387,7 +3629,7 @@ void wpas_dbus_signal_peer_device_found(struct wpa_supplicant *wpa_s,
 /**
  * wpas_dbus_signal_peer_lost - Send a peer lost signal
  * @wpa_s: %wpa_supplicant network interface data
- * @dev: peer device object
+ * @dev_addr: Peer P2P Device Address
  *
  * Notify listeners about lost a p2p peer device
  */
@@ -3402,7 +3644,7 @@ void wpas_dbus_signal_peer_device_lost(struct wpa_supplicant *wpa_s,
 /**
  * wpas_dbus_register_peer - Register a discovered peer object with dbus
  * @wpa_s: wpa_supplicant interface structure
- * @ssid: network configuration data
+ * @dev_addr: P2P Device Address of the peer
  * Returns: 0 on success, -1 on failure
  *
  * Registers network representing object with dbus
@@ -3422,8 +3664,9 @@ int wpas_dbus_register_peer(struct wpa_supplicant *wpa_s, const u8 *dev_addr)
 	if (ctrl_iface == NULL)
 		return 0;
 
-	if (wpa_s->p2p_mgmt)
-		wpa_s = wpa_s->parent;
+	wpa_s = wpa_s->parent->parent;
+	if (!wpa_s->dbus_new_path)
+		return 0;
 
 	os_snprintf(peer_obj_path, WPAS_DBUS_OBJECT_PATH_MAX,
 		    "%s/" WPAS_DBUS_NEW_P2P_PEERS_PART "/" COMPACT_MACSTR,
@@ -3481,12 +3724,12 @@ int wpas_dbus_unregister_peer(struct wpa_supplicant *wpa_s,
 	int ret;
 
 	/* Do nothing if the control interface is not turned on */
-	if (wpa_s == NULL || wpa_s->global == NULL ||
-	    wpa_s->dbus_new_path == NULL)
+	if (wpa_s == NULL || wpa_s->global == NULL)
 		return 0;
 
-	if (wpa_s->p2p_mgmt)
-		wpa_s = wpa_s->parent;
+	wpa_s = wpa_s->parent->parent;
+	if (!wpa_s->dbus_new_path)
+		return 0;
 
 	ctrl_iface = wpa_s->global->dbus;
 	if (ctrl_iface == NULL)
@@ -3504,6 +3747,42 @@ int wpas_dbus_unregister_peer(struct wpa_supplicant *wpa_s,
 }
 
 
+/**
+ * wpas_dbus_signal_p2p_find_stopped - Send P2P Find stopped signal
+ * @wpa_s: %wpa_supplicant network interface data
+ *
+ * Notify listeners about P2P Find stopped
+ */
+void wpas_dbus_signal_p2p_find_stopped(struct wpa_supplicant *wpa_s)
+{
+	struct wpas_dbus_priv *iface;
+	DBusMessage *msg;
+
+	iface = wpa_s->global->dbus;
+
+	/* Do nothing if the control interface is not turned on */
+	if (iface == NULL || !wpa_s->dbus_new_path)
+		return;
+
+	msg = dbus_message_new_signal(wpa_s->dbus_new_path,
+				      WPAS_DBUS_NEW_IFACE_P2PDEVICE,
+				      "FindStopped");
+	if (msg == NULL)
+		return;
+
+	dbus_connection_send(iface->con, msg, NULL);
+
+	dbus_message_unref(msg);
+}
+
+
+/**
+ * wpas_dbus_signal_peer_groups_changed - Send peer group change property signal
+ * @wpa_s: %wpa_supplicant network interface data
+ * @dev_addr: P2P Device Address
+ *
+ * Notify listeners about peer Groups property changes.
+ */
 void wpas_dbus_signal_peer_groups_changed(struct wpa_supplicant *wpa_s,
 					  const u8 *dev_addr)
 {
@@ -3512,6 +3791,8 @@ void wpas_dbus_signal_peer_groups_changed(struct wpa_supplicant *wpa_s,
 	if (wpa_s->p2p_mgmt)
 		wpa_s = wpa_s->parent;
 
+	if (!wpa_s->dbus_new_path)
+		return;
 	os_snprintf(peer_obj_path, WPAS_DBUS_OBJECT_PATH_MAX,
 		    "%s/" WPAS_DBUS_NEW_P2P_PEERS_PART "/" COMPACT_MACSTR,
 		    wpa_s->dbus_new_path, MAC2STR(dev_addr));
@@ -3713,6 +3994,9 @@ int wpas_dbus_register_persistent_group(struct wpa_supplicant *wpa_s,
 	/* Do nothing if the control interface is not turned on */
 	if (wpa_s == NULL || wpa_s->global == NULL)
 		return 0;
+	wpa_s = wpa_s->parent->parent;
+	if (!wpa_s->dbus_new_path)
+		return 0;
 
 	/* Make sure ssid is a persistent group */
 	if (ssid->disabled != 2 && !ssid->p2p_persistent_group)
@@ -3797,15 +4081,13 @@ int wpas_dbus_unregister_persistent_group(struct wpa_supplicant *wpa_s,
 	int ret;
 
 	/* Do nothing if the control interface is not turned on */
-	if (wpa_s == NULL || wpa_s->global == NULL ||
-	    wpa_s->dbus_new_path == NULL)
+	if (wpa_s == NULL || wpa_s->global == NULL)
 		return 0;
 
-	if (wpa_s->p2p_mgmt)
-		wpa_s = wpa_s->parent;
+	wpa_s = wpa_s->parent->parent;
 
 	ctrl_iface = wpa_s->global->dbus;
-	if (ctrl_iface == NULL)
+	if (ctrl_iface == NULL || !wpa_s->dbus_new_path)
 		return 0;
 
 	os_snprintf(pgrp_obj_path, WPAS_DBUS_OBJECT_PATH_MAX,
diff --git a/wpa_supplicant/dbus/dbus_new.h b/wpa_supplicant/dbus/dbus_new.h
index d162d2b663df..6d240fffce78 100644
--- a/wpa_supplicant/dbus/dbus_new.h
+++ b/wpa_supplicant/dbus/dbus_new.h
@@ -152,6 +152,7 @@ void wpas_dbus_signal_wps_event_m2d(struct wpa_supplicant *wpa_s,
 void wpas_dbus_signal_wps_event_fail(struct wpa_supplicant *wpa_s,
 				     struct wps_event_fail *fail);
 void wpas_dbus_signal_wps_event_success(struct wpa_supplicant *wpa_s);
+void wpas_dbus_signal_wps_event_pbc_overlap(struct wpa_supplicant *wpa_s);
 int wpas_dbus_register_network(struct wpa_supplicant *wpa_s,
 			       struct wpa_ssid *ssid);
 int wpas_dbus_unregister_network(struct wpa_supplicant *wpa_s, int nid);
@@ -168,6 +169,7 @@ void wpas_dbus_signal_debug_timestamp_changed(struct wpa_global *global);
 void wpas_dbus_signal_debug_show_keys_changed(struct wpa_global *global);
 
 int wpas_dbus_register_peer(struct wpa_supplicant *wpa_s, const u8 *dev_addr);
+void wpas_dbus_signal_p2p_find_stopped(struct wpa_supplicant *wpa_s);
 void wpas_dbus_signal_peer_device_found(struct wpa_supplicant *wpa_s,
 					   const u8 *dev_addr);
 int wpas_dbus_unregister_peer(struct wpa_supplicant *wpa_s,
@@ -184,10 +186,13 @@ void wpas_dbus_signal_p2p_provision_discovery(struct wpa_supplicant *wpa_s,
 					      u16 config_methods,
 					      unsigned int generated_pin);
 void wpas_dbus_signal_p2p_go_neg_req(struct wpa_supplicant *wpa_s,
-				     const u8 *src, u16 dev_passwd_id);
+				     const u8 *src, u16 dev_passwd_id,
+				     u8 go_intent);
 void wpas_dbus_signal_p2p_group_started(struct wpa_supplicant *wpa_s,
 					const struct wpa_ssid *ssid,
 					int client, int network_id);
+void wpas_dbus_signal_p2p_group_formation_failure(struct wpa_supplicant *wpa_s,
+						  const char *reason);
 void wpas_dbus_register_p2p_group(struct wpa_supplicant *wpa_s,
 				  struct wpa_ssid *ssid);
 void wpas_dbus_signal_p2p_go_neg_resp(struct wpa_supplicant *wpa_s,
@@ -228,6 +233,10 @@ void wpas_dbus_signal_sta_authorized(struct wpa_supplicant *wpa_s,
 				     const u8 *sta);
 void wpas_dbus_signal_sta_deauthorized(struct wpa_supplicant *wpa_s,
 				       const u8 *sta);
+void wpas_dbus_signal_p2p_invitation_received(struct wpa_supplicant *wpa_s,
+					      const u8 *sa, const u8 *dev_addr,
+					      const u8 *bssid, int id,
+					      int op_freq);
 
 #else /* CONFIG_CTRL_IFACE_DBUS_NEW */
 
@@ -295,6 +304,11 @@ static inline void wpas_dbus_signal_wps_event_success(
 {
 }
 
+static inline void wpas_dbus_signal_wps_event_pbc_overlap(
+	struct wpa_supplicant *wpa_s)
+{
+}
+
 static inline int wpas_dbus_register_network(struct wpa_supplicant *wpa_s,
 					     struct wpa_ssid *ssid)
 {
@@ -377,10 +391,10 @@ wpas_dbus_signal_p2p_provision_discovery(struct wpa_supplicant *wpa_s,
 {
 }
 
-static inline void wpas_dbus_signal_p2p_go_neg_req(
-				struct wpa_supplicant *wpa_s,
-				const u8 *src,
-				u16 dev_passwd_id)
+static inline void wpas_dbus_signal_p2p_go_neg_req(struct wpa_supplicant *wpa_s,
+						   const u8 *src,
+						   u16 dev_passwd_id,
+						   u8 go_intent)
 {
 }
 
@@ -391,6 +405,12 @@ wpas_dbus_signal_p2p_group_started(struct wpa_supplicant *wpa_s,
 {
 }
 
+static inline void
+wpas_dbus_signal_p2p_group_formation_failure(struct wpa_supplicant *wpa_s,
+					     const char *reason)
+{
+}
+
 static inline void
 wpas_dbus_register_p2p_group(struct wpa_supplicant *wpa_s,
 			     struct wpa_ssid *ssid)
@@ -459,6 +479,11 @@ wpas_dbus_signal_p2p_peer_joined(struct wpa_supplicant *wpa_s,
 {
 }
 
+static inline void
+wpas_dbus_signal_p2p_find_stopped(struct wpa_supplicant *wpa_s)
+{
+}
+
 static inline void
 wpas_dbus_signal_peer_device_found(struct wpa_supplicant *wpa_s,
 				   const u8 *dev_addr)
@@ -519,6 +544,14 @@ void wpas_dbus_signal_sta_deauthorized(struct wpa_supplicant *wpa_s,
 {
 }
 
+static inline
+void wpas_dbus_signal_p2p_invitation_received(struct wpa_supplicant *wpa_s,
+					      const u8 *sa, const u8 *dev_addr,
+					      const u8 *bssid, int id,
+					      int op_freq)
+{
+}
+
 #endif /* CONFIG_CTRL_IFACE_DBUS_NEW */
 
 #endif /* CTRL_IFACE_DBUS_H_NEW */
diff --git a/wpa_supplicant/dbus/dbus_new_handlers.c b/wpa_supplicant/dbus/dbus_new_handlers.c
index f2e62ca96386..67562a547172 100644
--- a/wpa_supplicant/dbus/dbus_new_handlers.c
+++ b/wpa_supplicant/dbus/dbus_new_handlers.c
@@ -157,7 +157,8 @@ static struct wpa_supplicant * get_iface_by_dbus_path(
 	struct wpa_supplicant *wpa_s;
 
 	for (wpa_s = global->ifaces; wpa_s; wpa_s = wpa_s->next) {
-		if (os_strcmp(wpa_s->dbus_new_path, path) == 0)
+		if (wpa_s->dbus_new_path &&
+		    os_strcmp(wpa_s->dbus_new_path, path) == 0)
 			return wpa_s;
 	}
 	return NULL;
@@ -213,7 +214,7 @@ dbus_bool_t set_network_properties(struct wpa_supplicant *wpa_s,
 		} else if (entry.type == DBUS_TYPE_STRING) {
 			if (should_quote_opt(entry.key)) {
 				size = os_strlen(entry.str_value);
-				if (size <= 0)
+				if (size == 0)
 					goto error;
 
 				size += 3;
@@ -600,7 +601,7 @@ DBusMessage * wpas_dbus_handler_create_interface(DBusMessage *message,
 		iface.bridge_ifname = bridge_ifname;
 		/* Otherwise, have wpa_supplicant attach to it. */
 		wpa_s = wpa_supplicant_add_iface(global, &iface, NULL);
-		if (wpa_s) {
+		if (wpa_s && wpa_s->dbus_new_path) {
 			const char *path = wpa_s->dbus_new_path;
 
 			reply = dbus_message_new_method_return(message);
@@ -684,7 +685,7 @@ DBusMessage * wpas_dbus_handler_get_interface(DBusMessage *message,
 			      DBUS_TYPE_INVALID);
 
 	wpa_s = wpa_supplicant_get_iface(global, ifname);
-	if (wpa_s == NULL)
+	if (wpa_s == NULL || wpa_s->dbus_new_path == NULL)
 		return wpas_dbus_error_iface_unknown(message);
 
 	path = wpa_s->dbus_new_path;
@@ -876,8 +877,10 @@ dbus_bool_t wpas_dbus_getter_interfaces(DBusMessageIter *iter,
 	unsigned int i = 0, num = 0;
 	dbus_bool_t success;
 
-	for (wpa_s = global->ifaces; wpa_s; wpa_s = wpa_s->next)
-		num++;
+	for (wpa_s = global->ifaces; wpa_s; wpa_s = wpa_s->next) {
+		if (wpa_s->dbus_new_path)
+			num++;
+	}
 
 	paths = os_calloc(num, sizeof(char *));
 	if (!paths) {
@@ -885,8 +888,10 @@ dbus_bool_t wpas_dbus_getter_interfaces(DBusMessageIter *iter,
 		return FALSE;
 	}
 
-	for (wpa_s = global->ifaces; wpa_s; wpa_s = wpa_s->next)
-		paths[i++] = wpa_s->dbus_new_path;
+	for (wpa_s = global->ifaces; wpa_s; wpa_s = wpa_s->next) {
+		if (wpa_s->dbus_new_path)
+			paths[i++] = wpa_s->dbus_new_path;
+	}
 
 	success = wpas_dbus_simple_array_property_getter(iter,
 							 DBUS_TYPE_OBJECT_PATH,
@@ -1034,10 +1039,10 @@ static int wpas_dbus_get_scan_ssids(DBusMessage *message, DBusMessageIter *var,
 
 		dbus_message_iter_get_fixed_array(&sub_array_iter, &val, &len);
 
-		if (len > MAX_SSID_LEN) {
+		if (len > SSID_MAX_LEN) {
 			wpa_printf(MSG_DEBUG,
 				   "%s[dbus]: SSID too long (len=%d max_len=%d)",
-				   __func__, len, MAX_SSID_LEN);
+				   __func__, len, SSID_MAX_LEN);
 			*reply = wpas_dbus_error_invalid_args(
 				message, "Invalid SSID: too long");
 			return -1;
@@ -1327,14 +1332,26 @@ DBusMessage * wpas_dbus_handler_scan(DBusMessage *message,
 				message,
 				"You can specify only Channels in passive scan");
 			goto out;
-		} else if (params.freqs && params.freqs[0]) {
-			if (wpa_supplicant_trigger_scan(wpa_s, &params)) {
-				reply = wpas_dbus_error_scan_error(
-					message, "Scan request rejected");
-			}
 		} else {
-			wpa_s->scan_req = MANUAL_SCAN_REQ;
-			wpa_supplicant_req_scan(wpa_s, 0, 0);
+			if (wpa_s->sched_scanning) {
+				wpa_printf(MSG_DEBUG,
+					   "%s[dbus]: Stop ongoing sched_scan to allow requested scan to proceed",
+					   __func__);
+				wpa_supplicant_cancel_sched_scan(wpa_s);
+			}
+
+			if (params.freqs && params.freqs[0]) {
+				wpa_s->last_scan_req = MANUAL_SCAN_REQ;
+				if (wpa_supplicant_trigger_scan(wpa_s,
+								&params)) {
+					reply = wpas_dbus_error_scan_error(
+						message,
+						"Scan request rejected");
+				}
+			} else {
+				wpa_s->scan_req = MANUAL_SCAN_REQ;
+				wpa_supplicant_req_scan(wpa_s, 0, 0);
+			}
 		}
 	} else if (os_strcmp(type, "active") == 0) {
 		if (!params.num_ssids) {
@@ -1344,6 +1361,14 @@ DBusMessage * wpas_dbus_handler_scan(DBusMessage *message,
 #ifdef CONFIG_AUTOSCAN
 		autoscan_deinit(wpa_s);
 #endif /* CONFIG_AUTOSCAN */
+		if (wpa_s->sched_scanning) {
+			wpa_printf(MSG_DEBUG,
+				   "%s[dbus]: Stop ongoing sched_scan to allow requested scan to proceed",
+				   __func__);
+			wpa_supplicant_cancel_sched_scan(wpa_s);
+		}
+
+		wpa_s->last_scan_req = MANUAL_SCAN_REQ;
 		if (wpa_supplicant_trigger_scan(wpa_s, &params)) {
 			reply = wpas_dbus_error_scan_error(
 				message, "Scan request rejected");
@@ -1478,7 +1503,8 @@ DBusMessage * wpas_dbus_handler_add_network(DBusMessage *message,
 
 	dbus_message_iter_init(message, &iter);
 
-	ssid = wpa_config_add_network(wpa_s->conf);
+	if (wpa_s->dbus_new_path)
+		ssid = wpa_config_add_network(wpa_s->conf);
 	if (ssid == NULL) {
 		wpa_printf(MSG_ERROR, "%s[dbus]: can't add new interface.",
 			   __func__);
@@ -1576,6 +1602,30 @@ DBusMessage * wpas_dbus_handler_reattach(DBusMessage *message,
 }
 
 
+/**
+ * wpas_dbus_handler_reconnect - Reconnect if disconnected
+ * @message: Pointer to incoming dbus message
+ * @wpa_s: wpa_supplicant structure for a network interface
+ * Returns: InterfaceDisabled DBus error message if disabled
+ * or NULL otherwise.
+ *
+ * Handler function for "Reconnect" method call of network interface.
+ */
+DBusMessage * wpas_dbus_handler_reconnect(DBusMessage *message,
+		struct wpa_supplicant *wpa_s)
+{
+	if (wpa_s->wpa_state == WPA_INTERFACE_DISABLED) {
+		return dbus_message_new_error(message,
+					      WPAS_DBUS_ERROR_IFACE_DISABLED,
+					      "This interface is disabled");
+	}
+
+	if (wpa_s->disconnected)
+		wpas_request_connection(wpa_s);
+	return NULL;
+}
+
+
 /**
  * wpas_dbus_handler_remove_network - Remove a configured network
  * @message: Pointer to incoming dbus message
@@ -1602,7 +1652,7 @@ DBusMessage * wpas_dbus_handler_remove_network(DBusMessage *message,
 	iface = wpas_dbus_new_decompose_object_path(op,
 						    WPAS_DBUS_NEW_NETWORKS_PART,
 						    &net_id);
-	if (iface == NULL || net_id == NULL ||
+	if (iface == NULL || net_id == NULL || !wpa_s->dbus_new_path ||
 	    os_strcmp(iface, wpa_s->dbus_new_path) != 0) {
 		reply = wpas_dbus_error_invalid_args(message, op);
 		goto out;
@@ -1715,7 +1765,7 @@ DBusMessage * wpas_dbus_handler_select_network(DBusMessage *message,
 	iface = wpas_dbus_new_decompose_object_path(op,
 						    WPAS_DBUS_NEW_NETWORKS_PART,
 						    &net_id);
-	if (iface == NULL || net_id == NULL ||
+	if (iface == NULL || net_id == NULL || !wpa_s->dbus_new_path ||
 	    os_strcmp(iface, wpa_s->dbus_new_path) != 0) {
 		reply = wpas_dbus_error_invalid_args(message, op);
 		goto out;
@@ -1773,7 +1823,7 @@ DBusMessage * wpas_dbus_handler_network_reply(DBusMessage *message,
 	iface = wpas_dbus_new_decompose_object_path(op,
 						    WPAS_DBUS_NEW_NETWORKS_PART,
 						    &net_id);
-	if (iface == NULL || net_id == NULL ||
+	if (iface == NULL || net_id == NULL || !wpa_s->dbus_new_path ||
 	    os_strcmp(iface, wpa_s->dbus_new_path) != 0) {
 		reply = wpas_dbus_error_invalid_args(message, op);
 		goto out;
@@ -2266,12 +2316,14 @@ DBusMessage * wpas_dbus_handler_set_pkcs11_engine_and_module_path(
 			message, DBUS_ERROR_FAILED,
 			"Reinit of the EAPOL state machine with the new PKCS #11 engine and module path failed.");
 
-	wpa_dbus_mark_property_changed(
-		wpa_s->global->dbus, wpa_s->dbus_new_path,
-		WPAS_DBUS_NEW_IFACE_INTERFACE, "PKCS11EnginePath");
-	wpa_dbus_mark_property_changed(
-		wpa_s->global->dbus, wpa_s->dbus_new_path,
-		WPAS_DBUS_NEW_IFACE_INTERFACE, "PKCS11ModulePath");
+	if (wpa_s->dbus_new_path) {
+		wpa_dbus_mark_property_changed(
+			wpa_s->global->dbus, wpa_s->dbus_new_path,
+			WPAS_DBUS_NEW_IFACE_INTERFACE, "PKCS11EnginePath");
+		wpa_dbus_mark_property_changed(
+			wpa_s->global->dbus, wpa_s->dbus_new_path,
+			WPAS_DBUS_NEW_IFACE_INTERFACE, "PKCS11ModulePath");
+	}
 
 	return NULL;
 }
@@ -3024,7 +3076,7 @@ dbus_bool_t wpas_dbus_getter_current_bss(DBusMessageIter *iter,
 	struct wpa_supplicant *wpa_s = user_data;
 	char path_buf[WPAS_DBUS_OBJECT_PATH_MAX], *bss_obj_path = path_buf;
 
-	if (wpa_s->current_bss)
+	if (wpa_s->current_bss && wpa_s->dbus_new_path)
 		os_snprintf(bss_obj_path, WPAS_DBUS_OBJECT_PATH_MAX,
 			    "%s/" WPAS_DBUS_NEW_BSSIDS_PART "/%u",
 			    wpa_s->dbus_new_path, wpa_s->current_bss->id);
@@ -3052,7 +3104,7 @@ dbus_bool_t wpas_dbus_getter_current_network(DBusMessageIter *iter,
 	struct wpa_supplicant *wpa_s = user_data;
 	char path_buf[WPAS_DBUS_OBJECT_PATH_MAX], *net_obj_path = path_buf;
 
-	if (wpa_s->current_ssid)
+	if (wpa_s->current_ssid && wpa_s->dbus_new_path)
 		os_snprintf(net_obj_path, WPAS_DBUS_OBJECT_PATH_MAX,
 			    "%s/" WPAS_DBUS_NEW_NETWORKS_PART "/%u",
 			    wpa_s->dbus_new_path, wpa_s->current_ssid->id);
@@ -3140,6 +3192,12 @@ dbus_bool_t wpas_dbus_getter_bsss(DBusMessageIter *iter, DBusError *error,
 	unsigned int i = 0;
 	dbus_bool_t success = FALSE;
 
+	if (!wpa_s->dbus_new_path) {
+		dbus_set_error(error, DBUS_ERROR_FAILED,
+			       "%s: no D-Bus interface", __func__);
+		return FALSE;
+	}
+
 	paths = os_calloc(wpa_s->num_bss, sizeof(char *));
 	if (!paths) {
 		dbus_set_error_const(error, DBUS_ERROR_NO_MEMORY, "no memory");
@@ -3191,6 +3249,12 @@ dbus_bool_t wpas_dbus_getter_networks(DBusMessageIter *iter, DBusError *error,
 	unsigned int i = 0, num = 0;
 	dbus_bool_t success = FALSE;
 
+	if (!wpa_s->dbus_new_path) {
+		dbus_set_error(error, DBUS_ERROR_FAILED,
+			       "%s: no D-Bus interface", __func__);
+		return FALSE;
+	}
+
 	for (ssid = wpa_s->conf->ssid; ssid; ssid = ssid->next)
 		if (!network_is_persistent_group(ssid))
 			num++;
@@ -3791,6 +3855,7 @@ dbus_bool_t wpas_dbus_getter_bss_wps(DBusMessageIter *iter, DBusError *error,
 	struct wpabuf *wps_ie;
 #endif /* CONFIG_WPS */
 	DBusMessageIter iter_dict, variant_iter;
+	int wps_support = 0;
 	const char *type = "";
 
 	res = get_bss_helper(args, error, __func__);
@@ -3805,6 +3870,7 @@ dbus_bool_t wpas_dbus_getter_bss_wps(DBusMessageIter *iter, DBusError *error,
 #ifdef CONFIG_WPS
 	wps_ie = wpa_bss_get_vendor_ie_multi(res, WPS_IE_VENDOR_TYPE);
 	if (wps_ie) {
+		wps_support = 1;
 		if (wps_is_selected_pbc_registrar(wps_ie))
 			type = "pbc";
 		else if (wps_is_selected_pin_registrar(wps_ie))
@@ -3814,7 +3880,7 @@ dbus_bool_t wpas_dbus_getter_bss_wps(DBusMessageIter *iter, DBusError *error,
 	}
 #endif /* CONFIG_WPS */
 
-	if (!wpa_dbus_dict_append_string(&iter_dict, "Type", type) ||
+	if ((wps_support && !wpa_dbus_dict_append_string(&iter_dict, "Type", type)) ||
 	    !wpa_dbus_dict_close_write(&variant_iter, &iter_dict) ||
 	    !dbus_message_iter_close_container(iter, &variant_iter))
 		goto nomem;
@@ -4102,7 +4168,7 @@ void wpas_dbus_signal_preq(struct wpa_supplicant *wpa_s,
 	struct wpas_dbus_priv *priv = wpa_s->global->dbus;
 
 	/* Do nothing if the control interface is not turned on */
-	if (priv == NULL)
+	if (priv == NULL || !wpa_s->dbus_new_path)
 		return;
 
 	if (wpa_s->preq_notify_peer == NULL)
diff --git a/wpa_supplicant/dbus/dbus_new_handlers.h b/wpa_supplicant/dbus/dbus_new_handlers.h
index 6113db500390..50f72ec507bf 100644
--- a/wpa_supplicant/dbus/dbus_new_handlers.h
+++ b/wpa_supplicant/dbus/dbus_new_handlers.h
@@ -107,6 +107,9 @@ DBusMessage * wpas_dbus_handler_reassociate(DBusMessage *message,
 DBusMessage * wpas_dbus_handler_reattach(DBusMessage *message,
 					 struct wpa_supplicant *wpa_s);
 
+DBusMessage * wpas_dbus_handler_reconnect(DBusMessage *message,
+					  struct wpa_supplicant *wpa_s);
+
 DBusMessage * wpas_dbus_handler_remove_network(DBusMessage *message,
 					       struct wpa_supplicant *wpa_s);
 
@@ -291,6 +294,9 @@ dbus_bool_t wpas_dbus_setter_network_properties(DBusMessageIter *iter,
 DBusMessage * wpas_dbus_handler_wps_start(DBusMessage *message,
 					  struct wpa_supplicant *wpa_s);
 
+DBusMessage * wpas_dbus_handler_wps_cancel(DBusMessage *message,
+					   struct wpa_supplicant *wpa_s);
+
 dbus_bool_t wpas_dbus_getter_process_credentials(DBusMessageIter *iter,
 	DBusError *error, void *user_data);
 
diff --git a/wpa_supplicant/dbus/dbus_new_handlers_p2p.c b/wpa_supplicant/dbus/dbus_new_handlers_p2p.c
index 0eff76386fa4..67c079e7506d 100644
--- a/wpa_supplicant/dbus/dbus_new_handlers_p2p.c
+++ b/wpa_supplicant/dbus/dbus_new_handlers_p2p.c
@@ -127,8 +127,7 @@ DBusMessage * wpas_dbus_handler_p2p_find(DBusMessage *message,
 		wpa_dbus_dict_entry_clear(&entry);
 	}
 
-	if (wpa_s->p2p_dev)
-		wpa_s = wpa_s->p2p_dev;
+	wpa_s = wpa_s->global->p2p_init_wpa_s;
 
 	wpas_p2p_find(wpa_s, timeout, type, num_req_dev_types, req_dev_types,
 		      NULL, 0, 0, NULL, 0);
@@ -147,10 +146,7 @@ error:
 DBusMessage * wpas_dbus_handler_p2p_stop_find(DBusMessage *message,
 					      struct wpa_supplicant *wpa_s)
 {
-	if (wpa_s->p2p_dev)
-		wpa_s = wpa_s->p2p_dev;
-
-	wpas_p2p_stop_find(wpa_s);
+	wpas_p2p_stop_find(wpa_s->global->p2p_init_wpa_s);
 	return NULL;
 }
 
@@ -168,8 +164,7 @@ DBusMessage * wpas_dbus_handler_p2p_rejectpeer(DBusMessage *message,
 	if (parse_peer_object_path(peer_object_path, peer_addr) < 0)
 		return wpas_dbus_error_invalid_args(message, NULL);
 
-	if (wpa_s->p2p_dev)
-		wpa_s = wpa_s->p2p_dev;
+	wpa_s = wpa_s->global->p2p_init_wpa_s;
 
 	if (wpas_p2p_reject(wpa_s, peer_addr) < 0)
 		return wpas_dbus_error_unknown_error(message,
@@ -188,8 +183,7 @@ DBusMessage * wpas_dbus_handler_p2p_listen(DBusMessage *message,
 				   DBUS_TYPE_INVALID))
 		return wpas_dbus_error_no_memory(message);
 
-	if (wpa_s->p2p_dev)
-		wpa_s = wpa_s->p2p_dev;
+	wpa_s = wpa_s->global->p2p_init_wpa_s;
 
 	if (wpas_p2p_listen(wpa_s, (unsigned int) timeout)) {
 		return dbus_message_new_error(message,
@@ -230,8 +224,7 @@ DBusMessage * wpas_dbus_handler_p2p_extendedlisten(
 		wpa_dbus_dict_entry_clear(&entry);
 	}
 
-	if (wpa_s->p2p_dev)
-		wpa_s = wpa_s->p2p_dev;
+	wpa_s = wpa_s->global->p2p_init_wpa_s;
 
 	if (wpas_p2p_ext_listen(wpa_s, period, interval))
 		return wpas_dbus_error_unknown_error(
@@ -282,9 +275,6 @@ DBusMessage * wpas_dbus_handler_p2p_presence_request(
 		wpa_dbus_dict_entry_clear(&entry);
 	}
 
-	if (wpa_s->p2p_dev)
-		wpa_s = wpa_s->p2p_dev;
-
 	if (wpas_p2p_presence_req(wpa_s, dur1, int1, dur2, int2) < 0)
 		return wpas_dbus_error_unknown_error(message,
 				"Failed to invoke presence request.");
@@ -339,8 +329,7 @@ DBusMessage * wpas_dbus_handler_p2p_group_add(DBusMessage *message,
 		wpa_dbus_dict_entry_clear(&entry);
 	}
 
-	if (wpa_s->p2p_dev)
-		wpa_s = wpa_s->p2p_dev;
+	wpa_s = wpa_s->global->p2p_init_wpa_s;
 
 	if (pg_object_path != NULL) {
 		char *net_id_str;
@@ -354,7 +343,8 @@ DBusMessage * wpas_dbus_handler_p2p_group_add(DBusMessage *message,
 			pg_object_path, WPAS_DBUS_NEW_PERSISTENT_GROUPS_PART,
 			&net_id_str);
 		if (iface == NULL || net_id_str == NULL ||
-		    os_strcmp(iface, wpa_s->dbus_new_path) != 0) {
+		    !wpa_s->parent->dbus_new_path ||
+		    os_strcmp(iface, wpa_s->parent->dbus_new_path) != 0) {
 			reply =
 			    wpas_dbus_error_invalid_args(message,
 							 pg_object_path);
@@ -374,7 +364,7 @@ DBusMessage * wpas_dbus_handler_p2p_group_add(DBusMessage *message,
 			goto inv_args;
 
 		if (wpas_p2p_group_add_persistent(wpa_s, ssid, 0, freq, 0, 0, 0,
-						  NULL, 0)) {
+						  NULL, 0, 0)) {
 			reply = wpas_dbus_error_unknown_error(
 				message,
 				"Failed to reinvoke a persistent group");
@@ -426,6 +416,64 @@ static dbus_bool_t wpa_dbus_p2p_check_enabled(struct wpa_supplicant *wpa_s,
 }
 
 
+DBusMessage * wpas_dbus_handler_p2p_remove_client(DBusMessage *message,
+						  struct wpa_supplicant *wpa_s)
+{
+	DBusMessageIter iter_dict;
+	DBusMessage *reply = NULL;
+	DBusMessageIter iter;
+	struct wpa_dbus_dict_entry entry;
+	char *peer_object_path = NULL;
+	char *interface_addr = NULL;
+	u8 peer_addr[ETH_ALEN];
+
+	if (!wpa_dbus_p2p_check_enabled(wpa_s, message, &reply, NULL))
+		return reply;
+
+	dbus_message_iter_init(message, &iter);
+
+	if (!wpa_dbus_dict_open_read(&iter, &iter_dict, NULL))
+		goto err;
+
+	while (wpa_dbus_dict_has_dict_entry(&iter_dict)) {
+		if (!wpa_dbus_dict_get_entry(&iter_dict, &entry))
+			goto err;
+
+		if (os_strcmp(entry.key, "peer") == 0 &&
+		    entry.type == DBUS_TYPE_OBJECT_PATH) {
+			os_free(peer_object_path);
+			peer_object_path = os_strdup(entry.str_value);
+			wpa_dbus_dict_entry_clear(&entry);
+		} else if (os_strcmp(entry.key, "iface") == 0 &&
+			   entry.type == DBUS_TYPE_STRING) {
+			os_free(interface_addr);
+			interface_addr = os_strdup(entry.str_value);
+			wpa_dbus_dict_entry_clear(&entry);
+		} else {
+			wpa_dbus_dict_entry_clear(&entry);
+			goto err;
+		}
+	}
+
+	if ((!peer_object_path && !interface_addr) ||
+	    (peer_object_path &&
+	     (parse_peer_object_path(peer_object_path, peer_addr) < 0 ||
+	      !p2p_peer_known(wpa_s->global->p2p, peer_addr))) ||
+	    (interface_addr && hwaddr_aton(interface_addr, peer_addr) < 0))
+		goto err;
+
+	wpas_p2p_remove_client(wpa_s, peer_addr, interface_addr != NULL);
+	reply = NULL;
+out:
+	os_free(peer_object_path);
+	os_free(interface_addr);
+	return reply;
+err:
+	reply = wpas_dbus_error_invalid_args(message, "Invalid address format");
+	goto out;
+}
+
+
 DBusMessage * wpas_dbus_handler_p2p_flush(DBusMessage *message,
 					  struct wpa_supplicant *wpa_s)
 {
@@ -434,8 +482,7 @@ DBusMessage * wpas_dbus_handler_p2p_flush(DBusMessage *message,
 	if (!wpa_dbus_p2p_check_enabled(wpa_s, message, &reply, NULL))
 		return reply;
 
-	if (wpa_s->p2p_dev)
-		wpa_s = wpa_s->p2p_dev;
+	wpa_s = wpa_s->global->p2p_init_wpa_s;
 
 	os_memset(wpa_s->p2p_auth_invite, 0, ETH_ALEN);
 	wpa_s->force_long_sd = 0;
@@ -531,8 +578,7 @@ DBusMessage * wpas_dbus_handler_p2p_connect(DBusMessage *message,
 	if ((!pin || !pin[0]) && wps_method == WPS_PIN_KEYPAD)
 		goto inv_args;
 
-	if (wpa_s->p2p_dev)
-		wpa_s = wpa_s->p2p_dev;
+	wpa_s = wpa_s->global->p2p_init_wpa_s;
 
 	new_pin = wpas_p2p_connect(wpa_s, addr, pin, wps_method,
 				   persistent_group, 0, join, authorize_only,
@@ -587,6 +633,26 @@ inv_args:
 }
 
 
+/**
+ * wpas_dbus_handler_p2p_cancel - Cancel P2P group formation
+ * @message: Pointer to incoming dbus message
+ * @wpa_s: %wpa_supplicant data structure
+ * Returns: NULL on success or DBus error on failure
+ *
+ * Handler for "Cancel" method call. Returns NULL if P2P cancel succeeds or DBus
+ * error on P2P cancel failure
+ */
+DBusMessage * wpas_dbus_handler_p2p_cancel(DBusMessage *message,
+					   struct wpa_supplicant *wpa_s)
+{
+	if (wpas_p2p_cancel(wpa_s))
+		return wpas_dbus_error_unknown_error(message,
+						     "P2P cancel failed");
+
+	return NULL;
+}
+
+
 DBusMessage * wpas_dbus_handler_p2p_invite(DBusMessage *message,
 					   struct wpa_supplicant *wpa_s)
 {
@@ -634,8 +700,7 @@ DBusMessage * wpas_dbus_handler_p2p_invite(DBusMessage *message,
 	    !p2p_peer_known(wpa_s->global->p2p, peer_addr))
 		goto err;
 
-	if (wpa_s->p2p_dev)
-		wpa_s = wpa_s->p2p_dev;
+	wpa_s = wpa_s->global->p2p_init_wpa_s;
 
 	if (persistent) {
 		char *net_id_str;
@@ -649,7 +714,8 @@ DBusMessage * wpas_dbus_handler_p2p_invite(DBusMessage *message,
 			WPAS_DBUS_NEW_PERSISTENT_GROUPS_PART,
 			&net_id_str);
 		if (iface == NULL || net_id_str == NULL ||
-		    os_strcmp(iface, wpa_s->dbus_new_path) != 0) {
+		    !wpa_s->parent->dbus_new_path ||
+		    os_strcmp(iface, wpa_s->parent->dbus_new_path) != 0) {
 			reply = wpas_dbus_error_invalid_args(message,
 							     pg_object_path);
 			goto out;
@@ -726,8 +792,7 @@ DBusMessage * wpas_dbus_handler_p2p_prov_disc_req(DBusMessage *message,
 	    os_strcmp(config_method, "pushbutton"))
 		return wpas_dbus_error_invalid_args(message, NULL);
 
-	if (wpa_s->p2p_dev)
-		wpa_s = wpa_s->p2p_dev;
+	wpa_s = wpa_s->global->p2p_init_wpa_s;
 
 	if (wpas_p2p_prov_disc(wpa_s, peer_addr, config_method,
 			       WPAS_P2P_PD_FOR_GO_NEG, NULL) < 0)
@@ -758,8 +823,7 @@ dbus_bool_t wpas_dbus_getter_p2p_device_config(DBusMessageIter *iter,
 	if (!wpa_dbus_p2p_check_enabled(wpa_s, NULL, NULL, error))
 		return FALSE;
 
-	if (wpa_s->p2p_dev)
-		wpa_s = wpa_s->p2p_dev;
+	wpa_s = wpa_s->global->p2p_init_wpa_s;
 
 	if (!dbus_message_iter_open_container(iter, DBUS_TYPE_VARIANT,
 					      "a{sv}", &variant_iter) ||
@@ -864,8 +928,7 @@ dbus_bool_t wpas_dbus_setter_p2p_device_config(DBusMessageIter *iter,
 	if (!wpa_dbus_p2p_check_enabled(wpa_s, NULL, NULL, error))
 		return FALSE;
 
-	if (wpa_s->p2p_dev)
-		wpa_s = wpa_s->p2p_dev;
+	wpa_s = wpa_s->global->p2p_init_wpa_s;
 
 	dbus_message_iter_recurse(iter, &variant_iter);
 	if (!wpa_dbus_dict_open_read(&variant_iter, &iter_dict, error))
@@ -1043,7 +1106,8 @@ dbus_bool_t wpas_dbus_getter_p2p_peers(DBusMessageIter *iter, DBusError *error,
 
 	char **peer_obj_paths = NULL;
 
-	if (!wpa_dbus_p2p_check_enabled(wpa_s, NULL, NULL, error))
+	if (!wpa_dbus_p2p_check_enabled(wpa_s, NULL, NULL, error) ||
+	    !wpa_s->parent->parent->dbus_new_path)
 		return FALSE;
 
 	dl_list_init(&peer_objpath_list);
@@ -1064,7 +1128,8 @@ dbus_bool_t wpas_dbus_getter_p2p_peers(DBusMessageIter *iter, DBusError *error,
 		os_snprintf(node->path, WPAS_DBUS_OBJECT_PATH_MAX,
 			    "%s/" WPAS_DBUS_NEW_P2P_PEERS_PART
 			    "/" COMPACT_MACSTR,
-			    wpa_s->dbus_new_path, MAC2STR(addr));
+			    wpa_s->parent->parent->dbus_new_path,
+			    MAC2STR(addr));
 		dl_list_add_tail(&peer_objpath_list, &node->list);
 		num++;
 
@@ -1184,13 +1249,17 @@ dbus_bool_t wpas_dbus_getter_p2p_peergo(DBusMessageIter *iter,
 	struct wpa_supplicant *wpa_s = user_data;
 	char go_peer_obj_path[WPAS_DBUS_OBJECT_PATH_MAX], *path;
 
+	if (!wpa_s->parent->parent->dbus_new_path)
+		return FALSE;
+
 	if (wpas_get_p2p_role(wpa_s) != WPAS_P2P_ROLE_CLIENT)
 		os_snprintf(go_peer_obj_path, WPAS_DBUS_OBJECT_PATH_MAX, "/");
 	else
 		os_snprintf(go_peer_obj_path, WPAS_DBUS_OBJECT_PATH_MAX,
 			    "%s/" WPAS_DBUS_NEW_P2P_PEERS_PART "/"
 			    COMPACT_MACSTR,
-			    wpa_s->dbus_new_path, MAC2STR(wpa_s->go_dev_addr));
+			    wpa_s->parent->parent->dbus_new_path,
+			    MAC2STR(wpa_s->go_dev_addr));
 
 	path = go_peer_obj_path;
 	return wpas_dbus_simple_property_getter(iter, DBUS_TYPE_OBJECT_PATH,
@@ -1240,6 +1309,154 @@ dbus_bool_t wpas_dbus_getter_p2p_peer_device_name(DBusMessageIter *iter,
 }
 
 
+dbus_bool_t wpas_dbus_getter_p2p_peer_manufacturer(DBusMessageIter *iter,
+						   DBusError *error,
+						   void *user_data)
+{
+	struct peer_handler_args *peer_args = user_data;
+	const struct p2p_peer_info *info;
+	char *tmp;
+
+	if (!wpa_dbus_p2p_check_enabled(peer_args->wpa_s, NULL, NULL, error))
+		return FALSE;
+
+	/* get the peer info */
+	info = p2p_get_peer_found(peer_args->wpa_s->global->p2p,
+				  peer_args->p2p_device_addr, 0);
+	if (info == NULL) {
+		dbus_set_error(error, DBUS_ERROR_FAILED, "failed to find peer");
+		return FALSE;
+	}
+
+	tmp = os_strdup(info->manufacturer);
+	if (!tmp) {
+		dbus_set_error_const(error, DBUS_ERROR_NO_MEMORY, "no memory");
+		return FALSE;
+	}
+
+	if (!wpas_dbus_simple_property_getter(iter, DBUS_TYPE_STRING, &tmp,
+					      error)) {
+		dbus_set_error_const(error, DBUS_ERROR_NO_MEMORY, "no memory");
+		os_free(tmp);
+		return FALSE;
+	}
+
+	os_free(tmp);
+	return TRUE;
+}
+
+
+dbus_bool_t wpas_dbus_getter_p2p_peer_modelname(DBusMessageIter *iter,
+						DBusError *error,
+						void *user_data)
+{
+	struct peer_handler_args *peer_args = user_data;
+	const struct p2p_peer_info *info;
+	char *tmp;
+
+	if (!wpa_dbus_p2p_check_enabled(peer_args->wpa_s, NULL, NULL, error))
+		return FALSE;
+
+	/* get the peer info */
+	info = p2p_get_peer_found(peer_args->wpa_s->global->p2p,
+				  peer_args->p2p_device_addr, 0);
+	if (info == NULL) {
+		dbus_set_error(error, DBUS_ERROR_FAILED, "failed to find peer");
+		return FALSE;
+	}
+
+	tmp = os_strdup(info->model_name);
+	if (!tmp) {
+		dbus_set_error_const(error, DBUS_ERROR_NO_MEMORY, "no memory");
+		return FALSE;
+	}
+
+	if (!wpas_dbus_simple_property_getter(iter, DBUS_TYPE_STRING, &tmp,
+					      error)) {
+		dbus_set_error_const(error, DBUS_ERROR_NO_MEMORY, "no memory");
+		os_free(tmp);
+		return FALSE;
+	}
+
+	os_free(tmp);
+	return TRUE;
+}
+
+
+dbus_bool_t wpas_dbus_getter_p2p_peer_modelnumber(DBusMessageIter *iter,
+						  DBusError *error,
+						  void *user_data)
+{
+	struct peer_handler_args *peer_args = user_data;
+	const struct p2p_peer_info *info;
+	char *tmp;
+
+	if (!wpa_dbus_p2p_check_enabled(peer_args->wpa_s, NULL, NULL, error))
+		return FALSE;
+
+	/* get the peer info */
+	info = p2p_get_peer_found(peer_args->wpa_s->global->p2p,
+				  peer_args->p2p_device_addr, 0);
+	if (info == NULL) {
+		dbus_set_error(error, DBUS_ERROR_FAILED, "failed to find peer");
+		return FALSE;
+	}
+
+	tmp = os_strdup(info->model_number);
+	if (!tmp) {
+		dbus_set_error_const(error, DBUS_ERROR_NO_MEMORY, "no memory");
+		return FALSE;
+	}
+
+	if (!wpas_dbus_simple_property_getter(iter, DBUS_TYPE_STRING, &tmp,
+					      error)) {
+		dbus_set_error_const(error, DBUS_ERROR_NO_MEMORY, "no memory");
+		os_free(tmp);
+		return FALSE;
+	}
+
+	os_free(tmp);
+	return TRUE;
+}
+
+
+dbus_bool_t wpas_dbus_getter_p2p_peer_serialnumber(DBusMessageIter *iter,
+						   DBusError *error,
+						   void *user_data)
+{
+	struct peer_handler_args *peer_args = user_data;
+	const struct p2p_peer_info *info;
+	char *tmp;
+
+	if (!wpa_dbus_p2p_check_enabled(peer_args->wpa_s, NULL, NULL, error))
+		return FALSE;
+
+	/* get the peer info */
+	info = p2p_get_peer_found(peer_args->wpa_s->global->p2p,
+				  peer_args->p2p_device_addr, 0);
+	if (info == NULL) {
+		dbus_set_error(error, DBUS_ERROR_FAILED, "failed to find peer");
+		return FALSE;
+	}
+
+	tmp = os_strdup(info->serial_number);
+	if (!tmp) {
+		dbus_set_error_const(error, DBUS_ERROR_NO_MEMORY, "no memory");
+		return FALSE;
+	}
+
+	if (!wpas_dbus_simple_property_getter(iter, DBUS_TYPE_STRING, &tmp,
+					      error)) {
+		dbus_set_error_const(error, DBUS_ERROR_NO_MEMORY, "no memory");
+		os_free(tmp);
+		return FALSE;
+	}
+
+	os_free(tmp);
+	return TRUE;
+}
+
+
 dbus_bool_t wpas_dbus_getter_p2p_peer_primary_device_type(
 	DBusMessageIter *iter, DBusError *error, void *user_data)
 {
@@ -1578,8 +1795,7 @@ dbus_bool_t wpas_dbus_getter_p2p_peer_groups(DBusMessageIter *iter,
 	os_memset(&data, 0, sizeof(data));
 
 	wpa_s = peer_args->wpa_s;
-	if (wpa_s->p2p_dev)
-		wpa_s = wpa_s->p2p_dev;
+	wpa_s = wpa_s->global->p2p_init_wpa_s;
 
 	wpa_s_go = wpas_get_p2p_client_iface(wpa_s, info->p2p_device_addr);
 	if (wpa_s_go) {
@@ -1636,6 +1852,10 @@ dbus_bool_t wpas_dbus_getter_persistent_groups(DBusMessageIter *iter,
 	unsigned int i = 0, num = 0;
 	dbus_bool_t success = FALSE;
 
+	wpa_s = wpa_s->global->p2p_init_wpa_s;
+	if (!wpa_s->parent->dbus_new_path)
+		return FALSE;
+
 	for (ssid = wpa_s->conf->ssid; ssid; ssid = ssid->next)
 		if (network_is_persistent_group(ssid))
 			num++;
@@ -1659,7 +1879,7 @@ dbus_bool_t wpas_dbus_getter_persistent_groups(DBusMessageIter *iter,
 		/* Construct the object path for this network. */
 		os_snprintf(paths[i++], WPAS_DBUS_OBJECT_PATH_MAX,
 			    "%s/" WPAS_DBUS_NEW_PERSISTENT_GROUPS_PART "/%d",
-			    wpa_s->dbus_new_path, ssid->id);
+			    wpa_s->parent->dbus_new_path, ssid->id);
 	}
 
 	success = wpas_dbus_simple_array_property_getter(iter,
@@ -1698,7 +1918,7 @@ dbus_bool_t wpas_dbus_getter_persistent_group_properties(DBusMessageIter *iter,
 
 
 /**
- * wpas_dbus_setter_persistent_group_properties - Get options for a persistent
+ * wpas_dbus_setter_persistent_group_properties - Set options for a persistent
  *	group
  * @iter: Pointer to incoming dbus message iter
  * @error: Location to store error on failure
@@ -1746,7 +1966,9 @@ DBusMessage * wpas_dbus_handler_add_persistent_group(
 
 	dbus_message_iter_init(message, &iter);
 
-	ssid = wpa_config_add_network(wpa_s->conf);
+	wpa_s = wpa_s->global->p2p_init_wpa_s;
+	if (wpa_s->parent->dbus_new_path)
+		ssid = wpa_config_add_network(wpa_s->conf);
 	if (ssid == NULL) {
 		wpa_printf(MSG_ERROR,
 			   "dbus: %s: Cannot add new persistent group",
@@ -1779,7 +2001,7 @@ DBusMessage * wpas_dbus_handler_add_persistent_group(
 	/* Construct the object path for this network. */
 	os_snprintf(path, WPAS_DBUS_OBJECT_PATH_MAX,
 		    "%s/" WPAS_DBUS_NEW_PERSISTENT_GROUPS_PART "/%d",
-		    wpa_s->dbus_new_path, ssid->id);
+		    wpa_s->parent->dbus_new_path, ssid->id);
 
 	reply = dbus_message_new_method_return(message);
 	if (reply == NULL) {
@@ -1826,6 +2048,8 @@ DBusMessage * wpas_dbus_handler_remove_persistent_group(
 	dbus_message_get_args(message, NULL, DBUS_TYPE_OBJECT_PATH, &op,
 			      DBUS_TYPE_INVALID);
 
+	wpa_s = wpa_s->global->p2p_init_wpa_s;
+
 	/*
 	 * Extract the network ID and ensure the network is actually a child of
 	 * this interface.
@@ -1834,7 +2058,8 @@ DBusMessage * wpas_dbus_handler_remove_persistent_group(
 		op, WPAS_DBUS_NEW_PERSISTENT_GROUPS_PART,
 		&persistent_group_id);
 	if (iface == NULL || persistent_group_id == NULL ||
-	    os_strcmp(iface, wpa_s->dbus_new_path) != 0) {
+	    !wpa_s->parent->dbus_new_path ||
+	    os_strcmp(iface, wpa_s->parent->dbus_new_path) != 0) {
 		reply = wpas_dbus_error_invalid_args(message, op);
 		goto out;
 	}
@@ -1899,6 +2124,8 @@ DBusMessage * wpas_dbus_handler_remove_all_persistent_groups(
 	struct wpa_ssid *ssid, *next;
 	struct wpa_config *config;
 
+	wpa_s = wpa_s->global->p2p_init_wpa_s;
+
 	config = wpa_s->conf;
 	ssid = config->ssid;
 	while (ssid) {
@@ -1928,6 +2155,9 @@ dbus_bool_t wpas_dbus_getter_p2p_group_members(DBusMessageIter *iter,
 	const u8 *addr;
 	dbus_bool_t success = FALSE;
 
+	if (!wpa_s->parent->parent->dbus_new_path)
+		return FALSE;
+
 	/* Verify correct role for this property */
 	if (wpas_get_p2p_role(wpa_s) != WPAS_P2P_ROLE_GO) {
 		return wpas_dbus_simple_array_property_getter(
@@ -1955,7 +2185,8 @@ dbus_bool_t wpas_dbus_getter_p2p_group_members(DBusMessageIter *iter,
 		os_snprintf(paths[i], WPAS_DBUS_OBJECT_PATH_MAX,
 			    "%s/" WPAS_DBUS_NEW_P2P_PEERS_PART
 			    "/" COMPACT_MACSTR,
-			    wpa_s->parent->dbus_new_path, MAC2STR(addr));
+			    wpa_s->parent->parent->dbus_new_path,
+			    MAC2STR(addr));
 		i++;
 	}
 
diff --git a/wpa_supplicant/dbus/dbus_new_handlers_p2p.h b/wpa_supplicant/dbus/dbus_new_handlers_p2p.h
index fdaccbafb143..2aecbbe46507 100644
--- a/wpa_supplicant/dbus/dbus_new_handlers_p2p.h
+++ b/wpa_supplicant/dbus/dbus_new_handlers_p2p.h
@@ -46,6 +46,9 @@ DBusMessage *wpas_dbus_handler_p2p_connect(
 		DBusMessage *message,
 		struct wpa_supplicant *wpa_s);
 
+DBusMessage * wpas_dbus_handler_p2p_cancel(DBusMessage *message,
+					   struct wpa_supplicant *wpa_s);
+
 DBusMessage *wpas_dbus_handler_p2p_invite(
 		DBusMessage *message,
 		struct wpa_supplicant *wpa_s);
@@ -53,6 +56,9 @@ DBusMessage *wpas_dbus_handler_p2p_invite(
 DBusMessage *wpas_dbus_handler_p2p_disconnect(
 	DBusMessage *message, struct wpa_supplicant *wpa_s);
 
+DBusMessage * wpas_dbus_handler_p2p_remove_client(
+	DBusMessage *message, struct wpa_supplicant *wpa_s);
+
 DBusMessage *wpas_dbus_handler_p2p_flush(
 	DBusMessage *message, struct wpa_supplicant *wpa_s);
 
@@ -112,6 +118,22 @@ dbus_bool_t wpas_dbus_getter_p2p_peer_device_name(DBusMessageIter *iter,
 						  DBusError *error,
 						  void *user_data);
 
+dbus_bool_t wpas_dbus_getter_p2p_peer_manufacturer(DBusMessageIter *iter,
+						   DBusError *error,
+						   void *user_data);
+
+dbus_bool_t wpas_dbus_getter_p2p_peer_modelname(DBusMessageIter *iter,
+						DBusError *error,
+						void *user_data);
+
+dbus_bool_t wpas_dbus_getter_p2p_peer_modelnumber(DBusMessageIter *iter,
+						  DBusError *error,
+						  void *user_data);
+
+dbus_bool_t wpas_dbus_getter_p2p_peer_serialnumber(DBusMessageIter *iter,
+						   DBusError *error,
+						   void *user_data);
+
 dbus_bool_t wpas_dbus_getter_p2p_peer_primary_device_type(
 	DBusMessageIter *iter, DBusError *error, void *user_data);
 
diff --git a/wpa_supplicant/dbus/dbus_new_handlers_wps.c b/wpa_supplicant/dbus/dbus_new_handlers_wps.c
index a94a0e51fc29..b2251baa3fe5 100644
--- a/wpa_supplicant/dbus/dbus_new_handlers_wps.c
+++ b/wpa_supplicant/dbus/dbus_new_handlers_wps.c
@@ -53,7 +53,7 @@ static int wpas_dbus_handler_wps_role(DBusMessage *message,
 	else if (os_strcmp(val, "registrar") == 0)
 		params->role = 2;
 	else {
-		wpa_printf(MSG_DEBUG, "dbus: WPS.Start - Uknown role %s", val);
+		wpa_printf(MSG_DEBUG, "dbus: WPS.Start - Unknown role %s", val);
 		*reply = wpas_dbus_error_invalid_args(message, val);
 		return -1;
 	}
@@ -113,7 +113,7 @@ static int wpas_dbus_handler_wps_bssid(DBusMessage *message,
 	dbus_message_iter_recurse(&variant_iter, &array_iter);
 	dbus_message_iter_get_fixed_array(&array_iter, &params->bssid, &len);
 	if (len != ETH_ALEN) {
-		wpa_printf(MSG_DEBUG, "dbus: WPS.Stsrt - Wrong Bssid length %d",
+		wpa_printf(MSG_DEBUG, "dbus: WPS.Start - Wrong Bssid length %d",
 			   len);
 		*reply = wpas_dbus_error_invalid_args(message,
 						      "Bssid is wrong length");
@@ -319,6 +319,26 @@ DBusMessage * wpas_dbus_handler_wps_start(DBusMessage *message,
 }
 
 
+/**
+ * wpas_dbus_handler_wps_cancel - Cancel ongoing WPS configuration
+ * @message: Pointer to incoming dbus message
+ * @wpa_s: %wpa_supplicant data structure
+ * Returns: NULL on success or DBus error on failure
+ *
+ * Handler for "Cancel" method call. Returns NULL if WPS cancel successfull
+ * or DBus error on WPS cancel failure
+ */
+DBusMessage * wpas_dbus_handler_wps_cancel(DBusMessage *message,
+					   struct wpa_supplicant *wpa_s)
+{
+	if (wpas_wps_cancel(wpa_s))
+		return wpas_dbus_error_unknown_error(message,
+						     "WPS cancel failed");
+
+	return NULL;
+}
+
+
 /**
  * wpas_dbus_getter_process_credentials - Check if credentials are processed
  * @message: Pointer to incoming dbus message
@@ -358,6 +378,8 @@ dbus_bool_t wpas_dbus_setter_process_credentials(DBusMessageIter *iter,
 	struct wpa_supplicant *wpa_s = user_data;
 	dbus_bool_t process_credentials, old_pc;
 
+	if (!wpa_s->dbus_new_path)
+		return FALSE;
 	if (!wpas_dbus_simple_property_setter(iter, error, DBUS_TYPE_BOOLEAN,
 					      &process_credentials))
 		return FALSE;
diff --git a/wpa_supplicant/dbus/dbus_new_helpers.c b/wpa_supplicant/dbus/dbus_new_helpers.c
index 15b090141c97..45623f346465 100644
--- a/wpa_supplicant/dbus/dbus_new_helpers.c
+++ b/wpa_supplicant/dbus/dbus_new_helpers.c
@@ -46,8 +46,13 @@ static dbus_bool_t fill_dict_with_properties(
 			goto error;
 
 		/* An error getting a property fails the request entirely */
-		if (!dsc->getter(&entry_iter, error, user_data))
+		if (!dsc->getter(&entry_iter, error, user_data)) {
+			wpa_printf(MSG_INFO,
+				   "dbus: %s dbus_interface=%s dbus_property=%s getter failed",
+				   __func__, dsc->dbus_interface,
+				   dsc->dbus_property);
 			return FALSE;
+		}
 
 		if (!dbus_message_iter_close_container(dict_iter, &entry_iter))
 			goto error;
diff --git a/wpa_supplicant/dbus/dbus_new_introspect.c b/wpa_supplicant/dbus/dbus_new_introspect.c
index 6209c67856bb..fba57e6361ae 100644
--- a/wpa_supplicant/dbus/dbus_new_introspect.c
+++ b/wpa_supplicant/dbus/dbus_new_introspect.c
@@ -257,7 +257,7 @@ DBusMessage * wpa_dbus_introspect(DBusMessage *message,
 	DBusMessage *reply;
 	struct wpabuf *xml;
 
-	xml = wpabuf_alloc(10000);
+	xml = wpabuf_alloc(15000);
 	if (xml == NULL)
 		return NULL;
 
diff --git a/wpa_supplicant/dbus/dbus_old.c b/wpa_supplicant/dbus/dbus_old.c
index 45bb4022702f..88227af7c03b 100644
--- a/wpa_supplicant/dbus/dbus_old.c
+++ b/wpa_supplicant/dbus/dbus_old.c
@@ -383,7 +383,7 @@ void wpa_supplicant_dbus_notify_scan_results(struct wpa_supplicant *wpa_s)
 	DBusMessage *_signal;
 
 	/* Do nothing if the control interface is not turned on */
-	if (iface == NULL)
+	if (iface == NULL || !wpa_s->dbus_path)
 		return;
 
 	_signal = dbus_message_new_signal(wpa_s->dbus_path,
@@ -474,7 +474,7 @@ void wpa_supplicant_dbus_notify_scanning(struct wpa_supplicant *wpa_s)
 	dbus_bool_t scanning = wpa_s->scanning ? TRUE : FALSE;
 
 	/* Do nothing if the control interface is not turned on */
-	if (iface == NULL)
+	if (iface == NULL || !wpa_s->dbus_path)
 		return;
 
 	_signal = dbus_message_new_signal(wpa_s->dbus_path,
@@ -509,7 +509,7 @@ void wpa_supplicant_dbus_notify_wps_cred(struct wpa_supplicant *wpa_s,
 	if (wpa_s->global == NULL)
 		return;
 	iface = wpa_s->global->dbus;
-	if (iface == NULL)
+	if (iface == NULL || !wpa_s->dbus_path)
 		return;
 
 	_signal = dbus_message_new_signal(wpa_s->dbus_path,
@@ -559,7 +559,7 @@ void wpa_supplicant_dbus_notify_certification(struct wpa_supplicant *wpa_s,
 	if (wpa_s->global == NULL)
 		return;
 	iface = wpa_s->global->dbus;
-	if (iface == NULL)
+	if (iface == NULL || !wpa_s->dbus_path)
 		return;
 
 	_signal = dbus_message_new_signal(wpa_s->dbus_path,
@@ -738,7 +738,7 @@ struct wpa_supplicant * wpa_supplicant_get_iface_by_dbus_path(
 	struct wpa_supplicant *wpa_s;
 
 	for (wpa_s = global->ifaces; wpa_s; wpa_s = wpa_s->next) {
-		if (strcmp(wpa_s->dbus_path, path) == 0)
+		if (wpa_s->dbus_path && strcmp(wpa_s->dbus_path, path) == 0)
 			return wpa_s;
 	}
 	return NULL;
diff --git a/wpa_supplicant/dbus/dbus_old_handlers.c b/wpa_supplicant/dbus/dbus_old_handlers.c
index 773ee8b49a2d..e8f62ef6bdc3 100644
--- a/wpa_supplicant/dbus/dbus_old_handlers.c
+++ b/wpa_supplicant/dbus/dbus_old_handlers.c
@@ -166,7 +166,7 @@ DBusMessage * wpas_dbus_global_add_interface(DBusMessage *message,
 		iface.bridge_ifname = bridge_ifname;
 		/* Otherwise, have wpa_supplicant attach to it. */
 		wpa_s = wpa_supplicant_add_iface(global, &iface, NULL);
-		if (wpa_s) {
+		if (wpa_s && wpa_s->dbus_path) {
 			const char *path = wpa_s->dbus_path;
 
 			reply = dbus_message_new_method_return(message);
@@ -262,7 +262,7 @@ DBusMessage * wpas_dbus_global_get_interface(DBusMessage *message,
 	}
 
 	wpa_s = wpa_supplicant_get_iface(global, ifname);
-	if (wpa_s == NULL) {
+	if (wpa_s == NULL || !wpa_s->dbus_path) {
 		reply = wpas_dbus_new_invalid_iface_error(message);
 		goto out;
 	}
@@ -354,6 +354,11 @@ DBusMessage * wpas_dbus_iface_scan_results(DBusMessage *message,
 	DBusMessageIter sub_iter;
 	struct wpa_bss *bss;
 
+	if (!wpa_s->dbus_path)
+		return dbus_message_new_error(message,
+					      WPAS_ERROR_INTERNAL_ERROR,
+					      "no D-Bus interface available");
+
 	/* Create and initialize the return message */
 	reply = dbus_message_new_method_return(message);
 	dbus_message_iter_init_append(reply, &iter);
@@ -495,7 +500,7 @@ DBusMessage * wpas_dbus_iface_capabilities(DBusMessage *message,
 	/* EAP methods */
 	eap_methods = eap_get_names_as_string_array(&num_items);
 	if (eap_methods) {
-		dbus_bool_t success = FALSE;
+		dbus_bool_t success;
 		size_t i = 0;
 
 		success = wpa_dbus_dict_append_string_array(
@@ -708,10 +713,11 @@ DBusMessage * wpas_dbus_iface_add_network(DBusMessage *message,
 					  struct wpa_supplicant *wpa_s)
 {
 	DBusMessage *reply = NULL;
-	struct wpa_ssid *ssid;
+	struct wpa_ssid *ssid = NULL;
 	char path_buf[WPAS_DBUS_OBJECT_PATH_MAX], *path = path_buf;
 
-	ssid = wpa_config_add_network(wpa_s->conf);
+	if (wpa_s->dbus_path)
+		ssid = wpa_config_add_network(wpa_s->conf);
 	if (ssid == NULL) {
 		reply = dbus_message_new_error(
 			message, WPAS_ERROR_ADD_NETWORK_ERROR,
@@ -769,7 +775,7 @@ DBusMessage * wpas_dbus_iface_remove_network(DBusMessage *message,
 	}
 
 	/* Ensure the network is actually a child of this interface */
-	if (os_strcmp(iface, wpa_s->dbus_path) != 0) {
+	if (!wpa_s->dbus_path || os_strcmp(iface, wpa_s->dbus_path) != 0) {
 		reply = wpas_dbus_new_invalid_network_error(message);
 		goto out;
 	}
@@ -803,10 +809,10 @@ out:
 }
 
 
-static const char  const *dont_quote[] = {
+static const char * const dont_quote[] = {
 	"key_mgmt", "proto", "pairwise", "auth_alg", "group", "eap",
 	"opensc_engine_path", "pkcs11_engine_path", "pkcs11_module_path",
-	"bssid", NULL
+	"bssid", "scan_freq", "freq_list", NULL
 };
 
 
@@ -878,7 +884,7 @@ DBusMessage * wpas_dbus_iface_set_network(DBusMessage *message,
 			if (should_quote_opt(entry.key)) {
 				size = os_strlen(entry.str_value);
 				/* Zero-length option check */
-				if (size <= 0)
+				if (size == 0)
 					goto error;
 				size += 3;  /* For quotes and terminator */
 				value = os_zalloc(size);
@@ -1020,7 +1026,7 @@ DBusMessage * wpas_dbus_iface_select_network(DBusMessage *message,
 			goto out;
 		}
 		/* Ensure the object path really points to this interface */
-		if (network == NULL ||
+		if (network == NULL || !wpa_s->dbus_path ||
 		    os_strcmp(iface_obj_path, wpa_s->dbus_path) != 0) {
 			reply = wpas_dbus_new_invalid_network_error(message);
 			goto out;
diff --git a/wpa_supplicant/defconfig b/wpa_supplicant/defconfig
index 7f627fdd6146..01a8c2ccb00b 100644
--- a/wpa_supplicant/defconfig
+++ b/wpa_supplicant/defconfig
@@ -495,3 +495,12 @@ CONFIG_PEERKEY=y
 #
 # External password backend for testing purposes (developer use)
 #CONFIG_EXT_PASSWORD_TEST=y
+
+# Enable Fast Session Transfer (FST)
+#CONFIG_FST=y
+
+# Enable CLI commands for FST testing
+#CONFIG_FST_TEST=y
+
+# OS X builds. This is only for building eapol_test.
+#CONFIG_OSX=y
diff --git a/wpa_supplicant/doc/docbook/eapol_test.8 b/wpa_supplicant/doc/docbook/eapol_test.8
index d06727c0b406..6361fc8eabb7 100644
--- a/wpa_supplicant/doc/docbook/eapol_test.8
+++ b/wpa_supplicant/doc/docbook/eapol_test.8
@@ -3,7 +3,7 @@
 .\" <http://shell.ipoline.com/~elmert/comp/docbook2X/> 
 .\" Please send any bug reports, improvements, comments, patches, 
 .\" etc. to Steve Cheng <steve@ggi-project.org>.
-.TH "EAPOL_TEST" "8" "15 March 2015" "" ""
+.TH "EAPOL_TEST" "8" "27 September 2015" "" ""
 
 .SH NAME
 eapol_test \- EAP peer and RADIUS client testing
diff --git a/wpa_supplicant/doc/docbook/wpa_background.8 b/wpa_supplicant/doc/docbook/wpa_background.8
index d532ddc28635..8beced14a086 100644
--- a/wpa_supplicant/doc/docbook/wpa_background.8
+++ b/wpa_supplicant/doc/docbook/wpa_background.8
@@ -3,7 +3,7 @@
 .\" <http://shell.ipoline.com/~elmert/comp/docbook2X/> 
 .\" Please send any bug reports, improvements, comments, patches, 
 .\" etc. to Steve Cheng <steve@ggi-project.org>.
-.TH "WPA_BACKGROUND" "8" "15 March 2015" "" ""
+.TH "WPA_BACKGROUND" "8" "27 September 2015" "" ""
 
 .SH NAME
 wpa_background \- Background information on Wi-Fi Protected Access and IEEE 802.11i
diff --git a/wpa_supplicant/doc/docbook/wpa_cli.8 b/wpa_supplicant/doc/docbook/wpa_cli.8
index f0f1d979299f..b00b266a54d7 100644
--- a/wpa_supplicant/doc/docbook/wpa_cli.8
+++ b/wpa_supplicant/doc/docbook/wpa_cli.8
@@ -3,7 +3,7 @@
 .\" <http://shell.ipoline.com/~elmert/comp/docbook2X/> 
 .\" Please send any bug reports, improvements, comments, patches, 
 .\" etc. to Steve Cheng <steve@ggi-project.org>.
-.TH "WPA_CLI" "8" "15 March 2015" "" ""
+.TH "WPA_CLI" "8" "27 September 2015" "" ""
 
 .SH NAME
 wpa_cli \- WPA command line client
diff --git a/wpa_supplicant/doc/docbook/wpa_gui.8 b/wpa_supplicant/doc/docbook/wpa_gui.8
index 1a172186b8d7..6284db29a150 100644
--- a/wpa_supplicant/doc/docbook/wpa_gui.8
+++ b/wpa_supplicant/doc/docbook/wpa_gui.8
@@ -3,13 +3,13 @@
 .\" <http://shell.ipoline.com/~elmert/comp/docbook2X/> 
 .\" Please send any bug reports, improvements, comments, patches, 
 .\" etc. to Steve Cheng <steve@ggi-project.org>.
-.TH "WPA_GUI" "8" "15 March 2015" "" ""
+.TH "WPA_GUI" "8" "27 September 2015" "" ""
 
 .SH NAME
 wpa_gui \- WPA Graphical User Interface
 .SH SYNOPSIS
 
-\fBwpa_gui\fR [ \fB-p \fIpath to ctrl sockets\fB\fR ] [ \fB-i \fIifname\fB\fR ] [ \fB-t\fR ]
+\fBwpa_gui\fR [ \fB-p \fIpath to ctrl sockets\fB\fR ] [ \fB-i \fIifname\fB\fR ] [ \fB-m \fIseconds\fB\fR ] [ \fB-t\fR ] [ \fB-q\fR ]
 
 .SH "OVERVIEW"
 .PP
@@ -32,10 +32,19 @@ Specify the interface that is being
 configured. By default, choose the first interface found with
 a control socket in the socket path.
 .TP
+\fB-m seconds\fR
+Set the update interval in seconds for the signal
+strength meter. This value must be a positive integer, otherwise
+meter is not enabled (default behavior).
+.TP
 \fB-t\fR
 Start program in the system tray only (if the window
 manager supports it). By default the main status window is
 shown.
+.TP
+\fB-q\fR
+Run program in the quiet mode - do not display tray
+icon pop-up messages.
 .SH "SEE ALSO"
 .PP
 \fBwpa_cli\fR(8)
diff --git a/wpa_supplicant/doc/docbook/wpa_gui.sgml b/wpa_supplicant/doc/docbook/wpa_gui.sgml
index 84766db3c459..5f7b49da0bcc 100644
--- a/wpa_supplicant/doc/docbook/wpa_gui.sgml
+++ b/wpa_supplicant/doc/docbook/wpa_gui.sgml
@@ -16,7 +16,9 @@
       <command>wpa_gui</command>
       <arg>-p <replaceable>path to ctrl sockets</replaceable></arg>
       <arg>-i <replaceable>ifname</replaceable></arg>
+      <arg>-m <replaceable>seconds</replaceable></arg>
       <arg>-t</arg>
+      <arg>-q</arg>
     </cmdsynopsis>
   </refsynopsisdiv>
 
@@ -50,6 +52,14 @@
 	a control socket in the socket path.</para></listitem>
       </varlistentry>
 
+      <varlistentry>
+	<term>-m seconds</term>
+
+	<listitem><para>Set the update interval in seconds for the signal
+	strength meter. This value must be a positive integer, otherwise
+	meter is not enabled (default behavior).</para></listitem>
+      </varlistentry>
+
       <varlistentry>
 	<term>-t</term>
 
@@ -57,6 +67,13 @@
 	manager supports it). By default the main status window is
 	shown.</para></listitem>
       </varlistentry>
+
+      <varlistentry>
+	<term>-q</term>
+
+        <listitem><para>Run program in the quiet mode - do not display tray
+	icon pop-up messages.</para></listitem>
+      </varlistentry>
     </variablelist>
   </refsect1>
   <refsect1>
diff --git a/wpa_supplicant/doc/docbook/wpa_passphrase.8 b/wpa_supplicant/doc/docbook/wpa_passphrase.8
index 60fba4302d8c..27cf1da94b65 100644
--- a/wpa_supplicant/doc/docbook/wpa_passphrase.8
+++ b/wpa_supplicant/doc/docbook/wpa_passphrase.8
@@ -3,7 +3,7 @@
 .\" <http://shell.ipoline.com/~elmert/comp/docbook2X/> 
 .\" Please send any bug reports, improvements, comments, patches, 
 .\" etc. to Steve Cheng <steve@ggi-project.org>.
-.TH "WPA_PASSPHRASE" "8" "15 March 2015" "" ""
+.TH "WPA_PASSPHRASE" "8" "27 September 2015" "" ""
 
 .SH NAME
 wpa_passphrase \- Generate a WPA PSK from an ASCII passphrase for a SSID
diff --git a/wpa_supplicant/doc/docbook/wpa_priv.8 b/wpa_supplicant/doc/docbook/wpa_priv.8
index 8b6f6946e04f..a9c2b6a2ce72 100644
--- a/wpa_supplicant/doc/docbook/wpa_priv.8
+++ b/wpa_supplicant/doc/docbook/wpa_priv.8
@@ -3,7 +3,7 @@
 .\" <http://shell.ipoline.com/~elmert/comp/docbook2X/> 
 .\" Please send any bug reports, improvements, comments, patches, 
 .\" etc. to Steve Cheng <steve@ggi-project.org>.
-.TH "WPA_PRIV" "8" "15 March 2015" "" ""
+.TH "WPA_PRIV" "8" "27 September 2015" "" ""
 
 .SH NAME
 wpa_priv \- wpa_supplicant privilege separation helper
diff --git a/wpa_supplicant/doc/docbook/wpa_supplicant.8 b/wpa_supplicant/doc/docbook/wpa_supplicant.8
index 35192a3ae42f..ce973c5cebb1 100644
--- a/wpa_supplicant/doc/docbook/wpa_supplicant.8
+++ b/wpa_supplicant/doc/docbook/wpa_supplicant.8
@@ -3,7 +3,7 @@
 .\" <http://shell.ipoline.com/~elmert/comp/docbook2X/> 
 .\" Please send any bug reports, improvements, comments, patches, 
 .\" etc. to Steve Cheng <steve@ggi-project.org>.
-.TH "WPA_SUPPLICANT" "8" "15 March 2015" "" ""
+.TH "WPA_SUPPLICANT" "8" "27 September 2015" "" ""
 
 .SH NAME
 wpa_supplicant \- Wi-Fi Protected Access client and IEEE 802.1X supplicant
diff --git a/wpa_supplicant/doc/docbook/wpa_supplicant.conf.5 b/wpa_supplicant/doc/docbook/wpa_supplicant.conf.5
index 851311c38f3b..304ef238d72e 100644
--- a/wpa_supplicant/doc/docbook/wpa_supplicant.conf.5
+++ b/wpa_supplicant/doc/docbook/wpa_supplicant.conf.5
@@ -3,7 +3,7 @@
 .\" <http://shell.ipoline.com/~elmert/comp/docbook2X/> 
 .\" Please send any bug reports, improvements, comments, patches, 
 .\" etc. to Steve Cheng <steve@ggi-project.org>.
-.TH "WPA_SUPPLICANT.CONF" "5" "15 March 2015" "" ""
+.TH "WPA_SUPPLICANT.CONF" "5" "27 September 2015" "" ""
 
 .SH NAME
 wpa_supplicant.conf \- configuration file for wpa_supplicant
diff --git a/wpa_supplicant/driver_i.h b/wpa_supplicant/driver_i.h
index 65b430d4a6c0..73768c756f0a 100644
--- a/wpa_supplicant/driver_i.h
+++ b/wpa_supplicant/driver_i.h
@@ -286,11 +286,13 @@ static inline int wpa_drv_set_country(struct wpa_supplicant *wpa_s,
 }
 
 static inline int wpa_drv_send_mlme(struct wpa_supplicant *wpa_s,
-				    const u8 *data, size_t data_len, int noack)
+				    const u8 *data, size_t data_len, int noack,
+				    unsigned int freq)
 {
 	if (wpa_s->driver->send_mlme)
 		return wpa_s->driver->send_mlme(wpa_s->drv_priv,
-						data, data_len, noack);
+						data, data_len, noack,
+						freq);
 	return -1;
 }
 
@@ -503,13 +505,6 @@ static inline int wpa_drv_set_ap_wps_ie(struct wpa_supplicant *wpa_s,
 					    proberesp, assocresp);
 }
 
-static inline int wpa_drv_shared_freq(struct wpa_supplicant *wpa_s)
-{
-	if (!wpa_s->driver->shared_freq)
-		return -1;
-	return wpa_s->driver->shared_freq(wpa_s->drv_priv);
-}
-
 static inline int wpa_drv_get_noa(struct wpa_supplicant *wpa_s,
 				  u8 *buf, size_t buf_len)
 {
@@ -890,4 +885,31 @@ static inline int wpa_drv_disable_transmit_sa(struct wpa_supplicant *wpa_s,
 }
 #endif /* CONFIG_MACSEC */
 
+static inline int wpa_drv_setband(struct wpa_supplicant *wpa_s,
+				  enum set_band band)
+{
+	if (!wpa_s->driver->set_band)
+		return -1;
+	return wpa_s->driver->set_band(wpa_s->drv_priv, band);
+}
+
+static inline int wpa_drv_get_pref_freq_list(struct wpa_supplicant *wpa_s,
+					     enum wpa_driver_if_type if_type,
+					     unsigned int *num,
+					     unsigned int *freq_list)
+{
+	if (!wpa_s->driver->get_pref_freq_list)
+		return -1;
+	return wpa_s->driver->get_pref_freq_list(wpa_s->drv_priv, if_type,
+						 num, freq_list);
+}
+
+static inline int wpa_drv_set_prob_oper_freq(struct wpa_supplicant *wpa_s,
+					     unsigned int freq)
+{
+	if (!wpa_s->driver->set_prob_oper_freq)
+		return 0;
+	return wpa_s->driver->set_prob_oper_freq(wpa_s->drv_priv, freq);
+}
+
 #endif /* DRIVER_I_H */
diff --git a/wpa_supplicant/eapol_test.c b/wpa_supplicant/eapol_test.c
index 9b7af30550bd..dce7d1fadd44 100644
--- a/wpa_supplicant/eapol_test.c
+++ b/wpa_supplicant/eapol_test.c
@@ -30,7 +30,7 @@
 #include "wpas_glue.h"
 
 
-struct wpa_driver_ops *wpa_drivers[] = { NULL };
+const struct wpa_driver_ops *const wpa_drivers[] = { NULL };
 
 
 struct extra_radius_attr {
@@ -76,6 +76,9 @@ struct eapol_test_data {
 
 	const char *pcsc_reader;
 	const char *pcsc_pin;
+
+	unsigned int ctrl_iface:1;
+	unsigned int id_req_sent:1;
 };
 
 static struct eapol_test_data eapol_test;
@@ -329,7 +332,11 @@ eapol_test_get_config_blob(void *ctx, const char *name)
 
 static void eapol_test_eapol_done_cb(void *ctx)
 {
+	struct eapol_test_data *e = ctx;
+
 	printf("WPA: EAPOL processing complete\n");
+	wpa_supplicant_cancel_auth_timeout(e->wpa_s);
+	wpa_supplicant_set_state(e->wpa_s, WPA_COMPLETED);
 }
 
 
@@ -407,6 +414,9 @@ static void eapol_sm_cb(struct eapol_sm *eapol, enum eapol_supp_result result,
 {
 	struct eapol_test_data *e = ctx;
 	printf("eapol_sm_cb: result=%d\n", result);
+	e->id_req_sent = 0;
+	if (e->ctrl_iface)
+		return;
 	e->eapol_test_num_reauths--;
 	if (e->eapol_test_num_reauths < 0)
 		eloop_terminate();
@@ -552,11 +562,21 @@ static void eapol_test_set_anon_id(void *ctx, const u8 *id, size_t len)
 }
 
 
+static enum wpa_states eapol_test_get_state(void *ctx)
+{
+	struct eapol_test_data *e = ctx;
+	struct wpa_supplicant *wpa_s = e->wpa_s;
+
+	return wpa_s->wpa_state;
+}
+
+
 static int test_eapol(struct eapol_test_data *e, struct wpa_supplicant *wpa_s,
 		      struct wpa_ssid *ssid)
 {
 	struct eapol_config eapol_conf;
 	struct eapol_ctx *ctx;
+	struct wpa_sm_ctx *wctx;
 
 	ctx = os_zalloc(sizeof(*ctx));
 	if (ctx == NULL) {
@@ -590,6 +610,25 @@ static int test_eapol(struct eapol_test_data *e, struct wpa_supplicant *wpa_s,
 		return -1;
 	}
 
+	wpa_s->key_mgmt = WPA_KEY_MGMT_IEEE8021X_NO_WPA;
+	wctx = os_zalloc(sizeof(*wctx));
+	if (wctx == NULL) {
+		os_free(ctx);
+		return -1;
+	}
+	wctx->ctx = e;
+	wctx->msg_ctx = wpa_s;
+	wctx->get_state = eapol_test_get_state;
+	wpa_s->wpa = wpa_sm_init(wctx);
+	if (!wpa_s->wpa) {
+		os_free(ctx);
+		os_free(wctx);
+		return -1;
+	}
+
+	if (!ssid)
+		return 0;
+
 	wpa_s->current_ssid = ssid;
 	os_memset(&eapol_conf, 0, sizeof(eapol_conf));
 	eapol_conf.accept_802_1x_keys = 1;
@@ -614,6 +653,8 @@ static void test_eapol_clean(struct eapol_test_data *e,
 {
 	struct extra_radius_attr *p, *prev;
 
+	wpa_sm_deinit(wpa_s->wpa);
+	wpa_s->wpa = NULL;
 	radius_client_deinit(e->radius);
 	wpabuf_free(e->last_eap_radius);
 	radius_msg_free(e->last_recv_radius);
@@ -757,6 +798,8 @@ static void ieee802_1x_decapsulate_radius(struct eapol_test_data *e)
 		break;
 	case EAP_CODE_FAILURE:
 		os_strlcpy(buf, "EAP Failure", sizeof(buf));
+		if (e->ctrl_iface)
+			break;
 		eloop_terminate();
 		break;
 	default:
@@ -901,25 +944,66 @@ ieee802_1x_receive_auth(struct radius_msg *msg, struct radius_msg *req,
 	if ((hdr->code == RADIUS_CODE_ACCESS_ACCEPT &&
 	     e->eapol_test_num_reauths < 0) ||
 	    hdr->code == RADIUS_CODE_ACCESS_REJECT) {
-		eloop_terminate();
+		if (!e->ctrl_iface)
+			eloop_terminate();
 	}
 
 	return RADIUS_RX_QUEUED;
 }
 
 
+static int driver_get_ssid(void *priv, u8 *ssid)
+{
+	ssid[0] = 0;
+	return 0;
+}
+
+
+static int driver_get_bssid(void *priv, u8 *bssid)
+{
+	struct eapol_test_data *e = priv;
+
+	if (e->ctrl_iface && !e->id_req_sent) {
+		eloop_register_timeout(0, 0, send_eap_request_identity,
+				       e->wpa_s, NULL);
+		e->id_req_sent = 1;
+	}
+
+	os_memset(bssid, 0, ETH_ALEN);
+	bssid[5] = 1;
+	return 0;
+}
+
+
+static int driver_get_capa(void *priv, struct wpa_driver_capa *capa)
+{
+	os_memset(capa, 0, sizeof(*capa));
+	capa->flags = WPA_DRIVER_FLAGS_WIRED;
+	return 0;
+}
+
+
+struct wpa_driver_ops eapol_test_drv_ops = {
+	.name = "test",
+	.get_ssid = driver_get_ssid,
+	.get_bssid = driver_get_bssid,
+	.get_capa = driver_get_capa,
+};
+
 static void wpa_init_conf(struct eapol_test_data *e,
 			  struct wpa_supplicant *wpa_s, const char *authsrv,
 			  int port, const char *secret,
-			  const char *cli_addr)
+			  const char *cli_addr, const char *ifname)
 {
 	struct hostapd_radius_server *as;
 	int res;
 
+	wpa_s->driver = &eapol_test_drv_ops;
+	wpa_s->drv_priv = e;
 	wpa_s->bssid[5] = 1;
 	os_memcpy(wpa_s->own_addr, e->own_addr, ETH_ALEN);
 	e->own_ip_addr.s_addr = htonl((127 << 24) | 1);
-	os_strlcpy(wpa_s->ifname, "test", sizeof(wpa_s->ifname));
+	os_strlcpy(wpa_s->ifname, ifname, sizeof(wpa_s->ifname));
 
 	e->radius_conf = os_zalloc(sizeof(struct hostapd_radius_servers));
 	assert(e->radius_conf != NULL);
@@ -938,13 +1022,12 @@ static void wpa_init_conf(struct eapol_test_data *e,
 		*pos++ = a[3];
 	}
 #else /* CONFIG_NATIVE_WINDOWS or CONFIG_ANSI_C_EXTRA */
-	if (inet_aton(authsrv, &as->addr.u.v4) < 0) {
+	if (hostapd_parse_ip_addr(authsrv, &as->addr) < 0) {
 		wpa_printf(MSG_ERROR, "Invalid IP address '%s'",
 			   authsrv);
 		assert(0);
 	}
 #endif /* CONFIG_NATIVE_WINDOWS or CONFIG_ANSI_C_EXTRA */
-	as->addr.af = AF_INET;
 	as->port = port;
 	as->shared_secret = (u8 *) os_strdup(secret);
 	as->shared_secret_len = os_strlen(secret);
@@ -1162,7 +1245,7 @@ static void usage(void)
 	       "           [-M<client MAC address>] [-o<server cert file] \\\n"
 	       "           [-N<attr spec>] [-R<PC/SC reader>] "
 	       "[-P<PC/SC PIN>] \\\n"
-	       "           [-A<client IP>]\n"
+	       "           [-A<client IP>] [-i<ifname>] [-T<ctrl_iface>]\n"
 	       "eapol_test scard\n"
 	       "eapol_test sim <PIN> <num triplets> [debug]\n"
 	       "\n");
@@ -1217,6 +1300,8 @@ int main(int argc, char *argv[])
 	int timeout = 30;
 	char *pos;
 	struct extra_radius_attr *p = NULL, *p1;
+	const char *ifname = "test";
+	const char *ctrl_iface = NULL;
 
 	if (os_program_init())
 		return -1;
@@ -1232,7 +1317,7 @@ int main(int argc, char *argv[])
 	wpa_debug_show_keys = 1;
 
 	for (;;) {
-		c = getopt(argc, argv, "a:A:c:C:eM:nN:o:p:P:r:R:s:St:W");
+		c = getopt(argc, argv, "a:A:c:C:ei:M:nN:o:p:P:r:R:s:St:T:W");
 		if (c < 0)
 			break;
 		switch (c) {
@@ -1251,6 +1336,9 @@ int main(int argc, char *argv[])
 		case 'e':
 			eapol_test.req_eap_key_name = 1;
 			break;
+		case 'i':
+			ifname = optarg;
+			break;
 		case 'M':
 			if (hwaddr_aton(optarg, eapol_test.own_addr)) {
 				usage();
@@ -1291,6 +1379,10 @@ int main(int argc, char *argv[])
 		case 't':
 			timeout = atoi(optarg);
 			break;
+		case 'T':
+			ctrl_iface = optarg;
+			eapol_test.ctrl_iface = 1;
+			break;
 		case 'W':
 			wait_for_monitor++;
 			break;
@@ -1337,7 +1429,7 @@ int main(int argc, char *argv[])
 					  &argv[optind + 1]);
 	}
 
-	if (conf == NULL) {
+	if (conf == NULL && !ctrl_iface) {
 		usage();
 		printf("Configuration file is required.\n");
 		return -1;
@@ -1359,12 +1451,15 @@ int main(int argc, char *argv[])
 	eapol_test.wpa_s = &wpa_s;
 	dl_list_init(&wpa_s.bss);
 	dl_list_init(&wpa_s.bss_id);
-	wpa_s.conf = wpa_config_read(conf, NULL);
+	if (conf)
+		wpa_s.conf = wpa_config_read(conf, NULL);
+	else
+		wpa_s.conf = wpa_config_alloc_empty(ctrl_iface, NULL);
 	if (wpa_s.conf == NULL) {
 		printf("Failed to parse configuration file '%s'.\n", conf);
 		return -1;
 	}
-	if (wpa_s.conf->ssid == NULL) {
+	if (!ctrl_iface && wpa_s.conf->ssid == NULL) {
 		printf("No networks defined.\n");
 		return -1;
 	}
@@ -1375,7 +1470,7 @@ int main(int argc, char *argv[])
 	}
 
 	wpa_init_conf(&eapol_test, &wpa_s, as_addr, as_port, as_secret,
-		      cli_addr);
+		      cli_addr, ifname);
 	wpa_s.ctrl_iface = wpa_supplicant_ctrl_iface_init(&wpa_s);
 	if (wpa_s.ctrl_iface == NULL) {
 		printf("Failed to initialize control interface '%s'.\n"
@@ -1388,7 +1483,8 @@ int main(int argc, char *argv[])
 		       wpa_s.conf->ctrl_interface);
 		return -1;
 	}
-	if (wpa_supplicant_scard_init(&wpa_s, wpa_s.conf->ssid))
+	if (wpa_s.conf->ssid &&
+	    wpa_supplicant_scard_init(&wpa_s, wpa_s.conf->ssid))
 		return -1;
 
 	if (test_eapol(&eapol_test, &wpa_s, wpa_s.conf->ssid))
@@ -1400,9 +1496,12 @@ int main(int argc, char *argv[])
 	if (wait_for_monitor)
 		wpa_supplicant_ctrl_iface_wait(wpa_s.ctrl_iface);
 
-	eloop_register_timeout(timeout, 0, eapol_test_timeout, &eapol_test,
-			       NULL);
-	eloop_register_timeout(0, 0, send_eap_request_identity, &wpa_s, NULL);
+	if (!ctrl_iface) {
+		eloop_register_timeout(timeout, 0, eapol_test_timeout,
+				       &eapol_test, NULL);
+		eloop_register_timeout(0, 0, send_eap_request_identity, &wpa_s,
+				       NULL);
+	}
 	eloop_register_signal_terminate(eapol_test_terminate, &wpa_s);
 	eloop_register_signal_reconfig(eapol_test_terminate, &wpa_s);
 	eloop_run();
diff --git a/wpa_supplicant/eapol_test.py b/wpa_supplicant/eapol_test.py
new file mode 100755
index 000000000000..80e7dfcf531d
--- /dev/null
+++ b/wpa_supplicant/eapol_test.py
@@ -0,0 +1,142 @@
+#!/usr/bin/env python2
+#
+# eapol_test controller
+# Copyright (c) 2015, Jouni Malinen <j@w1.fi>
+#
+# This software may be distributed under the terms of the BSD license.
+# See README for more details.
+
+import argparse
+import logging
+import os
+import Queue
+import sys
+import threading
+
+logger = logging.getLogger()
+dir = os.path.dirname(os.path.realpath(sys.modules[__name__].__file__))
+sys.path.append(os.path.join(dir, '..', 'wpaspy'))
+import wpaspy
+wpas_ctrl = '/tmp/eapol_test'
+
+class eapol_test:
+    def __init__(self, ifname):
+        self.ifname = ifname
+        self.ctrl = wpaspy.Ctrl(os.path.join(wpas_ctrl, ifname))
+        if "PONG" not in self.ctrl.request("PING"):
+            raise Exception("Failed to connect to eapol_test (%s)" % ifname)
+        self.mon = wpaspy.Ctrl(os.path.join(wpas_ctrl, ifname))
+        self.mon.attach()
+
+    def add_network(self):
+        id = self.request("ADD_NETWORK")
+        if "FAIL" in id:
+            raise Exception("ADD_NETWORK failed")
+        return int(id)
+
+    def remove_network(self, id):
+        id = self.request("REMOVE_NETWORK " + str(id))
+        if "FAIL" in id:
+            raise Exception("REMOVE_NETWORK failed")
+        return None
+
+    def set_network(self, id, field, value):
+        res = self.request("SET_NETWORK " + str(id) + " " + field + " " + value)
+        if "FAIL" in res:
+            raise Exception("SET_NETWORK failed")
+        return None
+
+    def set_network_quoted(self, id, field, value):
+        res = self.request("SET_NETWORK " + str(id) + " " + field + ' "' + value + '"')
+        if "FAIL" in res:
+            raise Exception("SET_NETWORK failed")
+        return None
+
+    def request(self, cmd, timeout=10):
+        return self.ctrl.request(cmd, timeout=timeout)
+
+    def wait_event(self, events, timeout=10):
+        start = os.times()[4]
+        while True:
+            while self.mon.pending():
+                ev = self.mon.recv()
+                logger.debug(self.ifname + ": " + ev)
+                for event in events:
+                    if event in ev:
+                        return ev
+            now = os.times()[4]
+            remaining = start + timeout - now
+            if remaining <= 0:
+                break
+            if not self.mon.pending(timeout=remaining):
+                break
+        return None
+
+def run(ifname, count, no_fast_reauth, res):
+    et = eapol_test(ifname)
+
+    et.request("AP_SCAN 0")
+    if no_fast_reauth:
+        et.request("SET fast_reauth 0")
+    else:
+        et.request("SET fast_reauth 1")
+    id = et.add_network()
+    et.set_network(id, "key_mgmt", "IEEE8021X")
+    et.set_network(id, "eapol_flags", "0")
+    et.set_network(id, "eap", "TLS")
+    et.set_network_quoted(id, "identity", "user")
+    et.set_network_quoted(id, "ca_cert", 'ca.pem')
+    et.set_network_quoted(id, "client_cert", 'client.pem')
+    et.set_network_quoted(id, "private_key", 'client.key')
+    et.set_network_quoted(id, "private_key_passwd", 'whatever')
+    et.set_network(id, "disabled", "0")
+
+    fail = False
+    for i in range(count):
+        et.request("REASSOCIATE")
+        ev = et.wait_event(["CTRL-EVENT-CONNECTED", "CTRL-EVENT-EAP-FAILURE"])
+        if ev is None or "CTRL-EVENT-CONNECTED" not in ev:
+            fail = True
+            break
+
+    et.remove_network(id)
+
+    if fail:
+        res.put("FAIL (%d OK)" % i)
+    else:
+        res.put("PASS %d" % (i + 1))
+
+def main():
+    parser = argparse.ArgumentParser(description='eapol_test controller')
+    parser.add_argument('--ctrl', help='control interface directory')
+    parser.add_argument('--num', help='number of processes')
+    parser.add_argument('--iter', help='number of iterations')
+    parser.add_argument('--no-fast-reauth', action='store_true',
+                        dest='no_fast_reauth',
+                        help='disable TLS session resumption')
+    args = parser.parse_args()
+
+    num = int(args.num)
+    iter = int(args.iter)
+    if args.ctrl:
+        global wpas_ctrl
+        wpas_ctrl = args.ctrl
+
+    t = {}
+    res = {}
+    for i in range(num):
+        res[i] = Queue.Queue()
+        t[i] = threading.Thread(target=run, args=(str(i), iter,
+                                                  args.no_fast_reauth, res[i]))
+    for i in range(num):
+        t[i].start()
+    for i in range(num):
+        t[i].join()
+        try:
+            results = res[i].get(False)
+        except:
+            results = "N/A"
+        print "%d: %s" % (i, results)
+
+if __name__ == "__main__":
+    main()
diff --git a/wpa_supplicant/events.c b/wpa_supplicant/events.c
index d275ca424e6e..3af1c7d89c64 100644
--- a/wpa_supplicant/events.c
+++ b/wpa_supplicant/events.c
@@ -23,6 +23,7 @@
 #include "eap_peer/eap.h"
 #include "ap/hostapd.h"
 #include "p2p/p2p.h"
+#include "fst/fst.h"
 #include "wnm_sta.h"
 #include "notify.h"
 #include "common/ieee802_11_defs.h"
@@ -71,6 +72,59 @@ static int wpas_temp_disabled(struct wpa_supplicant *wpa_s,
 }
 
 
+/**
+ * wpas_reenabled_network_time - Time until first network is re-enabled
+ * @wpa_s: Pointer to wpa_supplicant data
+ * Returns: If all enabled networks are temporarily disabled, returns the time
+ *	(in sec) until the first network is re-enabled. Otherwise returns 0.
+ *
+ * This function is used in case all enabled networks are temporarily disabled,
+ * in which case it returns the time (in sec) that the first network will be
+ * re-enabled. The function assumes that at least one network is enabled.
+ */
+static int wpas_reenabled_network_time(struct wpa_supplicant *wpa_s)
+{
+	struct wpa_ssid *ssid;
+	int disabled_for, res = 0;
+
+#ifdef CONFIG_INTERWORKING
+	if (wpa_s->conf->auto_interworking && wpa_s->conf->interworking &&
+	    wpa_s->conf->cred)
+		return 0;
+#endif /* CONFIG_INTERWORKING */
+
+	for (ssid = wpa_s->conf->ssid; ssid; ssid = ssid->next) {
+		if (ssid->disabled)
+			continue;
+
+		disabled_for = wpas_temp_disabled(wpa_s, ssid);
+		if (!disabled_for)
+			return 0;
+
+		if (!res || disabled_for < res)
+			res = disabled_for;
+	}
+
+	return res;
+}
+
+
+void wpas_network_reenabled(void *eloop_ctx, void *timeout_ctx)
+{
+	struct wpa_supplicant *wpa_s = eloop_ctx;
+
+	if (wpa_s->disconnected || wpa_s->wpa_state != WPA_SCANNING)
+		return;
+
+	wpa_dbg(wpa_s, MSG_DEBUG,
+		"Try to associate due to network getting re-enabled");
+	if (wpa_supplicant_fast_associate(wpa_s) != 1) {
+		wpa_supplicant_cancel_sched_scan(wpa_s);
+		wpa_supplicant_req_scan(wpa_s, 0, 0);
+	}
+}
+
+
 static struct wpa_bss * wpa_supplicant_get_new_bss(
 	struct wpa_supplicant *wpa_s, const u8 *bssid)
 {
@@ -105,11 +159,32 @@ static void wpa_supplicant_update_current_bss(struct wpa_supplicant *wpa_s)
 static int wpa_supplicant_select_config(struct wpa_supplicant *wpa_s)
 {
 	struct wpa_ssid *ssid, *old_ssid;
+	u8 drv_ssid[SSID_MAX_LEN];
+	size_t drv_ssid_len;
 	int res;
 
 	if (wpa_s->conf->ap_scan == 1 && wpa_s->current_ssid) {
 		wpa_supplicant_update_current_bss(wpa_s);
-		return 0;
+
+		if (wpa_s->current_ssid->ssid_len == 0)
+			return 0; /* current profile still in use */
+		res = wpa_drv_get_ssid(wpa_s, drv_ssid);
+		if (res < 0) {
+			wpa_msg(wpa_s, MSG_INFO,
+				"Failed to read SSID from driver");
+			return 0; /* try to use current profile */
+		}
+		drv_ssid_len = res;
+
+		if (drv_ssid_len == wpa_s->current_ssid->ssid_len &&
+		    os_memcmp(drv_ssid, wpa_s->current_ssid->ssid,
+			      drv_ssid_len) == 0)
+			return 0; /* current profile still in use */
+
+		wpa_msg(wpa_s, MSG_DEBUG,
+			"Driver-initiated BSS selection changed the SSID to %s",
+			wpa_ssid_txt(drv_ssid, drv_ssid_len));
+		/* continue selecting a new network profile */
 	}
 
 	wpa_dbg(wpa_s, MSG_DEBUG, "Select network based on association "
@@ -212,9 +287,6 @@ void wpa_supplicant_mark_disassoc(struct wpa_supplicant *wpa_s)
 	os_memset(wpa_s->bssid, 0, ETH_ALEN);
 	os_memset(wpa_s->pending_bssid, 0, ETH_ALEN);
 	sme_clear_on_disassoc(wpa_s);
-#ifdef CONFIG_P2P
-	os_memset(wpa_s->go_dev_addr, 0, ETH_ALEN);
-#endif /* CONFIG_P2P */
 	wpa_s->current_bss = NULL;
 	wpa_s->assoc_freq = 0;
 
@@ -756,9 +828,9 @@ static struct wpa_ssid * wpa_scan_res_match(struct wpa_supplicant *wpa_s,
 	osen = ie != NULL;
 
 	wpa_dbg(wpa_s, MSG_DEBUG, "%d: " MACSTR " ssid='%s' "
-		"wpa_ie_len=%u rsn_ie_len=%u caps=0x%x level=%d%s%s%s",
+		"wpa_ie_len=%u rsn_ie_len=%u caps=0x%x level=%d freq=%d %s%s%s",
 		i, MAC2STR(bss->bssid), wpa_ssid_txt(bss->ssid, bss->ssid_len),
-		wpa_ie_len, rsn_ie_len, bss->caps, bss->level,
+		wpa_ie_len, rsn_ie_len, bss->caps, bss->level, bss->freq,
 		wpa_bss_get_vendor_ie(bss, WPS_IE_VENDOR_TYPE) ? " wps" : "",
 		(wpa_bss_get_vendor_ie(bss, P2P_IE_VENDOR_TYPE) ||
 		 wpa_bss_get_vendor_ie_beacon(bss, P2P_IE_VENDOR_TYPE)) ?
@@ -964,6 +1036,19 @@ static struct wpa_ssid * wpa_scan_res_match(struct wpa_supplicant *wpa_s,
 		 */
 #endif /* CONFIG_P2P */
 
+		if (os_reltime_before(&bss->last_update, &wpa_s->scan_min_time))
+		{
+			struct os_reltime diff;
+
+			os_reltime_sub(&wpa_s->scan_min_time,
+				       &bss->last_update, &diff);
+			wpa_dbg(wpa_s, MSG_DEBUG,
+				"   skip - scan result not recent enough (%u.%06u seconds too old)",
+				(unsigned int) diff.sec,
+				(unsigned int) diff.usec);
+			continue;
+		}
+
 		/* Matching configuration found */
 		return ssid;
 	}
@@ -1011,14 +1096,13 @@ struct wpa_bss * wpa_supplicant_pick_network(struct wpa_supplicant *wpa_s,
 	struct wpa_bss *selected = NULL;
 	int prio;
 	struct wpa_ssid *next_ssid = NULL;
+	struct wpa_ssid *ssid;
 
 	if (wpa_s->last_scan_res == NULL ||
 	    wpa_s->last_scan_res_used == 0)
 		return NULL; /* no scan results from last update */
 
 	if (wpa_s->next_ssid) {
-		struct wpa_ssid *ssid;
-
 		/* check that next_ssid is still valid */
 		for (ssid = wpa_s->conf->ssid; ssid; ssid = ssid->next) {
 			if (ssid == wpa_s->next_ssid)
@@ -1054,6 +1138,27 @@ struct wpa_bss * wpa_supplicant_pick_network(struct wpa_supplicant *wpa_s,
 			break;
 	}
 
+	ssid = *selected_ssid;
+	if (selected && ssid && ssid->mem_only_psk && !ssid->psk_set &&
+	    !ssid->passphrase && !ssid->ext_psk) {
+		const char *field_name, *txt = NULL;
+
+		wpa_dbg(wpa_s, MSG_DEBUG,
+			"PSK/passphrase not yet available for the selected network");
+
+		wpas_notify_network_request(wpa_s, ssid,
+					    WPA_CTRL_REQ_PSK_PASSPHRASE, NULL);
+
+		field_name = wpa_supplicant_ctrl_req_to_string(
+			WPA_CTRL_REQ_PSK_PASSPHRASE, NULL, &txt);
+		if (field_name == NULL)
+			return NULL;
+
+		wpas_send_ctrl_req(wpa_s, ssid, field_name, txt);
+
+		selected = NULL;
+	}
+
 	return selected;
 }
 
@@ -1085,6 +1190,7 @@ int wpa_supplicant_connect(struct wpa_supplicant *wpa_s,
 	if (wpas_wps_scan_pbc_overlap(wpa_s, selected, ssid)) {
 		wpa_msg(wpa_s, MSG_INFO, WPS_EVENT_OVERLAP
 			"PBC session overlap");
+		wpas_notify_wps_event_pbc_overlap(wpa_s);
 #ifdef CONFIG_P2P
 		if (wpa_s->p2p_group_interface == P2P_GROUP_INTERFACE_CLIENT ||
 		    wpa_s->p2p_in_provisioning) {
@@ -1095,6 +1201,7 @@ int wpa_supplicant_connect(struct wpa_supplicant *wpa_s,
 #endif /* CONFIG_P2P */
 
 #ifdef CONFIG_WPS
+		wpas_wps_pbc_overlap(wpa_s);
 		wpas_wps_cancel(wpa_s);
 #endif /* CONFIG_WPS */
 		return -1;
@@ -1192,7 +1299,9 @@ static int wpa_supplicant_need_to_roam(struct wpa_supplicant *wpa_s,
 				       struct wpa_ssid *ssid)
 {
 	struct wpa_bss *current_bss = NULL;
+#ifndef CONFIG_NO_ROAMING
 	int min_diff;
+#endif /* CONFIG_NO_ROAMING */
 
 	if (wpa_s->reassociate)
 		return 1; /* explicit request to reassociate */
@@ -1421,6 +1530,17 @@ static int wpas_select_network_from_last_scan(struct wpa_supplicant *wpa_s,
 {
 	struct wpa_bss *selected;
 	struct wpa_ssid *ssid = NULL;
+	int time_to_reenable = wpas_reenabled_network_time(wpa_s);
+
+	if (time_to_reenable > 0) {
+		wpa_dbg(wpa_s, MSG_DEBUG,
+			"Postpone network selection by %d seconds since all networks are disabled",
+			time_to_reenable);
+		eloop_cancel_timeout(wpas_network_reenabled, wpa_s, NULL);
+		eloop_register_timeout(time_to_reenable, 0,
+				       wpas_network_reenabled, wpa_s, NULL);
+		return 0;
+	}
 
 	if (wpa_s->p2p_mgmt)
 		return 0; /* no normal connection on p2p_mgmt interface */
@@ -1520,6 +1640,9 @@ static int wpas_select_network_from_last_scan(struct wpa_supplicant *wpa_s,
 			if (wpa_supplicant_req_sched_scan(wpa_s))
 				wpa_supplicant_req_new_scan(wpa_s, timeout_sec,
 							    timeout_usec);
+
+			wpa_msg_ctrl(wpa_s, MSG_INFO,
+				     WPA_EVENT_NETWORK_NOT_FOUND);
 		}
 	}
 	return 0;
@@ -1577,7 +1700,7 @@ int wpa_supplicant_fast_associate(struct wpa_supplicant *wpa_s)
 #else /* CONFIG_NO_SCAN_PROCESSING */
 	struct os_reltime now;
 
-	if (wpa_s->last_scan_res_used <= 0)
+	if (wpa_s->last_scan_res_used == 0)
 		return -1;
 
 	os_get_reltime(&now);
@@ -1889,6 +2012,19 @@ static int wpa_supplicant_event_associnfo(struct wpa_supplicant *wpa_s,
 	if (wpa_found || rsn_found)
 		wpa_s->ap_ies_from_associnfo = 1;
 
+#ifdef CONFIG_FST
+	wpabuf_free(wpa_s->received_mb_ies);
+	wpa_s->received_mb_ies = NULL;
+	if (wpa_s->fst) {
+		struct mb_ies_info mb_ies;
+
+		wpa_printf(MSG_DEBUG, "Looking for MB IE");
+		if (!mb_ies_info_by_ies(&mb_ies, data->assoc_info.resp_ies,
+					data->assoc_info.resp_ies_len))
+			wpa_s->received_mb_ies = mb_ies_by_info(&mb_ies);
+	}
+#endif /* CONFIG_FST */
+
 	if (wpa_s->assoc_freq && data->assoc_info.freq &&
 	    wpa_s->assoc_freq != data->assoc_info.freq) {
 		wpa_printf(MSG_DEBUG, "Operating frequency changed from "
@@ -1932,6 +2068,7 @@ static void wpa_supplicant_event_assoc(struct wpa_supplicant *wpa_s,
 {
 	u8 bssid[ETH_ALEN];
 	int ft_completed;
+	int new_bss = 0;
 
 #ifdef CONFIG_AP
 	if (wpa_s->ap_iface) {
@@ -1946,6 +2083,8 @@ static void wpa_supplicant_event_assoc(struct wpa_supplicant *wpa_s,
 	}
 #endif /* CONFIG_AP */
 
+	eloop_cancel_timeout(wpas_network_reenabled, wpa_s, NULL);
+
 	ft_completed = wpa_ft_is_completed(wpa_s->wpa);
 	if (data && wpa_supplicant_event_associnfo(wpa_s, data) < 0)
 		return;
@@ -1961,6 +2100,7 @@ static void wpa_supplicant_event_assoc(struct wpa_supplicant *wpa_s,
 	if (os_memcmp(bssid, wpa_s->bssid, ETH_ALEN) != 0) {
 		wpa_dbg(wpa_s, MSG_DEBUG, "Associated to a new BSS: BSSID="
 			MACSTR, MAC2STR(bssid));
+		new_bss = 1;
 		random_add_randomness(bssid, ETH_ALEN);
 		os_memcpy(wpa_s->bssid, bssid, ETH_ALEN);
 		os_memset(wpa_s->pending_bssid, 0, ETH_ALEN);
@@ -1974,13 +2114,13 @@ static void wpa_supplicant_event_assoc(struct wpa_supplicant *wpa_s,
 				wpa_s, WLAN_REASON_DEAUTH_LEAVING);
 			return;
 		}
+	}
 
-		if (wpa_s->conf->ap_scan == 1 &&
-		    wpa_s->drv_flags & WPA_DRIVER_FLAGS_BSS_SELECTION) {
-			if (wpa_supplicant_assoc_update_ie(wpa_s) < 0)
-				wpa_msg(wpa_s, MSG_WARNING,
-					"WPA/RSN IEs not updated");
-		}
+	if (wpa_s->conf->ap_scan == 1 &&
+	    wpa_s->drv_flags & WPA_DRIVER_FLAGS_BSS_SELECTION) {
+		if (wpa_supplicant_assoc_update_ie(wpa_s) < 0 && new_bss)
+			wpa_msg(wpa_s, MSG_WARNING,
+				"WPA/RSN IEs not updated");
 	}
 
 #ifdef CONFIG_SME
@@ -2253,7 +2393,8 @@ static void wpa_supplicant_event_disassoc_finish(struct wpa_supplicant *wpa_s,
 			"try to re-connect");
 		wpa_s->reassociate = 0;
 		wpa_s->disconnected = 1;
-		wpa_supplicant_cancel_sched_scan(wpa_s);
+		if (!wpa_s->pno)
+			wpa_supplicant_cancel_sched_scan(wpa_s);
 	}
 	bssid = wpa_s->bssid;
 	if (is_zero_ether_addr(bssid))
@@ -2443,6 +2584,21 @@ wpa_supplicant_event_interface_status(struct wpa_supplicant *wpa_s,
 			wpa_msg(wpa_s, MSG_INFO, "Failed to initialize the "
 				"driver after interface was added");
 		}
+
+#ifdef CONFIG_P2P
+		if (!wpa_s->global->p2p &&
+		    !wpa_s->global->p2p_disabled &&
+		    !wpa_s->conf->p2p_disabled &&
+		    (wpa_s->drv_flags &
+		     WPA_DRIVER_FLAGS_DEDICATED_P2P_DEVICE) &&
+		    wpas_p2p_add_p2pdev_interface(
+			    wpa_s, wpa_s->global->params.conf_p2p_dev) < 0) {
+			wpa_printf(MSG_INFO,
+				   "P2P: Failed to enable P2P Device interface");
+			/* Try to continue without. P2P will be disabled. */
+		}
+#endif /* CONFIG_P2P */
+
 		break;
 	case EVENT_INTERFACE_REMOVED:
 		wpa_dbg(wpa_s, MSG_DEBUG, "Configured interface was removed");
@@ -2451,6 +2607,21 @@ wpa_supplicant_event_interface_status(struct wpa_supplicant *wpa_s,
 		wpa_supplicant_set_state(wpa_s, WPA_INTERFACE_DISABLED);
 		l2_packet_deinit(wpa_s->l2);
 		wpa_s->l2 = NULL;
+
+#ifdef CONFIG_P2P
+		if (wpa_s->global->p2p &&
+		    wpa_s->global->p2p_init_wpa_s->parent == wpa_s &&
+		    (wpa_s->drv_flags &
+		     WPA_DRIVER_FLAGS_DEDICATED_P2P_DEVICE)) {
+			wpa_dbg(wpa_s, MSG_DEBUG,
+				"Removing P2P Device interface");
+			wpa_supplicant_remove_iface(
+				wpa_s->global, wpa_s->global->p2p_init_wpa_s,
+				0);
+			wpa_s->global->p2p_init_wpa_s = NULL;
+		}
+#endif /* CONFIG_P2P */
+
 #ifdef CONFIG_TERMINATE_ONLASTIF
 		/* check if last interface */
 		if (!any_interfaces(wpa_s->global->ifaces))
@@ -2844,26 +3015,24 @@ static void wpa_supplicant_update_channel_list(
 	if (wpa_s->drv_priv == NULL)
 		return; /* Ignore event during drv initialization */
 
-	free_hw_features(wpa_s);
-	wpa_s->hw.modes = wpa_drv_get_hw_feature_data(
-		wpa_s, &wpa_s->hw.num_modes, &wpa_s->hw.flags);
-
-	wpas_p2p_update_channel_list(wpa_s);
-
-	/*
-	 * Check other interfaces to see if they share the same radio. If
-	 * so, they get updated with this same hw mode info.
-	 */
 	dl_list_for_each(ifs, &wpa_s->radio->ifaces, struct wpa_supplicant,
 			 radio_list) {
-		if (ifs != wpa_s) {
-			wpa_printf(MSG_DEBUG, "%s: Updating hw mode",
-				   ifs->ifname);
-			free_hw_features(ifs);
-			ifs->hw.modes = wpa_drv_get_hw_feature_data(
-				ifs, &ifs->hw.num_modes, &ifs->hw.flags);
-		}
+		wpa_printf(MSG_DEBUG, "%s: Updating hw mode",
+			   ifs->ifname);
+		free_hw_features(ifs);
+		ifs->hw.modes = wpa_drv_get_hw_feature_data(
+			ifs, &ifs->hw.num_modes, &ifs->hw.flags);
 	}
+
+	/* Restart sched_scan with updated channel list */
+	if (wpa_s->sched_scanning) {
+		wpa_dbg(wpa_s, MSG_DEBUG,
+			"Channel list changed restart sched scan.");
+		wpa_supplicant_cancel_sched_scan(wpa_s);
+		wpa_supplicant_req_scan(wpa_s, 0, 0);
+	}
+
+	wpas_p2p_update_channel_list(wpa_s, WPAS_P2P_CHANNEL_UPDATE_DRIVER);
 }
 
 
@@ -2964,6 +3133,13 @@ static void wpas_event_rx_mgmt_action(struct wpa_supplicant *wpa_s,
 		return;
 	}
 
+#ifdef CONFIG_FST
+	if (mgmt->u.action.category == WLAN_ACTION_FST && wpa_s->fst) {
+		fst_rx_action(wpa_s->fst, mgmt, len);
+		return;
+	}
+#endif /* CONFIG_FST */
+
 	wpas_p2p_rx_action(wpa_s, mgmt->da, mgmt->sa, mgmt->bssid,
 			   category, payload, plen, freq);
 	if (wpa_s->ifmsh)
@@ -2974,9 +3150,6 @@ static void wpas_event_rx_mgmt_action(struct wpa_supplicant *wpa_s,
 static void wpa_supplicant_notify_avoid_freq(struct wpa_supplicant *wpa_s,
 					     union wpa_event_data *event)
 {
-#ifdef CONFIG_P2P
-	struct wpa_supplicant *ifs;
-#endif /* CONFIG_P2P */
 	struct wpa_freq_range_list *list;
 	char *str = NULL;
 
@@ -2993,29 +3166,13 @@ static void wpa_supplicant_notify_avoid_freq(struct wpa_supplicant *wpa_s,
 			__func__);
 	} else {
 		wpa_dbg(wpa_s, MSG_DEBUG, "P2P: Update channel list based on frequency avoid event");
-		wpas_p2p_update_channel_list(wpa_s);
-	}
 
-	for (ifs = wpa_s->global->ifaces; ifs; ifs = ifs->next) {
-		int freq;
-		if (!ifs->current_ssid ||
-		    !ifs->current_ssid->p2p_group ||
-		    (ifs->current_ssid->mode != WPAS_MODE_P2P_GO &&
-		     ifs->current_ssid->mode != WPAS_MODE_P2P_GROUP_FORMATION))
-			continue;
-
-		freq = ifs->current_ssid->frequency;
-		if (!freq_range_list_includes(list, freq)) {
-			wpa_dbg(ifs, MSG_DEBUG, "P2P GO operating frequency %d MHz in safe range",
-				freq);
-			continue;
-		}
-
-		wpa_dbg(ifs, MSG_DEBUG, "P2P GO operating in unsafe frequency %d MHz",
-			freq);
-		/* TODO: Consider using CSA or removing the group within
-		 * wpa_supplicant */
-		wpa_msg(ifs, MSG_INFO, P2P_EVENT_REMOVE_AND_REFORM_GROUP);
+		/*
+		 * The update channel flow will also take care of moving a GO
+		 * from the unsafe frequency if needed.
+		 */
+		wpas_p2p_update_channel_list(wpa_s,
+					     WPAS_P2P_CHANNEL_UPDATE_AVOID);
 	}
 #endif /* CONFIG_P2P */
 
@@ -3047,6 +3204,7 @@ void wpa_supplicant_event(void *ctx, enum wpa_event_type event,
 			  union wpa_event_data *data)
 {
 	struct wpa_supplicant *wpa_s = ctx;
+	int resched;
 
 	if (wpa_s->wpa_state == WPA_INTERFACE_DISABLED &&
 	    event != EVENT_INTERFACE_ENABLED &&
@@ -3372,6 +3530,7 @@ void wpa_supplicant_event(void *ctx, enum wpa_event_type event,
 				wpas_p2p_probe_req_rx(
 					wpa_s, src, mgmt->da,
 					mgmt->bssid, ie, ie_len,
+					data->rx_mgmt.freq,
 					data->rx_mgmt.ssi_signal);
 				break;
 			}
@@ -3443,6 +3602,7 @@ void wpa_supplicant_event(void *ctx, enum wpa_event_type event,
 				      data->rx_probe_req.bssid,
 				      data->rx_probe_req.ie,
 				      data->rx_probe_req.ie_len,
+				      0,
 				      data->rx_probe_req.ssi_signal);
 		break;
 	case EVENT_REMAIN_ON_CHANNEL:
@@ -3613,6 +3773,7 @@ void wpa_supplicant_event(void *ctx, enum wpa_event_type event,
 	case EVENT_SCHED_SCAN_STOPPED:
 		wpa_s->pno = 0;
 		wpa_s->sched_scanning = 0;
+		resched = wpa_s->scanning;
 		wpa_supplicant_notify_scanning(wpa_s, 0);
 
 		if (wpa_s->wpa_state == WPA_INTERFACE_DISABLED)
@@ -3627,6 +3788,8 @@ void wpa_supplicant_event(void *ctx, enum wpa_event_type event,
 		} else if (wpa_s->pno_sched_pending) {
 			wpa_s->pno_sched_pending = 0;
 			wpas_start_pno(wpa_s);
+		} else if (resched) {
+			wpa_supplicant_req_scan(wpa_s, 0, 0);
 		}
 
 		break;
diff --git a/wpa_supplicant/hs20_supplicant.c b/wpa_supplicant/hs20_supplicant.c
index b9cd68193b2d..a1afc85ff9bb 100644
--- a/wpa_supplicant/hs20_supplicant.c
+++ b/wpa_supplicant/hs20_supplicant.c
@@ -46,7 +46,7 @@ struct osu_icon {
 
 struct osu_provider {
 	u8 bssid[ETH_ALEN];
-	u8 osu_ssid[32];
+	u8 osu_ssid[SSID_MAX_LEN];
 	u8 osu_ssid_len;
 	char server_uri[256];
 	u32 osu_methods; /* bit 0 = OMA-DM, bit 1 = SOAP-XML SPP */
@@ -188,14 +188,16 @@ int hs20_anqp_send_req(struct wpa_supplicant *wpa_s, const u8 *dst, u32 stypes,
 	struct wpa_bss *bss;
 	int res;
 
-	freq = wpa_s->assoc_freq;
 	bss = wpa_bss_get_bssid(wpa_s, dst);
-	if (bss) {
-		wpa_bss_anqp_unshare_alloc(bss);
-		freq = bss->freq;
-	}
-	if (freq <= 0)
+	if (!bss) {
+		wpa_printf(MSG_WARNING,
+			   "ANQP: Cannot send query to unknown BSS "
+			   MACSTR, MAC2STR(dst));
 		return -1;
+	}
+
+	wpa_bss_anqp_unshare_alloc(bss);
+	freq = bss->freq;
 
 	wpa_printf(MSG_DEBUG, "HS20: ANQP Query Request to " MACSTR " for "
 		   "subtypes 0x%x", MAC2STR(dst), stypes);
@@ -822,7 +824,7 @@ void hs20_osu_icon_fetch(struct wpa_supplicant *wpa_s)
 			continue;
 		}
 		osu_ssid_len = *pos++;
-		if (osu_ssid_len > 32) {
+		if (osu_ssid_len > SSID_MAX_LEN) {
 			wpa_printf(MSG_DEBUG, "HS 2.0: Invalid OSU SSID "
 				   "Length %u", osu_ssid_len);
 			continue;
diff --git a/wpa_supplicant/ibss_rsn.c b/wpa_supplicant/ibss_rsn.c
index d0ae135bdf72..d9d0ae7f10dd 100644
--- a/wpa_supplicant/ibss_rsn.c
+++ b/wpa_supplicant/ibss_rsn.c
@@ -571,6 +571,9 @@ int ibss_rsn_start(struct ibss_rsn *ibss_rsn, const u8 *addr)
 	struct ibss_rsn_peer *peer;
 	int res;
 
+	if (!ibss_rsn)
+		return -1;
+
 	/* if the peer already exists, exit immediately */
 	peer = ibss_rsn_get_peer(ibss_rsn, addr);
 	if (peer)
@@ -694,7 +697,8 @@ void ibss_rsn_deinit(struct ibss_rsn *ibss_rsn)
 		ibss_rsn_free(prev);
 	}
 
-	wpa_deinit(ibss_rsn->auth_group);
+	if (ibss_rsn->auth_group)
+		wpa_deinit(ibss_rsn->auth_group);
 	os_free(ibss_rsn);
 
 }
diff --git a/wpa_supplicant/interworking.c b/wpa_supplicant/interworking.c
index 4a396654487e..fd47c179ea4b 100644
--- a/wpa_supplicant/interworking.c
+++ b/wpa_supplicant/interworking.c
@@ -2058,7 +2058,7 @@ static struct wpa_cred * interworking_credentials_available_helper(
 	int *excluded)
 {
 	struct wpa_cred *cred, *cred2;
-	int excluded1, excluded2;
+	int excluded1, excluded2 = 0;
 
 	if (disallowed_bssid(wpa_s, bss->bssid) ||
 	    disallowed_ssid(wpa_s, bss->ssid, bss->ssid_len)) {
@@ -2673,14 +2673,16 @@ int anqp_send_req(struct wpa_supplicant *wpa_s, const u8 *dst,
 	struct wpa_bss *bss;
 	int res;
 
-	freq = wpa_s->assoc_freq;
 	bss = wpa_bss_get_bssid(wpa_s, dst);
-	if (bss) {
-		wpa_bss_anqp_unshare_alloc(bss);
-		freq = bss->freq;
-	}
-	if (freq <= 0)
+	if (!bss) {
+		wpa_printf(MSG_WARNING,
+			   "ANQP: Cannot send query to unknown BSS "
+			   MACSTR, MAC2STR(dst));
 		return -1;
+	}
+
+	wpa_bss_anqp_unshare_alloc(bss);
+	freq = bss->freq;
 
 	wpa_msg(wpa_s, MSG_DEBUG,
 		"ANQP: Query Request to " MACSTR " for %u id(s)",
diff --git a/wpa_supplicant/main.c b/wpa_supplicant/main.c
index 22827479c643..d5d47e1d77b2 100644
--- a/wpa_supplicant/main.c
+++ b/wpa_supplicant/main.c
@@ -12,6 +12,7 @@
 #endif /* __linux__ */
 
 #include "common.h"
+#include "fst/fst.h"
 #include "wpa_supplicant_i.h"
 #include "driver_i.h"
 #include "p2p_supplicant.h"
@@ -237,7 +238,7 @@ int main(int argc, char *argv[])
 			goto out;
 #ifdef CONFIG_P2P
 		case 'm':
-			iface->conf_p2p_dev = optarg;
+			params.conf_p2p_dev = optarg;
 			break;
 #endif /* CONFIG_P2P */
 		case 'o':
@@ -288,7 +289,7 @@ int main(int argc, char *argv[])
 			if (iface == NULL)
 				goto out;
 			ifaces = iface;
-			iface = &ifaces[iface_count - 1]; 
+			iface = &ifaces[iface_count - 1];
 			os_memset(iface, 0, sizeof(*iface));
 			break;
 		default:
@@ -309,6 +310,17 @@ int main(int argc, char *argv[])
 			   "wpa_supplicant");
 	}
 
+	if (fst_global_init()) {
+		wpa_printf(MSG_ERROR, "Failed to initialize FST");
+		exitcode = -1;
+		goto out;
+	}
+
+#if defined(CONFIG_FST) && defined(CONFIG_CTRL_IFACE)
+	if (!fst_global_add_ctrl(fst_ctrl_cli))
+		wpa_printf(MSG_WARNING, "Failed to add CLI FST ctrl");
+#endif
+
 	for (i = 0; exitcode == 0 && i < iface_count; i++) {
 		struct wpa_supplicant *wpa_s;
 
@@ -334,6 +346,8 @@ int main(int argc, char *argv[])
 
 	wpa_supplicant_deinit(global);
 
+	fst_global_deinit();
+
 out:
 	wpa_supplicant_fd_workaround(0);
 	os_free(ifaces);
diff --git a/wpa_supplicant/mesh.c b/wpa_supplicant/mesh.c
index 33b4af38faa8..77f708b42daa 100644
--- a/wpa_supplicant/mesh.c
+++ b/wpa_supplicant/mesh.c
@@ -47,8 +47,8 @@ void wpa_supplicant_mesh_iface_deinit(struct wpa_supplicant *wpa_s,
 
 	if (ifmsh->mconf) {
 		mesh_mpm_deinit(wpa_s, ifmsh);
-		if (ifmsh->mconf->ies) {
-			ifmsh->mconf->ies = NULL;
+		if (ifmsh->mconf->rsn_ie) {
+			ifmsh->mconf->rsn_ie = NULL;
 			/* We cannot free this struct
 			 * because wpa_authenticator on
 			 * hostapd side is also using it
@@ -171,6 +171,8 @@ static int wpa_supplicant_mesh_init(struct wpa_supplicant *wpa_s,
 	ifmsh->conf = conf;
 
 	ifmsh->bss[0]->max_plinks = wpa_s->conf->max_peer_links;
+	ifmsh->bss[0]->dot11RSNASAERetransPeriod =
+		wpa_s->conf->dot11RSNASAERetransPeriod;
 	os_strlcpy(bss->conf->iface, wpa_s->ifname, sizeof(bss->conf->iface));
 
 	mconf = mesh_config_create(ssid);
@@ -350,8 +352,8 @@ int wpa_supplicant_join_mesh(struct wpa_supplicant *wpa_s,
 	}
 
 	if (wpa_s->ifmsh) {
-		params.ies = wpa_s->ifmsh->mconf->ies;
-		params.ie_len = wpa_s->ifmsh->mconf->ie_len;
+		params.ies = wpa_s->ifmsh->mconf->rsn_ie;
+		params.ie_len = wpa_s->ifmsh->mconf->rsn_ie_len;
 		params.basic_rates = wpa_s->ifmsh->basic_rates;
 	}
 
@@ -453,22 +455,23 @@ static int mesh_attr_text(const u8 *ies, size_t ies_len, char *buf, char *end)
 		ret = os_snprintf(pos, end - pos, "bss_basic_rate_set=%d",
 				  bss_basic_rate_set[0]);
 		if (os_snprintf_error(end - pos, ret))
-			return pos - buf;
+			goto fail;
 		pos += ret;
 
 		for (i = 1; i < bss_basic_rate_set_len; i++) {
 			ret = os_snprintf(pos, end - pos, " %d",
 					  bss_basic_rate_set[i]);
 			if (os_snprintf_error(end - pos, ret))
-				return pos - buf;
+				goto fail;
 			pos += ret;
 		}
 
 		ret = os_snprintf(pos, end - pos, "\n");
 		if (os_snprintf_error(end - pos, ret))
-			return pos - buf;
+			goto fail;
 		pos += ret;
 	}
+fail:
 	os_free(bss_basic_rate_set);
 
 	return pos - buf;
diff --git a/wpa_supplicant/mesh_mpm.c b/wpa_supplicant/mesh_mpm.c
index 1d6f2be2b50b..f81b88c89401 100644
--- a/wpa_supplicant/mesh_mpm.c
+++ b/wpa_supplicant/mesh_mpm.c
@@ -239,6 +239,9 @@ static void mesh_mpm_send_plink_action(struct wpa_supplicant *wpa_s,
 			   2 + 22;  /* HT operation */
 	}
 #endif /* CONFIG_IEEE80211N */
+	if (type != PLINK_CLOSE)
+		buf_len += conf->rsn_ie_len; /* RSN IE */
+
 	buf = wpabuf_alloc(buf_len);
 	if (!buf)
 		return;
@@ -262,6 +265,9 @@ static void mesh_mpm_send_plink_action(struct wpa_supplicant *wpa_s,
 		pos = hostapd_eid_ext_supp_rates(bss, pos);
 		wpabuf_put_data(buf, supp_rates, pos - supp_rates);
 
+		/* IE: RSN IE */
+		wpabuf_put_data(buf, conf->rsn_ie, conf->rsn_ie_len);
+
 		/* IE: Mesh ID */
 		wpabuf_put_u8(buf, WLAN_EID_MESH_ID);
 		wpabuf_put_u8(buf, conf->meshid_len);
@@ -551,8 +557,7 @@ static struct sta_info * mesh_mpm_add_peer(struct wpa_supplicant *wpa_s,
 	mesh_mpm_init_link(wpa_s, sta);
 
 #ifdef CONFIG_IEEE80211N
-	copy_sta_ht_capab(data, sta, elems->ht_capabilities,
-			elems->ht_capabilities_len);
+	copy_sta_ht_capab(data, sta, elems->ht_capabilities);
 	update_ht_state(data, sta);
 #endif /* CONFIG_IEEE80211N */
 
diff --git a/wpa_supplicant/mesh_rsn.c b/wpa_supplicant/mesh_rsn.c
index 936002d954ec..747f1ae6968b 100644
--- a/wpa_supplicant/mesh_rsn.c
+++ b/wpa_supplicant/mesh_rsn.c
@@ -190,7 +190,8 @@ static int __mesh_rsn_auth_init(struct mesh_rsn *rsn, const u8 *addr)
 static void mesh_rsn_deinit(struct mesh_rsn *rsn)
 {
 	os_memset(rsn->mgtk, 0, sizeof(rsn->mgtk));
-	wpa_deinit(rsn->auth);
+	if (rsn->auth)
+		wpa_deinit(rsn->auth);
 }
 
 
@@ -209,14 +210,15 @@ struct mesh_rsn *mesh_rsn_auth_init(struct wpa_supplicant *wpa_s,
 
 	if (__mesh_rsn_auth_init(mesh_rsn, wpa_s->own_addr) < 0) {
 		mesh_rsn_deinit(mesh_rsn);
+		os_free(mesh_rsn);
 		return NULL;
 	}
 
 	bss->wpa_auth = mesh_rsn->auth;
 
 	ie = wpa_auth_get_wpa_ie(mesh_rsn->auth, &ie_len);
-	conf->ies = (u8 *) ie;
-	conf->ie_len = ie_len;
+	conf->rsn_ie = (u8 *) ie;
+	conf->rsn_ie_len = ie_len;
 
 	wpa_supplicant_rsn_supp_set_config(wpa_s, wpa_s->current_ssid);
 
diff --git a/wpa_supplicant/notify.c b/wpa_supplicant/notify.c
index ea7dbdb15bf2..45d06bf35744 100644
--- a/wpa_supplicant/notify.c
+++ b/wpa_supplicant/notify.c
@@ -17,6 +17,7 @@
 #include "dbus/dbus_old.h"
 #include "dbus/dbus_new.h"
 #include "rsn_supp/wpa.h"
+#include "fst/fst.h"
 #include "driver_i.h"
 #include "scan.h"
 #include "p2p_supplicant.h"
@@ -88,6 +89,16 @@ void wpas_notify_state_changed(struct wpa_supplicant *wpa_s,
 	/* notify the new DBus API */
 	wpas_dbus_signal_prop_changed(wpa_s, WPAS_DBUS_PROP_STATE);
 
+#ifdef CONFIG_FST
+	if (wpa_s->fst && !is_zero_ether_addr(wpa_s->bssid)) {
+		if (new_state == WPA_COMPLETED)
+			fst_notify_peer_connected(wpa_s->fst, wpa_s->bssid);
+		else if (old_state >= WPA_ASSOCIATED &&
+			 new_state < WPA_ASSOCIATED)
+			fst_notify_peer_disconnected(wpa_s->fst, wpa_s->bssid);
+	}
+#endif /* CONFIG_FST */
+
 	if (new_state == WPA_COMPLETED)
 		wpas_p2p_notif_connected(wpa_s);
 	else if (old_state >= WPA_ASSOCIATED && new_state < WPA_ASSOCIATED)
@@ -268,6 +279,16 @@ void wpas_notify_wps_event_success(struct wpa_supplicant *wpa_s)
 #endif /* CONFIG_WPS */
 }
 
+void wpas_notify_wps_event_pbc_overlap(struct wpa_supplicant *wpa_s)
+{
+	if (wpa_s->p2p_mgmt)
+		return;
+
+#ifdef CONFIG_WPS
+	wpas_dbus_signal_wps_event_pbc_overlap(wpa_s);
+#endif /* CONFIG_WPS */
+}
+
 
 void wpas_notify_network_added(struct wpa_supplicant *wpa_s,
 			       struct wpa_ssid *ssid)
@@ -307,14 +328,12 @@ void wpas_notify_persistent_group_removed(struct wpa_supplicant *wpa_s,
 void wpas_notify_network_removed(struct wpa_supplicant *wpa_s,
 				 struct wpa_ssid *ssid)
 {
-	if (wpa_s->p2p_mgmt)
-		return;
-
 	if (wpa_s->next_ssid == ssid)
 		wpa_s->next_ssid = NULL;
 	if (wpa_s->wpa)
 		wpa_sm_pmksa_cache_flush(wpa_s->wpa, ssid);
-	if (!ssid->p2p_group && wpa_s->global->p2p_group_formation != wpa_s)
+	if (!ssid->p2p_group && wpa_s->global->p2p_group_formation != wpa_s &&
+	    !wpa_s->p2p_mgmt)
 		wpas_dbus_unregister_network(wpa_s, ssid->id);
 	if (network_is_persistent_group(ssid))
 		wpas_notify_persistent_group_removed(wpa_s, ssid);
@@ -522,6 +541,13 @@ void wpas_notify_resume(struct wpa_global *global)
 
 #ifdef CONFIG_P2P
 
+void wpas_notify_p2p_find_stopped(struct wpa_supplicant *wpa_s)
+{
+	/* Notify P2P find has stopped */
+	wpas_dbus_signal_p2p_find_stopped(wpa_s);
+}
+
+
 void wpas_notify_p2p_device_found(struct wpa_supplicant *wpa_s,
 				  const u8 *dev_addr, int new_device)
 {
@@ -556,9 +582,9 @@ void wpas_notify_p2p_group_removed(struct wpa_supplicant *wpa_s,
 
 
 void wpas_notify_p2p_go_neg_req(struct wpa_supplicant *wpa_s,
-				const u8 *src, u16 dev_passwd_id)
+				const u8 *src, u16 dev_passwd_id, u8 go_intent)
 {
-	wpas_dbus_signal_p2p_go_neg_req(wpa_s, src, dev_passwd_id);
+	wpas_dbus_signal_p2p_go_neg_req(wpa_s, src, dev_passwd_id, go_intent);
 }
 
 
@@ -631,12 +657,30 @@ void wpas_notify_p2p_group_started(struct wpa_supplicant *wpa_s,
 }
 
 
+void wpas_notify_p2p_group_formation_failure(struct wpa_supplicant *wpa_s,
+					     const char *reason)
+{
+	/* Notify a group formation failed */
+	wpas_dbus_signal_p2p_group_formation_failure(wpa_s, reason);
+}
+
+
 void wpas_notify_p2p_wps_failed(struct wpa_supplicant *wpa_s,
 				struct wps_event_fail *fail)
 {
 	wpas_dbus_signal_p2p_wps_failed(wpa_s, fail);
 }
 
+
+void wpas_notify_p2p_invitation_received(struct wpa_supplicant *wpa_s,
+					 const u8 *sa, const u8 *go_dev_addr,
+					 const u8 *bssid, int id, int op_freq)
+{
+	/* Notify a P2P Invitation Request */
+	wpas_dbus_signal_p2p_invitation_received(wpa_s, sa, go_dev_addr, bssid,
+						 id, op_freq);
+}
+
 #endif /* CONFIG_P2P */
 
 
@@ -775,10 +819,12 @@ void wpas_notify_network_type_changed(struct wpa_supplicant *wpa_s,
 		ssid->disabled = 0;
 		wpas_dbus_unregister_network(wpa_s, ssid->id);
 		ssid->disabled = 2;
+		ssid->p2p_persistent_group = 1;
 		wpas_dbus_register_persistent_group(wpa_s, ssid);
 	} else {
 		/* Changed from persistent group to normal network profile */
 		wpas_dbus_unregister_persistent_group(wpa_s, ssid->id);
+		ssid->p2p_persistent_group = 0;
 		wpas_dbus_register_network(wpa_s, ssid);
 	}
 #endif /* CONFIG_P2P */
diff --git a/wpa_supplicant/notify.h b/wpa_supplicant/notify.h
index b268332ffc33..d9f0f5a96732 100644
--- a/wpa_supplicant/notify.h
+++ b/wpa_supplicant/notify.h
@@ -45,6 +45,7 @@ void wpas_notify_wps_event_m2d(struct wpa_supplicant *wpa_s,
 void wpas_notify_wps_event_fail(struct wpa_supplicant *wpa_s,
 				struct wps_event_fail *fail);
 void wpas_notify_wps_event_success(struct wpa_supplicant *wpa_s);
+void wpas_notify_wps_event_pbc_overlap(struct wpa_supplicant *wpa_s);
 void wpas_notify_network_added(struct wpa_supplicant *wpa_s,
 			       struct wpa_ssid *ssid);
 void wpas_notify_network_removed(struct wpa_supplicant *wpa_s,
@@ -84,6 +85,7 @@ void wpas_notify_resume(struct wpa_global *global);
 void wpas_notify_sta_authorized(struct wpa_supplicant *wpa_s,
 				const u8 *mac_addr, int authorized,
 				const u8 *p2p_dev_addr);
+void wpas_notify_p2p_find_stopped(struct wpa_supplicant *wpa_s);
 void wpas_notify_p2p_device_found(struct wpa_supplicant *wpa_s,
 				  const u8 *dev_addr, int new_device);
 void wpas_notify_p2p_device_lost(struct wpa_supplicant *wpa_s,
@@ -92,7 +94,7 @@ void wpas_notify_p2p_group_removed(struct wpa_supplicant *wpa_s,
 				   const struct wpa_ssid *ssid,
 				   const char *role);
 void wpas_notify_p2p_go_neg_req(struct wpa_supplicant *wpa_s,
-				const u8 *src, u16 dev_passwd_id);
+				const u8 *src, u16 dev_passwd_id, u8 go_intent);
 void wpas_notify_p2p_go_neg_completed(struct wpa_supplicant *wpa_s,
 				      struct p2p_go_neg_results *res);
 void wpas_notify_p2p_invitation_result(struct wpa_supplicant *wpa_s,
@@ -112,6 +114,8 @@ void wpas_notify_p2p_provision_discovery(struct wpa_supplicant *wpa_s,
 void wpas_notify_p2p_group_started(struct wpa_supplicant *wpa_s,
 				   struct wpa_ssid *ssid, int network_id,
 				   int client);
+void wpas_notify_p2p_group_formation_failure(struct wpa_supplicant *wpa_s,
+					     const char *reason);
 void wpas_notify_persistent_group_added(struct wpa_supplicant *wpa_s,
 					struct wpa_ssid *ssid);
 void wpas_notify_persistent_group_removed(struct wpa_supplicant *wpa_s,
@@ -133,5 +137,8 @@ void wpas_notify_network_bssid_set_changed(struct wpa_supplicant *wpa_s,
 					   struct wpa_ssid *ssid);
 void wpas_notify_network_type_changed(struct wpa_supplicant *wpa_s,
 				      struct wpa_ssid *ssid);
+void wpas_notify_p2p_invitation_received(struct wpa_supplicant *wpa_s,
+					 const u8 *sa, const u8 *go_dev_addr,
+					 const u8 *bssid, int id, int op_freq);
 
 #endif /* NOTIFY_H */
diff --git a/wpa_supplicant/p2p_supplicant.c b/wpa_supplicant/p2p_supplicant.c
index b200ca010262..78bdd0837e8b 100644
--- a/wpa_supplicant/p2p_supplicant.c
+++ b/wpa_supplicant/p2p_supplicant.c
@@ -47,6 +47,12 @@
 
 #define P2P_AUTO_PD_SCAN_ATTEMPTS 5
 
+/**
+ * Defines time interval in seconds when a GO needs to evacuate a frequency that
+ * it is currently using, but is no longer valid for P2P use cases.
+ */
+#define P2P_GO_FREQ_CHANGE_TIME 5
+
 #ifndef P2P_MAX_CLIENT_IDLE
 /*
  * How many seconds to try to reconnect to the GO when connection in P2P client
@@ -85,6 +91,12 @@
 
 #define P2P_MGMT_DEVICE_PREFIX		"p2p-dev-"
 
+/*
+ * How many seconds to wait to re-attempt to move GOs, in case previous attempt
+ * was not possible.
+ */
+#define P2P_RECONSIDER_GO_MOVE_DELAY 30
+
 enum p2p_group_removal_reason {
 	P2P_GROUP_REMOVAL_UNKNOWN,
 	P2P_GROUP_REMOVAL_SILENT,
@@ -94,7 +106,8 @@ enum p2p_group_removal_reason {
 	P2P_GROUP_REMOVAL_UNAVAILABLE,
 	P2P_GROUP_REMOVAL_GO_ENDING_SESSION,
 	P2P_GROUP_REMOVAL_PSK_FAILURE,
-	P2P_GROUP_REMOVAL_FREQ_CONFLICT
+	P2P_GROUP_REMOVAL_FREQ_CONFLICT,
+	P2P_GROUP_REMOVAL_GO_LEAVE_CHANNEL
 };
 
 
@@ -126,6 +139,18 @@ static void wpas_p2p_psk_failure_removal(void *eloop_ctx, void *timeout_ctx);
 static void wpas_p2p_group_deinit(struct wpa_supplicant *wpa_s);
 static int wpas_p2p_add_group_interface(struct wpa_supplicant *wpa_s,
 					enum wpa_driver_if_type type);
+static void wpas_p2p_group_formation_failed(struct wpa_supplicant *wpa_s,
+					    int already_deleted);
+static void wpas_p2p_optimize_listen_channel(struct wpa_supplicant *wpa_s,
+					     struct wpa_used_freq_data *freqs,
+					     unsigned int num);
+static void wpas_p2p_move_go(void *eloop_ctx, void *timeout_ctx);
+static int wpas_p2p_go_is_peer_freq(struct wpa_supplicant *wpa_s, int freq);
+static void
+wpas_p2p_consider_moving_gos(struct wpa_supplicant *wpa_s,
+			     struct wpa_used_freq_data *freqs, unsigned int num,
+			     enum wpas_p2p_channel_update_trig trig);
+static void wpas_p2p_reconsider_moving_go(void *eloop_ctx, void *timeout_ctx);
 
 
 /*
@@ -191,7 +216,11 @@ static void wpas_p2p_set_own_freq_preference(struct wpa_supplicant *wpa_s,
 {
 	if (wpa_s->global->p2p_disabled || wpa_s->global->p2p == NULL)
 		return;
-	if (wpa_s->parent->conf->p2p_ignore_shared_freq &&
+
+	/* Use the wpa_s used to control the P2P Device operation */
+	wpa_s = wpa_s->global->p2p_init_wpa_s;
+
+	if (wpa_s->conf->p2p_ignore_shared_freq &&
 	    freq > 0 && wpa_s->num_multichan_concurrent > 1 &&
 	    wpas_p2p_num_unused_channels(wpa_s) > 0) {
 		wpa_printf(MSG_DEBUG, "P2P: Ignore own channel preference %d MHz due to p2p_ignore_shared_freq=1 configuration",
@@ -600,7 +629,7 @@ static u8 p2ps_group_capability(void *ctx, u8 incoming, u8 role)
 	 */
 	go_wpa_s = wpas_p2p_get_go_group(wpa_s);
 	persistent_go = wpas_p2p_get_persistent_go(wpa_s);
-	p2p_no_group_iface = wpa_s->conf->p2p_no_group_iface;
+	p2p_no_group_iface = !wpas_p2p_create_iface(wpa_s);
 
 	wpa_printf(MSG_DEBUG, "P2P: GO(iface)=%p persistent(ssid)=%p",
 		   go_wpa_s, persistent_go);
@@ -866,9 +895,12 @@ static int wpas_p2p_group_delete(struct wpa_supplicant *wpa_s,
 		wpa_printf(MSG_DEBUG, "P2P: Cancelled P2P group formation "
 			   "timeout");
 		wpa_s->p2p_in_provisioning = 0;
+		wpas_p2p_group_formation_failed(wpa_s, 1);
 	}
 
 	wpa_s->p2p_in_invitation = 0;
+	eloop_cancel_timeout(wpas_p2p_move_go, wpa_s, NULL);
+	eloop_cancel_timeout(wpas_p2p_reconsider_moving_go, wpa_s, NULL);
 
 	/*
 	 * Make sure wait for the first client does not remain active after the
@@ -940,6 +972,8 @@ static int wpas_p2p_group_delete(struct wpa_supplicant *wpa_s,
 	else
 		wpa_drv_deinit_p2p_cli(wpa_s);
 
+	os_memset(wpa_s->go_dev_addr, 0, ETH_ALEN);
+
 	return 0;
 }
 
@@ -1109,13 +1143,14 @@ static void wpas_p2p_add_persistent_group_client(struct wpa_supplicant *wpa_s,
 	u8 *n;
 	size_t i;
 	int found = 0;
+	struct wpa_supplicant *p2p_wpa_s = wpa_s->global->p2p_init_wpa_s;
 
 	ssid = wpa_s->current_ssid;
 	if (ssid == NULL || ssid->mode != WPAS_MODE_P2P_GO ||
 	    !ssid->p2p_persistent_group)
 		return;
 
-	for (s = wpa_s->parent->conf->ssid; s; s = s->next) {
+	for (s = p2p_wpa_s->conf->ssid; s; s = s->next) {
 		if (s->disabled != 2 || s->mode != WPAS_MODE_P2P_GO)
 			continue;
 
@@ -1174,8 +1209,8 @@ static void wpas_p2p_add_persistent_group_client(struct wpa_supplicant *wpa_s,
 			  0xff, ETH_ALEN);
 	}
 
-	if (wpa_s->parent->conf->update_config &&
-	    wpa_config_write(wpa_s->parent->confname, wpa_s->parent->conf))
+	if (p2p_wpa_s->conf->update_config &&
+	    wpa_config_write(p2p_wpa_s->confname, p2p_wpa_s->conf))
 		wpa_printf(MSG_DEBUG, "P2P: Failed to update configuration");
 }
 
@@ -1226,7 +1261,7 @@ static void wpas_p2p_group_started(struct wpa_supplicant *wpa_s,
 
 
 static void wpas_group_formation_completed(struct wpa_supplicant *wpa_s,
-					   int success)
+					   int success, int already_deleted)
 {
 	struct wpa_ssid *ssid;
 	int client;
@@ -1251,6 +1286,9 @@ static void wpas_group_formation_completed(struct wpa_supplicant *wpa_s,
 	if (!success) {
 		wpa_msg_global(wpa_s->parent, MSG_INFO,
 			       P2P_EVENT_GROUP_FORMATION_FAILURE);
+		wpas_notify_p2p_group_formation_failure(wpa_s, "");
+		if (already_deleted)
+			return;
 		wpas_p2p_group_delete(wpa_s,
 				      P2P_GROUP_REMOVAL_FORMATION_FAILED);
 		return;
@@ -1677,14 +1715,22 @@ static void p2p_go_configured(void *ctx, void *data)
 				       params->persistent_group, "");
 		wpa_s->group_formation_reported = 1;
 
-		if (wpa_s->parent->p2ps_join_addr_valid) {
-			wpa_dbg(wpa_s, MSG_DEBUG,
-				"P2PS: Setting default PIN for " MACSTR,
-				MAC2STR(wpa_s->parent->p2ps_join_addr));
-			wpa_supplicant_ap_wps_pin(wpa_s,
-						  wpa_s->parent->p2ps_join_addr,
-						  "12345670", NULL, 0, 0);
-			wpa_s->parent->p2ps_join_addr_valid = 0;
+		if (wpa_s->parent->p2ps_method_config_any) {
+			if (is_zero_ether_addr(wpa_s->parent->p2ps_join_addr)) {
+				wpa_dbg(wpa_s, MSG_DEBUG,
+					"P2PS: Setting default PIN for ANY");
+				wpa_supplicant_ap_wps_pin(wpa_s, NULL,
+							  "12345670", NULL, 0,
+							  0);
+			} else {
+				wpa_dbg(wpa_s, MSG_DEBUG,
+					"P2PS: Setting default PIN for " MACSTR,
+					MAC2STR(wpa_s->parent->p2ps_join_addr));
+				wpa_supplicant_ap_wps_pin(
+					wpa_s, wpa_s->parent->p2ps_join_addr,
+					"12345670", NULL, 0, 0);
+			}
+			wpa_s->parent->p2ps_method_config_any = 0;
 		}
 
 		os_get_reltime(&wpa_s->global->p2p_go_wait_client);
@@ -1771,6 +1817,7 @@ static void wpas_start_wps_go(struct wpa_supplicant *wpa_s,
 	wpa_s->show_group_started = 0;
 	wpa_s->p2p_go_group_formation_completed = 0;
 	wpa_s->group_formation_reported = 0;
+	os_memset(wpa_s->go_dev_addr, 0, ETH_ALEN);
 
 	wpa_config_set_network_defaults(ssid);
 	ssid->temporary = 1;
@@ -1853,6 +1900,7 @@ static void wpas_p2p_clone_config(struct wpa_supplicant *dst,
 	d->num_sec_device_types = s->num_sec_device_types;
 
 	d->p2p_group_idle = s->p2p_group_idle;
+	d->p2p_go_freq_change_policy = s->p2p_go_freq_change_policy;
 	d->p2p_intra_bss = s->p2p_intra_bss;
 	d->persistent_reconnect = s->persistent_reconnect;
 	d->max_num_sta = s->max_num_sta;
@@ -1869,6 +1917,7 @@ static void wpas_p2p_clone_config(struct wpa_supplicant *dst,
 		d->wps_nfc_dh_privkey = wpabuf_dup(s->wps_nfc_dh_privkey);
 		d->wps_nfc_dh_pubkey = wpabuf_dup(s->wps_nfc_dh_pubkey);
 	}
+	d->p2p_cli_probe = s->p2p_cli_probe;
 }
 
 
@@ -2014,17 +2063,18 @@ static void wpas_p2p_group_formation_timeout(void *eloop_ctx,
 {
 	struct wpa_supplicant *wpa_s = eloop_ctx;
 	wpa_printf(MSG_DEBUG, "P2P: Group Formation timed out");
-	wpas_p2p_group_formation_failed(wpa_s);
+	wpas_p2p_group_formation_failed(wpa_s, 0);
 }
 
 
-void wpas_p2p_group_formation_failed(struct wpa_supplicant *wpa_s)
+static void wpas_p2p_group_formation_failed(struct wpa_supplicant *wpa_s,
+					    int already_deleted)
 {
 	eloop_cancel_timeout(wpas_p2p_group_formation_timeout,
 			     wpa_s->parent, NULL);
 	if (wpa_s->global->p2p)
 		p2p_group_formation_failed(wpa_s->global->p2p);
-	wpas_group_formation_completed(wpa_s, 0);
+	wpas_group_formation_completed(wpa_s, 0, already_deleted);
 }
 
 
@@ -2070,6 +2120,11 @@ static void wpas_go_neg_completed(void *ctx, struct p2p_go_neg_results *res)
 		return;
 	}
 
+	if (!res->role_go) {
+		/* Inform driver of the operating channel of GO. */
+		wpa_drv_set_prob_oper_freq(wpa_s, res->freq);
+	}
+
 	if (wpa_s->p2p_go_ht40)
 		res->ht40 = 1;
 	if (wpa_s->p2p_go_vht)
@@ -2105,7 +2160,7 @@ static void wpas_go_neg_completed(void *ctx, struct p2p_go_neg_results *res)
 			wpas_p2p_remove_pending_group_interface(wpa_s);
 			eloop_cancel_timeout(wpas_p2p_long_listen_timeout,
 					     wpa_s, NULL);
-			wpas_p2p_group_formation_failed(wpa_s);
+			wpas_p2p_group_formation_failed(wpa_s, 1);
 			return;
 		}
 		if (group_wpa_s != wpa_s) {
@@ -2117,18 +2172,22 @@ static void wpas_go_neg_completed(void *ctx, struct p2p_go_neg_results *res)
 		wpa_s->pending_interface_name[0] = '\0';
 		group_wpa_s->p2p_in_provisioning = 1;
 
-		if (res->role_go)
+		if (res->role_go) {
 			wpas_start_wps_go(group_wpa_s, res, 1);
-		else
+		} else {
+			os_get_reltime(&group_wpa_s->scan_min_time);
 			wpas_start_wps_enrollee(group_wpa_s, res);
+		}
 	} else {
 		wpa_s->p2p_in_provisioning = 1;
 		wpa_s->global->p2p_group_formation = wpa_s;
 
-		if (res->role_go)
+		if (res->role_go) {
 			wpas_start_wps_go(wpa_s, res, 1);
-		else
+		} else {
+			os_get_reltime(&wpa_s->scan_min_time);
 			wpas_start_wps_enrollee(ctx, res);
+		}
 	}
 
 	wpa_s->p2p_long_listen = 0;
@@ -2141,13 +2200,15 @@ static void wpas_go_neg_completed(void *ctx, struct p2p_go_neg_results *res)
 }
 
 
-static void wpas_go_neg_req_rx(void *ctx, const u8 *src, u16 dev_passwd_id)
+static void wpas_go_neg_req_rx(void *ctx, const u8 *src, u16 dev_passwd_id,
+			       u8 go_intent)
 {
 	struct wpa_supplicant *wpa_s = ctx;
 	wpa_msg_global(wpa_s, MSG_INFO, P2P_EVENT_GO_NEG_REQUEST MACSTR
-		       " dev_passwd_id=%u", MAC2STR(src), dev_passwd_id);
+		       " dev_passwd_id=%u go_intent=%u", MAC2STR(src),
+		       dev_passwd_id, go_intent);
 
-	wpas_notify_p2p_go_neg_req(wpa_s, src, dev_passwd_id);
+	wpas_notify_p2p_go_neg_req(wpa_s, src, dev_passwd_id, go_intent);
 }
 
 
@@ -2248,6 +2309,7 @@ static void wpas_find_stopped(void *ctx)
 {
 	struct wpa_supplicant *wpa_s = ctx;
 	wpa_msg_global(wpa_s, MSG_INFO, P2P_EVENT_FIND_STOPPED);
+	wpas_notify_p2p_find_stopped(wpa_s);
 }
 
 
@@ -2376,1236 +2438,24 @@ static void wpas_stop_listen(void *ctx)
 		wpa_s->roc_waiting_drv_freq = 0;
 	}
 	wpa_drv_set_ap_wps_ie(wpa_s, NULL, NULL, NULL);
-	wpa_drv_probe_req_report(wpa_s, 0);
+
+	/*
+	 * Don't cancel Probe Request RX reporting for a connected P2P Client
+	 * handling Probe Request frames.
+	 */
+	if (!wpa_s->p2p_cli_probe)
+		wpa_drv_probe_req_report(wpa_s, 0);
+
 	wpas_p2p_listen_work_done(wpa_s);
 }
 
 
-static int wpas_send_probe_resp(void *ctx, const struct wpabuf *buf)
+static int wpas_send_probe_resp(void *ctx, const struct wpabuf *buf,
+				unsigned int freq)
 {
 	struct wpa_supplicant *wpa_s = ctx;
-	return wpa_drv_send_mlme(wpa_s, wpabuf_head(buf), wpabuf_len(buf), 1);
-}
-
-
-/*
- * DNS Header section is used only to calculate compression pointers, so the
- * contents of this data does not matter, but the length needs to be reserved
- * in the virtual packet.
- */
-#define DNS_HEADER_LEN 12
-
-/*
- * 27-octet in-memory packet from P2P specification containing two implied
- * queries for _tcp.lcoal. PTR IN and _udp.local. PTR IN
- */
-#define P2P_SD_IN_MEMORY_LEN 27
-
-static int p2p_sd_dns_uncompress_label(char **upos, char *uend, u8 *start,
-				       u8 **spos, const u8 *end)
-{
-	while (*spos < end) {
-		u8 val = ((*spos)[0] & 0xc0) >> 6;
-		int len;
-
-		if (val == 1 || val == 2) {
-			/* These are reserved values in RFC 1035 */
-			wpa_printf(MSG_DEBUG, "P2P: Invalid domain name "
-				   "sequence starting with 0x%x", val);
-			return -1;
-		}
-
-		if (val == 3) {
-			u16 offset;
-			u8 *spos_tmp;
-
-			/* Offset */
-			if (*spos + 2 > end) {
-				wpa_printf(MSG_DEBUG, "P2P: No room for full "
-					   "DNS offset field");
-				return -1;
-			}
-
-			offset = (((*spos)[0] & 0x3f) << 8) | (*spos)[1];
-			if (offset >= *spos - start) {
-				wpa_printf(MSG_DEBUG, "P2P: Invalid DNS "
-					   "pointer offset %u", offset);
-				return -1;
-			}
-
-			(*spos) += 2;
-			spos_tmp = start + offset;
-			return p2p_sd_dns_uncompress_label(upos, uend, start,
-							   &spos_tmp,
-							   *spos - 2);
-		}
-
-		/* Label */
-		len = (*spos)[0] & 0x3f;
-		if (len == 0)
-			return 0;
-
-		(*spos)++;
-		if (*spos + len > end) {
-			wpa_printf(MSG_DEBUG, "P2P: Invalid domain name "
-				   "sequence - no room for label with length "
-				   "%u", len);
-			return -1;
-		}
-
-		if (*upos + len + 2 > uend)
-			return -2;
-
-		os_memcpy(*upos, *spos, len);
-		*spos += len;
-		*upos += len;
-		(*upos)[0] = '.';
-		(*upos)++;
-		(*upos)[0] = '\0';
-	}
-
-	return 0;
-}
-
-
-/* Uncompress domain names per RFC 1035 using the P2P SD in-memory packet.
- * Returns -1 on parsing error (invalid input sequence), -2 if output buffer is
- * not large enough */
-static int p2p_sd_dns_uncompress(char *buf, size_t buf_len, const u8 *msg,
-				 size_t msg_len, size_t offset)
-{
-	/* 27-octet in-memory packet from P2P specification */
-	const char *prefix = "\x04_tcp\x05local\x00\x00\x0C\x00\x01"
-		"\x04_udp\xC0\x11\x00\x0C\x00\x01";
-	u8 *tmp, *end, *spos;
-	char *upos, *uend;
-	int ret = 0;
-
-	if (buf_len < 2)
-		return -1;
-	if (offset > msg_len)
-		return -1;
-
-	tmp = os_malloc(DNS_HEADER_LEN + P2P_SD_IN_MEMORY_LEN + msg_len);
-	if (tmp == NULL)
-		return -1;
-	spos = tmp + DNS_HEADER_LEN + P2P_SD_IN_MEMORY_LEN;
-	end = spos + msg_len;
-	spos += offset;
-
-	os_memset(tmp, 0, DNS_HEADER_LEN);
-	os_memcpy(tmp + DNS_HEADER_LEN, prefix, P2P_SD_IN_MEMORY_LEN);
-	os_memcpy(tmp + DNS_HEADER_LEN + P2P_SD_IN_MEMORY_LEN, msg, msg_len);
-
-	upos = buf;
-	uend = buf + buf_len;
-
-	ret = p2p_sd_dns_uncompress_label(&upos, uend, tmp, &spos, end);
-	if (ret) {
-		os_free(tmp);
-		return ret;
-	}
-
-	if (upos == buf) {
-		upos[0] = '.';
-		upos[1] = '\0';
-	} else if (upos[-1] == '.')
-		upos[-1] = '\0';
-
-	os_free(tmp);
-	return 0;
-}
-
-
-static struct p2p_srv_bonjour *
-wpas_p2p_service_get_bonjour(struct wpa_supplicant *wpa_s,
-			     const struct wpabuf *query)
-{
-	struct p2p_srv_bonjour *bsrv;
-	size_t len;
-
-	len = wpabuf_len(query);
-	dl_list_for_each(bsrv, &wpa_s->global->p2p_srv_bonjour,
-			 struct p2p_srv_bonjour, list) {
-		if (len == wpabuf_len(bsrv->query) &&
-		    os_memcmp(wpabuf_head(query), wpabuf_head(bsrv->query),
-			      len) == 0)
-			return bsrv;
-	}
-	return NULL;
-}
-
-
-static struct p2p_srv_upnp *
-wpas_p2p_service_get_upnp(struct wpa_supplicant *wpa_s, u8 version,
-			  const char *service)
-{
-	struct p2p_srv_upnp *usrv;
-
-	dl_list_for_each(usrv, &wpa_s->global->p2p_srv_upnp,
-			 struct p2p_srv_upnp, list) {
-		if (version == usrv->version &&
-		    os_strcmp(service, usrv->service) == 0)
-			return usrv;
-	}
-	return NULL;
-}
-
-
-static void wpas_sd_add_empty(struct wpabuf *resp, u8 srv_proto,
-			      u8 srv_trans_id, u8 status)
-{
-	u8 *len_pos;
-
-	if (wpabuf_tailroom(resp) < 5)
-		return;
-
-	/* Length (to be filled) */
-	len_pos = wpabuf_put(resp, 2);
-	wpabuf_put_u8(resp, srv_proto);
-	wpabuf_put_u8(resp, srv_trans_id);
-	/* Status Code */
-	wpabuf_put_u8(resp, status);
-	/* Response Data: empty */
-	WPA_PUT_LE16(len_pos, (u8 *) wpabuf_put(resp, 0) - len_pos - 2);
-}
-
-
-static void wpas_sd_add_proto_not_avail(struct wpabuf *resp, u8 srv_proto,
-					u8 srv_trans_id)
-{
-	wpas_sd_add_empty(resp, srv_proto, srv_trans_id,
-			  P2P_SD_PROTO_NOT_AVAILABLE);
-}
-
-
-static void wpas_sd_add_bad_request(struct wpabuf *resp, u8 srv_proto,
-				    u8 srv_trans_id)
-{
-	wpas_sd_add_empty(resp, srv_proto, srv_trans_id, P2P_SD_BAD_REQUEST);
-}
-
-
-static void wpas_sd_add_not_found(struct wpabuf *resp, u8 srv_proto,
-				  u8 srv_trans_id)
-{
-	wpas_sd_add_empty(resp, srv_proto, srv_trans_id,
-			  P2P_SD_REQUESTED_INFO_NOT_AVAILABLE);
-}
-
-
-static void wpas_sd_all_bonjour(struct wpa_supplicant *wpa_s,
-				struct wpabuf *resp, u8 srv_trans_id)
-{
-	struct p2p_srv_bonjour *bsrv;
-	u8 *len_pos;
-
-	wpa_printf(MSG_DEBUG, "P2P: SD Request for all Bonjour services");
-
-	if (dl_list_empty(&wpa_s->global->p2p_srv_bonjour)) {
-		wpa_printf(MSG_DEBUG, "P2P: Bonjour protocol not available");
-		return;
-	}
-
-	dl_list_for_each(bsrv, &wpa_s->global->p2p_srv_bonjour,
-			 struct p2p_srv_bonjour, list) {
-		if (wpabuf_tailroom(resp) <
-		    5 + wpabuf_len(bsrv->query) + wpabuf_len(bsrv->resp))
-			return;
-		/* Length (to be filled) */
-		len_pos = wpabuf_put(resp, 2);
-		wpabuf_put_u8(resp, P2P_SERV_BONJOUR);
-		wpabuf_put_u8(resp, srv_trans_id);
-		/* Status Code */
-		wpabuf_put_u8(resp, P2P_SD_SUCCESS);
-		wpa_hexdump_ascii(MSG_DEBUG, "P2P: Matching Bonjour service",
-				  wpabuf_head(bsrv->resp),
-				  wpabuf_len(bsrv->resp));
-		/* Response Data */
-		wpabuf_put_buf(resp, bsrv->query); /* Key */
-		wpabuf_put_buf(resp, bsrv->resp); /* Value */
-		WPA_PUT_LE16(len_pos, (u8 *) wpabuf_put(resp, 0) - len_pos -
-			     2);
-	}
-}
-
-
-static int match_bonjour_query(struct p2p_srv_bonjour *bsrv, const u8 *query,
-			       size_t query_len)
-{
-	char str_rx[256], str_srv[256];
-
-	if (query_len < 3 || wpabuf_len(bsrv->query) < 3)
-		return 0; /* Too short to include DNS Type and Version */
-	if (os_memcmp(query + query_len - 3,
-		      wpabuf_head_u8(bsrv->query) + wpabuf_len(bsrv->query) - 3,
-		      3) != 0)
-		return 0; /* Mismatch in DNS Type or Version */
-	if (query_len == wpabuf_len(bsrv->query) &&
-	    os_memcmp(query, wpabuf_head(bsrv->query), query_len - 3) == 0)
-		return 1; /* Binary match */
-
-	if (p2p_sd_dns_uncompress(str_rx, sizeof(str_rx), query, query_len - 3,
-				  0))
-		return 0; /* Failed to uncompress query */
-	if (p2p_sd_dns_uncompress(str_srv, sizeof(str_srv),
-				  wpabuf_head(bsrv->query),
-				  wpabuf_len(bsrv->query) - 3, 0))
-		return 0; /* Failed to uncompress service */
-
-	return os_strcmp(str_rx, str_srv) == 0;
-}
-
-
-static void wpas_sd_req_bonjour(struct wpa_supplicant *wpa_s,
-				struct wpabuf *resp, u8 srv_trans_id,
-				const u8 *query, size_t query_len)
-{
-	struct p2p_srv_bonjour *bsrv;
-	u8 *len_pos;
-	int matches = 0;
-
-	wpa_hexdump_ascii(MSG_DEBUG, "P2P: SD Request for Bonjour",
-			  query, query_len);
-	if (dl_list_empty(&wpa_s->global->p2p_srv_bonjour)) {
-		wpa_printf(MSG_DEBUG, "P2P: Bonjour protocol not available");
-		wpas_sd_add_proto_not_avail(resp, P2P_SERV_BONJOUR,
-					    srv_trans_id);
-		return;
-	}
-
-	if (query_len == 0) {
-		wpas_sd_all_bonjour(wpa_s, resp, srv_trans_id);
-		return;
-	}
-
-	dl_list_for_each(bsrv, &wpa_s->global->p2p_srv_bonjour,
-			 struct p2p_srv_bonjour, list) {
-		if (!match_bonjour_query(bsrv, query, query_len))
-			continue;
-
-		if (wpabuf_tailroom(resp) <
-		    5 + query_len + wpabuf_len(bsrv->resp))
-			return;
-
-		matches++;
-
-		/* Length (to be filled) */
-		len_pos = wpabuf_put(resp, 2);
-		wpabuf_put_u8(resp, P2P_SERV_BONJOUR);
-		wpabuf_put_u8(resp, srv_trans_id);
-
-		/* Status Code */
-		wpabuf_put_u8(resp, P2P_SD_SUCCESS);
-		wpa_hexdump_ascii(MSG_DEBUG, "P2P: Matching Bonjour service",
-				  wpabuf_head(bsrv->resp),
-				  wpabuf_len(bsrv->resp));
-
-		/* Response Data */
-		wpabuf_put_data(resp, query, query_len); /* Key */
-		wpabuf_put_buf(resp, bsrv->resp); /* Value */
-
-		WPA_PUT_LE16(len_pos, (u8 *) wpabuf_put(resp, 0) - len_pos - 2);
-	}
-
-	if (matches == 0) {
-		wpa_printf(MSG_DEBUG, "P2P: Requested Bonjour service not "
-			   "available");
-		if (wpabuf_tailroom(resp) < 5)
-			return;
-
-		/* Length (to be filled) */
-		len_pos = wpabuf_put(resp, 2);
-		wpabuf_put_u8(resp, P2P_SERV_BONJOUR);
-		wpabuf_put_u8(resp, srv_trans_id);
-
-		/* Status Code */
-		wpabuf_put_u8(resp, P2P_SD_REQUESTED_INFO_NOT_AVAILABLE);
-		/* Response Data: empty */
-		WPA_PUT_LE16(len_pos, (u8 *) wpabuf_put(resp, 0) - len_pos -
-			     2);
-	}
-}
-
-
-static void wpas_sd_all_upnp(struct wpa_supplicant *wpa_s,
-			     struct wpabuf *resp, u8 srv_trans_id)
-{
-	struct p2p_srv_upnp *usrv;
-	u8 *len_pos;
-
-	wpa_printf(MSG_DEBUG, "P2P: SD Request for all UPnP services");
-
-	if (dl_list_empty(&wpa_s->global->p2p_srv_upnp)) {
-		wpa_printf(MSG_DEBUG, "P2P: UPnP protocol not available");
-		return;
-	}
-
-	dl_list_for_each(usrv, &wpa_s->global->p2p_srv_upnp,
-			 struct p2p_srv_upnp, list) {
-		if (wpabuf_tailroom(resp) < 5 + 1 + os_strlen(usrv->service))
-			return;
-
-		/* Length (to be filled) */
-		len_pos = wpabuf_put(resp, 2);
-		wpabuf_put_u8(resp, P2P_SERV_UPNP);
-		wpabuf_put_u8(resp, srv_trans_id);
-
-		/* Status Code */
-		wpabuf_put_u8(resp, P2P_SD_SUCCESS);
-		/* Response Data */
-		wpabuf_put_u8(resp, usrv->version);
-		wpa_printf(MSG_DEBUG, "P2P: Matching UPnP Service: %s",
-			   usrv->service);
-		wpabuf_put_str(resp, usrv->service);
-		WPA_PUT_LE16(len_pos, (u8 *) wpabuf_put(resp, 0) - len_pos -
-			     2);
-	}
-}
-
-
-static void wpas_sd_req_upnp(struct wpa_supplicant *wpa_s,
-			     struct wpabuf *resp, u8 srv_trans_id,
-			     const u8 *query, size_t query_len)
-{
-	struct p2p_srv_upnp *usrv;
-	u8 *len_pos;
-	u8 version;
-	char *str;
-	int count = 0;
-
-	wpa_hexdump_ascii(MSG_DEBUG, "P2P: SD Request for UPnP",
-			  query, query_len);
-
-	if (dl_list_empty(&wpa_s->global->p2p_srv_upnp)) {
-		wpa_printf(MSG_DEBUG, "P2P: UPnP protocol not available");
-		wpas_sd_add_proto_not_avail(resp, P2P_SERV_UPNP,
-					    srv_trans_id);
-		return;
-	}
-
-	if (query_len == 0) {
-		wpas_sd_all_upnp(wpa_s, resp, srv_trans_id);
-		return;
-	}
-
-	if (wpabuf_tailroom(resp) < 5)
-		return;
-
-	/* Length (to be filled) */
-	len_pos = wpabuf_put(resp, 2);
-	wpabuf_put_u8(resp, P2P_SERV_UPNP);
-	wpabuf_put_u8(resp, srv_trans_id);
-
-	version = query[0];
-	str = os_malloc(query_len);
-	if (str == NULL)
-		return;
-	os_memcpy(str, query + 1, query_len - 1);
-	str[query_len - 1] = '\0';
-
-	dl_list_for_each(usrv, &wpa_s->global->p2p_srv_upnp,
-			 struct p2p_srv_upnp, list) {
-		if (version != usrv->version)
-			continue;
-
-		if (os_strcmp(str, "ssdp:all") != 0 &&
-		    os_strstr(usrv->service, str) == NULL)
-			continue;
-
-		if (wpabuf_tailroom(resp) < 2)
-			break;
-		if (count == 0) {
-			/* Status Code */
-			wpabuf_put_u8(resp, P2P_SD_SUCCESS);
-			/* Response Data */
-			wpabuf_put_u8(resp, version);
-		} else
-			wpabuf_put_u8(resp, ',');
-
-		count++;
-
-		wpa_printf(MSG_DEBUG, "P2P: Matching UPnP Service: %s",
-			   usrv->service);
-		if (wpabuf_tailroom(resp) < os_strlen(usrv->service))
-			break;
-		wpabuf_put_str(resp, usrv->service);
-	}
-	os_free(str);
-
-	if (count == 0) {
-		wpa_printf(MSG_DEBUG, "P2P: Requested UPnP service not "
-			   "available");
-		/* Status Code */
-		wpabuf_put_u8(resp, P2P_SD_REQUESTED_INFO_NOT_AVAILABLE);
-		/* Response Data: empty */
-	}
-
-	WPA_PUT_LE16(len_pos, (u8 *) wpabuf_put(resp, 0) - len_pos - 2);
-}
-
-
-#ifdef CONFIG_WIFI_DISPLAY
-static void wpas_sd_req_wfd(struct wpa_supplicant *wpa_s,
-			    struct wpabuf *resp, u8 srv_trans_id,
-			    const u8 *query, size_t query_len)
-{
-	const u8 *pos;
-	u8 role;
-	u8 *len_pos;
-
-	wpa_hexdump(MSG_DEBUG, "P2P: SD Request for WFD", query, query_len);
-
-	if (!wpa_s->global->wifi_display) {
-		wpa_printf(MSG_DEBUG, "P2P: WFD protocol not available");
-		wpas_sd_add_proto_not_avail(resp, P2P_SERV_WIFI_DISPLAY,
-					    srv_trans_id);
-		return;
-	}
-
-	if (query_len < 1) {
-		wpa_printf(MSG_DEBUG, "P2P: Missing WFD Requested Device "
-			   "Role");
-		return;
-	}
-
-	if (wpabuf_tailroom(resp) < 5)
-		return;
-
-	pos = query;
-	role = *pos++;
-	wpa_printf(MSG_DEBUG, "P2P: WSD for device role 0x%x", role);
-
-	/* TODO: role specific handling */
-
-	/* Length (to be filled) */
-	len_pos = wpabuf_put(resp, 2);
-	wpabuf_put_u8(resp, P2P_SERV_WIFI_DISPLAY);
-	wpabuf_put_u8(resp, srv_trans_id);
-	wpabuf_put_u8(resp, P2P_SD_SUCCESS); /* Status Code */
-
-	while (pos < query + query_len) {
-		if (*pos < MAX_WFD_SUBELEMS &&
-		    wpa_s->global->wfd_subelem[*pos] &&
-		    wpabuf_tailroom(resp) >=
-		    wpabuf_len(wpa_s->global->wfd_subelem[*pos])) {
-			wpa_printf(MSG_DEBUG, "P2P: Add WSD response "
-				   "subelement %u", *pos);
-			wpabuf_put_buf(resp, wpa_s->global->wfd_subelem[*pos]);
-		}
-		pos++;
-	}
-
-	WPA_PUT_LE16(len_pos, (u8 *) wpabuf_put(resp, 0) - len_pos - 2);
-}
-#endif /* CONFIG_WIFI_DISPLAY */
-
-
-static int find_p2ps_substr(struct p2ps_advertisement *adv_data,
-			    const u8 *needle, size_t needle_len)
-{
-	const u8 *haystack = (const u8 *) adv_data->svc_info;
-	size_t haystack_len, i;
-
-	/* Allow search term to be empty */
-	if (!needle || !needle_len)
-		return 1;
-
-	if (!haystack)
-		return 0;
-
-	haystack_len = os_strlen(adv_data->svc_info);
-	for (i = 0; i < haystack_len; i++) {
-		if (haystack_len - i < needle_len)
-			break;
-		if (os_memcmp(haystack + i, needle, needle_len) == 0)
-			return 1;
-	}
-
-	return 0;
-}
-
-
-static void wpas_sd_req_asp(struct wpa_supplicant *wpa_s,
-			    struct wpabuf *resp, u8 srv_trans_id,
-			    const u8 *query, size_t query_len)
-{
-	struct p2ps_advertisement *adv_data;
-	const u8 *svc = &query[1];
-	const u8 *info = NULL;
-	size_t svc_len = query[0];
-	size_t info_len = 0;
-	int prefix = 0;
-	u8 *count_pos = NULL;
-	u8 *len_pos = NULL;
-
-	wpa_hexdump(MSG_DEBUG, "P2P: SD Request for ASP", query, query_len);
-
-	if (!wpa_s->global->p2p) {
-		wpa_printf(MSG_DEBUG, "P2P: ASP protocol not available");
-		wpas_sd_add_proto_not_avail(resp, P2P_SERV_P2PS, srv_trans_id);
-		return;
-	}
-
-	/* Info block is optional */
-	if (svc_len + 1 < query_len) {
-		info = &svc[svc_len];
-		info_len = *info++;
-	}
-
-	/* Range check length of svc string and info block */
-	if (svc_len + (info_len ? info_len + 2 : 1) > query_len) {
-		wpa_printf(MSG_DEBUG, "P2P: ASP bad request");
-		wpas_sd_add_bad_request(resp, P2P_SERV_P2PS, srv_trans_id);
-		return;
-	}
-
-	/* Detect and correct for prefix search */
-	if (svc_len && svc[svc_len - 1] == '*') {
-		prefix = 1;
-		svc_len--;
-	}
-
-	for (adv_data = p2p_get_p2ps_adv_list(wpa_s->global->p2p);
-	     adv_data; adv_data = adv_data->next) {
-		/* If not a prefix match, reject length mismatches */
-		if (!prefix && svc_len != os_strlen(adv_data->svc_name))
-			continue;
-
-		/* Search each service for request */
-		if (os_memcmp(adv_data->svc_name, svc, svc_len) == 0 &&
-		    find_p2ps_substr(adv_data, info, info_len)) {
-			size_t len = os_strlen(adv_data->svc_name);
-			size_t svc_info_len = 0;
-
-			if (adv_data->svc_info)
-				svc_info_len = os_strlen(adv_data->svc_info);
-
-			if (len > 0xff || svc_info_len > 0xffff)
-				return;
-
-			/* Length & Count to be filled as we go */
-			if (!len_pos && !count_pos) {
-				if (wpabuf_tailroom(resp) <
-				    len + svc_info_len + 16)
-					return;
-
-				len_pos = wpabuf_put(resp, 2);
-				wpabuf_put_u8(resp, P2P_SERV_P2PS);
-				wpabuf_put_u8(resp, srv_trans_id);
-				/* Status Code */
-				wpabuf_put_u8(resp, P2P_SD_SUCCESS);
-				count_pos = wpabuf_put(resp, 1);
-				*count_pos = 0;
-			} else if (wpabuf_tailroom(resp) <
-				   len + svc_info_len + 10)
-				return;
-
-			if (svc_info_len) {
-				wpa_printf(MSG_DEBUG,
-					   "P2P: Add Svc: %s info: %s",
-					   adv_data->svc_name,
-					   adv_data->svc_info);
-			} else {
-				wpa_printf(MSG_DEBUG, "P2P: Add Svc: %s",
-					   adv_data->svc_name);
-			}
-
-			/* Advertisement ID */
-			wpabuf_put_le32(resp, adv_data->id);
-
-			/* Config Methods */
-			wpabuf_put_be16(resp, adv_data->config_methods);
-
-			/* Service Name */
-			wpabuf_put_u8(resp, (u8) len);
-			wpabuf_put_data(resp, adv_data->svc_name, len);
-
-			/* Service State */
-			wpabuf_put_u8(resp, adv_data->state);
-
-			/* Service Information */
-			wpabuf_put_le16(resp, (u16) svc_info_len);
-			wpabuf_put_data(resp, adv_data->svc_info, svc_info_len);
-
-			/* Update length and count */
-			(*count_pos)++;
-			WPA_PUT_LE16(len_pos,
-				     (u8 *) wpabuf_put(resp, 0) - len_pos - 2);
-		}
-	}
-
-	/* Return error if no matching svc found */
-	if (count_pos == NULL) {
-		wpa_printf(MSG_DEBUG, "P2P: ASP service not found");
-		wpas_sd_add_not_found(resp, P2P_SERV_P2PS, srv_trans_id);
-	}
-}
-
-
-static void wpas_sd_request(void *ctx, int freq, const u8 *sa, u8 dialog_token,
-			    u16 update_indic, const u8 *tlvs, size_t tlvs_len)
-{
-	struct wpa_supplicant *wpa_s = ctx;
-	const u8 *pos = tlvs;
-	const u8 *end = tlvs + tlvs_len;
-	const u8 *tlv_end;
-	u16 slen;
-	struct wpabuf *resp;
-	u8 srv_proto, srv_trans_id;
-	size_t buf_len;
-	char *buf;
-
-	wpa_hexdump(MSG_MSGDUMP, "P2P: Service Discovery Request TLVs",
-		    tlvs, tlvs_len);
-	buf_len = 2 * tlvs_len + 1;
-	buf = os_malloc(buf_len);
-	if (buf) {
-		wpa_snprintf_hex(buf, buf_len, tlvs, tlvs_len);
-		wpa_msg_ctrl(wpa_s, MSG_INFO, P2P_EVENT_SERV_DISC_REQ "%d "
-			     MACSTR " %u %u %s",
-			     freq, MAC2STR(sa), dialog_token, update_indic,
-			     buf);
-		os_free(buf);
-	}
-
-	if (wpa_s->p2p_sd_over_ctrl_iface) {
-		wpas_notify_p2p_sd_request(wpa_s, freq, sa, dialog_token,
-					   update_indic, tlvs, tlvs_len);
-		return; /* to be processed by an external program */
-	}
-
-	resp = wpabuf_alloc(10000);
-	if (resp == NULL)
-		return;
-
-	while (pos + 1 < end) {
-		wpa_printf(MSG_DEBUG, "P2P: Service Request TLV");
-		slen = WPA_GET_LE16(pos);
-		pos += 2;
-		if (pos + slen > end || slen < 2) {
-			wpa_printf(MSG_DEBUG, "P2P: Unexpected Query Data "
-				   "length");
-			wpabuf_free(resp);
-			return;
-		}
-		tlv_end = pos + slen;
-
-		srv_proto = *pos++;
-		wpa_printf(MSG_DEBUG, "P2P: Service Protocol Type %u",
-			   srv_proto);
-		srv_trans_id = *pos++;
-		wpa_printf(MSG_DEBUG, "P2P: Service Transaction ID %u",
-			   srv_trans_id);
-
-		wpa_hexdump(MSG_MSGDUMP, "P2P: Query Data",
-			    pos, tlv_end - pos);
-
-
-		if (wpa_s->force_long_sd) {
-			wpa_printf(MSG_DEBUG, "P2P: SD test - force long "
-				   "response");
-			wpas_sd_all_bonjour(wpa_s, resp, srv_trans_id);
-			wpas_sd_all_upnp(wpa_s, resp, srv_trans_id);
-			goto done;
-		}
-
-		switch (srv_proto) {
-		case P2P_SERV_ALL_SERVICES:
-			wpa_printf(MSG_DEBUG, "P2P: Service Discovery Request "
-				   "for all services");
-			if (dl_list_empty(&wpa_s->global->p2p_srv_upnp) &&
-			    dl_list_empty(&wpa_s->global->p2p_srv_bonjour)) {
-				wpa_printf(MSG_DEBUG, "P2P: No service "
-					   "discovery protocols available");
-				wpas_sd_add_proto_not_avail(
-					resp, P2P_SERV_ALL_SERVICES,
-					srv_trans_id);
-				break;
-			}
-			wpas_sd_all_bonjour(wpa_s, resp, srv_trans_id);
-			wpas_sd_all_upnp(wpa_s, resp, srv_trans_id);
-			break;
-		case P2P_SERV_BONJOUR:
-			wpas_sd_req_bonjour(wpa_s, resp, srv_trans_id,
-					    pos, tlv_end - pos);
-			break;
-		case P2P_SERV_UPNP:
-			wpas_sd_req_upnp(wpa_s, resp, srv_trans_id,
-					 pos, tlv_end - pos);
-			break;
-#ifdef CONFIG_WIFI_DISPLAY
-		case P2P_SERV_WIFI_DISPLAY:
-			wpas_sd_req_wfd(wpa_s, resp, srv_trans_id,
-					pos, tlv_end - pos);
-			break;
-#endif /* CONFIG_WIFI_DISPLAY */
-		case P2P_SERV_P2PS:
-			wpas_sd_req_asp(wpa_s, resp, srv_trans_id,
-					pos, tlv_end - pos);
-			break;
-		default:
-			wpa_printf(MSG_DEBUG, "P2P: Unavailable service "
-				   "protocol %u", srv_proto);
-			wpas_sd_add_proto_not_avail(resp, srv_proto,
-						    srv_trans_id);
-			break;
-		}
-
-		pos = tlv_end;
-	}
-
-done:
-	wpas_notify_p2p_sd_request(wpa_s, freq, sa, dialog_token,
-				   update_indic, tlvs, tlvs_len);
-
-	wpas_p2p_sd_response(wpa_s, freq, sa, dialog_token, resp);
-
-	wpabuf_free(resp);
-}
-
-
-static void wpas_sd_p2ps_serv_response(struct wpa_supplicant *wpa_s,
-				       const u8 *sa, u8 srv_trans_id,
-				       const u8 *pos, const u8 *tlv_end)
-{
-	u8 left = *pos++;
-	u32 adv_id;
-	u8 svc_status;
-	u16 config_methods;
-	char svc_str[256];
-
-	while (left-- && pos < tlv_end) {
-		char *buf = NULL;
-		size_t buf_len;
-		u8 svc_len;
-
-		/* Sanity check fixed length+svc_str */
-		if (pos + 6 >= tlv_end)
-			break;
-		svc_len = pos[6];
-		if (pos + svc_len + 10 > tlv_end)
-			break;
-
-		/* Advertisement ID */
-		adv_id = WPA_GET_LE32(pos);
-		pos += sizeof(u32);
-
-		/* Config Methods */
-		config_methods = WPA_GET_BE16(pos);
-		pos += sizeof(u16);
-
-		/* Service Name */
-		pos++; /* svc_len */
-		os_memcpy(svc_str, pos, svc_len);
-		svc_str[svc_len] = '\0';
-		pos += svc_len;
-
-		/* Service Status */
-		svc_status = *pos++;
-
-		/* Service Information Length */
-		buf_len = WPA_GET_LE16(pos);
-		pos += sizeof(u16);
-
-		/* Sanity check buffer length */
-		if (buf_len > (unsigned int) (tlv_end - pos))
-			break;
-
-		if (buf_len) {
-			buf = os_zalloc(2 * buf_len + 1);
-			if (buf) {
-				utf8_escape((const char *) pos, buf_len, buf,
-					    2 * buf_len + 1);
-			}
-		}
-
-		pos += buf_len;
-
-		if (buf) {
-			wpa_msg_global(wpa_s, MSG_INFO, P2P_EVENT_SERV_ASP_RESP
-				       MACSTR " %x %x %x %x %s '%s'",
-				       MAC2STR(sa), srv_trans_id, adv_id,
-				       svc_status, config_methods, svc_str,
-				       buf);
-			os_free(buf);
-		} else {
-			wpa_msg_global(wpa_s, MSG_INFO, P2P_EVENT_SERV_ASP_RESP
-				       MACSTR " %x %x %x %x %s",
-				       MAC2STR(sa), srv_trans_id, adv_id,
-				       svc_status, config_methods, svc_str);
-		}
-	}
-}
-
-
-static void wpas_sd_response(void *ctx, const u8 *sa, u16 update_indic,
-			     const u8 *tlvs, size_t tlvs_len)
-{
-	struct wpa_supplicant *wpa_s = ctx;
-	const u8 *pos = tlvs;
-	const u8 *end = tlvs + tlvs_len;
-	const u8 *tlv_end;
-	u16 slen;
-	size_t buf_len;
-	char *buf;
-
-	wpa_hexdump(MSG_MSGDUMP, "P2P: Service Discovery Response TLVs",
-		    tlvs, tlvs_len);
-	if (tlvs_len > 1500) {
-		/* TODO: better way for handling this */
-		wpa_msg_ctrl(wpa_s, MSG_INFO,
-			     P2P_EVENT_SERV_DISC_RESP MACSTR
-			     " %u <long response: %u bytes>",
-			     MAC2STR(sa), update_indic,
-			     (unsigned int) tlvs_len);
-	} else {
-		buf_len = 2 * tlvs_len + 1;
-		buf = os_malloc(buf_len);
-		if (buf) {
-			wpa_snprintf_hex(buf, buf_len, tlvs, tlvs_len);
-			wpa_msg_ctrl(wpa_s, MSG_INFO,
-				     P2P_EVENT_SERV_DISC_RESP MACSTR " %u %s",
-				     MAC2STR(sa), update_indic, buf);
-			os_free(buf);
-		}
-	}
-
-	while (pos < end) {
-		u8 srv_proto, srv_trans_id, status;
-
-		wpa_printf(MSG_DEBUG, "P2P: Service Response TLV");
-		slen = WPA_GET_LE16(pos);
-		pos += 2;
-		if (pos + slen > end || slen < 3) {
-			wpa_printf(MSG_DEBUG, "P2P: Unexpected Response Data "
-				   "length");
-			return;
-		}
-		tlv_end = pos + slen;
-
-		srv_proto = *pos++;
-		wpa_printf(MSG_DEBUG, "P2P: Service Protocol Type %u",
-			   srv_proto);
-		srv_trans_id = *pos++;
-		wpa_printf(MSG_DEBUG, "P2P: Service Transaction ID %u",
-			   srv_trans_id);
-		status = *pos++;
-		wpa_printf(MSG_DEBUG, "P2P: Status Code ID %u",
-			   status);
-
-		wpa_hexdump(MSG_MSGDUMP, "P2P: Response Data",
-			    pos, tlv_end - pos);
-
-		if (srv_proto == P2P_SERV_P2PS && pos < tlv_end) {
-			wpas_sd_p2ps_serv_response(wpa_s, sa, srv_trans_id,
-						   pos, tlv_end);
-		}
-
-		pos = tlv_end;
-	}
-
-	wpas_notify_p2p_sd_response(wpa_s, sa, update_indic, tlvs, tlvs_len);
-}
-
-
-u64 wpas_p2p_sd_request(struct wpa_supplicant *wpa_s, const u8 *dst,
-			const struct wpabuf *tlvs)
-{
-	if (wpa_s->global->p2p_disabled || wpa_s->global->p2p == NULL)
-		return 0;
-	return (uintptr_t) p2p_sd_request(wpa_s->global->p2p, dst, tlvs);
-}
-
-
-u64 wpas_p2p_sd_request_upnp(struct wpa_supplicant *wpa_s, const u8 *dst,
-			     u8 version, const char *query)
-{
-	struct wpabuf *tlvs;
-	u64 ret;
-
-	tlvs = wpabuf_alloc(2 + 1 + 1 + 1 + os_strlen(query));
-	if (tlvs == NULL)
-		return 0;
-	wpabuf_put_le16(tlvs, 1 + 1 + 1 + os_strlen(query));
-	wpabuf_put_u8(tlvs, P2P_SERV_UPNP); /* Service Protocol Type */
-	wpabuf_put_u8(tlvs, 1); /* Service Transaction ID */
-	wpabuf_put_u8(tlvs, version);
-	wpabuf_put_str(tlvs, query);
-	ret = wpas_p2p_sd_request(wpa_s, dst, tlvs);
-	wpabuf_free(tlvs);
-	return ret;
-}
-
-
-u64 wpas_p2p_sd_request_asp(struct wpa_supplicant *wpa_s, const u8 *dst, u8 id,
-			    const char *svc_str, const char *info_substr)
-{
-	struct wpabuf *tlvs;
-	size_t plen, svc_len, substr_len = 0;
-	u64 ret;
-
-	svc_len = os_strlen(svc_str);
-	if (info_substr)
-		substr_len = os_strlen(info_substr);
-
-	if (svc_len > 0xff || substr_len > 0xff)
-		return 0;
-
-	plen = 1 + 1 + 1 + svc_len + 1 + substr_len;
-	tlvs = wpabuf_alloc(2 + plen);
-	if (tlvs == NULL)
-		return 0;
-
-	wpabuf_put_le16(tlvs, plen);
-	wpabuf_put_u8(tlvs, P2P_SERV_P2PS);
-	wpabuf_put_u8(tlvs, id); /* Service Transaction ID */
-	wpabuf_put_u8(tlvs, (u8) svc_len); /* Service String Length */
-	wpabuf_put_data(tlvs, svc_str, svc_len);
-	wpabuf_put_u8(tlvs, (u8) substr_len); /* Info Substring Length */
-	wpabuf_put_data(tlvs, info_substr, substr_len);
-	ret = wpas_p2p_sd_request(wpa_s, dst, tlvs);
-	wpabuf_free(tlvs);
-
-	return ret;
-}
-
-
-#ifdef CONFIG_WIFI_DISPLAY
-
-static u64 wpas_p2p_sd_request_wfd(struct wpa_supplicant *wpa_s, const u8 *dst,
-				   const struct wpabuf *tlvs)
-{
-	if (wpa_s->global->p2p_disabled || wpa_s->global->p2p == NULL)
-		return 0;
-	return (uintptr_t) p2p_sd_request_wfd(wpa_s->global->p2p, dst, tlvs);
-}
-
-
-#define MAX_WFD_SD_SUBELEMS 20
-
-static void wfd_add_sd_req_role(struct wpabuf *tlvs, u8 id, u8 role,
-				const char *subelems)
-{
-	u8 *len;
-	const char *pos;
-	int val;
-	int count = 0;
-
-	len = wpabuf_put(tlvs, 2);
-	wpabuf_put_u8(tlvs, P2P_SERV_WIFI_DISPLAY); /* Service Protocol Type */
-	wpabuf_put_u8(tlvs, id); /* Service Transaction ID */
-
-	wpabuf_put_u8(tlvs, role);
-
-	pos = subelems;
-	while (*pos) {
-		val = atoi(pos);
-		if (val >= 0 && val < 256) {
-			wpabuf_put_u8(tlvs, val);
-			count++;
-			if (count == MAX_WFD_SD_SUBELEMS)
-				break;
-		}
-		pos = os_strchr(pos + 1, ',');
-		if (pos == NULL)
-			break;
-		pos++;
-	}
-
-	WPA_PUT_LE16(len, (u8 *) wpabuf_put(tlvs, 0) - len - 2);
-}
-
-
-u64 wpas_p2p_sd_request_wifi_display(struct wpa_supplicant *wpa_s,
-				     const u8 *dst, const char *role)
-{
-	struct wpabuf *tlvs;
-	u64 ret;
-	const char *subelems;
-	u8 id = 1;
-
-	subelems = os_strchr(role, ' ');
-	if (subelems == NULL)
-		return 0;
-	subelems++;
-
-	tlvs = wpabuf_alloc(4 * (2 + 1 + 1 + 1 + MAX_WFD_SD_SUBELEMS));
-	if (tlvs == NULL)
-		return 0;
-
-	if (os_strstr(role, "[source]"))
-		wfd_add_sd_req_role(tlvs, id++, 0x00, subelems);
-	if (os_strstr(role, "[pri-sink]"))
-		wfd_add_sd_req_role(tlvs, id++, 0x01, subelems);
-	if (os_strstr(role, "[sec-sink]"))
-		wfd_add_sd_req_role(tlvs, id++, 0x02, subelems);
-	if (os_strstr(role, "[source+sink]"))
-		wfd_add_sd_req_role(tlvs, id++, 0x03, subelems);
-
-	ret = wpas_p2p_sd_request_wfd(wpa_s, dst, tlvs);
-	wpabuf_free(tlvs);
-	return ret;
-}
-
-#endif /* CONFIG_WIFI_DISPLAY */
-
-
-int wpas_p2p_sd_cancel_request(struct wpa_supplicant *wpa_s, u64 req)
-{
-	if (wpa_s->global->p2p_disabled || wpa_s->global->p2p == NULL)
-		return -1;
-	return p2p_sd_cancel_request(wpa_s->global->p2p,
-				     (void *) (uintptr_t) req);
-}
-
-
-void wpas_p2p_sd_response(struct wpa_supplicant *wpa_s, int freq,
-			  const u8 *dst, u8 dialog_token,
-			  const struct wpabuf *resp_tlvs)
-{
-	if (wpa_s->global->p2p_disabled || wpa_s->global->p2p == NULL)
-		return;
-	p2p_sd_response(wpa_s->global->p2p, freq, dst, dialog_token,
-			resp_tlvs);
-}
-
-
-void wpas_p2p_sd_service_update(struct wpa_supplicant *wpa_s)
-{
-	if (wpa_s->global->p2p)
-		p2p_sd_service_update(wpa_s->global->p2p);
-}
-
-
-static void wpas_p2p_srv_bonjour_free(struct p2p_srv_bonjour *bsrv)
-{
-	dl_list_del(&bsrv->list);
-	wpabuf_free(bsrv->query);
-	wpabuf_free(bsrv->resp);
-	os_free(bsrv);
-}
-
-
-static void wpas_p2p_srv_upnp_free(struct p2p_srv_upnp *usrv)
-{
-	dl_list_del(&usrv->list);
-	os_free(usrv->service);
-	os_free(usrv);
-}
-
-
-void wpas_p2p_service_flush(struct wpa_supplicant *wpa_s)
-{
-	struct p2p_srv_bonjour *bsrv, *bn;
-	struct p2p_srv_upnp *usrv, *un;
-
-	dl_list_for_each_safe(bsrv, bn, &wpa_s->global->p2p_srv_bonjour,
-			      struct p2p_srv_bonjour, list)
-		wpas_p2p_srv_bonjour_free(bsrv);
-
-	dl_list_for_each_safe(usrv, un, &wpa_s->global->p2p_srv_upnp,
-			      struct p2p_srv_upnp, list)
-		wpas_p2p_srv_upnp_free(usrv);
-
-	wpas_p2p_sd_service_update(wpa_s);
-}
-
-
-int wpas_p2p_service_p2ps_id_exists(struct wpa_supplicant *wpa_s, u32 adv_id)
-{
-	if (adv_id == 0)
-		return 1;
-
-	if (p2p_service_p2ps_id(wpa_s->global->p2p, adv_id))
-		return 1;
-
-	return 0;
-}
-
-
-int wpas_p2p_service_del_asp(struct wpa_supplicant *wpa_s, u32 adv_id)
-{
-	return p2p_service_del_asp(wpa_s->global->p2p, adv_id);
-}
-
-
-int wpas_p2p_service_add_asp(struct wpa_supplicant *wpa_s,
-			     int auto_accept, u32 adv_id,
-			     const char *adv_str, u8 svc_state,
-			     u16 config_methods, const char *svc_info)
-{
-	return p2p_service_add_asp(wpa_s->global->p2p, auto_accept, adv_id,
-				   adv_str, svc_state, config_methods,
-				   svc_info);
-}
-
-
-int wpas_p2p_service_add_bonjour(struct wpa_supplicant *wpa_s,
-				 struct wpabuf *query, struct wpabuf *resp)
-{
-	struct p2p_srv_bonjour *bsrv;
-
-	bsrv = os_zalloc(sizeof(*bsrv));
-	if (bsrv == NULL)
-		return -1;
-	bsrv->query = query;
-	bsrv->resp = resp;
-	dl_list_add(&wpa_s->global->p2p_srv_bonjour, &bsrv->list);
-
-	wpas_p2p_sd_service_update(wpa_s);
-	return 0;
-}
-
-
-int wpas_p2p_service_del_bonjour(struct wpa_supplicant *wpa_s,
-				 const struct wpabuf *query)
-{
-	struct p2p_srv_bonjour *bsrv;
-
-	bsrv = wpas_p2p_service_get_bonjour(wpa_s, query);
-	if (bsrv == NULL)
-		return -1;
-	wpas_p2p_srv_bonjour_free(bsrv);
-	wpas_p2p_sd_service_update(wpa_s);
-	return 0;
-}
-
-
-int wpas_p2p_service_add_upnp(struct wpa_supplicant *wpa_s, u8 version,
-			      const char *service)
-{
-	struct p2p_srv_upnp *usrv;
-
-	if (wpas_p2p_service_get_upnp(wpa_s, version, service))
-		return 0; /* Already listed */
-	usrv = os_zalloc(sizeof(*usrv));
-	if (usrv == NULL)
-		return -1;
-	usrv->version = version;
-	usrv->service = os_strdup(service);
-	if (usrv->service == NULL) {
-		os_free(usrv);
-		return -1;
-	}
-	dl_list_add(&wpa_s->global->p2p_srv_upnp, &usrv->list);
-
-	wpas_p2p_sd_service_update(wpa_s);
-	return 0;
-}
-
-
-int wpas_p2p_service_del_upnp(struct wpa_supplicant *wpa_s, u8 version,
-			      const char *service)
-{
-	struct p2p_srv_upnp *usrv;
-
-	usrv = wpas_p2p_service_get_upnp(wpa_s, version, service);
-	if (usrv == NULL)
-		return -1;
-	wpas_p2p_srv_upnp_free(usrv);
-	wpas_p2p_sd_service_update(wpa_s);
-	return 0;
+	return wpa_drv_send_mlme(wpa_s, wpabuf_head(buf), wpabuf_len(buf), 1,
+				 freq);
 }
 
 
@@ -3776,12 +2626,62 @@ static void wpas_prov_disc_fail(void *ctx, const u8 *peer,
 }
 
 
-static int freq_included(const struct p2p_channels *channels, unsigned int freq)
+static int freq_included(struct wpa_supplicant *wpa_s,
+			 const struct p2p_channels *channels,
+			 unsigned int freq)
 {
-	if (channels == NULL)
-		return 1; /* Assume no restrictions */
-	return p2p_channels_includes_freq(channels, freq);
+	if ((channels == NULL || p2p_channels_includes_freq(channels, freq)) &&
+	    wpas_p2p_go_is_peer_freq(wpa_s, freq))
+		return 1;
+	return 0;
+}
 
+
+static void wpas_p2p_go_update_common_freqs(struct wpa_supplicant *wpa_s)
+{
+	unsigned int num = P2P_MAX_CHANNELS;
+	int *common_freqs;
+	int ret;
+
+	p2p_go_dump_common_freqs(wpa_s);
+	common_freqs = os_calloc(num, sizeof(int));
+	if (!common_freqs)
+		return;
+
+	ret = p2p_group_get_common_freqs(wpa_s->p2p_group, common_freqs, &num);
+	if (ret < 0) {
+		wpa_dbg(wpa_s, MSG_DEBUG,
+			"P2P: Failed to get group common freqs");
+		os_free(common_freqs);
+		return;
+	}
+
+	os_free(wpa_s->p2p_group_common_freqs);
+	wpa_s->p2p_group_common_freqs = common_freqs;
+	wpa_s->p2p_group_common_freqs_num = num;
+	p2p_go_dump_common_freqs(wpa_s);
+}
+
+
+/*
+ * Check if the given frequency is one of the possible operating frequencies
+ * set after the completion of the GO Negotiation.
+ */
+static int wpas_p2p_go_is_peer_freq(struct wpa_supplicant *wpa_s, int freq)
+{
+	unsigned int i;
+
+	p2p_go_dump_common_freqs(wpa_s);
+
+	/* assume no restrictions */
+	if (!wpa_s->p2p_group_common_freqs_num)
+		return 1;
+
+	for (i = 0; i < wpa_s->p2p_group_common_freqs_num; i++) {
+		if (wpa_s->p2p_group_common_freqs[i] == freq)
+			return 1;
+	}
+	return 0;
 }
 
 
@@ -3957,7 +2857,7 @@ accept_inv:
 				   "running a GO but we are capable of MCC, "
 				   "figure out the best channel to use");
 			*force_freq = 0;
-		} else if (!freq_included(channels, *force_freq)) {
+		} else if (!freq_included(wpa_s, channels, *force_freq)) {
 			/* We are the GO, and *force_freq is not in the
 			 * intersection */
 			wpa_printf(MSG_DEBUG, "P2P: Forced GO freq %d MHz not "
@@ -3995,7 +2895,8 @@ static void wpas_invitation_received(void *ctx, const u8 *sa, const u8 *bssid,
 			int go = s->mode == WPAS_MODE_P2P_GO;
 			wpas_p2p_group_add_persistent(
 				wpa_s, s, go, 0, op_freq, 0, 0, NULL,
-				go ? P2P_MAX_INITIAL_CONN_WAIT_GO_REINVOKE : 0);
+				go ? P2P_MAX_INITIAL_CONN_WAIT_GO_REINVOKE : 0,
+				1);
 		} else if (bssid) {
 			wpa_s->user_initiated_pd = 0;
 			wpas_p2p_join(wpa_s, bssid, go_dev_addr,
@@ -4026,6 +2927,8 @@ static void wpas_invitation_received(void *ctx, const u8 *sa, const u8 *bssid,
 				       " unknown-network",
 				       MAC2STR(sa), MAC2STR(go_dev_addr));
 		}
+		wpas_notify_p2p_invitation_received(wpa_s, sa, go_dev_addr,
+						    bssid, 0, op_freq);
 		return;
 	}
 
@@ -4038,6 +2941,8 @@ static void wpas_invitation_received(void *ctx, const u8 *sa, const u8 *bssid,
 			       "sa=" MACSTR " persistent=%d",
 			       MAC2STR(sa), s->id);
 	}
+	wpas_notify_p2p_invitation_received(wpa_s, sa, go_dev_addr, bssid,
+					    s->id, op_freq);
 }
 
 
@@ -4046,6 +2951,7 @@ static void wpas_remove_persistent_peer(struct wpa_supplicant *wpa_s,
 					const u8 *peer, int inv)
 {
 	size_t i;
+	struct wpa_supplicant *p2p_wpa_s = wpa_s->global->p2p_init_wpa_s;
 
 	if (ssid == NULL)
 		return;
@@ -4075,8 +2981,8 @@ static void wpas_remove_persistent_peer(struct wpa_supplicant *wpa_s,
 		   ssid->p2p_client_list + (i + 1) * 2 * ETH_ALEN,
 		   (ssid->num_p2p_clients - i - 1) * 2 * ETH_ALEN);
 	ssid->num_p2p_clients--;
-	if (wpa_s->parent->conf->update_config &&
-	    wpa_config_write(wpa_s->parent->confname, wpa_s->parent->conf))
+	if (p2p_wpa_s->conf->update_config &&
+	    wpa_config_write(p2p_wpa_s->confname, p2p_wpa_s->conf))
 		wpa_printf(MSG_DEBUG, "P2P: Failed to update configuration");
 }
 
@@ -4163,10 +3069,10 @@ static void wpas_invitation_result(void *ctx, int status, const u8 *bssid,
 	os_sleep(0, 50000);
 
 	if (neg_freq > 0 && ssid->mode == WPAS_MODE_P2P_GO &&
-	    freq_included(channels, neg_freq))
+	    freq_included(wpa_s, channels, neg_freq))
 		freq = neg_freq;
 	else if (peer_oper_freq > 0 && ssid->mode != WPAS_MODE_P2P_GO &&
-		 freq_included(channels, peer_oper_freq))
+		 freq_included(wpa_s, channels, peer_oper_freq))
 		freq = peer_oper_freq;
 	else
 		freq = 0;
@@ -4181,7 +3087,7 @@ static void wpas_invitation_result(void *ctx, int status, const u8 *bssid,
 				      channels,
 				      ssid->mode == WPAS_MODE_P2P_GO ?
 				      P2P_MAX_INITIAL_CONN_WAIT_GO_REINVOKE :
-				      0);
+				      0, 1);
 }
 
 
@@ -4320,7 +3226,7 @@ struct p2p_oper_class_map {
 	enum { BW20, BW40PLUS, BW40MINUS, BW80, BW2160 } bw;
 };
 
-static struct p2p_oper_class_map op_class[] = {
+static const struct p2p_oper_class_map op_class[] = {
 	{ HOSTAPD_MODE_IEEE80211G, 81, 1, 13, 1, BW20 },
 #if 0 /* Do not enable HT40 on 2 GHz for now */
 	{ HOSTAPD_MODE_IEEE80211G, 83, 1, 9, 1, BW40PLUS },
@@ -4328,6 +3234,7 @@ static struct p2p_oper_class_map op_class[] = {
 #endif
 	{ HOSTAPD_MODE_IEEE80211A, 115, 36, 48, 4, BW20 },
 	{ HOSTAPD_MODE_IEEE80211A, 124, 149, 161, 4, BW20 },
+	{ HOSTAPD_MODE_IEEE80211A, 125, 149, 169, 4, BW20 },
 	{ HOSTAPD_MODE_IEEE80211A, 116, 36, 44, 8, BW40PLUS },
 	{ HOSTAPD_MODE_IEEE80211A, 117, 40, 48, 8, BW40MINUS },
 	{ HOSTAPD_MODE_IEEE80211A, 126, 149, 157, 8, BW40PLUS },
@@ -4453,7 +3360,7 @@ static int wpas_p2p_setup_channels(struct wpa_supplicant *wpa_s,
 	cla = cli_cla = 0;
 
 	for (op = 0; op_class[op].op_class; op++) {
-		struct p2p_oper_class_map *o = &op_class[op];
+		const struct p2p_oper_class_map *o = &op_class[op];
 		u8 ch;
 		struct p2p_reg_class *reg = NULL, *cli_reg = NULL;
 
@@ -4512,12 +3419,13 @@ int wpas_p2p_get_ht40_mode(struct wpa_supplicant *wpa_s,
 	enum chan_allowed ret;
 
 	for (op = 0; op_class[op].op_class; op++) {
-		struct p2p_oper_class_map *o = &op_class[op];
+		const struct p2p_oper_class_map *o = &op_class[op];
 		u8 ch;
 
 		for (ch = o->min_chan; ch <= o->max_chan; ch += o->inc) {
 			if (o->mode != HOSTAPD_MODE_IEEE80211A ||
-			    o->bw == BW20 || ch != channel)
+			    (o->bw != BW40PLUS && o->bw != BW40MINUS) ||
+			    ch != channel)
 				continue;
 			ret = wpas_p2p_verify_channel(wpa_s, mode, ch, o->bw);
 			if (ret == ALLOWED)
@@ -4581,12 +3489,7 @@ struct wpa_supplicant * wpas_get_p2p_client_iface(struct wpa_supplicant *wpa_s,
 {
 	for (wpa_s = wpa_s->global->ifaces; wpa_s; wpa_s = wpa_s->next) {
 		struct wpa_ssid *ssid = wpa_s->current_ssid;
-		if (ssid == NULL)
-			continue;
-		if (ssid->mode != WPAS_MODE_INFRA)
-			continue;
-		if (wpa_s->wpa_state != WPA_COMPLETED &&
-		    wpa_s->wpa_state != WPA_GROUP_HANDSHAKE)
+		if (ssid && (ssid->mode != WPAS_MODE_INFRA || !ssid->p2p_group))
 			continue;
 		if (os_memcmp(wpa_s->go_dev_addr, peer_dev_addr, ETH_ALEN) == 0)
 			return wpa_s;
@@ -4667,14 +3570,12 @@ int wpas_p2p_add_p2pdev_interface(struct wpa_supplicant *wpa_s,
 		iface.confname = wpa_s->confname;
 		iface.ctrl_interface = wpa_s->conf->ctrl_interface;
 	}
-	iface.conf_p2p_dev = NULL;
 
 	p2pdev_wpa_s = wpa_supplicant_add_iface(wpa_s->global, &iface, wpa_s);
 	if (!p2pdev_wpa_s) {
 		wpa_printf(MSG_DEBUG, "P2P: Failed to add P2P Device interface");
 		return -1;
 	}
-	wpa_s->p2p_dev = p2pdev_wpa_s;
 
 	wpa_s->pending_interface_name[0] = '\0';
 	return 0;
@@ -4705,7 +3606,8 @@ static void wpas_presence_resp(void *ctx, const u8 *src, u8 status,
 
 static int wpas_get_persistent_group(void *ctx, const u8 *addr, const u8 *ssid,
 				     size_t ssid_len, u8 *go_dev_addr,
-				     u8 *ret_ssid, size_t *ret_ssid_len)
+				     u8 *ret_ssid, size_t *ret_ssid_len,
+				     u8 *intended_iface_addr)
 {
 	struct wpa_supplicant *wpa_s = ctx;
 	struct wpa_ssid *s;
@@ -4715,6 +3617,19 @@ static int wpas_get_persistent_group(void *ctx, const u8 *addr, const u8 *ssid,
 		os_memcpy(ret_ssid, s->ssid, s->ssid_len);
 		*ret_ssid_len = s->ssid_len;
 		os_memcpy(go_dev_addr, s->bssid, ETH_ALEN);
+
+		if (s->mode != WPAS_MODE_P2P_GO) {
+			os_memset(intended_iface_addr, 0, ETH_ALEN);
+		} else if (wpas_p2p_create_iface(wpa_s)) {
+			if (wpas_p2p_add_group_interface(wpa_s, WPA_IF_P2P_GO))
+				return 0;
+
+			os_memcpy(intended_iface_addr,
+				  wpa_s->pending_interface_addr, ETH_ALEN);
+		} else {
+			os_memcpy(intended_iface_addr, wpa_s->own_addr,
+				  ETH_ALEN);
+		}
 		return 1;
 	}
 
@@ -4729,17 +3644,24 @@ static int wpas_get_go_info(void *ctx, u8 *intended_addr,
 	struct wpa_ssid *s;
 	u8 bssid[ETH_ALEN];
 
+	/*
+	 * group_iface will be set to 1 only if a dedicated interface for P2P
+	 * role is required. First, we try to reuse an active GO. However,
+	 * if it is not present, we will try to reactivate an existing
+	 * persistent group and set group_iface to 1, so the caller will know
+	 * that the pending interface should be used.
+	 */
+	*group_iface = 0;
 	s = wpas_p2p_group_go_ssid(wpa_s, bssid);
 	if (!s) {
 		s = wpas_p2p_get_persistent_go(wpa_s);
+		*group_iface = wpas_p2p_create_iface(wpa_s);
 		if (s)
 			os_memcpy(bssid, s->bssid, ETH_ALEN);
+		else
+			return 0;
 	}
 
-	*group_iface = wpas_p2p_create_iface(wpa_s);
-	if (!s)
-		return 0;
-
 	os_memcpy(intended_addr, bssid, ETH_ALEN);
 	os_memcpy(ssid, s->ssid, s->ssid_len);
 	*ssid_len = s->ssid_len;
@@ -4793,19 +3715,49 @@ static int wpas_remove_stale_groups(void *ctx, const u8 *peer, const u8 *go,
 }
 
 
+static void wpas_p2ps_get_feat_cap_str(char *buf, size_t buf_len,
+				       const u8 *feat_cap, size_t feat_cap_len)
+{
+	static const char pref[] = " feature_cap=";
+	int ret;
+
+	buf[0] = '\0';
+
+	/*
+	 * We expect a feature capability to contain at least one byte to be
+	 * reported. The string buffer provided by the caller function is
+	 * expected to be big enough to contain all bytes of the attribute for
+	 * known specifications. This function truncates the reported bytes if
+	 * the feature capability data exceeds the string buffer size.
+	 */
+	if (!feat_cap || !feat_cap_len || buf_len < sizeof(pref) + 2)
+		return;
+
+	os_memcpy(buf, pref, sizeof(pref));
+	ret = wpa_snprintf_hex(&buf[sizeof(pref) - 1],
+			       buf_len - sizeof(pref) + 1,
+			       feat_cap, feat_cap_len);
+
+	if (ret != (2 * (int) feat_cap_len))
+		wpa_printf(MSG_WARNING, "P2PS feature_cap bytes truncated");
+}
+
+
 static void wpas_p2ps_prov_complete(void *ctx, u8 status, const u8 *dev,
 				    const u8 *adv_mac, const u8 *ses_mac,
 				    const u8 *grp_mac, u32 adv_id, u32 ses_id,
 				    u8 conncap, int passwd_id,
 				    const u8 *persist_ssid,
 				    size_t persist_ssid_size, int response_done,
-				    int prov_start, const char *session_info)
+				    int prov_start, const char *session_info,
+				    const u8 *feat_cap, size_t feat_cap_len)
 {
 	struct wpa_supplicant *wpa_s = ctx;
 	u8 mac[ETH_ALEN];
 	struct wpa_ssid *persistent_go, *stale, *s;
 	int save_config = 0;
 	struct wpa_supplicant *go_wpa_s;
+	char feat_cap_str[256];
 
 	if (!dev)
 		return;
@@ -4818,6 +3770,9 @@ static void wpas_p2ps_prov_complete(void *ctx, u8 status, const u8 *dev,
 	if (!grp_mac)
 		grp_mac = mac;
 
+	wpas_p2ps_get_feat_cap_str(feat_cap_str, sizeof(feat_cap_str),
+				   feat_cap, feat_cap_len);
+
 	if (prov_start) {
 		if (session_info == NULL) {
 			wpa_msg_global(wpa_s, MSG_INFO,
@@ -4825,22 +3780,22 @@ static void wpas_p2ps_prov_complete(void *ctx, u8 status, const u8 *dev,
 				       " adv_id=%x conncap=%x"
 				       " adv_mac=" MACSTR
 				       " session=%x mac=" MACSTR
-				       " dev_passwd_id=%d",
+				       " dev_passwd_id=%d%s",
 				       MAC2STR(dev), adv_id, conncap,
 				       MAC2STR(adv_mac),
 				       ses_id, MAC2STR(ses_mac),
-				       passwd_id);
+				       passwd_id, feat_cap_str);
 		} else {
 			wpa_msg_global(wpa_s, MSG_INFO,
 				       P2P_EVENT_P2PS_PROVISION_START MACSTR
 				       " adv_id=%x conncap=%x"
 				       " adv_mac=" MACSTR
 				       " session=%x mac=" MACSTR
-				       " dev_passwd_id=%d info='%s'",
+				       " dev_passwd_id=%d info='%s'%s",
 				       MAC2STR(dev), adv_id, conncap,
 				       MAC2STR(adv_mac),
 				       ses_id, MAC2STR(ses_mac),
-				       passwd_id, session_info);
+				       passwd_id, session_info, feat_cap_str);
 		}
 		return;
 	}
@@ -4862,16 +3817,24 @@ static void wpas_p2ps_prov_complete(void *ctx, u8 status, const u8 *dev,
 			       P2P_EVENT_P2PS_PROVISION_DONE MACSTR
 			       " status=%d"
 			       " adv_id=%x adv_mac=" MACSTR
-			       " session=%x mac=" MACSTR,
+			       " session=%x mac=" MACSTR "%s",
 			       MAC2STR(dev), status,
 			       adv_id, MAC2STR(adv_mac),
-			       ses_id, MAC2STR(ses_mac));
+			       ses_id, MAC2STR(ses_mac), feat_cap_str);
 		return;
 	}
 
 	/* Clean up stale persistent groups with this device */
 	s = wpas_p2p_get_persistent(wpa_s, dev, persist_ssid,
 				    persist_ssid_size);
+
+	if (persist_ssid && s && s->mode != WPAS_MODE_P2P_GO &&
+	    is_zero_ether_addr(grp_mac)) {
+		wpa_dbg(wpa_s, MSG_ERROR,
+			"P2P: Peer device is a GO in a persistent group, but it did not provide the intended MAC address");
+		return;
+	}
+
 	for (;;) {
 		stale = wpas_p2p_get_persistent(wpa_s, dev, NULL, 0);
 		if (!stale)
@@ -4927,29 +3890,39 @@ static void wpas_p2ps_prov_complete(void *ctx, u8 status, const u8 *dev,
 			       " status=%d"
 			       " adv_id=%x adv_mac=" MACSTR
 			       " session=%x mac=" MACSTR
-			       " persist=%d",
+			       " persist=%d%s",
 			       MAC2STR(dev), status,
 			       adv_id, MAC2STR(adv_mac),
-			       ses_id, MAC2STR(ses_mac), s->id);
+			       ses_id, MAC2STR(ses_mac), s->id, feat_cap_str);
 		return;
 	}
 
 	if (conncap == P2PS_SETUP_GROUP_OWNER) {
-		const char *go_ifname = NULL;
+		/*
+		 * We need to copy the interface name. Simply saving a
+		 * pointer isn't enough, since if we use pending_interface_name
+		 * it will be overwritten when the group is added.
+		 */
+		char go_ifname[100];
+
+		go_ifname[0] = '\0';
 		if (!go_wpa_s) {
 			wpa_s->global->pending_p2ps_group = 1;
 
-			if (wpa_s->conf->p2p_no_group_iface)
-				go_ifname = wpa_s->ifname;
+			if (!wpas_p2p_create_iface(wpa_s))
+				os_memcpy(go_ifname, wpa_s->ifname,
+					  sizeof(go_ifname));
 			else if (wpa_s->pending_interface_name[0])
-				go_ifname = wpa_s->pending_interface_name;
+				os_memcpy(go_ifname,
+					  wpa_s->pending_interface_name,
+					  sizeof(go_ifname));
 
-			if (!go_ifname) {
+			if (!go_ifname[0]) {
 				wpas_p2ps_prov_complete(
 					wpa_s, P2P_SC_FAIL_UNKNOWN_GROUP,
 					dev, adv_mac, ses_mac,
-					NULL, adv_id, ses_id, 0, 0,
-					NULL, 0, 0, 0, NULL);
+					grp_mac, adv_id, ses_id, 0, 0,
+					NULL, 0, 0, 0, NULL, NULL, 0);
 				return;
 			}
 
@@ -4961,30 +3934,37 @@ static void wpas_p2ps_prov_complete(void *ctx, u8 status, const u8 *dev,
 					persistent_go->mode ==
 					WPAS_MODE_P2P_GO ?
 					P2P_MAX_INITIAL_CONN_WAIT_GO_REINVOKE :
-					0);
+					0, 0);
 			} else if (response_done) {
 				wpas_p2p_group_add(wpa_s, 1, 0, 0, 0);
 			}
 
 			if (passwd_id == DEV_PW_P2PS_DEFAULT) {
-				os_memcpy(wpa_s->p2ps_join_addr, dev, ETH_ALEN);
-				wpa_s->p2ps_join_addr_valid = 1;
-				wpa_dbg(wpa_s, MSG_DEBUG,
-					"P2PS: Saving PIN for " MACSTR,
-					MAC2STR(dev));
+				os_memcpy(wpa_s->p2ps_join_addr, grp_mac,
+					  ETH_ALEN);
+				wpa_s->p2ps_method_config_any = 1;
 			}
 		} else if (passwd_id == DEV_PW_P2PS_DEFAULT) {
-			go_ifname = go_wpa_s->ifname;
+			os_memcpy(go_ifname, go_wpa_s->ifname,
+				  sizeof(go_ifname));
 
-			wpa_dbg(go_wpa_s, MSG_DEBUG,
-				"P2P: Setting PIN-1 For " MACSTR, MAC2STR(dev));
-			wpa_supplicant_ap_wps_pin(go_wpa_s, dev, "12345670",
-						  NULL, 0, 0);
+			if (is_zero_ether_addr(grp_mac)) {
+				wpa_dbg(go_wpa_s, MSG_DEBUG,
+					"P2P: Setting PIN-1 for ANY");
+				wpa_supplicant_ap_wps_pin(go_wpa_s, NULL,
+							  "12345670", NULL, 0,
+							  0);
+			} else {
+				wpa_dbg(go_wpa_s, MSG_DEBUG,
+					"P2P: Setting PIN-1 for " MACSTR,
+					MAC2STR(grp_mac));
+				wpa_supplicant_ap_wps_pin(go_wpa_s, grp_mac,
+							  "12345670", NULL, 0,
+							  0);
+			}
 
-			os_memcpy(wpa_s->p2ps_join_addr, dev, ETH_ALEN);
-			wpa_s->p2ps_join_addr_valid = 1;
-			wpa_dbg(wpa_s, MSG_DEBUG,
-				"P2PS: Saving PIN for " MACSTR, MAC2STR(dev));
+			os_memcpy(wpa_s->p2ps_join_addr, grp_mac, ETH_ALEN);
+			wpa_s->p2ps_method_config_any = 1;
 		}
 
 		wpa_msg_global(wpa_s, MSG_INFO,
@@ -4992,11 +3972,11 @@ static void wpas_p2ps_prov_complete(void *ctx, u8 status, const u8 *dev,
 			       " status=%d conncap=%x"
 			       " adv_id=%x adv_mac=" MACSTR
 			       " session=%x mac=" MACSTR
-			       " dev_passwd_id=%d go=%s",
+			       " dev_passwd_id=%d go=%s%s",
 			       MAC2STR(dev), status, conncap,
 			       adv_id, MAC2STR(adv_mac),
 			       ses_id, MAC2STR(ses_mac),
-			       passwd_id, go_ifname);
+			       passwd_id, go_ifname, feat_cap_str);
 		return;
 	}
 
@@ -5014,22 +3994,22 @@ static void wpas_p2ps_prov_complete(void *ctx, u8 status, const u8 *dev,
 			       " status=%d conncap=%x"
 			       " adv_id=%x adv_mac=" MACSTR
 			       " session=%x mac=" MACSTR
-			       " dev_passwd_id=%d join=" MACSTR,
+			       " dev_passwd_id=%d join=" MACSTR "%s",
 			       MAC2STR(dev), status, conncap,
 			       adv_id, MAC2STR(adv_mac),
 			       ses_id, MAC2STR(ses_mac),
-			       passwd_id, MAC2STR(grp_mac));
+			       passwd_id, MAC2STR(grp_mac), feat_cap_str);
 	} else {
 		wpa_msg_global(wpa_s, MSG_INFO,
 			       P2P_EVENT_P2PS_PROVISION_DONE MACSTR
 			       " status=%d conncap=%x"
 			       " adv_id=%x adv_mac=" MACSTR
 			       " session=%x mac=" MACSTR
-			       " dev_passwd_id=%d",
+			       " dev_passwd_id=%d%s",
 			       MAC2STR(dev), status, conncap,
 			       adv_id, MAC2STR(adv_mac),
 			       ses_id, MAC2STR(ses_mac),
-			       passwd_id);
+			       passwd_id, feat_cap_str);
 	}
 }
 
@@ -5059,7 +4039,7 @@ static int wpas_prov_disc_resp_cb(void *ctx)
 		wpas_p2p_group_add_persistent(
 			wpa_s, persistent_go, 0, 0, 0, 0, 0, NULL,
 			persistent_go->mode == WPAS_MODE_P2P_GO ?
-			P2P_MAX_INITIAL_CONN_WAIT_GO_REINVOKE : 0);
+			P2P_MAX_INITIAL_CONN_WAIT_GO_REINVOKE : 0, 0);
 	} else {
 		wpas_p2p_group_add(wpa_s, 1, 0, 0, 0);
 	}
@@ -5068,6 +4048,17 @@ static int wpas_prov_disc_resp_cb(void *ctx)
 }
 
 
+static int wpas_p2p_get_pref_freq_list(void *ctx, int go,
+				       unsigned int *len,
+				       unsigned int *freq_list)
+{
+	struct wpa_supplicant *wpa_s = ctx;
+
+	return wpa_drv_get_pref_freq_list(wpa_s, go ? WPA_IF_P2P_GO :
+					  WPA_IF_P2P_CLIENT, len, freq_list);
+}
+
+
 /**
  * wpas_p2p_init - Initialize P2P module for %wpa_supplicant
  * @global: Pointer to global data from wpa_supplicant_init()
@@ -5121,6 +4112,7 @@ int wpas_p2p_init(struct wpa_global *global, struct wpa_supplicant *wpa_s)
 	p2p.p2ps_prov_complete = wpas_p2ps_prov_complete;
 	p2p.prov_disc_resp_cb = wpas_prov_disc_resp_cb;
 	p2p.p2ps_group_capability = p2ps_group_capability;
+	p2p.get_pref_freq_list = wpas_p2p_get_pref_freq_list;
 
 	os_memcpy(wpa_s->global->p2p_dev_addr, wpa_s->own_addr, ETH_ALEN);
 	os_memcpy(p2p.dev_addr, wpa_s->global->p2p_dev_addr, ETH_ALEN);
@@ -5152,9 +4144,9 @@ int wpas_p2p_init(struct wpa_global *global, struct wpa_supplicant *wpa_s)
 		 */
 		if (p2p_config_get_random_social(&p2p, &p2p.reg_class,
 						 &p2p.channel) != 0) {
-			wpa_printf(MSG_ERROR,
-				   "P2P: Failed to select random social channel as listen channel");
-			return -1;
+			wpa_printf(MSG_INFO,
+				   "P2P: No social channels supported by the driver - do not enable P2P");
+			return 0;
 		}
 		p2p.channel_forced = 0;
 	}
@@ -5425,6 +4417,7 @@ static void wpas_p2p_check_join_scan_limit(struct wpa_supplicant *wpa_s)
 		}
 		wpa_msg_global(wpa_s->parent, MSG_INFO,
 			       P2P_EVENT_GROUP_FORMATION_FAILURE);
+		wpas_notify_p2p_group_formation_failure(wpa_s, "");
 	}
 }
 
@@ -5623,10 +4616,25 @@ static void wpas_p2p_scan_res_join(struct wpa_supplicant *wpa_s,
 					       wpa_s->pending_join_iface_addr);
 	}
 	if (bss) {
+		u8 dev_addr[ETH_ALEN];
+
 		freq = bss->freq;
 		wpa_printf(MSG_DEBUG, "P2P: Target GO operating frequency "
 			   "from BSS table: %d MHz (SSID %s)", freq,
 			   wpa_ssid_txt(bss->ssid, bss->ssid_len));
+		if (p2p_parse_dev_addr((const u8 *) (bss + 1), bss->ie_len,
+				       dev_addr) == 0 &&
+		    os_memcmp(wpa_s->pending_join_dev_addr,
+			      wpa_s->pending_join_iface_addr, ETH_ALEN) == 0 &&
+		    os_memcmp(dev_addr, wpa_s->pending_join_dev_addr,
+			      ETH_ALEN) != 0) {
+			wpa_printf(MSG_DEBUG,
+				   "P2P: Update target GO device address based on BSS entry: " MACSTR " (was " MACSTR ")",
+				   MAC2STR(dev_addr),
+				   MAC2STR(wpa_s->pending_join_dev_addr));
+			os_memcpy(wpa_s->pending_join_dev_addr, dev_addr,
+				  ETH_ALEN);
+		}
 	}
 	if (freq > 0) {
 		u16 method;
@@ -5635,6 +4643,8 @@ static void wpas_p2p_scan_res_join(struct wpa_supplicant *wpa_s,
 			wpa_msg_global(wpa_s->parent, MSG_INFO,
 				       P2P_EVENT_GROUP_FORMATION_FAILURE
 				       "reason=FREQ_CONFLICT");
+			wpas_notify_p2p_group_formation_failure(
+				wpa_s, "FREQ_CONFLICT");
 			return;
 		}
 
@@ -5654,6 +4664,9 @@ static void wpas_p2p_scan_res_join(struct wpa_supplicant *wpa_s,
 		case WPS_PBC:
 			method = WPS_CONFIG_PUSHBUTTON;
 			break;
+		case WPS_P2PS:
+			method = WPS_CONFIG_P2PS;
+			break;
 		default:
 			method = 0;
 			break;
@@ -5896,11 +4909,16 @@ static int wpas_p2p_join_start(struct wpa_supplicant *wpa_s, int freq,
 
 
 static int wpas_p2p_setup_freqs(struct wpa_supplicant *wpa_s, int freq,
-				int *force_freq, int *pref_freq, int go)
+				int *force_freq, int *pref_freq, int go,
+				unsigned int *pref_freq_list,
+				unsigned int *num_pref_freq)
 {
 	struct wpa_used_freq_data *freqs;
 	int res, best_freq, num_unused;
-	unsigned int freq_in_use = 0, num, i;
+	unsigned int freq_in_use = 0, num, i, max_pref_freq;
+
+	max_pref_freq = *num_pref_freq;
+	*num_pref_freq = 0;
 
 	freqs = os_calloc(wpa_s->num_multichan_concurrent,
 			  sizeof(struct wpa_used_freq_data));
@@ -5965,6 +4983,47 @@ static int wpas_p2p_setup_freqs(struct wpa_supplicant *wpa_s, int freq,
 
 	best_freq = wpas_p2p_pick_best_used_freq(wpa_s, freqs, num);
 
+	if (!wpa_s->conf->num_p2p_pref_chan && *pref_freq == 0) {
+		enum wpa_driver_if_type iface_type;
+
+		if (go)
+			iface_type = WPA_IF_P2P_GO;
+		else
+			iface_type = WPA_IF_P2P_CLIENT;
+
+		wpa_printf(MSG_DEBUG, "P2P: best_freq=%d, go=%d",
+			   best_freq, go);
+
+		res = wpa_drv_get_pref_freq_list(wpa_s, iface_type,
+						 &max_pref_freq,
+						 pref_freq_list);
+		if (!res && max_pref_freq > 0) {
+			*num_pref_freq = max_pref_freq;
+			i = 0;
+			while (wpas_p2p_disallowed_freq(wpa_s->global,
+							pref_freq_list[i]) &&
+			       i < *num_pref_freq) {
+				wpa_printf(MSG_DEBUG,
+					   "P2P: preferred_freq_list[%d]=%d is disallowed",
+					   i, pref_freq_list[i]);
+				i++;
+			}
+			if (i != *num_pref_freq) {
+				best_freq = pref_freq_list[i];
+				wpa_printf(MSG_DEBUG,
+					   "P2P: Using preferred_freq_list[%d]=%d",
+					   i, best_freq);
+			} else {
+				wpa_printf(MSG_DEBUG,
+					   "P2P: All driver preferred frequencies are disallowed for P2P use");
+				*num_pref_freq = 0;
+			}
+		} else {
+			wpa_printf(MSG_DEBUG,
+				   "P2P: No preferred frequency list available");
+		}
+	}
+
 	/* We have a candidate frequency to use */
 	if (best_freq > 0) {
 		if (*pref_freq == 0 && num_unused > 0) {
@@ -6029,6 +5088,7 @@ int wpas_p2p_connect(struct wpa_supplicant *wpa_s, const u8 *peer_addr,
 	enum wpa_driver_if_type iftype;
 	const u8 *if_addr;
 	struct wpa_ssid *ssid = NULL;
+	unsigned int pref_freq_list[P2P_MAX_PREF_CHANNELS], size;
 
 	if (wpa_s->global->p2p_disabled || wpa_s->global->p2p == NULL)
 		return -1;
@@ -6045,6 +5105,7 @@ int wpas_p2p_connect(struct wpa_supplicant *wpa_s, const u8 *peer_addr,
 
 	wpa_s->global->p2p_fail_on_wps_complete = 0;
 	wpa_s->global->pending_p2ps_group = 0;
+	wpa_s->p2ps_method_config_any = 0;
 
 	if (go_intent < 0)
 		go_intent = wpa_s->conf->p2p_go_intent;
@@ -6105,13 +5166,16 @@ int wpas_p2p_connect(struct wpa_supplicant *wpa_s, const u8 *peer_addr,
 		return ret;
 	}
 
+	size = P2P_MAX_PREF_CHANNELS;
 	res = wpas_p2p_setup_freqs(wpa_s, freq, &force_freq, &pref_freq,
-				   go_intent == 15);
+				   go_intent == 15, pref_freq_list, &size);
 	if (res)
 		return res;
 	wpas_p2p_set_own_freq_preference(wpa_s,
 					 force_freq ? force_freq : pref_freq);
 
+	p2p_set_own_pref_freq_list(wpa_s->global->p2p, pref_freq_list, size);
+
 	wpa_s->create_p2p_iface = wpas_p2p_create_iface(wpa_s);
 
 	if (wpa_s->create_p2p_iface) {
@@ -6126,8 +5190,10 @@ int wpas_p2p_connect(struct wpa_supplicant *wpa_s, const u8 *peer_addr,
 		}
 
 		if_addr = wpa_s->pending_interface_addr;
-	} else
+	} else {
 		if_addr = wpa_s->own_addr;
+		os_memset(wpa_s->go_dev_addr, 0, ETH_ALEN);
+	}
 
 	if (auth) {
 		if (wpas_p2p_auth_go_neg(wpa_s, peer_addr, wps_method,
@@ -6272,6 +5338,38 @@ static int wpas_p2p_select_go_freq(struct wpa_supplicant *wpa_s, int freq)
 {
 	unsigned int r;
 
+	if (!wpa_s->conf->num_p2p_pref_chan && !freq) {
+		unsigned int i, size = P2P_MAX_PREF_CHANNELS;
+		unsigned int pref_freq_list[P2P_MAX_PREF_CHANNELS];
+		int res;
+
+		res = wpa_drv_get_pref_freq_list(wpa_s, WPA_IF_P2P_GO,
+						 &size, pref_freq_list);
+		if (!res && size > 0) {
+			i = 0;
+			while (wpas_p2p_disallowed_freq(wpa_s->global,
+							pref_freq_list[i]) &&
+			       i < size) {
+				wpa_printf(MSG_DEBUG,
+					   "P2P: preferred_freq_list[%d]=%d is disallowed",
+					   i, pref_freq_list[i]);
+				i++;
+			}
+			if (i != size) {
+				freq = pref_freq_list[i];
+				wpa_printf(MSG_DEBUG,
+					   "P2P: Using preferred_freq_list[%d]=%d",
+					   i, freq);
+			} else {
+				wpa_printf(MSG_DEBUG,
+					   "P2P: All driver preferred frequencies are disallowed for P2P use");
+			}
+		} else {
+			wpa_printf(MSG_DEBUG,
+				   "P2P: No preferred frequency list available");
+		}
+	}
+
 	if (freq == 2) {
 		wpa_printf(MSG_DEBUG, "P2P: Request to start GO on 2.4 GHz "
 			   "band");
@@ -6335,30 +5433,45 @@ static int wpas_p2p_select_go_freq(struct wpa_supplicant *wpa_s, int freq)
 }
 
 
-static int wpas_p2p_select_freq_no_pref(struct wpa_supplicant *wpa_s,
-					struct p2p_go_neg_results *params,
-					const struct p2p_channels *channels)
+static int wpas_p2p_supported_freq_go(struct wpa_supplicant *wpa_s,
+				      const struct p2p_channels *channels,
+				      int freq)
+{
+	if (!wpas_p2p_disallowed_freq(wpa_s->global, freq) &&
+	    p2p_supported_freq_go(wpa_s->global->p2p, freq) &&
+	    freq_included(wpa_s, channels, freq))
+		return 1;
+	return 0;
+}
+
+
+static void wpas_p2p_select_go_freq_no_pref(struct wpa_supplicant *wpa_s,
+					    struct p2p_go_neg_results *params,
+					    const struct p2p_channels *channels)
 {
 	unsigned int i, r;
 
 	/* first try some random selection of the social channels */
 	if (os_get_random((u8 *) &r, sizeof(r)) < 0)
-		return -1;
+		return;
 
 	for (i = 0; i < 3; i++) {
 		params->freq = 2412 + ((r + i) % 3) * 25;
-		if (!wpas_p2p_disallowed_freq(wpa_s->global, params->freq) &&
-		    freq_included(channels, params->freq) &&
-		    p2p_supported_freq(wpa_s->global->p2p, params->freq))
+		if (wpas_p2p_supported_freq_go(wpa_s, channels, params->freq))
 			goto out;
 	}
 
-	/* try all channels in reg. class 81 */
+	/* try all other channels in operating class 81 */
 	for (i = 0; i < 11; i++) {
 		params->freq = 2412 + i * 5;
-		if (!wpas_p2p_disallowed_freq(wpa_s->global, params->freq) &&
-		    freq_included(channels, params->freq) &&
-		    p2p_supported_freq(wpa_s->global->p2p, params->freq))
+
+		/* skip social channels; covered in the previous loop */
+		if (params->freq == 2412 ||
+		    params->freq == 2437 ||
+		    params->freq == 2462)
+			continue;
+
+		if (wpas_p2p_supported_freq_go(wpa_s, channels, params->freq))
 			goto out;
 	}
 
@@ -6366,7 +5479,7 @@ static int wpas_p2p_select_freq_no_pref(struct wpa_supplicant *wpa_s,
 	for (i = 0; i < 4; i++) {
 		params->freq = 5180 + i * 20;
 		if (!wpas_p2p_disallowed_freq(wpa_s->global, params->freq) &&
-		    freq_included(channels, params->freq) &&
+		    freq_included(wpa_s, channels, params->freq) &&
 		    p2p_supported_freq(wpa_s->global->p2p, params->freq))
 			goto out;
 	}
@@ -6375,7 +5488,7 @@ static int wpas_p2p_select_freq_no_pref(struct wpa_supplicant *wpa_s,
 	for (i = 0; i < 4; i++) {
 		params->freq = 5745 + i * 20;
 		if (!wpas_p2p_disallowed_freq(wpa_s->global, params->freq) &&
-		    freq_included(channels, params->freq) &&
+		    freq_included(wpa_s, channels, params->freq) &&
 		    p2p_supported_freq(wpa_s->global->p2p, params->freq))
 			goto out;
 	}
@@ -6383,7 +5496,7 @@ static int wpas_p2p_select_freq_no_pref(struct wpa_supplicant *wpa_s,
 	/* try social channel class 180 channel 2 */
 	params->freq = 58320 + 1 * 2160;
 	if (!wpas_p2p_disallowed_freq(wpa_s->global, params->freq) &&
-	    freq_included(channels, params->freq) &&
+	    freq_included(wpa_s, channels, params->freq) &&
 	    p2p_supported_freq(wpa_s->global->p2p, params->freq))
 		goto out;
 
@@ -6391,17 +5504,17 @@ static int wpas_p2p_select_freq_no_pref(struct wpa_supplicant *wpa_s,
 	for (i = 0; i < 4; i++) {
 		params->freq = 58320 + i * 2160;
 		if (!wpas_p2p_disallowed_freq(wpa_s->global, params->freq) &&
-		    freq_included(channels, params->freq) &&
+		    freq_included(wpa_s, channels, params->freq) &&
 		    p2p_supported_freq(wpa_s->global->p2p, params->freq))
 			goto out;
 	}
 
+	params->freq = 0;
 	wpa_printf(MSG_DEBUG, "P2P: No 2.4, 5, or 60 GHz channel allowed");
-	return -1;
+	return;
 out:
 	wpa_printf(MSG_DEBUG, "P2P: Set GO freq %d MHz (no preference known)",
 		   params->freq);
-	return 0;
 }
 
 
@@ -6411,75 +5524,17 @@ static int wpas_p2p_init_go_params(struct wpa_supplicant *wpa_s,
 				   const struct p2p_channels *channels)
 {
 	struct wpa_used_freq_data *freqs;
-	unsigned int pref_freq, cand_freq;
+	unsigned int cand;
 	unsigned int num, i;
 
 	os_memset(params, 0, sizeof(*params));
 	params->role_go = 1;
 	params->ht40 = ht40;
 	params->vht = vht;
-	if (freq) {
-		if (!freq_included(channels, freq)) {
-			wpa_printf(MSG_DEBUG, "P2P: Forced GO freq %d MHz not "
-				   "accepted", freq);
-			return -1;
-		}
-		wpa_printf(MSG_DEBUG, "P2P: Set GO freq based on forced "
-			   "frequency %d MHz", freq);
-		params->freq = freq;
-	} else if (wpa_s->conf->p2p_oper_reg_class == 81 &&
-		   wpa_s->conf->p2p_oper_channel >= 1 &&
-		   wpa_s->conf->p2p_oper_channel <= 11 &&
-		   freq_included(channels,
-				 2407 + 5 * wpa_s->conf->p2p_oper_channel)) {
-		params->freq = 2407 + 5 * wpa_s->conf->p2p_oper_channel;
-		wpa_printf(MSG_DEBUG, "P2P: Set GO freq based on configured "
-			   "frequency %d MHz", params->freq);
-	} else if ((wpa_s->conf->p2p_oper_reg_class == 115 ||
-		    wpa_s->conf->p2p_oper_reg_class == 116 ||
-		    wpa_s->conf->p2p_oper_reg_class == 117 ||
-		    wpa_s->conf->p2p_oper_reg_class == 124 ||
-		    wpa_s->conf->p2p_oper_reg_class == 126 ||
-		    wpa_s->conf->p2p_oper_reg_class == 127) &&
-		   freq_included(channels,
-				 5000 + 5 * wpa_s->conf->p2p_oper_channel)) {
-		params->freq = 5000 + 5 * wpa_s->conf->p2p_oper_channel;
-		wpa_printf(MSG_DEBUG, "P2P: Set GO freq based on configured "
-			   "frequency %d MHz", params->freq);
-	} else if (wpa_s->conf->p2p_oper_channel == 0 &&
-		   wpa_s->best_overall_freq > 0 &&
-		   p2p_supported_freq_go(wpa_s->global->p2p,
-					 wpa_s->best_overall_freq) &&
-		   freq_included(channels, wpa_s->best_overall_freq)) {
-		params->freq = wpa_s->best_overall_freq;
-		wpa_printf(MSG_DEBUG, "P2P: Set GO freq based on best overall "
-			   "channel %d MHz", params->freq);
-	} else if (wpa_s->conf->p2p_oper_channel == 0 &&
-		   wpa_s->best_24_freq > 0 &&
-		   p2p_supported_freq_go(wpa_s->global->p2p,
-					 wpa_s->best_24_freq) &&
-		   freq_included(channels, wpa_s->best_24_freq)) {
-		params->freq = wpa_s->best_24_freq;
-		wpa_printf(MSG_DEBUG, "P2P: Set GO freq based on best 2.4 GHz "
-			   "channel %d MHz", params->freq);
-	} else if (wpa_s->conf->p2p_oper_channel == 0 &&
-		   wpa_s->best_5_freq > 0 &&
-		   p2p_supported_freq_go(wpa_s->global->p2p,
-					 wpa_s->best_5_freq) &&
-		   freq_included(channels, wpa_s->best_5_freq)) {
-		params->freq = wpa_s->best_5_freq;
-		wpa_printf(MSG_DEBUG, "P2P: Set GO freq based on best 5 GHz "
-			   "channel %d MHz", params->freq);
-	} else if ((pref_freq = p2p_get_pref_freq(wpa_s->global->p2p,
-						  channels))) {
-		params->freq = pref_freq;
-		wpa_printf(MSG_DEBUG, "P2P: Set GO freq %d MHz from preferred "
-			   "channels", params->freq);
-	} else {
-		/* no preference, select some channel */
-		if (wpas_p2p_select_freq_no_pref(wpa_s, params, channels) < 0)
-			return -1;
-	}
+
+	if (wpa_s->p2p_group_common_freqs_num)
+		wpa_printf(MSG_DEBUG, "P2P: %s called for an active GO",
+			   __func__);
 
 	freqs = os_calloc(wpa_s->num_multichan_concurrent,
 			  sizeof(struct wpa_used_freq_data));
@@ -6489,51 +5544,166 @@ static int wpas_p2p_init_go_params(struct wpa_supplicant *wpa_s,
 	num = wpas_p2p_valid_oper_freqs(wpa_s, freqs,
 					wpa_s->num_multichan_concurrent);
 
-	cand_freq = wpas_p2p_pick_best_used_freq(wpa_s, freqs, num);
+	/* try using the forced freq */
+	if (freq) {
+		if (!wpas_p2p_supported_freq_go(wpa_s, channels, freq)) {
+			wpa_printf(MSG_DEBUG,
+				   "P2P: Forced GO freq %d MHz not accepted",
+				   freq);
+			goto fail;
+		}
 
-	/* First try the best used frequency if possible */
-	if (!freq && cand_freq > 0 && freq_included(channels, cand_freq)) {
-		params->freq = cand_freq;
-	} else if (!freq) {
-		/* Try any of the used frequencies */
 		for (i = 0; i < num; i++) {
-			if (freq_included(channels, freqs[i].freq)) {
-				wpa_printf(MSG_DEBUG, "P2P: Force GO on a channel we are already using (%u MHz)",
-					   freqs[i].freq);
+			if (freqs[i].freq == freq) {
+				wpa_printf(MSG_DEBUG,
+					   "P2P: forced freq (%d MHz) is also shared",
+					   freq);
+				params->freq = freq;
+				goto success;
+			}
+		}
+
+		if (wpas_p2p_num_unused_channels(wpa_s) <= 0) {
+			wpa_printf(MSG_DEBUG,
+				   "P2P: Cannot force GO on freq (%d MHz) as all the channels are in use",
+				   freq);
+			goto fail;
+		}
+
+		wpa_printf(MSG_DEBUG,
+			   "P2P: force GO freq (%d MHz) on a free channel",
+			   freq);
+		params->freq = freq;
+		goto success;
+	}
+
+	/* consider using one of the shared frequencies */
+	if (num) {
+		cand = wpas_p2p_pick_best_used_freq(wpa_s, freqs, num);
+		if (wpas_p2p_supported_freq_go(wpa_s, channels, cand)) {
+			wpa_printf(MSG_DEBUG,
+				   "P2P: Use shared freq (%d MHz) for GO",
+				   freq);
+			params->freq = cand;
+			goto success;
+		}
+
+		/* try using one of the shared freqs */
+		for (i = 0; i < num; i++) {
+			if (wpas_p2p_supported_freq_go(wpa_s, channels,
+						       freqs[i].freq)) {
+				wpa_printf(MSG_DEBUG,
+					   "P2P: Use shared freq (%d MHz) for GO",
+					   freq);
 				params->freq = freqs[i].freq;
-				break;
-			}
-		}
-
-		if (i == num) {
-			if (wpas_p2p_num_unused_channels(wpa_s) <= 0) {
-				wpa_printf(MSG_DEBUG, "P2P: Cannot force GO on any of the channels we are already using");
-				os_free(freqs);
-				return -1;
-			} else {
-				wpa_printf(MSG_DEBUG, "P2P: Cannot force GO on any of the channels we are already using. Use one of the free channels");
-			}
-		}
-	} else {
-		for (i = 0; i < num; i++) {
-			if (freqs[i].freq == freq)
-				break;
-		}
-
-		if (i == num) {
-			if (wpas_p2p_num_unused_channels(wpa_s) <= 0) {
-				if (freq)
-					wpa_printf(MSG_DEBUG, "P2P: Cannot force GO on freq (%u MHz) as all the channels are in use", freq);
-				os_free(freqs);
-				return -1;
-			} else {
-				wpa_printf(MSG_DEBUG, "P2P: Use one of the free channels");
+				goto success;
 			}
 		}
 	}
 
+	if (wpas_p2p_num_unused_channels(wpa_s) <= 0) {
+		wpa_printf(MSG_DEBUG,
+			   "P2P: Cannot force GO on any of the channels we are already using");
+		goto fail;
+	}
+
+	/* try using the setting from the configuration file */
+	if (wpa_s->conf->p2p_oper_reg_class == 81 &&
+	    wpa_s->conf->p2p_oper_channel >= 1 &&
+	    wpa_s->conf->p2p_oper_channel <= 11 &&
+	    wpas_p2p_supported_freq_go(
+		    wpa_s, channels,
+		    2407 + 5 * wpa_s->conf->p2p_oper_channel)) {
+		params->freq = 2407 + 5 * wpa_s->conf->p2p_oper_channel;
+		wpa_printf(MSG_DEBUG, "P2P: Set GO freq based on configured "
+			   "frequency %d MHz", params->freq);
+		goto success;
+	}
+
+	if ((wpa_s->conf->p2p_oper_reg_class == 115 ||
+	     wpa_s->conf->p2p_oper_reg_class == 116 ||
+	     wpa_s->conf->p2p_oper_reg_class == 117 ||
+	     wpa_s->conf->p2p_oper_reg_class == 124 ||
+	     wpa_s->conf->p2p_oper_reg_class == 125 ||
+	     wpa_s->conf->p2p_oper_reg_class == 126 ||
+	     wpa_s->conf->p2p_oper_reg_class == 127) &&
+	    wpas_p2p_supported_freq_go(wpa_s, channels,
+				       5000 +
+				       5 * wpa_s->conf->p2p_oper_channel)) {
+		params->freq = 5000 + 5 * wpa_s->conf->p2p_oper_channel;
+		wpa_printf(MSG_DEBUG, "P2P: Set GO freq based on configured "
+			   "frequency %d MHz", params->freq);
+		goto success;
+	}
+
+	/* Try using best channels */
+	if (wpa_s->conf->p2p_oper_channel == 0 &&
+	    wpa_s->best_overall_freq > 0 &&
+	    wpas_p2p_supported_freq_go(wpa_s, channels,
+				       wpa_s->best_overall_freq)) {
+		params->freq = wpa_s->best_overall_freq;
+		wpa_printf(MSG_DEBUG, "P2P: Set GO freq based on best overall "
+			   "channel %d MHz", params->freq);
+		goto success;
+	}
+
+	if (wpa_s->conf->p2p_oper_channel == 0 &&
+	    wpa_s->best_24_freq > 0 &&
+	    wpas_p2p_supported_freq_go(wpa_s, channels,
+				       wpa_s->best_24_freq)) {
+		params->freq = wpa_s->best_24_freq;
+		wpa_printf(MSG_DEBUG, "P2P: Set GO freq based on best 2.4 GHz "
+			   "channel %d MHz", params->freq);
+		goto success;
+	}
+
+	if (wpa_s->conf->p2p_oper_channel == 0 &&
+	    wpa_s->best_5_freq > 0 &&
+	    wpas_p2p_supported_freq_go(wpa_s, channels,
+				       wpa_s->best_5_freq)) {
+		params->freq = wpa_s->best_5_freq;
+		wpa_printf(MSG_DEBUG, "P2P: Set GO freq based on best 5 GHz "
+			   "channel %d MHz", params->freq);
+		goto success;
+	}
+
+	/* try using preferred channels */
+	cand = p2p_get_pref_freq(wpa_s->global->p2p, channels);
+	if (cand && wpas_p2p_supported_freq_go(wpa_s, channels, cand)) {
+		params->freq = cand;
+		wpa_printf(MSG_DEBUG, "P2P: Set GO freq %d MHz from preferred "
+			   "channels", params->freq);
+		goto success;
+	}
+
+	/* Try using one of the group common freqs */
+	if (wpa_s->p2p_group_common_freqs) {
+		for (i = 0; i < wpa_s->p2p_group_common_freqs_num; i++) {
+			cand = wpa_s->p2p_group_common_freqs[i];
+			if (wpas_p2p_supported_freq_go(wpa_s, channels, cand)) {
+				params->freq = cand;
+				wpa_printf(MSG_DEBUG,
+					   "P2P: Use freq %d MHz common with the peer",
+					   params->freq);
+				goto success;
+			}
+		}
+	}
+
+	/* no preference, select some channel */
+	wpas_p2p_select_go_freq_no_pref(wpa_s, params, channels);
+
+	if (params->freq == 0) {
+		wpa_printf(MSG_DEBUG, "P2P: did not find a freq for GO use");
+		goto fail;
+	}
+
+success:
 	os_free(freqs);
 	return 0;
+fail:
+	os_free(freqs);
+	return -1;
 }
 
 
@@ -6636,13 +5806,15 @@ int wpas_p2p_group_add(struct wpa_supplicant *wpa_s, int persistent_group,
 
 static int wpas_start_p2p_client(struct wpa_supplicant *wpa_s,
 				 struct wpa_ssid *params, int addr_allocated,
-				 int freq)
+				 int freq, int force_scan)
 {
 	struct wpa_ssid *ssid;
 
 	wpa_s = wpas_p2p_get_group_iface(wpa_s, addr_allocated, 0);
 	if (wpa_s == NULL)
 		return -1;
+	if (force_scan)
+		os_get_reltime(&wpa_s->scan_min_time);
 	wpa_s->p2p_last_4way_hs_fail = NULL;
 
 	wpa_supplicant_ap_deinit(wpa_s);
@@ -6650,6 +5822,7 @@ static int wpas_start_p2p_client(struct wpa_supplicant *wpa_s,
 	ssid = wpa_config_add_network(wpa_s->conf);
 	if (ssid == NULL)
 		return -1;
+	os_memset(wpa_s->go_dev_addr, 0, ETH_ALEN);
 	wpa_config_set_network_defaults(ssid);
 	ssid->temporary = 1;
 	ssid->proto = WPA_PROTO_RSN;
@@ -6691,7 +5864,7 @@ int wpas_p2p_group_add_persistent(struct wpa_supplicant *wpa_s,
 				  struct wpa_ssid *ssid, int addr_allocated,
 				  int force_freq, int neg_freq, int ht40,
 				  int vht, const struct p2p_channels *channels,
-				  int connection_timeout)
+				  int connection_timeout, int force_scan)
 {
 	struct p2p_go_neg_results params;
 	int go = 0, freq;
@@ -6703,6 +5876,23 @@ int wpas_p2p_group_add_persistent(struct wpa_supplicant *wpa_s,
 	    go == (ssid->mode == WPAS_MODE_P2P_GO)) {
 		wpa_printf(MSG_DEBUG, "P2P: Requested persistent group is "
 			   "already running");
+		if (go == 0 &&
+		    eloop_cancel_timeout(wpas_p2p_group_formation_timeout,
+					 wpa_s->parent, NULL)) {
+			/*
+			 * This can happen if Invitation Response frame was lost
+			 * and the peer (GO of a persistent group) tries to
+			 * invite us again. Reschedule the timeout to avoid
+			 * terminating the wait for the connection too early
+			 * since we now know that the peer is still trying to
+			 * invite us instead of having already started the GO.
+			 */
+			wpa_printf(MSG_DEBUG,
+				   "P2P: Reschedule group formation timeout since peer is still trying to invite us");
+			eloop_register_timeout(P2P_MAX_INITIAL_CONN_WAIT, 0,
+					       wpas_p2p_group_formation_timeout,
+					       wpa_s->parent, NULL);
+		}
 		return 0;
 	}
 
@@ -6722,21 +5912,30 @@ int wpas_p2p_group_add_persistent(struct wpa_supplicant *wpa_s,
 		} else {
 			freq = wpas_p2p_select_go_freq(wpa_s, neg_freq);
 			if (freq < 0 ||
-			    (freq > 0 && !freq_included(channels, freq)))
+			    (freq > 0 && !freq_included(wpa_s, channels, freq)))
 				freq = 0;
 		}
-	} else {
+	} else if (ssid->mode == WPAS_MODE_INFRA) {
 		freq = neg_freq;
-		if (freq < 0 ||
-		    (freq > 0 && !freq_included(channels, freq)))
-			freq = 0;
-	}
+		if (freq <= 0 || !freq_included(wpa_s, channels, freq)) {
+			struct os_reltime now;
+			struct wpa_bss *bss =
+				wpa_bss_get_p2p_dev_addr(wpa_s, ssid->bssid);
 
-	if (ssid->mode == WPAS_MODE_INFRA)
-		return wpas_start_p2p_client(wpa_s, ssid, addr_allocated, freq);
+			os_get_reltime(&now);
+			if (bss &&
+			    !os_reltime_expired(&now, &bss->last_update, 5) &&
+			    freq_included(wpa_s, channels, bss->freq))
+				freq = bss->freq;
+			else
+				freq = 0;
+		}
 
-	if (ssid->mode != WPAS_MODE_P2P_GO)
+		return wpas_start_p2p_client(wpa_s, ssid, addr_allocated, freq,
+					     force_scan);
+	} else {
 		return -1;
+	}
 
 	if (wpas_p2p_init_go_params(wpa_s, &params, freq, ht40, vht, channels))
 		return -1;
@@ -6909,7 +6108,7 @@ void wpas_p2p_wps_success(struct wpa_supplicant *wpa_s, const u8 *peer_addr,
 	}
 	if (wpa_s->global->p2p)
 		p2p_wps_success_cb(wpa_s->global->p2p, peer_addr);
-	wpas_group_formation_completed(wpa_s, 1);
+	wpas_group_formation_completed(wpa_s, 1, 0);
 }
 
 
@@ -7163,7 +6362,11 @@ int wpas_p2p_assoc_req_ie(struct wpa_supplicant *wpa_s, struct wpa_bss *bss,
 
 	if (wpa_s->global->p2p_disabled)
 		return -1;
-	if (wpa_s->conf->p2p_disabled)
+	/*
+	 * Advertize mandatory cross connection capability even on
+	 * p2p_disabled=1 interface when associating with a P2P Manager WLAN AP.
+	 */
+	if (wpa_s->conf->p2p_disabled && p2p_group)
 		return -1;
 	if (wpa_s->global->p2p == NULL)
 		return -1;
@@ -7181,7 +6384,8 @@ int wpas_p2p_assoc_req_ie(struct wpa_supplicant *wpa_s, struct wpa_bss *bss,
 
 int wpas_p2p_probe_req_rx(struct wpa_supplicant *wpa_s, const u8 *addr,
 			  const u8 *dst, const u8 *bssid,
-			  const u8 *ie, size_t ie_len, int ssi_signal)
+			  const u8 *ie, size_t ie_len,
+			  unsigned int rx_freq, int ssi_signal)
 {
 	if (wpa_s->global->p2p_disabled)
 		return 0;
@@ -7189,7 +6393,7 @@ int wpas_p2p_probe_req_rx(struct wpa_supplicant *wpa_s, const u8 *addr,
 		return 0;
 
 	switch (p2p_probe_req_rx(wpa_s->global->p2p, addr, dst, bssid,
-				 ie, ie_len)) {
+				 ie, ie_len, rx_freq)) {
 	case P2P_PREQ_NOT_P2P:
 		wpas_notify_preq(wpa_s, addr, dst, bssid, ie, ie_len,
 				 ssi_signal);
@@ -7263,6 +6467,7 @@ int wpas_p2p_invite(struct wpa_supplicant *wpa_s, const u8 *peer_addr,
 	int force_freq = 0;
 	int res;
 	int no_pref_freq_given = pref_freq == 0;
+	unsigned int pref_freq_list[P2P_MAX_PREF_CHANNELS], size;
 
 	wpa_s->global->p2p_invite_group = NULL;
 	if (peer_addr)
@@ -7296,10 +6501,13 @@ int wpas_p2p_invite(struct wpa_supplicant *wpa_s, const u8 *peer_addr,
 	}
 	wpa_s->pending_invite_ssid_id = ssid->id;
 
+	size = P2P_MAX_PREF_CHANNELS;
 	res = wpas_p2p_setup_freqs(wpa_s, freq, &force_freq, &pref_freq,
-				   role == P2P_INVITE_ROLE_GO);
+				   role == P2P_INVITE_ROLE_GO,
+				   pref_freq_list, &size);
 	if (res)
 		return res;
+	p2p_set_own_pref_freq_list(wpa_s->global->p2p, pref_freq_list, size);
 
 	if (wpa_s->global->p2p_disabled || wpa_s->global->p2p == NULL)
 		return -1;
@@ -7336,6 +6544,7 @@ int wpas_p2p_invite_group(struct wpa_supplicant *wpa_s, const char *ifname,
 	int persistent;
 	int freq = 0, force_freq = 0, pref_freq = 0;
 	int res;
+	unsigned int pref_freq_list[P2P_MAX_PREF_CHANNELS], size;
 
 	wpa_s->p2p_persistent_go_freq = 0;
 	wpa_s->p2p_go_ht40 = 0;
@@ -7387,8 +6596,10 @@ int wpas_p2p_invite_group(struct wpa_supplicant *wpa_s, const char *ifname,
 	if (wpa_s->global->p2p_disabled || wpa_s->global->p2p == NULL)
 		return -1;
 
+	size = P2P_MAX_PREF_CHANNELS;
 	res = wpas_p2p_setup_freqs(wpa_s, freq, &force_freq, &pref_freq,
-				   role == P2P_INVITE_ROLE_ACTIVE_GO);
+				   role == P2P_INVITE_ROLE_ACTIVE_GO,
+				   pref_freq_list, &size);
 	if (res)
 		return res;
 	wpas_p2p_set_own_freq_preference(wpa_s, force_freq);
@@ -7921,7 +7132,7 @@ int wpas_p2p_notif_pbc_overlap(struct wpa_supplicant *wpa_s)
 		   "session overlap");
 	if (wpa_s != wpa_s->parent)
 		wpa_msg_ctrl(wpa_s->parent, MSG_INFO, WPS_EVENT_OVERLAP);
-	wpas_p2p_group_formation_failed(wpa_s);
+	wpas_p2p_group_formation_failed(wpa_s, 0);
 	return 1;
 }
 
@@ -7933,14 +7144,22 @@ void wpas_p2p_pbc_overlap_cb(void *eloop_ctx, void *timeout_ctx)
 }
 
 
-void wpas_p2p_update_channel_list(struct wpa_supplicant *wpa_s)
+void wpas_p2p_update_channel_list(struct wpa_supplicant *wpa_s,
+				  enum wpas_p2p_channel_update_trig trig)
 {
 	struct p2p_channels chan, cli_chan;
-	struct wpa_supplicant *ifs;
+	struct wpa_used_freq_data *freqs = NULL;
+	unsigned int num = wpa_s->num_multichan_concurrent;
 
 	if (wpa_s->global == NULL || wpa_s->global->p2p == NULL)
 		return;
 
+	freqs = os_calloc(num, sizeof(struct wpa_used_freq_data));
+	if (!freqs)
+		return;
+
+	num = get_shared_radio_freqs_data(wpa_s, freqs, num);
+
 	os_memset(&chan, 0, sizeof(chan));
 	os_memset(&cli_chan, 0, sizeof(cli_chan));
 	if (wpas_p2p_setup_channels(wpa_s, &chan, &cli_chan)) {
@@ -7951,27 +7170,17 @@ void wpas_p2p_update_channel_list(struct wpa_supplicant *wpa_s)
 
 	p2p_update_channel_list(wpa_s->global->p2p, &chan, &cli_chan);
 
-	for (ifs = wpa_s->global->ifaces; ifs; ifs = ifs->next) {
-		int freq;
-		if (!ifs->current_ssid ||
-		    !ifs->current_ssid->p2p_group ||
-		    (ifs->current_ssid->mode != WPAS_MODE_P2P_GO &&
-		     ifs->current_ssid->mode != WPAS_MODE_P2P_GROUP_FORMATION))
-				continue;
-		freq = ifs->current_ssid->frequency;
-		if (freq_included(&chan, freq)) {
-			wpa_dbg(ifs, MSG_DEBUG,
-				"P2P GO operating frequency %d MHz in valid range",
-				freq);
-			continue;
-		}
+	wpas_p2p_optimize_listen_channel(wpa_s, freqs, num);
 
-		wpa_dbg(ifs, MSG_DEBUG,
-			"P2P GO operating in invalid frequency %d MHz",	freq);
-		/* TODO: Consider using CSA or removing the group within
-		 * wpa_supplicant */
-		wpa_msg(ifs, MSG_INFO, P2P_EVENT_REMOVE_AND_REFORM_GROUP);
-	}
+	/*
+	 * The used frequencies map changed, so it is possible that a GO is
+	 * using a channel that is no longer valid for P2P use. It is also
+	 * possible that due to policy consideration, it would be preferable to
+	 * move it to a frequency already used by other station interfaces.
+	 */
+	wpas_p2p_consider_moving_gos(wpa_s, freqs, num, trig);
+
+	os_free(freqs);
 }
 
 
@@ -8031,7 +7240,7 @@ int wpas_p2p_cancel(struct wpa_supplicant *wpa_s)
 			eloop_cancel_timeout(wpas_p2p_group_formation_timeout,
 					     wpa_s->parent, NULL);
 			if (wpa_s->p2p_in_provisioning) {
-				wpas_group_formation_completed(wpa_s, 0);
+				wpas_group_formation_completed(wpa_s, 0, 0);
 				break;
 			}
 			wpas_p2p_group_delete(wpa_s,
@@ -8041,7 +7250,7 @@ int wpas_p2p_cancel(struct wpa_supplicant *wpa_s)
 			wpa_printf(MSG_DEBUG, "P2P: Interface %s in invitation found - cancelling",
 				   wpa_s->ifname);
 			found = 1;
-			wpas_p2p_group_formation_failed(wpa_s);
+			wpas_p2p_group_formation_failed(wpa_s, 0);
 		}
 	}
 
@@ -8237,7 +7446,7 @@ void wpas_p2p_notify_ap_sta_authorized(struct wpa_supplicant *wpa_s,
 			 */
 			if (wpa_s->global->p2p)
 				p2p_wps_success_cb(wpa_s->global->p2p, addr);
-			wpas_group_formation_completed(wpa_s, 1);
+			wpas_group_formation_completed(wpa_s, 1, 0);
 		}
 	}
 	if (!wpa_s->p2p_go_group_formation_completed) {
@@ -8262,7 +7471,7 @@ static int wpas_p2p_fallback_to_go_neg(struct wpa_supplicant *wpa_s,
 
 	if (wpa_s->global->p2p_group_formation)
 		group = wpa_s->global->p2p_group_formation;
-	wpa_s = wpa_s->parent;
+	wpa_s = wpa_s->global->p2p_init_wpa_s;
 	offchannel_send_action_done(wpa_s);
 	if (group_added)
 		ret = wpas_p2p_group_delete(group, P2P_GROUP_REMOVAL_SILENT);
@@ -8517,16 +7726,17 @@ void wpas_p2p_remove_client(struct wpa_supplicant *wpa_s, const u8 *peer,
 {
 	struct wpa_ssid *s;
 	struct wpa_supplicant *w;
+	struct wpa_supplicant *p2p_wpa_s = wpa_s->global->p2p_init_wpa_s;
 
 	wpa_dbg(wpa_s, MSG_DEBUG, "P2P: Remove client " MACSTR, MAC2STR(peer));
 
 	/* Remove from any persistent group */
-	for (s = wpa_s->parent->conf->ssid; s; s = s->next) {
+	for (s = p2p_wpa_s->conf->ssid; s; s = s->next) {
 		if (s->disabled != 2 || s->mode != WPAS_MODE_P2P_GO)
 			continue;
 		if (!iface_addr)
-			wpas_remove_persistent_peer(wpa_s, s, peer, 0);
-		wpas_p2p_remove_psk(wpa_s->parent, s, peer, iface_addr);
+			wpas_remove_persistent_peer(p2p_wpa_s, s, peer, 0);
+		wpas_p2p_remove_psk(p2p_wpa_s, s, peer, iface_addr);
 	}
 
 	/* Remove from any operating group */
@@ -9230,6 +8440,16 @@ static void wpas_p2p_optimize_listen_channel(struct wpa_supplicant *wpa_s,
 	u8 curr_chan, cand, chan;
 	unsigned int i;
 
+	/*
+	 * If possible, optimize the Listen channel to be a channel that is
+	 * already used by one of the other interfaces.
+	 */
+	if (!wpa_s->conf->p2p_optimize_listen_chan)
+		return;
+
+	if (!wpa_s->current_ssid || wpa_s->wpa_state != WPA_COMPLETED)
+		return;
+
 	curr_chan = p2p_get_listen_channel(wpa_s->global->p2p);
 	for (i = 0, cand = 0; i < num; i++) {
 		ieee80211_freq_to_chan(freqs[i].freq, &chan);
@@ -9251,23 +8471,86 @@ static void wpas_p2p_optimize_listen_channel(struct wpa_supplicant *wpa_s,
 }
 
 
-void wpas_p2p_indicate_state_change(struct wpa_supplicant *wpa_s)
+static int wpas_p2p_move_go_csa(struct wpa_supplicant *wpa_s)
 {
-	struct wpa_used_freq_data *freqs;
-	unsigned int num = wpa_s->num_multichan_concurrent;
+	if (!(wpa_s->drv_flags & WPA_DRIVER_FLAGS_AP_CSA)) {
+		wpa_dbg(wpa_s, MSG_DEBUG, "CSA is not enabled");
+		return -1;
+	}
 
-	if (wpa_s->global->p2p_disabled || wpa_s->global->p2p == NULL)
+	/* TODO: Add CSA support */
+	wpa_dbg(wpa_s, MSG_DEBUG, "Moving GO with CSA is not implemented");
+	return -1;
+}
+
+
+static void wpas_p2p_move_go_no_csa(struct wpa_supplicant *wpa_s)
+{
+	struct p2p_go_neg_results params;
+	struct wpa_ssid *current_ssid = wpa_s->current_ssid;
+
+	wpa_msg_global(wpa_s, MSG_INFO, P2P_EVENT_REMOVE_AND_REFORM_GROUP);
+
+	wpa_dbg(wpa_s, MSG_DEBUG, "P2P: Move GO from freq=%d MHz",
+		current_ssid->frequency);
+
+	/* Stop the AP functionality */
+	/* TODO: Should do this in a way that does not indicated to possible
+	 * P2P Clients in the group that the group is terminated. */
+	wpa_supplicant_ap_deinit(wpa_s);
+
+	/* Reselect the GO frequency */
+	if (wpas_p2p_init_go_params(wpa_s, &params, 0, 0, 0, NULL)) {
+		wpa_dbg(wpa_s, MSG_DEBUG, "P2P: Failed to reselect freq");
+		wpas_p2p_group_delete(wpa_s,
+				      P2P_GROUP_REMOVAL_GO_LEAVE_CHANNEL);
 		return;
+	}
+	wpa_dbg(wpa_s, MSG_DEBUG, "P2P: New freq selected for the GO (%u MHz)",
+		params.freq);
+
+	if (params.freq &&
+	    !p2p_supported_freq_go(wpa_s->global->p2p, params.freq)) {
+		wpa_printf(MSG_DEBUG,
+			   "P2P: Selected freq (%u MHz) is not valid for P2P",
+			   params.freq);
+		wpas_p2p_group_delete(wpa_s,
+				      P2P_GROUP_REMOVAL_GO_LEAVE_CHANNEL);
+		return;
+	}
+
+	/* Update the frequency */
+	current_ssid->frequency = params.freq;
+	wpa_s->connect_without_scan = current_ssid;
+	wpa_s->reassociate = 1;
+	wpa_s->disconnected = 0;
+	wpa_supplicant_req_scan(wpa_s, 0, 0);
+}
+
+
+static void wpas_p2p_move_go(void *eloop_ctx, void *timeout_ctx)
+{
+	struct wpa_supplicant *wpa_s = eloop_ctx;
+
+	if (!wpa_s->ap_iface || !wpa_s->current_ssid)
+		return;
+
+	wpas_p2p_go_update_common_freqs(wpa_s);
 
 	/*
-	 * If possible, optimize the Listen channel to be a channel that is
-	 * already used by one of the other interfaces.
+	 * First, try a channel switch flow. If it is not supported or fails,
+	 * take down the GO and bring it up again.
 	 */
-	if (!wpa_s->conf->p2p_optimize_listen_chan)
-		return;
+	if (wpas_p2p_move_go_csa(wpa_s) < 0)
+		wpas_p2p_move_go_no_csa(wpa_s);
+}
 
-	if (!wpa_s->current_ssid || wpa_s->wpa_state != WPA_COMPLETED)
-		return;
+
+static void wpas_p2p_reconsider_moving_go(void *eloop_ctx, void *timeout_ctx)
+{
+	struct wpa_supplicant *wpa_s = eloop_ctx;
+	struct wpa_used_freq_data *freqs = NULL;
+	unsigned int num = wpa_s->num_multichan_concurrent;
 
 	freqs = os_calloc(num, sizeof(struct wpa_used_freq_data));
 	if (!freqs)
@@ -9275,11 +8558,158 @@ void wpas_p2p_indicate_state_change(struct wpa_supplicant *wpa_s)
 
 	num = get_shared_radio_freqs_data(wpa_s, freqs, num);
 
-	wpas_p2p_optimize_listen_channel(wpa_s, freqs, num);
+	/* Previous attempt to move a GO was not possible -- try again. */
+	wpas_p2p_consider_moving_gos(wpa_s, freqs, num,
+				     WPAS_P2P_CHANNEL_UPDATE_ANY);
+
 	os_free(freqs);
 }
 
 
+/*
+ * Consider moving a GO from its currently used frequency:
+ * 1. It is possible that due to regulatory consideration the frequency
+ *    can no longer be used and there is a need to evacuate the GO.
+ * 2. It is possible that due to MCC considerations, it would be preferable
+ *    to move the GO to a channel that is currently used by some other
+ *    station interface.
+ *
+ * In case a frequency that became invalid is once again valid, cancel a
+ * previously initiated GO frequency change.
+ */
+static void wpas_p2p_consider_moving_one_go(struct wpa_supplicant *wpa_s,
+					    struct wpa_used_freq_data *freqs,
+					    unsigned int num)
+{
+	unsigned int i, invalid_freq = 0, policy_move = 0, flags = 0;
+	unsigned int timeout;
+	int freq;
+
+	wpas_p2p_go_update_common_freqs(wpa_s);
+
+	freq = wpa_s->current_ssid->frequency;
+	for (i = 0, invalid_freq = 0; i < num; i++) {
+		if (freqs[i].freq == freq) {
+			flags = freqs[i].flags;
+
+			/* The channel is invalid, must change it */
+			if (!p2p_supported_freq_go(wpa_s->global->p2p, freq)) {
+				wpa_dbg(wpa_s, MSG_DEBUG,
+					"P2P: Freq=%d MHz no longer valid for GO",
+					freq);
+				invalid_freq = 1;
+			}
+		} else if (freqs[i].flags == 0) {
+			/* Freq is not used by any other station interface */
+			continue;
+		} else if (!p2p_supported_freq(wpa_s->global->p2p,
+					       freqs[i].freq)) {
+			/* Freq is not valid for P2P use cases */
+			continue;
+		} else if (wpa_s->conf->p2p_go_freq_change_policy ==
+			   P2P_GO_FREQ_MOVE_SCM) {
+			policy_move = 1;
+		} else if (wpa_s->conf->p2p_go_freq_change_policy ==
+			   P2P_GO_FREQ_MOVE_SCM_PEER_SUPPORTS &&
+			   wpas_p2p_go_is_peer_freq(wpa_s, freqs[i].freq)) {
+			policy_move = 1;
+		}
+	}
+
+	wpa_dbg(wpa_s, MSG_DEBUG,
+		"P2P: GO move: invalid_freq=%u, policy_move=%u, flags=0x%X",
+		invalid_freq, policy_move, flags);
+
+	/*
+	 * The channel is valid, or we are going to have a policy move, so
+	 * cancel timeout.
+	 */
+	if (!invalid_freq || policy_move) {
+		wpa_dbg(wpa_s, MSG_DEBUG,
+			"P2P: Cancel a GO move from freq=%d MHz", freq);
+		eloop_cancel_timeout(wpas_p2p_move_go, wpa_s, NULL);
+
+		if (wpas_p2p_in_progress(wpa_s)) {
+			wpa_dbg(wpa_s, MSG_DEBUG,
+				"P2P: GO move: policy CS is not allowed - setting timeout to re-consider GO move");
+			eloop_cancel_timeout(wpas_p2p_reconsider_moving_go,
+					     wpa_s, NULL);
+			eloop_register_timeout(P2P_RECONSIDER_GO_MOVE_DELAY, 0,
+					       wpas_p2p_reconsider_moving_go,
+					       wpa_s, NULL);
+			return;
+		}
+	}
+
+	if (!invalid_freq && (!policy_move || flags != 0)) {
+		wpa_dbg(wpa_s, MSG_DEBUG,
+			"P2P: Not initiating a GO frequency change");
+		return;
+	}
+
+	if (invalid_freq && !wpas_p2p_disallowed_freq(wpa_s->global, freq))
+		timeout = P2P_GO_FREQ_CHANGE_TIME;
+	else
+		timeout = 0;
+
+	wpa_dbg(wpa_s, MSG_DEBUG, "P2P: Move GO from freq=%d MHz in %d secs",
+		freq, timeout);
+	eloop_cancel_timeout(wpas_p2p_move_go, wpa_s, NULL);
+	eloop_register_timeout(timeout, 0, wpas_p2p_move_go, wpa_s, NULL);
+}
+
+
+static void wpas_p2p_consider_moving_gos(struct wpa_supplicant *wpa_s,
+					 struct wpa_used_freq_data *freqs,
+					 unsigned int num,
+					 enum wpas_p2p_channel_update_trig trig)
+{
+	struct wpa_supplicant *ifs;
+
+	eloop_cancel_timeout(wpas_p2p_reconsider_moving_go, ELOOP_ALL_CTX,
+			     NULL);
+
+	/*
+	 * Travers all the radio interfaces, and for each GO interface, check
+	 * if there is a need to move the GO from the frequency it is using,
+	 * or in case the frequency is valid again, cancel the evacuation flow.
+	 */
+	dl_list_for_each(ifs, &wpa_s->radio->ifaces, struct wpa_supplicant,
+			 radio_list) {
+		if (ifs->current_ssid == NULL ||
+		    ifs->current_ssid->mode != WPAS_MODE_P2P_GO)
+			continue;
+
+		/*
+		 * The GO was just started or completed channel switch, no need
+		 * to move it.
+		 */
+		if (wpa_s == ifs &&
+		    (trig == WPAS_P2P_CHANNEL_UPDATE_STATE_CHANGE ||
+		     trig == WPAS_P2P_CHANNEL_UPDATE_CS)) {
+			wpa_dbg(wpa_s, MSG_DEBUG,
+				"P2P: GO move - schedule re-consideration");
+			eloop_register_timeout(P2P_RECONSIDER_GO_MOVE_DELAY, 0,
+					       wpas_p2p_reconsider_moving_go,
+					       wpa_s, NULL);
+			continue;
+		}
+
+		wpas_p2p_consider_moving_one_go(ifs, freqs, num);
+	}
+}
+
+
+void wpas_p2p_indicate_state_change(struct wpa_supplicant *wpa_s)
+{
+	if (wpa_s->global->p2p_disabled || wpa_s->global->p2p == NULL)
+		return;
+
+	wpas_p2p_update_channel_list(wpa_s,
+				     WPAS_P2P_CHANNEL_UPDATE_STATE_CHANGE);
+}
+
+
 void wpas_p2p_deinit_iface(struct wpa_supplicant *wpa_s)
 {
 	if (wpa_s == wpa_s->global->p2p_init_wpa_s && wpa_s->global->p2p) {
diff --git a/wpa_supplicant/p2p_supplicant.h b/wpa_supplicant/p2p_supplicant.h
index b7861786ca22..56e683498d66 100644
--- a/wpa_supplicant/p2p_supplicant.h
+++ b/wpa_supplicant/p2p_supplicant.h
@@ -17,6 +17,15 @@ struct p2p_channels;
 struct wps_event_fail;
 struct p2ps_provision;
 
+enum wpas_p2p_channel_update_trig {
+	WPAS_P2P_CHANNEL_UPDATE_ANY,
+	WPAS_P2P_CHANNEL_UPDATE_DRIVER,
+	WPAS_P2P_CHANNEL_UPDATE_STATE_CHANGE,
+	WPAS_P2P_CHANNEL_UPDATE_AVOID,
+	WPAS_P2P_CHANNEL_UPDATE_DISALLOW,
+	WPAS_P2P_CHANNEL_UPDATE_CS,
+};
+
 int wpas_p2p_add_p2pdev_interface(struct wpa_supplicant *wpa_s,
 				  const char *conf_p2p_dev);
 struct wpa_supplicant * wpas_get_p2p_go_iface(struct wpa_supplicant *wpa_s,
@@ -36,7 +45,7 @@ int wpas_p2p_group_add_persistent(struct wpa_supplicant *wpa_s,
 				  struct wpa_ssid *ssid, int addr_allocated,
 				  int force_freq, int neg_freq, int ht40,
 				  int vht, const struct p2p_channels *channels,
-				  int connection_timeout);
+				  int connection_timeout, int force_scan);
 struct p2p_group * wpas_p2p_group_init(struct wpa_supplicant *wpa_s,
 				       struct wpa_ssid *ssid);
 enum wpas_p2p_prov_disc_use {
@@ -66,7 +75,6 @@ int wpas_p2p_listen_start(struct wpa_supplicant *wpa_s, unsigned int timeout);
 int wpas_p2p_assoc_req_ie(struct wpa_supplicant *wpa_s, struct wpa_bss *bss,
 			  u8 *buf, size_t len, int p2p_group);
 void wpas_p2p_scan_ie(struct wpa_supplicant *wpa_s, struct wpabuf *ies);
-void wpas_p2p_group_formation_failed(struct wpa_supplicant *wpa_s);
 u64 wpas_p2p_sd_request(struct wpa_supplicant *wpa_s, const u8 *dst,
 			const struct wpabuf *tlvs);
 u64 wpas_p2p_sd_request_asp(struct wpa_supplicant *wpa_s, const u8 *dst, u8 id,
@@ -91,9 +99,15 @@ int wpas_p2p_service_del_upnp(struct wpa_supplicant *wpa_s, u8 version,
 			      const char *service);
 int wpas_p2p_service_add_asp(struct wpa_supplicant *wpa_s, int auto_accept,
 			     u32 adv_id, const char *adv_str, u8 svc_state,
-			     u16 config_methods, const char *svc_info);
+			     u16 config_methods, const char *svc_info,
+			     const u8 *cpt_priority);
 int wpas_p2p_service_del_asp(struct wpa_supplicant *wpa_s, u32 adv_id);
+void wpas_p2p_service_flush_asp(struct wpa_supplicant *wpa_s);
 int wpas_p2p_service_p2ps_id_exists(struct wpa_supplicant *wpa_s, u32 adv_id);
+void wpas_sd_request(void *ctx, int freq, const u8 *sa, u8 dialog_token,
+		     u16 update_indic, const u8 *tlvs, size_t tlvs_len);
+void wpas_sd_response(void *ctx, const u8 *sa, u16 update_indic,
+		      const u8 *tlvs, size_t tlvs_len);
 int wpas_p2p_reject(struct wpa_supplicant *wpa_s, const u8 *addr);
 int wpas_p2p_invite(struct wpa_supplicant *wpa_s, const u8 *peer_addr,
 		    struct wpa_ssid *ssid, const u8 *go_dev_addr, int freq,
@@ -153,10 +167,13 @@ void wpas_p2p_update_config(struct wpa_supplicant *wpa_s);
 int wpas_p2p_probe_req_rx(struct wpa_supplicant *wpa_s, const u8 *addr,
 			  const u8 *dst, const u8 *bssid,
 			  const u8 *ie, size_t ie_len,
-			  int ssi_signal);
+			  unsigned int rx_freq, int ssi_signal);
 void wpas_p2p_wps_success(struct wpa_supplicant *wpa_s, const u8 *peer_addr,
 			  int registrar);
-void wpas_p2p_update_channel_list(struct wpa_supplicant *wpa_s);
+
+void wpas_p2p_update_channel_list(struct wpa_supplicant *wpa_s,
+				  enum wpas_p2p_channel_update_trig trig);
+
 void wpas_p2p_update_best_channels(struct wpa_supplicant *wpa_s,
 				   int freq_24, int freq_5, int freq_overall);
 void wpas_p2p_rx_action(struct wpa_supplicant *wpa_s, const u8 *da,
@@ -207,7 +224,7 @@ static inline int wpas_p2p_probe_req_rx(struct wpa_supplicant *wpa_s,
 					const u8 *addr,
 					const u8 *dst, const u8 *bssid,
 					const u8 *ie, size_t ie_len,
-					int ssi_signal)
+					unsigned int rx_freq, int ssi_signal)
 {
 	return 0;
 }
@@ -217,7 +234,9 @@ static inline void wpas_p2p_wps_success(struct wpa_supplicant *wpa_s,
 {
 }
 
-static inline void wpas_p2p_update_channel_list(struct wpa_supplicant *wpa_s)
+static inline void
+wpas_p2p_update_channel_list(struct wpa_supplicant *wpa_s,
+			     enum wpas_p2p_channel_update_trig trig)
 {
 }
 
diff --git a/wpa_supplicant/p2p_supplicant_sd.c b/wpa_supplicant/p2p_supplicant_sd.c
new file mode 100644
index 000000000000..fc07b07462f5
--- /dev/null
+++ b/wpa_supplicant/p2p_supplicant_sd.c
@@ -0,0 +1,1273 @@
+/*
+ * wpa_supplicant - P2P service discovery
+ * Copyright (c) 2009-2010, Atheros Communications
+ * Copyright (c) 2010-2014, Jouni Malinen <j@w1.fi>
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#include "utils/includes.h"
+
+#include "utils/common.h"
+#include "p2p/p2p.h"
+#include "wpa_supplicant_i.h"
+#include "notify.h"
+#include "p2p_supplicant.h"
+
+
+/*
+ * DNS Header section is used only to calculate compression pointers, so the
+ * contents of this data does not matter, but the length needs to be reserved
+ * in the virtual packet.
+ */
+#define DNS_HEADER_LEN 12
+
+/*
+ * 27-octet in-memory packet from P2P specification containing two implied
+ * queries for _tcp.lcoal. PTR IN and _udp.local. PTR IN
+ */
+#define P2P_SD_IN_MEMORY_LEN 27
+
+static int p2p_sd_dns_uncompress_label(char **upos, char *uend, u8 *start,
+				       u8 **spos, const u8 *end)
+{
+	while (*spos < end) {
+		u8 val = ((*spos)[0] & 0xc0) >> 6;
+		int len;
+
+		if (val == 1 || val == 2) {
+			/* These are reserved values in RFC 1035 */
+			wpa_printf(MSG_DEBUG, "P2P: Invalid domain name "
+				   "sequence starting with 0x%x", val);
+			return -1;
+		}
+
+		if (val == 3) {
+			u16 offset;
+			u8 *spos_tmp;
+
+			/* Offset */
+			if (*spos + 2 > end) {
+				wpa_printf(MSG_DEBUG, "P2P: No room for full "
+					   "DNS offset field");
+				return -1;
+			}
+
+			offset = (((*spos)[0] & 0x3f) << 8) | (*spos)[1];
+			if (offset >= *spos - start) {
+				wpa_printf(MSG_DEBUG, "P2P: Invalid DNS "
+					   "pointer offset %u", offset);
+				return -1;
+			}
+
+			(*spos) += 2;
+			spos_tmp = start + offset;
+			return p2p_sd_dns_uncompress_label(upos, uend, start,
+							   &spos_tmp,
+							   *spos - 2);
+		}
+
+		/* Label */
+		len = (*spos)[0] & 0x3f;
+		if (len == 0)
+			return 0;
+
+		(*spos)++;
+		if (*spos + len > end) {
+			wpa_printf(MSG_DEBUG, "P2P: Invalid domain name "
+				   "sequence - no room for label with length "
+				   "%u", len);
+			return -1;
+		}
+
+		if (*upos + len + 2 > uend)
+			return -2;
+
+		os_memcpy(*upos, *spos, len);
+		*spos += len;
+		*upos += len;
+		(*upos)[0] = '.';
+		(*upos)++;
+		(*upos)[0] = '\0';
+	}
+
+	return 0;
+}
+
+
+/* Uncompress domain names per RFC 1035 using the P2P SD in-memory packet.
+ * Returns -1 on parsing error (invalid input sequence), -2 if output buffer is
+ * not large enough */
+static int p2p_sd_dns_uncompress(char *buf, size_t buf_len, const u8 *msg,
+				 size_t msg_len, size_t offset)
+{
+	/* 27-octet in-memory packet from P2P specification */
+	const char *prefix = "\x04_tcp\x05local\x00\x00\x0C\x00\x01"
+		"\x04_udp\xC0\x11\x00\x0C\x00\x01";
+	u8 *tmp, *end, *spos;
+	char *upos, *uend;
+	int ret = 0;
+
+	if (buf_len < 2)
+		return -1;
+	if (offset > msg_len)
+		return -1;
+
+	tmp = os_malloc(DNS_HEADER_LEN + P2P_SD_IN_MEMORY_LEN + msg_len);
+	if (tmp == NULL)
+		return -1;
+	spos = tmp + DNS_HEADER_LEN + P2P_SD_IN_MEMORY_LEN;
+	end = spos + msg_len;
+	spos += offset;
+
+	os_memset(tmp, 0, DNS_HEADER_LEN);
+	os_memcpy(tmp + DNS_HEADER_LEN, prefix, P2P_SD_IN_MEMORY_LEN);
+	os_memcpy(tmp + DNS_HEADER_LEN + P2P_SD_IN_MEMORY_LEN, msg, msg_len);
+
+	upos = buf;
+	uend = buf + buf_len;
+
+	ret = p2p_sd_dns_uncompress_label(&upos, uend, tmp, &spos, end);
+	if (ret) {
+		os_free(tmp);
+		return ret;
+	}
+
+	if (upos == buf) {
+		upos[0] = '.';
+		upos[1] = '\0';
+	} else if (upos[-1] == '.')
+		upos[-1] = '\0';
+
+	os_free(tmp);
+	return 0;
+}
+
+
+static struct p2p_srv_bonjour *
+wpas_p2p_service_get_bonjour(struct wpa_supplicant *wpa_s,
+			     const struct wpabuf *query)
+{
+	struct p2p_srv_bonjour *bsrv;
+	size_t len;
+
+	len = wpabuf_len(query);
+	dl_list_for_each(bsrv, &wpa_s->global->p2p_srv_bonjour,
+			 struct p2p_srv_bonjour, list) {
+		if (len == wpabuf_len(bsrv->query) &&
+		    os_memcmp(wpabuf_head(query), wpabuf_head(bsrv->query),
+			      len) == 0)
+			return bsrv;
+	}
+	return NULL;
+}
+
+
+static struct p2p_srv_upnp *
+wpas_p2p_service_get_upnp(struct wpa_supplicant *wpa_s, u8 version,
+			  const char *service)
+{
+	struct p2p_srv_upnp *usrv;
+
+	dl_list_for_each(usrv, &wpa_s->global->p2p_srv_upnp,
+			 struct p2p_srv_upnp, list) {
+		if (version == usrv->version &&
+		    os_strcmp(service, usrv->service) == 0)
+			return usrv;
+	}
+	return NULL;
+}
+
+
+static void wpas_sd_add_empty(struct wpabuf *resp, u8 srv_proto,
+			      u8 srv_trans_id, u8 status)
+{
+	u8 *len_pos;
+
+	if (wpabuf_tailroom(resp) < 5)
+		return;
+
+	/* Length (to be filled) */
+	len_pos = wpabuf_put(resp, 2);
+	wpabuf_put_u8(resp, srv_proto);
+	wpabuf_put_u8(resp, srv_trans_id);
+	/* Status Code */
+	wpabuf_put_u8(resp, status);
+	/* Response Data: empty */
+	WPA_PUT_LE16(len_pos, (u8 *) wpabuf_put(resp, 0) - len_pos - 2);
+}
+
+
+static void wpas_sd_add_proto_not_avail(struct wpabuf *resp, u8 srv_proto,
+					u8 srv_trans_id)
+{
+	wpas_sd_add_empty(resp, srv_proto, srv_trans_id,
+			  P2P_SD_PROTO_NOT_AVAILABLE);
+}
+
+
+static void wpas_sd_add_bad_request(struct wpabuf *resp, u8 srv_proto,
+				    u8 srv_trans_id)
+{
+	wpas_sd_add_empty(resp, srv_proto, srv_trans_id, P2P_SD_BAD_REQUEST);
+}
+
+
+static void wpas_sd_add_not_found(struct wpabuf *resp, u8 srv_proto,
+				  u8 srv_trans_id)
+{
+	wpas_sd_add_empty(resp, srv_proto, srv_trans_id,
+			  P2P_SD_REQUESTED_INFO_NOT_AVAILABLE);
+}
+
+
+static void wpas_sd_all_bonjour(struct wpa_supplicant *wpa_s,
+				struct wpabuf *resp, u8 srv_trans_id)
+{
+	struct p2p_srv_bonjour *bsrv;
+	u8 *len_pos;
+
+	wpa_printf(MSG_DEBUG, "P2P: SD Request for all Bonjour services");
+
+	if (dl_list_empty(&wpa_s->global->p2p_srv_bonjour)) {
+		wpa_printf(MSG_DEBUG, "P2P: Bonjour protocol not available");
+		return;
+	}
+
+	dl_list_for_each(bsrv, &wpa_s->global->p2p_srv_bonjour,
+			 struct p2p_srv_bonjour, list) {
+		if (wpabuf_tailroom(resp) <
+		    5 + wpabuf_len(bsrv->query) + wpabuf_len(bsrv->resp))
+			return;
+		/* Length (to be filled) */
+		len_pos = wpabuf_put(resp, 2);
+		wpabuf_put_u8(resp, P2P_SERV_BONJOUR);
+		wpabuf_put_u8(resp, srv_trans_id);
+		/* Status Code */
+		wpabuf_put_u8(resp, P2P_SD_SUCCESS);
+		wpa_hexdump_ascii(MSG_DEBUG, "P2P: Matching Bonjour service",
+				  wpabuf_head(bsrv->resp),
+				  wpabuf_len(bsrv->resp));
+		/* Response Data */
+		wpabuf_put_buf(resp, bsrv->query); /* Key */
+		wpabuf_put_buf(resp, bsrv->resp); /* Value */
+		WPA_PUT_LE16(len_pos, (u8 *) wpabuf_put(resp, 0) - len_pos -
+			     2);
+	}
+}
+
+
+static int match_bonjour_query(struct p2p_srv_bonjour *bsrv, const u8 *query,
+			       size_t query_len)
+{
+	char str_rx[256], str_srv[256];
+
+	if (query_len < 3 || wpabuf_len(bsrv->query) < 3)
+		return 0; /* Too short to include DNS Type and Version */
+	if (os_memcmp(query + query_len - 3,
+		      wpabuf_head_u8(bsrv->query) + wpabuf_len(bsrv->query) - 3,
+		      3) != 0)
+		return 0; /* Mismatch in DNS Type or Version */
+	if (query_len == wpabuf_len(bsrv->query) &&
+	    os_memcmp(query, wpabuf_head(bsrv->query), query_len - 3) == 0)
+		return 1; /* Binary match */
+
+	if (p2p_sd_dns_uncompress(str_rx, sizeof(str_rx), query, query_len - 3,
+				  0))
+		return 0; /* Failed to uncompress query */
+	if (p2p_sd_dns_uncompress(str_srv, sizeof(str_srv),
+				  wpabuf_head(bsrv->query),
+				  wpabuf_len(bsrv->query) - 3, 0))
+		return 0; /* Failed to uncompress service */
+
+	return os_strcmp(str_rx, str_srv) == 0;
+}
+
+
+static void wpas_sd_req_bonjour(struct wpa_supplicant *wpa_s,
+				struct wpabuf *resp, u8 srv_trans_id,
+				const u8 *query, size_t query_len)
+{
+	struct p2p_srv_bonjour *bsrv;
+	u8 *len_pos;
+	int matches = 0;
+
+	wpa_hexdump_ascii(MSG_DEBUG, "P2P: SD Request for Bonjour",
+			  query, query_len);
+	if (dl_list_empty(&wpa_s->global->p2p_srv_bonjour)) {
+		wpa_printf(MSG_DEBUG, "P2P: Bonjour protocol not available");
+		wpas_sd_add_proto_not_avail(resp, P2P_SERV_BONJOUR,
+					    srv_trans_id);
+		return;
+	}
+
+	if (query_len == 0) {
+		wpas_sd_all_bonjour(wpa_s, resp, srv_trans_id);
+		return;
+	}
+
+	dl_list_for_each(bsrv, &wpa_s->global->p2p_srv_bonjour,
+			 struct p2p_srv_bonjour, list) {
+		if (!match_bonjour_query(bsrv, query, query_len))
+			continue;
+
+		if (wpabuf_tailroom(resp) <
+		    5 + query_len + wpabuf_len(bsrv->resp))
+			return;
+
+		matches++;
+
+		/* Length (to be filled) */
+		len_pos = wpabuf_put(resp, 2);
+		wpabuf_put_u8(resp, P2P_SERV_BONJOUR);
+		wpabuf_put_u8(resp, srv_trans_id);
+
+		/* Status Code */
+		wpabuf_put_u8(resp, P2P_SD_SUCCESS);
+		wpa_hexdump_ascii(MSG_DEBUG, "P2P: Matching Bonjour service",
+				  wpabuf_head(bsrv->resp),
+				  wpabuf_len(bsrv->resp));
+
+		/* Response Data */
+		wpabuf_put_data(resp, query, query_len); /* Key */
+		wpabuf_put_buf(resp, bsrv->resp); /* Value */
+
+		WPA_PUT_LE16(len_pos, (u8 *) wpabuf_put(resp, 0) - len_pos - 2);
+	}
+
+	if (matches == 0) {
+		wpa_printf(MSG_DEBUG, "P2P: Requested Bonjour service not "
+			   "available");
+		if (wpabuf_tailroom(resp) < 5)
+			return;
+
+		/* Length (to be filled) */
+		len_pos = wpabuf_put(resp, 2);
+		wpabuf_put_u8(resp, P2P_SERV_BONJOUR);
+		wpabuf_put_u8(resp, srv_trans_id);
+
+		/* Status Code */
+		wpabuf_put_u8(resp, P2P_SD_REQUESTED_INFO_NOT_AVAILABLE);
+		/* Response Data: empty */
+		WPA_PUT_LE16(len_pos, (u8 *) wpabuf_put(resp, 0) - len_pos -
+			     2);
+	}
+}
+
+
+static void wpas_sd_all_upnp(struct wpa_supplicant *wpa_s,
+			     struct wpabuf *resp, u8 srv_trans_id)
+{
+	struct p2p_srv_upnp *usrv;
+	u8 *len_pos;
+
+	wpa_printf(MSG_DEBUG, "P2P: SD Request for all UPnP services");
+
+	if (dl_list_empty(&wpa_s->global->p2p_srv_upnp)) {
+		wpa_printf(MSG_DEBUG, "P2P: UPnP protocol not available");
+		return;
+	}
+
+	dl_list_for_each(usrv, &wpa_s->global->p2p_srv_upnp,
+			 struct p2p_srv_upnp, list) {
+		if (wpabuf_tailroom(resp) < 5 + 1 + os_strlen(usrv->service))
+			return;
+
+		/* Length (to be filled) */
+		len_pos = wpabuf_put(resp, 2);
+		wpabuf_put_u8(resp, P2P_SERV_UPNP);
+		wpabuf_put_u8(resp, srv_trans_id);
+
+		/* Status Code */
+		wpabuf_put_u8(resp, P2P_SD_SUCCESS);
+		/* Response Data */
+		wpabuf_put_u8(resp, usrv->version);
+		wpa_printf(MSG_DEBUG, "P2P: Matching UPnP Service: %s",
+			   usrv->service);
+		wpabuf_put_str(resp, usrv->service);
+		WPA_PUT_LE16(len_pos, (u8 *) wpabuf_put(resp, 0) - len_pos -
+			     2);
+	}
+}
+
+
+static void wpas_sd_req_upnp(struct wpa_supplicant *wpa_s,
+			     struct wpabuf *resp, u8 srv_trans_id,
+			     const u8 *query, size_t query_len)
+{
+	struct p2p_srv_upnp *usrv;
+	u8 *len_pos;
+	u8 version;
+	char *str;
+	int count = 0;
+
+	wpa_hexdump_ascii(MSG_DEBUG, "P2P: SD Request for UPnP",
+			  query, query_len);
+
+	if (dl_list_empty(&wpa_s->global->p2p_srv_upnp)) {
+		wpa_printf(MSG_DEBUG, "P2P: UPnP protocol not available");
+		wpas_sd_add_proto_not_avail(resp, P2P_SERV_UPNP,
+					    srv_trans_id);
+		return;
+	}
+
+	if (query_len == 0) {
+		wpas_sd_all_upnp(wpa_s, resp, srv_trans_id);
+		return;
+	}
+
+	if (wpabuf_tailroom(resp) < 5)
+		return;
+
+	/* Length (to be filled) */
+	len_pos = wpabuf_put(resp, 2);
+	wpabuf_put_u8(resp, P2P_SERV_UPNP);
+	wpabuf_put_u8(resp, srv_trans_id);
+
+	version = query[0];
+	str = os_malloc(query_len);
+	if (str == NULL)
+		return;
+	os_memcpy(str, query + 1, query_len - 1);
+	str[query_len - 1] = '\0';
+
+	dl_list_for_each(usrv, &wpa_s->global->p2p_srv_upnp,
+			 struct p2p_srv_upnp, list) {
+		if (version != usrv->version)
+			continue;
+
+		if (os_strcmp(str, "ssdp:all") != 0 &&
+		    os_strstr(usrv->service, str) == NULL)
+			continue;
+
+		if (wpabuf_tailroom(resp) < 2)
+			break;
+		if (count == 0) {
+			/* Status Code */
+			wpabuf_put_u8(resp, P2P_SD_SUCCESS);
+			/* Response Data */
+			wpabuf_put_u8(resp, version);
+		} else
+			wpabuf_put_u8(resp, ',');
+
+		count++;
+
+		wpa_printf(MSG_DEBUG, "P2P: Matching UPnP Service: %s",
+			   usrv->service);
+		if (wpabuf_tailroom(resp) < os_strlen(usrv->service))
+			break;
+		wpabuf_put_str(resp, usrv->service);
+	}
+	os_free(str);
+
+	if (count == 0) {
+		wpa_printf(MSG_DEBUG, "P2P: Requested UPnP service not "
+			   "available");
+		/* Status Code */
+		wpabuf_put_u8(resp, P2P_SD_REQUESTED_INFO_NOT_AVAILABLE);
+		/* Response Data: empty */
+	}
+
+	WPA_PUT_LE16(len_pos, (u8 *) wpabuf_put(resp, 0) - len_pos - 2);
+}
+
+
+#ifdef CONFIG_WIFI_DISPLAY
+static void wpas_sd_req_wfd(struct wpa_supplicant *wpa_s,
+			    struct wpabuf *resp, u8 srv_trans_id,
+			    const u8 *query, size_t query_len)
+{
+	const u8 *pos;
+	u8 role;
+	u8 *len_pos;
+
+	wpa_hexdump(MSG_DEBUG, "P2P: SD Request for WFD", query, query_len);
+
+	if (!wpa_s->global->wifi_display) {
+		wpa_printf(MSG_DEBUG, "P2P: WFD protocol not available");
+		wpas_sd_add_proto_not_avail(resp, P2P_SERV_WIFI_DISPLAY,
+					    srv_trans_id);
+		return;
+	}
+
+	if (query_len < 1) {
+		wpa_printf(MSG_DEBUG, "P2P: Missing WFD Requested Device "
+			   "Role");
+		return;
+	}
+
+	if (wpabuf_tailroom(resp) < 5)
+		return;
+
+	pos = query;
+	role = *pos++;
+	wpa_printf(MSG_DEBUG, "P2P: WSD for device role 0x%x", role);
+
+	/* TODO: role specific handling */
+
+	/* Length (to be filled) */
+	len_pos = wpabuf_put(resp, 2);
+	wpabuf_put_u8(resp, P2P_SERV_WIFI_DISPLAY);
+	wpabuf_put_u8(resp, srv_trans_id);
+	wpabuf_put_u8(resp, P2P_SD_SUCCESS); /* Status Code */
+
+	while (pos < query + query_len) {
+		if (*pos < MAX_WFD_SUBELEMS &&
+		    wpa_s->global->wfd_subelem[*pos] &&
+		    wpabuf_tailroom(resp) >=
+		    wpabuf_len(wpa_s->global->wfd_subelem[*pos])) {
+			wpa_printf(MSG_DEBUG, "P2P: Add WSD response "
+				   "subelement %u", *pos);
+			wpabuf_put_buf(resp, wpa_s->global->wfd_subelem[*pos]);
+		}
+		pos++;
+	}
+
+	WPA_PUT_LE16(len_pos, (u8 *) wpabuf_put(resp, 0) - len_pos - 2);
+}
+#endif /* CONFIG_WIFI_DISPLAY */
+
+
+static int find_p2ps_substr(struct p2ps_advertisement *adv_data,
+			    const u8 *needle, size_t needle_len)
+{
+	const u8 *haystack = (const u8 *) adv_data->svc_info;
+	size_t haystack_len, i;
+
+	/* Allow search term to be empty */
+	if (!needle || !needle_len)
+		return 1;
+
+	if (!haystack)
+		return 0;
+
+	haystack_len = os_strlen(adv_data->svc_info);
+	for (i = 0; i < haystack_len; i++) {
+		if (haystack_len - i < needle_len)
+			break;
+		if (os_memcmp(haystack + i, needle, needle_len) == 0)
+			return 1;
+	}
+
+	return 0;
+}
+
+
+static void wpas_sd_req_asp(struct wpa_supplicant *wpa_s,
+			    struct wpabuf *resp, u8 srv_trans_id,
+			    const u8 *query, size_t query_len)
+{
+	struct p2ps_advertisement *adv_data;
+	const u8 *svc = &query[1];
+	const u8 *info = NULL;
+	size_t svc_len = query[0];
+	size_t info_len = 0;
+	int prefix = 0;
+	u8 *count_pos = NULL;
+	u8 *len_pos = NULL;
+
+	wpa_hexdump(MSG_DEBUG, "P2P: SD Request for ASP", query, query_len);
+
+	if (!wpa_s->global->p2p) {
+		wpa_printf(MSG_DEBUG, "P2P: ASP protocol not available");
+		wpas_sd_add_proto_not_avail(resp, P2P_SERV_P2PS, srv_trans_id);
+		return;
+	}
+
+	/* Info block is optional */
+	if (svc_len + 1 < query_len) {
+		info = &svc[svc_len];
+		info_len = *info++;
+	}
+
+	/* Range check length of svc string and info block */
+	if (svc_len + (info_len ? info_len + 2 : 1) > query_len) {
+		wpa_printf(MSG_DEBUG, "P2P: ASP bad request");
+		wpas_sd_add_bad_request(resp, P2P_SERV_P2PS, srv_trans_id);
+		return;
+	}
+
+	/* Detect and correct for prefix search */
+	if (svc_len && svc[svc_len - 1] == '*') {
+		prefix = 1;
+		svc_len--;
+	}
+
+	for (adv_data = p2p_get_p2ps_adv_list(wpa_s->global->p2p);
+	     adv_data; adv_data = adv_data->next) {
+		/* If not a prefix match, reject length mismatches */
+		if (!prefix && svc_len != os_strlen(adv_data->svc_name))
+			continue;
+
+		/* Search each service for request */
+		if (os_memcmp(adv_data->svc_name, svc, svc_len) == 0 &&
+		    find_p2ps_substr(adv_data, info, info_len)) {
+			size_t len = os_strlen(adv_data->svc_name);
+			size_t svc_info_len = 0;
+
+			if (adv_data->svc_info)
+				svc_info_len = os_strlen(adv_data->svc_info);
+
+			if (len > 0xff || svc_info_len > 0xffff)
+				return;
+
+			/* Length & Count to be filled as we go */
+			if (!len_pos && !count_pos) {
+				if (wpabuf_tailroom(resp) <
+				    len + svc_info_len + 16)
+					return;
+
+				len_pos = wpabuf_put(resp, 2);
+				wpabuf_put_u8(resp, P2P_SERV_P2PS);
+				wpabuf_put_u8(resp, srv_trans_id);
+				/* Status Code */
+				wpabuf_put_u8(resp, P2P_SD_SUCCESS);
+				count_pos = wpabuf_put(resp, 1);
+				*count_pos = 0;
+			} else if (wpabuf_tailroom(resp) <
+				   len + svc_info_len + 10)
+				return;
+
+			if (svc_info_len) {
+				wpa_printf(MSG_DEBUG,
+					   "P2P: Add Svc: %s info: %s",
+					   adv_data->svc_name,
+					   adv_data->svc_info);
+			} else {
+				wpa_printf(MSG_DEBUG, "P2P: Add Svc: %s",
+					   adv_data->svc_name);
+			}
+
+			/* Advertisement ID */
+			wpabuf_put_le32(resp, adv_data->id);
+
+			/* Config Methods */
+			wpabuf_put_be16(resp, adv_data->config_methods);
+
+			/* Service Name */
+			wpabuf_put_u8(resp, (u8) len);
+			wpabuf_put_data(resp, adv_data->svc_name, len);
+
+			/* Service State */
+			wpabuf_put_u8(resp, adv_data->state);
+
+			/* Service Information */
+			wpabuf_put_le16(resp, (u16) svc_info_len);
+			wpabuf_put_data(resp, adv_data->svc_info, svc_info_len);
+
+			/* Update length and count */
+			(*count_pos)++;
+			WPA_PUT_LE16(len_pos,
+				     (u8 *) wpabuf_put(resp, 0) - len_pos - 2);
+		}
+	}
+
+	/* Return error if no matching svc found */
+	if (count_pos == NULL) {
+		wpa_printf(MSG_DEBUG, "P2P: ASP service not found");
+		wpas_sd_add_not_found(resp, P2P_SERV_P2PS, srv_trans_id);
+	}
+}
+
+
+static void wpas_sd_all_asp(struct wpa_supplicant *wpa_s,
+			    struct wpabuf *resp, u8 srv_trans_id)
+{
+	/* Query data to add all P2PS advertisements:
+	 *  - Service name length: 1
+	 *  - Service name: '*'
+	 *  - Service Information Request Length: 0
+	 */
+	const u8 q[] = { 1, (const u8) '*', 0 };
+
+	if (p2p_get_p2ps_adv_list(wpa_s->global->p2p))
+		wpas_sd_req_asp(wpa_s, resp, srv_trans_id, q, sizeof(q));
+}
+
+
+void wpas_sd_request(void *ctx, int freq, const u8 *sa, u8 dialog_token,
+		     u16 update_indic, const u8 *tlvs, size_t tlvs_len)
+{
+	struct wpa_supplicant *wpa_s = ctx;
+	const u8 *pos = tlvs;
+	const u8 *end = tlvs + tlvs_len;
+	const u8 *tlv_end;
+	u16 slen;
+	struct wpabuf *resp;
+	u8 srv_proto, srv_trans_id;
+	size_t buf_len;
+	char *buf;
+
+	wpa_hexdump(MSG_MSGDUMP, "P2P: Service Discovery Request TLVs",
+		    tlvs, tlvs_len);
+	buf_len = 2 * tlvs_len + 1;
+	buf = os_malloc(buf_len);
+	if (buf) {
+		wpa_snprintf_hex(buf, buf_len, tlvs, tlvs_len);
+		wpa_msg_ctrl(wpa_s, MSG_INFO, P2P_EVENT_SERV_DISC_REQ "%d "
+			     MACSTR " %u %u %s",
+			     freq, MAC2STR(sa), dialog_token, update_indic,
+			     buf);
+		os_free(buf);
+	}
+
+	if (wpa_s->p2p_sd_over_ctrl_iface) {
+		wpas_notify_p2p_sd_request(wpa_s, freq, sa, dialog_token,
+					   update_indic, tlvs, tlvs_len);
+		return; /* to be processed by an external program */
+	}
+
+	resp = wpabuf_alloc(10000);
+	if (resp == NULL)
+		return;
+
+	while (pos + 1 < end) {
+		wpa_printf(MSG_DEBUG, "P2P: Service Request TLV");
+		slen = WPA_GET_LE16(pos);
+		pos += 2;
+		if (pos + slen > end || slen < 2) {
+			wpa_printf(MSG_DEBUG, "P2P: Unexpected Query Data "
+				   "length");
+			wpabuf_free(resp);
+			return;
+		}
+		tlv_end = pos + slen;
+
+		srv_proto = *pos++;
+		wpa_printf(MSG_DEBUG, "P2P: Service Protocol Type %u",
+			   srv_proto);
+		srv_trans_id = *pos++;
+		wpa_printf(MSG_DEBUG, "P2P: Service Transaction ID %u",
+			   srv_trans_id);
+
+		wpa_hexdump(MSG_MSGDUMP, "P2P: Query Data",
+			    pos, tlv_end - pos);
+
+
+		if (wpa_s->force_long_sd) {
+			wpa_printf(MSG_DEBUG, "P2P: SD test - force long "
+				   "response");
+			wpas_sd_all_bonjour(wpa_s, resp, srv_trans_id);
+			wpas_sd_all_upnp(wpa_s, resp, srv_trans_id);
+			wpas_sd_all_asp(wpa_s, resp, srv_trans_id);
+			goto done;
+		}
+
+		switch (srv_proto) {
+		case P2P_SERV_ALL_SERVICES:
+			wpa_printf(MSG_DEBUG, "P2P: Service Discovery Request "
+				   "for all services");
+			if (dl_list_empty(&wpa_s->global->p2p_srv_upnp) &&
+			    dl_list_empty(&wpa_s->global->p2p_srv_bonjour) &&
+			    !p2p_get_p2ps_adv_list(wpa_s->global->p2p)) {
+				wpa_printf(MSG_DEBUG, "P2P: No service "
+					   "discovery protocols available");
+				wpas_sd_add_proto_not_avail(
+					resp, P2P_SERV_ALL_SERVICES,
+					srv_trans_id);
+				break;
+			}
+			wpas_sd_all_bonjour(wpa_s, resp, srv_trans_id);
+			wpas_sd_all_upnp(wpa_s, resp, srv_trans_id);
+			wpas_sd_all_asp(wpa_s, resp, srv_trans_id);
+			break;
+		case P2P_SERV_BONJOUR:
+			wpas_sd_req_bonjour(wpa_s, resp, srv_trans_id,
+					    pos, tlv_end - pos);
+			break;
+		case P2P_SERV_UPNP:
+			wpas_sd_req_upnp(wpa_s, resp, srv_trans_id,
+					 pos, tlv_end - pos);
+			break;
+#ifdef CONFIG_WIFI_DISPLAY
+		case P2P_SERV_WIFI_DISPLAY:
+			wpas_sd_req_wfd(wpa_s, resp, srv_trans_id,
+					pos, tlv_end - pos);
+			break;
+#endif /* CONFIG_WIFI_DISPLAY */
+		case P2P_SERV_P2PS:
+			wpas_sd_req_asp(wpa_s, resp, srv_trans_id,
+					pos, tlv_end - pos);
+			break;
+		default:
+			wpa_printf(MSG_DEBUG, "P2P: Unavailable service "
+				   "protocol %u", srv_proto);
+			wpas_sd_add_proto_not_avail(resp, srv_proto,
+						    srv_trans_id);
+			break;
+		}
+
+		pos = tlv_end;
+	}
+
+done:
+	wpas_notify_p2p_sd_request(wpa_s, freq, sa, dialog_token,
+				   update_indic, tlvs, tlvs_len);
+
+	wpas_p2p_sd_response(wpa_s, freq, sa, dialog_token, resp);
+
+	wpabuf_free(resp);
+}
+
+
+static void wpas_sd_p2ps_serv_response(struct wpa_supplicant *wpa_s,
+				       const u8 *sa, u8 srv_trans_id,
+				       const u8 *pos, const u8 *tlv_end)
+{
+	u8 left = *pos++;
+	u32 adv_id;
+	u8 svc_status;
+	u16 config_methods;
+	char svc_str[256];
+
+	while (left-- && pos < tlv_end) {
+		char *buf = NULL;
+		size_t buf_len;
+		u8 svc_len;
+
+		/* Sanity check fixed length+svc_str */
+		if (pos + 6 >= tlv_end)
+			break;
+		svc_len = pos[6];
+		if (pos + svc_len + 10 > tlv_end)
+			break;
+
+		/* Advertisement ID */
+		adv_id = WPA_GET_LE32(pos);
+		pos += sizeof(u32);
+
+		/* Config Methods */
+		config_methods = WPA_GET_BE16(pos);
+		pos += sizeof(u16);
+
+		/* Service Name */
+		pos++; /* svc_len */
+		os_memcpy(svc_str, pos, svc_len);
+		svc_str[svc_len] = '\0';
+		pos += svc_len;
+
+		/* Service Status */
+		svc_status = *pos++;
+
+		/* Service Information Length */
+		buf_len = WPA_GET_LE16(pos);
+		pos += sizeof(u16);
+
+		/* Sanity check buffer length */
+		if (buf_len > (unsigned int) (tlv_end - pos))
+			break;
+
+		if (buf_len) {
+			buf = os_zalloc(2 * buf_len + 1);
+			if (buf) {
+				utf8_escape((const char *) pos, buf_len, buf,
+					    2 * buf_len + 1);
+			}
+		}
+
+		pos += buf_len;
+
+		if (buf) {
+			wpa_msg_global(wpa_s, MSG_INFO, P2P_EVENT_SERV_ASP_RESP
+				       MACSTR " %x %x %x %x %s '%s'",
+				       MAC2STR(sa), srv_trans_id, adv_id,
+				       svc_status, config_methods, svc_str,
+				       buf);
+			os_free(buf);
+		} else {
+			wpa_msg_global(wpa_s, MSG_INFO, P2P_EVENT_SERV_ASP_RESP
+				       MACSTR " %x %x %x %x %s",
+				       MAC2STR(sa), srv_trans_id, adv_id,
+				       svc_status, config_methods, svc_str);
+		}
+	}
+}
+
+
+void wpas_sd_response(void *ctx, const u8 *sa, u16 update_indic,
+		      const u8 *tlvs, size_t tlvs_len)
+{
+	struct wpa_supplicant *wpa_s = ctx;
+	const u8 *pos = tlvs;
+	const u8 *end = tlvs + tlvs_len;
+	const u8 *tlv_end;
+	u16 slen;
+	size_t buf_len;
+	char *buf;
+
+	wpa_hexdump(MSG_MSGDUMP, "P2P: Service Discovery Response TLVs",
+		    tlvs, tlvs_len);
+	if (tlvs_len > 1500) {
+		/* TODO: better way for handling this */
+		wpa_msg_ctrl(wpa_s, MSG_INFO,
+			     P2P_EVENT_SERV_DISC_RESP MACSTR
+			     " %u <long response: %u bytes>",
+			     MAC2STR(sa), update_indic,
+			     (unsigned int) tlvs_len);
+	} else {
+		buf_len = 2 * tlvs_len + 1;
+		buf = os_malloc(buf_len);
+		if (buf) {
+			wpa_snprintf_hex(buf, buf_len, tlvs, tlvs_len);
+			wpa_msg_ctrl(wpa_s, MSG_INFO,
+				     P2P_EVENT_SERV_DISC_RESP MACSTR " %u %s",
+				     MAC2STR(sa), update_indic, buf);
+			os_free(buf);
+		}
+	}
+
+	while (pos < end) {
+		u8 srv_proto, srv_trans_id, status;
+
+		wpa_printf(MSG_DEBUG, "P2P: Service Response TLV");
+		slen = WPA_GET_LE16(pos);
+		pos += 2;
+		if (pos + slen > end || slen < 3) {
+			wpa_printf(MSG_DEBUG, "P2P: Unexpected Response Data "
+				   "length");
+			return;
+		}
+		tlv_end = pos + slen;
+
+		srv_proto = *pos++;
+		wpa_printf(MSG_DEBUG, "P2P: Service Protocol Type %u",
+			   srv_proto);
+		srv_trans_id = *pos++;
+		wpa_printf(MSG_DEBUG, "P2P: Service Transaction ID %u",
+			   srv_trans_id);
+		status = *pos++;
+		wpa_printf(MSG_DEBUG, "P2P: Status Code ID %u",
+			   status);
+
+		wpa_hexdump(MSG_MSGDUMP, "P2P: Response Data",
+			    pos, tlv_end - pos);
+
+		if (srv_proto == P2P_SERV_P2PS && pos < tlv_end) {
+			wpas_sd_p2ps_serv_response(wpa_s, sa, srv_trans_id,
+						   pos, tlv_end);
+		}
+
+		pos = tlv_end;
+	}
+
+	wpas_notify_p2p_sd_response(wpa_s, sa, update_indic, tlvs, tlvs_len);
+}
+
+
+u64 wpas_p2p_sd_request(struct wpa_supplicant *wpa_s, const u8 *dst,
+			const struct wpabuf *tlvs)
+{
+	if (wpa_s->global->p2p_disabled || wpa_s->global->p2p == NULL)
+		return 0;
+	return (uintptr_t) p2p_sd_request(wpa_s->global->p2p, dst, tlvs);
+}
+
+
+u64 wpas_p2p_sd_request_upnp(struct wpa_supplicant *wpa_s, const u8 *dst,
+			     u8 version, const char *query)
+{
+	struct wpabuf *tlvs;
+	u64 ret;
+
+	tlvs = wpabuf_alloc(2 + 1 + 1 + 1 + os_strlen(query));
+	if (tlvs == NULL)
+		return 0;
+	wpabuf_put_le16(tlvs, 1 + 1 + 1 + os_strlen(query));
+	wpabuf_put_u8(tlvs, P2P_SERV_UPNP); /* Service Protocol Type */
+	wpabuf_put_u8(tlvs, 1); /* Service Transaction ID */
+	wpabuf_put_u8(tlvs, version);
+	wpabuf_put_str(tlvs, query);
+	ret = wpas_p2p_sd_request(wpa_s, dst, tlvs);
+	wpabuf_free(tlvs);
+	return ret;
+}
+
+
+u64 wpas_p2p_sd_request_asp(struct wpa_supplicant *wpa_s, const u8 *dst, u8 id,
+			    const char *svc_str, const char *info_substr)
+{
+	struct wpabuf *tlvs;
+	size_t plen, svc_len, substr_len = 0;
+	u64 ret;
+
+	svc_len = os_strlen(svc_str);
+	if (info_substr)
+		substr_len = os_strlen(info_substr);
+
+	if (svc_len > 0xff || substr_len > 0xff)
+		return 0;
+
+	plen = 1 + 1 + 1 + svc_len + 1 + substr_len;
+	tlvs = wpabuf_alloc(2 + plen);
+	if (tlvs == NULL)
+		return 0;
+
+	wpabuf_put_le16(tlvs, plen);
+	wpabuf_put_u8(tlvs, P2P_SERV_P2PS);
+	wpabuf_put_u8(tlvs, id); /* Service Transaction ID */
+	wpabuf_put_u8(tlvs, (u8) svc_len); /* Service String Length */
+	wpabuf_put_data(tlvs, svc_str, svc_len);
+	wpabuf_put_u8(tlvs, (u8) substr_len); /* Info Substring Length */
+	wpabuf_put_data(tlvs, info_substr, substr_len);
+	ret = wpas_p2p_sd_request(wpa_s, dst, tlvs);
+	wpabuf_free(tlvs);
+
+	return ret;
+}
+
+
+#ifdef CONFIG_WIFI_DISPLAY
+
+static u64 wpas_p2p_sd_request_wfd(struct wpa_supplicant *wpa_s, const u8 *dst,
+				   const struct wpabuf *tlvs)
+{
+	if (wpa_s->global->p2p_disabled || wpa_s->global->p2p == NULL)
+		return 0;
+	return (uintptr_t) p2p_sd_request_wfd(wpa_s->global->p2p, dst, tlvs);
+}
+
+
+#define MAX_WFD_SD_SUBELEMS 20
+
+static void wfd_add_sd_req_role(struct wpabuf *tlvs, u8 id, u8 role,
+				const char *subelems)
+{
+	u8 *len;
+	const char *pos;
+	int val;
+	int count = 0;
+
+	len = wpabuf_put(tlvs, 2);
+	wpabuf_put_u8(tlvs, P2P_SERV_WIFI_DISPLAY); /* Service Protocol Type */
+	wpabuf_put_u8(tlvs, id); /* Service Transaction ID */
+
+	wpabuf_put_u8(tlvs, role);
+
+	pos = subelems;
+	while (*pos) {
+		val = atoi(pos);
+		if (val >= 0 && val < 256) {
+			wpabuf_put_u8(tlvs, val);
+			count++;
+			if (count == MAX_WFD_SD_SUBELEMS)
+				break;
+		}
+		pos = os_strchr(pos + 1, ',');
+		if (pos == NULL)
+			break;
+		pos++;
+	}
+
+	WPA_PUT_LE16(len, (u8 *) wpabuf_put(tlvs, 0) - len - 2);
+}
+
+
+u64 wpas_p2p_sd_request_wifi_display(struct wpa_supplicant *wpa_s,
+				     const u8 *dst, const char *role)
+{
+	struct wpabuf *tlvs;
+	u64 ret;
+	const char *subelems;
+	u8 id = 1;
+
+	subelems = os_strchr(role, ' ');
+	if (subelems == NULL)
+		return 0;
+	subelems++;
+
+	tlvs = wpabuf_alloc(4 * (2 + 1 + 1 + 1 + MAX_WFD_SD_SUBELEMS));
+	if (tlvs == NULL)
+		return 0;
+
+	if (os_strstr(role, "[source]"))
+		wfd_add_sd_req_role(tlvs, id++, 0x00, subelems);
+	if (os_strstr(role, "[pri-sink]"))
+		wfd_add_sd_req_role(tlvs, id++, 0x01, subelems);
+	if (os_strstr(role, "[sec-sink]"))
+		wfd_add_sd_req_role(tlvs, id++, 0x02, subelems);
+	if (os_strstr(role, "[source+sink]"))
+		wfd_add_sd_req_role(tlvs, id++, 0x03, subelems);
+
+	ret = wpas_p2p_sd_request_wfd(wpa_s, dst, tlvs);
+	wpabuf_free(tlvs);
+	return ret;
+}
+
+#endif /* CONFIG_WIFI_DISPLAY */
+
+
+int wpas_p2p_sd_cancel_request(struct wpa_supplicant *wpa_s, u64 req)
+{
+	if (wpa_s->global->p2p_disabled || wpa_s->global->p2p == NULL)
+		return -1;
+	return p2p_sd_cancel_request(wpa_s->global->p2p,
+				     (void *) (uintptr_t) req);
+}
+
+
+void wpas_p2p_sd_response(struct wpa_supplicant *wpa_s, int freq,
+			  const u8 *dst, u8 dialog_token,
+			  const struct wpabuf *resp_tlvs)
+{
+	if (wpa_s->global->p2p_disabled || wpa_s->global->p2p == NULL)
+		return;
+	p2p_sd_response(wpa_s->global->p2p, freq, dst, dialog_token,
+			resp_tlvs);
+}
+
+
+void wpas_p2p_sd_service_update(struct wpa_supplicant *wpa_s)
+{
+	if (wpa_s->global->p2p)
+		p2p_sd_service_update(wpa_s->global->p2p);
+}
+
+
+static void wpas_p2p_srv_bonjour_free(struct p2p_srv_bonjour *bsrv)
+{
+	dl_list_del(&bsrv->list);
+	wpabuf_free(bsrv->query);
+	wpabuf_free(bsrv->resp);
+	os_free(bsrv);
+}
+
+
+static void wpas_p2p_srv_upnp_free(struct p2p_srv_upnp *usrv)
+{
+	dl_list_del(&usrv->list);
+	os_free(usrv->service);
+	os_free(usrv);
+}
+
+
+void wpas_p2p_service_flush(struct wpa_supplicant *wpa_s)
+{
+	struct p2p_srv_bonjour *bsrv, *bn;
+	struct p2p_srv_upnp *usrv, *un;
+
+	dl_list_for_each_safe(bsrv, bn, &wpa_s->global->p2p_srv_bonjour,
+			      struct p2p_srv_bonjour, list)
+		wpas_p2p_srv_bonjour_free(bsrv);
+
+	dl_list_for_each_safe(usrv, un, &wpa_s->global->p2p_srv_upnp,
+			      struct p2p_srv_upnp, list)
+		wpas_p2p_srv_upnp_free(usrv);
+
+	wpas_p2p_service_flush_asp(wpa_s);
+	wpas_p2p_sd_service_update(wpa_s);
+}
+
+
+int wpas_p2p_service_p2ps_id_exists(struct wpa_supplicant *wpa_s, u32 adv_id)
+{
+	if (adv_id == 0)
+		return 1;
+
+	if (p2p_service_p2ps_id(wpa_s->global->p2p, adv_id))
+		return 1;
+
+	return 0;
+}
+
+
+int wpas_p2p_service_del_asp(struct wpa_supplicant *wpa_s, u32 adv_id)
+{
+	int ret;
+
+	ret = p2p_service_del_asp(wpa_s->global->p2p, adv_id);
+	if (ret == 0)
+		wpas_p2p_sd_service_update(wpa_s);
+	return ret;
+}
+
+
+int wpas_p2p_service_add_asp(struct wpa_supplicant *wpa_s,
+			     int auto_accept, u32 adv_id,
+			     const char *adv_str, u8 svc_state,
+			     u16 config_methods, const char *svc_info,
+			     const u8 *cpt_priority)
+{
+	int ret;
+
+	ret = p2p_service_add_asp(wpa_s->global->p2p, auto_accept, adv_id,
+				  adv_str, svc_state, config_methods,
+				  svc_info, cpt_priority);
+	if (ret == 0)
+		wpas_p2p_sd_service_update(wpa_s);
+	return ret;
+}
+
+
+void wpas_p2p_service_flush_asp(struct wpa_supplicant *wpa_s)
+{
+	p2p_service_flush_asp(wpa_s->global->p2p);
+}
+
+
+int wpas_p2p_service_add_bonjour(struct wpa_supplicant *wpa_s,
+				 struct wpabuf *query, struct wpabuf *resp)
+{
+	struct p2p_srv_bonjour *bsrv;
+
+	bsrv = os_zalloc(sizeof(*bsrv));
+	if (bsrv == NULL)
+		return -1;
+	bsrv->query = query;
+	bsrv->resp = resp;
+	dl_list_add(&wpa_s->global->p2p_srv_bonjour, &bsrv->list);
+
+	wpas_p2p_sd_service_update(wpa_s);
+	return 0;
+}
+
+
+int wpas_p2p_service_del_bonjour(struct wpa_supplicant *wpa_s,
+				 const struct wpabuf *query)
+{
+	struct p2p_srv_bonjour *bsrv;
+
+	bsrv = wpas_p2p_service_get_bonjour(wpa_s, query);
+	if (bsrv == NULL)
+		return -1;
+	wpas_p2p_srv_bonjour_free(bsrv);
+	wpas_p2p_sd_service_update(wpa_s);
+	return 0;
+}
+
+
+int wpas_p2p_service_add_upnp(struct wpa_supplicant *wpa_s, u8 version,
+			      const char *service)
+{
+	struct p2p_srv_upnp *usrv;
+
+	if (wpas_p2p_service_get_upnp(wpa_s, version, service))
+		return 0; /* Already listed */
+	usrv = os_zalloc(sizeof(*usrv));
+	if (usrv == NULL)
+		return -1;
+	usrv->version = version;
+	usrv->service = os_strdup(service);
+	if (usrv->service == NULL) {
+		os_free(usrv);
+		return -1;
+	}
+	dl_list_add(&wpa_s->global->p2p_srv_upnp, &usrv->list);
+
+	wpas_p2p_sd_service_update(wpa_s);
+	return 0;
+}
+
+
+int wpas_p2p_service_del_upnp(struct wpa_supplicant *wpa_s, u8 version,
+			      const char *service)
+{
+	struct p2p_srv_upnp *usrv;
+
+	usrv = wpas_p2p_service_get_upnp(wpa_s, version, service);
+	if (usrv == NULL)
+		return -1;
+	wpas_p2p_srv_upnp_free(usrv);
+	wpas_p2p_sd_service_update(wpa_s);
+	return 0;
+}
diff --git a/wpa_supplicant/preauth_test.c b/wpa_supplicant/preauth_test.c
index ed5708585be1..f4bba98e2a82 100644
--- a/wpa_supplicant/preauth_test.c
+++ b/wpa_supplicant/preauth_test.c
@@ -27,7 +27,7 @@
 #include "drivers/driver.h"
 
 
-struct wpa_driver_ops *wpa_drivers[] = { NULL };
+const struct wpa_driver_ops *const wpa_drivers[] = { NULL };
 
 
 struct preauth_test_data {
diff --git a/wpa_supplicant/scan.c b/wpa_supplicant/scan.c
index 805891a88005..d7049a1a8164 100644
--- a/wpa_supplicant/scan.c
+++ b/wpa_supplicant/scan.c
@@ -418,22 +418,6 @@ static void wpa_supplicant_optimize_freqs(
 static void wpas_add_interworking_elements(struct wpa_supplicant *wpa_s,
 					   struct wpabuf *buf)
 {
-	if (wpa_s->conf->interworking == 0)
-		return;
-
-	wpabuf_put_u8(buf, WLAN_EID_EXT_CAPAB);
-	wpabuf_put_u8(buf, 6);
-	wpabuf_put_u8(buf, 0x00);
-	wpabuf_put_u8(buf, 0x00);
-	wpabuf_put_u8(buf, 0x00);
-	wpabuf_put_u8(buf, 0x80); /* Bit 31 - Interworking */
-	wpabuf_put_u8(buf, 0x00);
-#ifdef CONFIG_HS20
-	wpabuf_put_u8(buf, 0x40); /* Bit 46 - WNM-Notification */
-#else /* CONFIG_HS20 */
-	wpabuf_put_u8(buf, 0x00);
-#endif /* CONFIG_HS20 */
-
 	wpabuf_put_u8(buf, WLAN_EID_INTERWORKING);
 	wpabuf_put_u8(buf, is_zero_ether_addr(wpa_s->conf->hessid) ? 1 :
 		      1 + ETH_ALEN);
@@ -448,11 +432,19 @@ static void wpas_add_interworking_elements(struct wpa_supplicant *wpa_s,
 static struct wpabuf * wpa_supplicant_extra_ies(struct wpa_supplicant *wpa_s)
 {
 	struct wpabuf *extra_ie = NULL;
+	u8 ext_capab[18];
+	int ext_capab_len;
 #ifdef CONFIG_WPS
 	int wps = 0;
 	enum wps_request_type req_type = WPS_REQ_ENROLLEE_INFO;
 #endif /* CONFIG_WPS */
 
+	ext_capab_len = wpas_build_ext_capab(wpa_s, ext_capab,
+					     sizeof(ext_capab));
+	if (ext_capab_len > 0 &&
+	    wpabuf_resize(&extra_ie, ext_capab_len) == 0)
+		wpabuf_put_data(extra_ie, ext_capab, ext_capab_len);
+
 #ifdef CONFIG_INTERWORKING
 	if (wpa_s->conf->interworking &&
 	    wpabuf_resize(&extra_ie, 100) == 0)
@@ -493,6 +485,12 @@ static struct wpabuf * wpa_supplicant_extra_ies(struct wpa_supplicant *wpa_s)
 		wpas_hs20_add_indication(extra_ie, -1);
 #endif /* CONFIG_HS20 */
 
+#ifdef CONFIG_FST
+	if (wpa_s->fst_ies &&
+	    wpabuf_resize(&extra_ie, wpabuf_len(wpa_s->fst_ies)) == 0)
+		wpabuf_put_buf(extra_ie, wpa_s->fst_ies);
+#endif /* CONFIG_FST */
+
 	return extra_ie;
 }
 
@@ -622,6 +620,37 @@ static void wpa_set_scan_ssids(struct wpa_supplicant *wpa_s,
 }
 
 
+static int wpa_set_ssids_from_scan_req(struct wpa_supplicant *wpa_s,
+				       struct wpa_driver_scan_params *params,
+				       size_t max_ssids)
+{
+	unsigned int i;
+
+	if (wpa_s->ssids_from_scan_req == NULL ||
+	    wpa_s->num_ssids_from_scan_req == 0)
+		return 0;
+
+	if (wpa_s->num_ssids_from_scan_req > max_ssids) {
+		wpa_s->num_ssids_from_scan_req = max_ssids;
+		wpa_printf(MSG_DEBUG, "Over max scan SSIDs from scan req: %u",
+			   (unsigned int) max_ssids);
+	}
+
+	for (i = 0; i < wpa_s->num_ssids_from_scan_req; i++) {
+		params->ssids[i].ssid = wpa_s->ssids_from_scan_req[i].ssid;
+		params->ssids[i].ssid_len =
+			wpa_s->ssids_from_scan_req[i].ssid_len;
+		wpa_hexdump_ascii(MSG_DEBUG, "specific SSID",
+				  params->ssids[i].ssid,
+				  params->ssids[i].ssid_len);
+	}
+
+	params->num_ssids = wpa_s->num_ssids_from_scan_req;
+	wpa_s->num_ssids_from_scan_req = 0;
+	return 1;
+}
+
+
 static void wpa_supplicant_scan(void *eloop_ctx, void *timeout_ctx)
 {
 	struct wpa_supplicant *wpa_s = eloop_ctx;
@@ -734,6 +763,12 @@ static void wpa_supplicant_scan(void *eloop_ctx, void *timeout_ctx)
 		goto scan;
 	}
 
+	if (wpa_s->last_scan_req == MANUAL_SCAN_REQ &&
+	    wpa_set_ssids_from_scan_req(wpa_s, &params, max_ssids)) {
+		wpa_printf(MSG_DEBUG, "Use specific SSIDs from SCAN command");
+		goto ssid_list_set;
+	}
+
 #ifdef CONFIG_P2P
 	if ((wpa_s->p2p_in_provisioning || wpa_s->show_group_started) &&
 	    wpa_s->go_params && !wpa_s->conf->passive_scan) {
@@ -773,6 +808,9 @@ static void wpa_supplicant_scan(void *eloop_ctx, void *timeout_ctx)
 	}
 
 	if (wpa_s->last_scan_req != MANUAL_SCAN_REQ &&
+#ifdef CONFIG_AP
+	    !wpa_s->ap_iface &&
+#endif /* CONFIG_AP */
 	    wpa_s->conf->ap_scan == 2) {
 		wpa_s->connect_without_scan = NULL;
 		wpa_s->prev_scan_wildcard = 0;
@@ -899,10 +937,8 @@ static void wpa_supplicant_scan(void *eloop_ctx, void *timeout_ctx)
 		wpa_dbg(wpa_s, MSG_DEBUG, "Starting AP scan for wildcard "
 			"SSID");
 	}
-#ifdef CONFIG_P2P
-ssid_list_set:
-#endif /* CONFIG_P2P */
 
+ssid_list_set:
 	wpa_supplicant_optimize_freqs(wpa_s, &params);
 	extra_ie = wpa_supplicant_extra_ies(wpa_s);
 
@@ -1652,7 +1688,7 @@ static int wpa_scan_result_compar(const void *a, const void *b)
 		snr_a_full = wa->snr;
 		snr_a = MIN(wa->snr, GREAT_SNR);
 		snr_b_full = wb->snr;
-		snr_b = MIN(wa->snr, GREAT_SNR);
+		snr_b = MIN(wb->snr, GREAT_SNR);
 	} else {
 		/* Level is not in dBm, so we can't calculate
 		 * SNR. Just use raw level (units unknown). */
diff --git a/wpa_supplicant/sme.c b/wpa_supplicant/sme.c
index 178811371409..f2e5a43b978f 100644
--- a/wpa_supplicant/sme.c
+++ b/wpa_supplicant/sme.c
@@ -67,7 +67,7 @@ static int sme_set_sae_group(struct wpa_supplicant *wpa_s)
 
 	for (;;) {
 		int group = groups[wpa_s->sme.sae_group_index];
-		if (group < 0)
+		if (group <= 0)
 			break;
 		if (sae_set_group(&wpa_s->sme.sae, group) == 0) {
 			wpa_dbg(wpa_s, MSG_DEBUG, "SME: Selected SAE group %d",
@@ -438,6 +438,21 @@ static void sme_send_authentication(struct wpa_supplicant *wpa_s,
 	}
 #endif /* CONFIG_HS20 */
 
+#ifdef CONFIG_FST
+	if (wpa_s->fst_ies) {
+		int fst_ies_len = wpabuf_len(wpa_s->fst_ies);
+
+		if (wpa_s->sme.assoc_req_ie_len + fst_ies_len <=
+		    sizeof(wpa_s->sme.assoc_req_ie)) {
+			os_memcpy(wpa_s->sme.assoc_req_ie +
+				  wpa_s->sme.assoc_req_ie_len,
+				  wpabuf_head(wpa_s->fst_ies),
+				  fst_ies_len);
+			wpa_s->sme.assoc_req_ie_len += fst_ies_len;
+		}
+	}
+#endif /* CONFIG_FST */
+
 	ext_capab_len = wpas_build_ext_capab(wpa_s, ext_capab,
 					     sizeof(ext_capab));
 	if (ext_capab_len > 0) {
@@ -583,7 +598,8 @@ static void sme_auth_start_cb(struct wpa_radio_work *work, int deinit)
 	wpa_s->connect_work = work;
 
 	if (cwork->bss_removed ||
-	    !wpas_valid_bss_ssid(wpa_s, cwork->bss, cwork->ssid)) {
+	    !wpas_valid_bss_ssid(wpa_s, cwork->bss, cwork->ssid) ||
+	    wpas_network_disabled(wpa_s, cwork->ssid)) {
 		wpa_dbg(wpa_s, MSG_DEBUG, "SME: BSS/SSID entry for authentication not valid anymore - drop connection attempt");
 		wpas_connect_work_done(wpa_s);
 		return;
@@ -698,6 +714,8 @@ static int sme_sae_auth(struct wpa_supplicant *wpa_s, u16 auth_transaction,
 		return -1;
 
 	if (auth_transaction == 1) {
+		u16 res;
+
 		groups = wpa_s->conf->sae_groups;
 
 		wpa_dbg(wpa_s, MSG_DEBUG, "SME SAE commit");
@@ -708,8 +726,14 @@ static int sme_sae_auth(struct wpa_supplicant *wpa_s, u16 auth_transaction,
 			return -1;
 		if (groups && groups[0] <= 0)
 			groups = NULL;
-		if (sae_parse_commit(&wpa_s->sme.sae, data, len, NULL, NULL,
-				     groups) != WLAN_STATUS_SUCCESS)
+		res = sae_parse_commit(&wpa_s->sme.sae, data, len, NULL, NULL,
+				       groups);
+		if (res == SAE_SILENTLY_DISCARD) {
+			wpa_printf(MSG_DEBUG,
+				   "SAE: Drop commit message due to reflection attack");
+			return 0;
+		}
+		if (res != WLAN_STATUS_SUCCESS)
 			return -1;
 
 		if (sae_process_commit(&wpa_s->sme.sae) < 0) {
@@ -793,8 +817,22 @@ void sme_event_auth(struct wpa_supplicant *wpa_s, union wpa_event_data *data)
 #endif /* CONFIG_SAE */
 
 	if (data->auth.status_code != WLAN_STATUS_SUCCESS) {
-		wpa_dbg(wpa_s, MSG_DEBUG, "SME: Authentication failed (status "
-			"code %d)", data->auth.status_code);
+		char *ie_txt = NULL;
+
+		if (data->auth.ies && data->auth.ies_len) {
+			size_t buflen = 2 * data->auth.ies_len + 1;
+			ie_txt = os_malloc(buflen);
+			if (ie_txt) {
+				wpa_snprintf_hex(ie_txt, buflen, data->auth.ies,
+						 data->auth.ies_len);
+			}
+		}
+		wpa_msg(wpa_s, MSG_INFO, WPA_EVENT_AUTH_REJECT MACSTR
+			" auth_type=%u auth_transaction=%u status_code=%u ie=%s",
+			MAC2STR(data->auth.peer), data->auth.auth_type,
+			data->auth.auth_transaction, data->auth.status_code,
+			ie_txt);
+		os_free(ie_txt);
 
 		if (data->auth.status_code !=
 		    WLAN_STATUS_NOT_SUPPORTED_AUTH_ALG ||
@@ -831,12 +869,20 @@ void sme_event_auth(struct wpa_supplicant *wpa_s, union wpa_event_data *data)
 
 #ifdef CONFIG_IEEE80211R
 	if (data->auth.auth_type == WLAN_AUTH_FT) {
-		union wpa_event_data edata;
-		os_memset(&edata, 0, sizeof(edata));
-		edata.ft_ies.ies = data->auth.ies;
-		edata.ft_ies.ies_len = data->auth.ies_len;
-		os_memcpy(edata.ft_ies.target_ap, data->auth.peer, ETH_ALEN);
-		wpa_supplicant_event(wpa_s, EVENT_FT_RESPONSE, &edata);
+		if (wpa_ft_process_response(wpa_s->wpa, data->auth.ies,
+					    data->auth.ies_len, 0,
+					    data->auth.peer, NULL, 0) < 0) {
+			wpa_dbg(wpa_s, MSG_DEBUG,
+				"SME: FT Authentication response processing failed");
+			wpa_msg(wpa_s, MSG_INFO, WPA_EVENT_DISCONNECTED "bssid="
+				MACSTR
+				" reason=%d locally_generated=1",
+				MAC2STR(wpa_s->pending_bssid),
+				WLAN_REASON_DEAUTH_LEAVING);
+			wpas_connection_failed(wpa_s, wpa_s->pending_bssid);
+			wpa_supplicant_mark_disassoc(wpa_s);
+			return;
+		}
 	}
 #endif /* CONFIG_IEEE80211R */
 
diff --git a/wpa_supplicant/wpa_cli.c b/wpa_supplicant/wpa_cli.c
index 5a0af0dc9bba..7ddae3d3b6b8 100644
--- a/wpa_supplicant/wpa_cli.c
+++ b/wpa_supplicant/wpa_cli.c
@@ -26,16 +26,16 @@
 #endif /* ANDROID */
 
 
-static const char *wpa_cli_version =
+static const char *const wpa_cli_version =
 "wpa_cli v" VERSION_STR "\n"
 "Copyright (c) 2004-2015, Jouni Malinen <j@w1.fi> and contributors";
 
 
-static const char *wpa_cli_license =
+static const char *const wpa_cli_license =
 "This software may be distributed under the terms of the BSD license.\n"
 "See README for more details.\n";
 
-static const char *wpa_cli_full_license =
+static const char *const wpa_cli_full_license =
 "This software may be distributed under the terms of the BSD license.\n"
 "\n"
 "Redistribution and use in source and binary forms, with or without\n"
@@ -76,6 +76,7 @@ static int wpa_cli_last_id = 0;
 #define CONFIG_CTRL_IFACE_DIR "/var/run/wpa_supplicant"
 #endif /* CONFIG_CTRL_IFACE_DIR */
 static const char *ctrl_iface_dir = CONFIG_CTRL_IFACE_DIR;
+static const char *client_socket_dir = NULL;
 static char *ctrl_ifname = NULL;
 static const char *pid_file = NULL;
 static const char *action_file = NULL;
@@ -92,6 +93,7 @@ static DEFINE_DL_LIST(bsses); /* struct cli_txt_entry */
 static DEFINE_DL_LIST(p2p_peers); /* struct cli_txt_entry */
 static DEFINE_DL_LIST(p2p_groups); /* struct cli_txt_entry */
 static DEFINE_DL_LIST(ifnames); /* struct cli_txt_entry */
+static DEFINE_DL_LIST(networks); /* struct cli_txt_entry */
 
 
 static void print_help(const char *cmd);
@@ -99,13 +101,16 @@ static void wpa_cli_mon_receive(int sock, void *eloop_ctx, void *sock_ctx);
 static void wpa_cli_close_connection(void);
 static char * wpa_cli_get_default_ifname(void);
 static char ** wpa_list_cmd_list(void);
+static void update_networks(struct wpa_ctrl *ctrl);
 
 
 static void usage(void)
 {
 	printf("wpa_cli [-p<path to ctrl sockets>] [-i<ifname>] [-hvB] "
 	       "[-a<action file>] \\\n"
-	       "        [-P<pid file>] [-g<global ctrl>] [-G<ping interval>]  "
+	       "        [-P<pid file>] [-g<global ctrl>] [-G<ping interval>] "
+	       "\\\n"
+	       "        [-s<wpa_client_socket_file_path>] "
 	       "[command..]\n"
 	       "  -h = help (show this usage text)\n"
 	       "  -v = shown version information\n"
@@ -168,11 +173,12 @@ static void cli_txt_list_del_addr(struct dl_list *txt_list, const char *txt)
 
 
 #ifdef CONFIG_P2P
-static void cli_txt_list_del_word(struct dl_list *txt_list, const char *txt)
+static void cli_txt_list_del_word(struct dl_list *txt_list, const char *txt,
+				  int separator)
 {
 	const char *end;
 	char *buf;
-	end = os_strchr(txt, ' ');
+	end = os_strchr(txt, separator);
 	if (end == NULL)
 		end = txt + os_strlen(txt);
 	buf = dup_binstr(txt, end - txt);
@@ -213,14 +219,16 @@ static int cli_txt_list_add_addr(struct dl_list *txt_list, const char *txt)
 	os_snprintf(buf, sizeof(buf), MACSTR, MAC2STR(addr));
 	return cli_txt_list_add(txt_list, buf);
 }
+#endif /* CONFIG_P2P */
 
 
-static int cli_txt_list_add_word(struct dl_list *txt_list, const char *txt)
+static int cli_txt_list_add_word(struct dl_list *txt_list, const char *txt,
+				 int separator)
 {
 	const char *end;
 	char *buf;
 	int ret;
-	end = os_strchr(txt, ' ');
+	end = os_strchr(txt, separator);
 	if (end == NULL)
 		end = txt + os_strlen(txt);
 	buf = dup_binstr(txt, end - txt);
@@ -230,7 +238,6 @@ static int cli_txt_list_add_word(struct dl_list *txt_list, const char *txt)
 	os_free(buf);
 	return ret;
 }
-#endif /* CONFIG_P2P */
 
 
 static char ** cli_txt_list_array(struct dl_list *txt_list)
@@ -326,6 +333,13 @@ static int wpa_cli_open_connection(const char *ifname, int attach)
 	}
 #endif /* ANDROID */
 
+	if (client_socket_dir && client_socket_dir[0] &&
+	    access(client_socket_dir, F_OK) < 0) {
+		perror(client_socket_dir);
+		os_free(cfile);
+		return -1;
+	}
+
 	if (cfile == NULL) {
 		flen = os_strlen(ctrl_iface_dir) + os_strlen(ifname) + 2;
 		cfile = os_malloc(flen);
@@ -339,14 +353,14 @@ static int wpa_cli_open_connection(const char *ifname, int attach)
 		}
 	}
 
-	ctrl_conn = wpa_ctrl_open(cfile);
+	ctrl_conn = wpa_ctrl_open2(cfile, client_socket_dir);
 	if (ctrl_conn == NULL) {
 		os_free(cfile);
 		return -1;
 	}
 
 	if (attach && interactive)
-		mon_conn = wpa_ctrl_open(cfile);
+		mon_conn = wpa_ctrl_open2(cfile, client_socket_dir);
 	else
 		mon_conn = NULL;
 	os_free(cfile);
@@ -498,6 +512,10 @@ static int wpa_cli_cmd_status(struct wpa_ctrl *ctrl, int argc, char *argv[])
 		return wpa_ctrl_command(ctrl, "STATUS-WPS");
 	if (argc > 0 && os_strcmp(argv[0], "driver") == 0)
 		return wpa_ctrl_command(ctrl, "STATUS-DRIVER");
+#ifdef ANDROID
+	if (argc > 0 && os_strcmp(argv[0], "no_events") == 0)
+		return wpa_ctrl_command(ctrl, "STATUS-NO_EVENTS");
+#endif /* ANDROID */
 	return wpa_ctrl_command(ctrl, "STATUS");
 }
 
@@ -608,35 +626,58 @@ static char ** wpa_cli_complete_set(const char *str, int pos)
 		"uapsd", "ps", "wifi_display", "bssid_filter", "disallow_aps",
 		"no_keep_alive",
 		/* global configuration parameters */
-		"eapol_version", "ap_scan", "disable_scan_offload",
-		"fast_reauth", "opensc_engine_path", "pkcs11_engine_path",
-		"pkcs11_module_path", "openssl_ciphers",
-		"pcsc_reader", "pcsc_pin",
-		"driver_param", "dot11RSNAConfigPMKLifetime",
+#ifdef CONFIG_CTRL_IFACE
+		"ctrl_interface", "no_ctrl_interface", "ctrl_interface_group",
+#endif /* CONFIG_CTRL_IFACE */
+		"eapol_version", "ap_scan", "bgscan",
+#ifdef CONFIG_MESH
+		"user_mpm", "max_peer_links", "mesh_max_inactivity",
+		"dot11RSNASAERetransPeriod",
+#endif /* CONFIG_MESH */
+		"disable_scan_offload", "fast_reauth", "opensc_engine_path",
+		"pkcs11_engine_path", "pkcs11_module_path", "openssl_ciphers",
+		"pcsc_reader", "pcsc_pin", "external_sim", "driver_param",
+		"dot11RSNAConfigPMKLifetime",
 		"dot11RSNAConfigPMKReauthThreshold",
 		"dot11RSNAConfigSATimeout",
-		"update_config", "load_dynamic_eap", "uuid", "device_name",
-		"manufacturer", "model_name", "model_number", "serial_number",
-		"device_type", "os_version", "config_methods",
-		"wps_cred_processing", "wps_vendor_ext_m1", "sec_device_type",
+#ifndef CONFIG_NO_CONFIG_WRITE
+		"update_config",
+#endif /* CONFIG_NO_CONFIG_WRITE */
+		"load_dynamic_eap",
+#ifdef CONFIG_WPS
+		"uuid", "device_name", "manufacturer", "model_name",
+		"model_number", "serial_number", "device_type", "os_version",
+		"config_methods", "wps_cred_processing", "wps_vendor_ext_m1",
+#endif /* CONFIG_WPS */
+#ifdef CONFIG_P2P
+		"sec_device_type",
 		"p2p_listen_reg_class", "p2p_listen_channel",
-		"p2p_oper_reg_class", "p2p_oper_channel",
-		"p2p_go_intent", "p2p_ssid_postfix", "persistent_reconnect",
-		"p2p_intra_bss", "p2p_group_idle", "p2p_pref_chan",
-		"p2p_no_go_freq",
-		"p2p_go_ht40", "p2p_disabled", "p2p_no_group_iface",
-		"p2p_go_vht",
-		"p2p_ignore_shared_freq", "country", "bss_max_count",
-		"bss_expiration_age", "bss_expiration_scan_count",
-		"filter_ssids", "filter_rssi", "max_num_sta",
-		"disassoc_low_ack", "hs20", "interworking", "hessid",
-		"access_network_type", "pbc_in_m1", "autoscan",
-		"wps_nfc_dev_pw_id", "wps_nfc_dh_pubkey", "wps_nfc_dh_privkey",
-		"wps_nfc_dev_pw", "ext_password_backend",
+		"p2p_oper_reg_class", "p2p_oper_channel", "p2p_go_intent",
+		"p2p_ssid_postfix", "persistent_reconnect", "p2p_intra_bss",
+		"p2p_group_idle", "p2p_passphrase_len", "p2p_pref_chan",
+		"p2p_no_go_freq", "p2p_add_cli_chan",
+		"p2p_optimize_listen_chan", "p2p_go_ht40", "p2p_go_vht",
+		"p2p_disabled", "p2p_go_ctwindow", "p2p_no_group_iface",
+		"p2p_ignore_shared_freq", "ip_addr_go", "ip_addr_mask",
+		"ip_addr_start", "ip_addr_end",
+#endif /* CONFIG_P2P */
+		"country", "bss_max_count", "bss_expiration_age",
+		"bss_expiration_scan_count", "filter_ssids", "filter_rssi",
+		"max_num_sta", "disassoc_low_ack",
+#ifdef CONFIG_HS20
+		"hs20",
+#endif /* CONFIG_HS20 */
+		"interworking", "hessid", "access_network_type", "pbc_in_m1",
+		"autoscan", "wps_nfc_dev_pw_id", "wps_nfc_dh_pubkey",
+		"wps_nfc_dh_privkey", "wps_nfc_dev_pw", "ext_password_backend",
 		"p2p_go_max_inactivity", "auto_interworking", "okc", "pmf",
-		"sae_groups", "dtim_period", "beacon_int", "ap_vendor_elements",
-		"ignore_old_scan_res", "freq_list", "external_sim",
-		"tdls_external_control", "p2p_search_delay"
+		"sae_groups", "dtim_period", "beacon_int",
+		"ap_vendor_elements", "ignore_old_scan_res", "freq_list",
+		"scan_cur_freq", "sched_scan_interval",
+		"tdls_external_control", "osu_dir", "wowlan_triggers",
+		"p2p_search_delay", "mac_addr", "rand_addr_lifetime",
+		"preassoc_mac_addr", "key_mgmt_offload", "passive_scan",
+		"reassoc_same_bss_optim", "wps_priority"
 	};
 	int i, num_fields = ARRAY_SIZE(fields);
 
@@ -670,6 +711,74 @@ static int wpa_cli_cmd_get(struct wpa_ctrl *ctrl, int argc, char *argv[])
 }
 
 
+static char ** wpa_cli_complete_get(const char *str, int pos)
+{
+	int arg = get_cmd_arg_num(str, pos);
+	const char *fields[] = {
+#ifdef CONFIG_CTRL_IFACE
+		"ctrl_interface", "ctrl_interface_group",
+#endif /* CONFIG_CTRL_IFACE */
+		"eapol_version", "ap_scan",
+#ifdef CONFIG_MESH
+		"user_mpm", "max_peer_links", "mesh_max_inactivity",
+#endif /* CONFIG_MESH */
+		"disable_scan_offload", "fast_reauth", "opensc_engine_path",
+		"pkcs11_engine_path", "pkcs11_module_path", "openssl_ciphers",
+		"pcsc_reader", "pcsc_pin", "external_sim", "driver_param",
+		"dot11RSNAConfigPMKLifetime",
+		"dot11RSNAConfigPMKReauthThreshold",
+		"dot11RSNAConfigSATimeout",
+#ifndef CONFIG_NO_CONFIG_WRITE
+		"update_config",
+#endif /* CONFIG_NO_CONFIG_WRITE */
+#ifdef CONFIG_WPS
+		"device_name", "manufacturer", "model_name", "model_number",
+		"serial_number", "config_methods", "wps_cred_processing",
+#endif /* CONFIG_WPS */
+#ifdef CONFIG_P2P
+		"p2p_listen_reg_class", "p2p_listen_channel",
+		"p2p_oper_reg_class", "p2p_oper_channel", "p2p_go_intent",
+		"p2p_ssid_postfix", "persistent_reconnect", "p2p_intra_bss",
+		"p2p_group_idle", "p2p_passphrase_len", "p2p_add_cli_chan",
+		"p2p_optimize_listen_chan", "p2p_go_ht40", "p2p_go_vht",
+		"p2p_disabled", "p2p_go_ctwindow", "p2p_no_group_iface",
+		"p2p_ignore_shared_freq", "ip_addr_go", "ip_addr_mask",
+		"ip_addr_start", "ip_addr_end",
+#endif /* CONFIG_P2P */
+		"bss_max_count", "bss_expiration_age",
+		"bss_expiration_scan_count", "filter_ssids", "filter_rssi",
+		"max_num_sta", "disassoc_low_ack",
+#ifdef CONFIG_HS20
+		"hs20",
+#endif /* CONFIG_HS20 */
+		"interworking", "access_network_type", "pbc_in_m1", "autoscan",
+		"wps_nfc_dev_pw_id", "ext_password_backend",
+		"p2p_go_max_inactivity", "auto_interworking", "okc", "pmf",
+		"dtim_period", "beacon_int", "ignore_old_scan_res",
+		"scan_cur_freq", "sched_scan_interval",
+		"tdls_external_control", "osu_dir", "wowlan_triggers",
+		"p2p_search_delay", "mac_addr", "rand_addr_lifetime",
+		"preassoc_mac_addr", "key_mgmt_offload", "passive_scan",
+		"reassoc_same_bss_optim"
+	};
+	int i, num_fields = ARRAY_SIZE(fields);
+
+	if (arg == 1) {
+		char **res = os_calloc(num_fields + 1, sizeof(char *));
+		if (res == NULL)
+			return NULL;
+		for (i = 0; i < num_fields; i++) {
+			res[i] = os_strdup(fields[i]);
+			if (res[i] == NULL)
+				return res;
+		}
+		return res;
+	}
+
+	return NULL;
+}
+
+
 static int wpa_cli_cmd_logoff(struct wpa_ctrl *ctrl, int argc, char *argv[])
 {
 	return wpa_ctrl_command(ctrl, "LOGOFF");
@@ -873,12 +982,12 @@ static int wpa_cli_cmd_wps_reg(struct wpa_ctrl *ctrl, int argc, char *argv[])
 		res = os_snprintf(cmd, sizeof(cmd), "WPS_REG %s %s",
 				  argv[0], argv[1]);
 	else if (argc == 5 || argc == 6) {
-		char ssid_hex[2 * 32 + 1];
+		char ssid_hex[2 * SSID_MAX_LEN + 1];
 		char key_hex[2 * 64 + 1];
 		int i;
 
 		ssid_hex[0] = '\0';
-		for (i = 0; i < 32; i++) {
+		for (i = 0; i < SSID_MAX_LEN; i++) {
 			if (argv[2][i] == '\0')
 				break;
 			os_snprintf(&ssid_hex[i * 2], 3, "%02x", argv[2][i]);
@@ -1002,12 +1111,12 @@ static int wpa_cli_cmd_wps_er_config(struct wpa_ctrl *ctrl, int argc,
 	int res;
 
 	if (argc == 5 || argc == 6) {
-		char ssid_hex[2 * 32 + 1];
+		char ssid_hex[2 * SSID_MAX_LEN + 1];
 		char key_hex[2 * 64 + 1];
 		int i;
 
 		ssid_hex[0] = '\0';
-		for (i = 0; i < 32; i++) {
+		for (i = 0; i < SSID_MAX_LEN; i++) {
 			if (argv[2][i] == '\0')
 				break;
 			os_snprintf(&ssid_hex[i * 2], 3, "%02x", argv[2][i]);
@@ -1361,14 +1470,20 @@ static int wpa_cli_cmd_disable_network(struct wpa_ctrl *ctrl, int argc,
 static int wpa_cli_cmd_add_network(struct wpa_ctrl *ctrl, int argc,
 				   char *argv[])
 {
-	return wpa_ctrl_command(ctrl, "ADD_NETWORK");
+	int res = wpa_ctrl_command(ctrl, "ADD_NETWORK");
+	if (interactive)
+		update_networks(ctrl);
+	return res;
 }
 
 
 static int wpa_cli_cmd_remove_network(struct wpa_ctrl *ctrl, int argc,
 				      char *argv[])
 {
-	return wpa_cli_cmd(ctrl, "REMOVE_NETWORK", 1, argc, argv);
+	int res = wpa_cli_cmd(ctrl, "REMOVE_NETWORK", 1, argc, argv);
+	if (interactive)
+		update_networks(ctrl);
+	return res;
 }
 
 
@@ -1429,6 +1544,105 @@ static int wpa_cli_cmd_get_network(struct wpa_ctrl *ctrl, int argc,
 }
 
 
+static const char *network_fields[] = {
+	"ssid", "scan_ssid", "bssid", "bssid_blacklist",
+	"bssid_whitelist", "psk", "proto", "key_mgmt",
+	"bg_scan_period", "pairwise", "group", "auth_alg", "scan_freq",
+	"freq_list",
+#ifdef IEEE8021X_EAPOL
+	"eap", "identity", "anonymous_identity", "password", "ca_cert",
+	"ca_path", "client_cert", "private_key", "private_key_passwd",
+	"dh_file", "subject_match", "altsubject_match",
+	"domain_suffix_match", "domain_match", "ca_cert2", "ca_path2",
+	"client_cert2", "private_key2", "private_key2_passwd",
+	"dh_file2", "subject_match2", "altsubject_match2",
+	"domain_suffix_match2", "domain_match2", "phase1", "phase2",
+	"pcsc", "pin", "engine_id", "key_id", "cert_id", "ca_cert_id",
+	"pin2", "engine2_id", "key2_id", "cert2_id", "ca_cert2_id",
+	"engine", "engine2", "eapol_flags", "sim_num",
+	"openssl_ciphers", "erp",
+#endif /* IEEE8021X_EAPOL */
+	"wep_key0", "wep_key1", "wep_key2", "wep_key3",
+	"wep_tx_keyidx", "priority",
+#ifdef IEEE8021X_EAPOL
+	"eap_workaround", "pac_file", "fragment_size", "ocsp",
+#endif /* IEEE8021X_EAPOL */
+#ifdef CONFIG_MESH
+	"mode", "no_auto_peer",
+#else /* CONFIG_MESH */
+	"mode",
+#endif /* CONFIG_MESH */
+	"proactive_key_caching", "disabled", "id_str",
+#ifdef CONFIG_IEEE80211W
+	"ieee80211w",
+#endif /* CONFIG_IEEE80211W */
+	"peerkey", "mixed_cell", "frequency", "fixed_freq",
+#ifdef CONFIG_MESH
+	"mesh_basic_rates", "dot11MeshMaxRetries",
+	"dot11MeshRetryTimeout", "dot11MeshConfirmTimeout",
+	"dot11MeshHoldingTimeout",
+#endif /* CONFIG_MESH */
+	"wpa_ptk_rekey", "bgscan", "ignore_broadcast_ssid",
+#ifdef CONFIG_P2P
+	"go_p2p_dev_addr", "p2p_client_list", "psk_list",
+#endif /* CONFIG_P2P */
+#ifdef CONFIG_HT_OVERRIDES
+	"disable_ht", "disable_ht40", "disable_sgi", "disable_ldpc",
+	"ht40_intolerant", "disable_max_amsdu", "ampdu_factor",
+	"ampdu_density", "ht_mcs",
+#endif /* CONFIG_HT_OVERRIDES */
+#ifdef CONFIG_VHT_OVERRIDES
+	"disable_vht", "vht_capa", "vht_capa_mask", "vht_rx_mcs_nss_1",
+	"vht_rx_mcs_nss_2", "vht_rx_mcs_nss_3", "vht_rx_mcs_nss_4",
+	"vht_rx_mcs_nss_5", "vht_rx_mcs_nss_6", "vht_rx_mcs_nss_7",
+	"vht_rx_mcs_nss_8", "vht_tx_mcs_nss_1", "vht_tx_mcs_nss_2",
+	"vht_tx_mcs_nss_3", "vht_tx_mcs_nss_4", "vht_tx_mcs_nss_5",
+	"vht_tx_mcs_nss_6", "vht_tx_mcs_nss_7", "vht_tx_mcs_nss_8",
+#endif /* CONFIG_VHT_OVERRIDES */
+	"ap_max_inactivity", "dtim_period", "beacon_int",
+#ifdef CONFIG_MACSEC
+	"macsec_policy",
+#endif /* CONFIG_MACSEC */
+#ifdef CONFIG_HS20
+	"update_identifier",
+#endif /* CONFIG_HS20 */
+	"mac_addr"
+};
+
+
+static char ** wpa_cli_complete_network(const char *str, int pos)
+{
+	int arg = get_cmd_arg_num(str, pos);
+	int i, num_fields = ARRAY_SIZE(network_fields);
+	char **res = NULL;
+
+	switch (arg) {
+	case 1:
+		res = cli_txt_list_array(&networks);
+		break;
+	case 2:
+		res = os_calloc(num_fields + 1, sizeof(char *));
+		if (res == NULL)
+			return NULL;
+		for (i = 0; i < num_fields; i++) {
+			res[i] = os_strdup(network_fields[i]);
+			if (res[i] == NULL)
+				break;
+		}
+	}
+	return res;
+}
+
+
+static char ** wpa_cli_complete_network_id(const char *str, int pos)
+{
+	int arg = get_cmd_arg_num(str, pos);
+	if (arg == 1)
+		return cli_txt_list_array(&networks);
+	return NULL;
+}
+
+
 static int wpa_cli_cmd_dup_network(struct wpa_ctrl *ctrl, int argc,
 				   char *argv[])
 {
@@ -1447,6 +1661,31 @@ static int wpa_cli_cmd_dup_network(struct wpa_ctrl *ctrl, int argc,
 }
 
 
+static char ** wpa_cli_complete_dup_network(const char *str, int pos)
+{
+	int arg = get_cmd_arg_num(str, pos);
+	int i, num_fields = ARRAY_SIZE(network_fields);
+	char **res = NULL;
+
+	switch (arg) {
+	case 1:
+	case 2:
+		res = cli_txt_list_array(&networks);
+		break;
+	case 3:
+		res = os_calloc(num_fields + 1, sizeof(char *));
+		if (res == NULL)
+			return NULL;
+		for (i = 0; i < num_fields; i++) {
+			res[i] = os_strdup(network_fields[i]);
+			if (res[i] == NULL)
+				break;
+		}
+	}
+	return res;
+}
+
+
 static int wpa_cli_cmd_list_creds(struct wpa_ctrl *ctrl, int argc,
 				  char *argv[])
 {
@@ -1621,20 +1860,20 @@ static int wpa_cli_cmd_interface_add(struct wpa_ctrl *ctrl, int argc,
 		printf("Invalid INTERFACE_ADD command: needs at least one "
 		       "argument (interface name)\n"
 		       "All arguments: ifname confname driver ctrl_interface "
-		       "driver_param bridge_name\n");
+		       "driver_param bridge_name [create]\n");
 		return -1;
 	}
 
 	/*
 	 * INTERFACE_ADD <ifname>TAB<confname>TAB<driver>TAB<ctrl_interface>TAB
-	 * <driver_param>TAB<bridge_name>
+	 * <driver_param>TAB<bridge_name>[TAB<create>]
 	 */
 	res = os_snprintf(cmd, sizeof(cmd),
-			  "INTERFACE_ADD %s\t%s\t%s\t%s\t%s\t%s",
+			  "INTERFACE_ADD %s\t%s\t%s\t%s\t%s\t%s\t%s",
 			  argv[0],
 			  argc > 1 ? argv[1] : "", argc > 2 ? argv[2] : "",
 			  argc > 3 ? argv[3] : "", argc > 4 ? argv[4] : "",
-			  argc > 5 ? argv[5] : "");
+			  argc > 5 ? argv[5] : "", argc > 6 ? argv[6] : "");
 	if (os_snprintf_error(sizeof(cmd), res))
 		return -1;
 	cmd[sizeof(cmd) - 1] = '\0';
@@ -2431,6 +2670,13 @@ static int wpa_cli_cmd_tdls_teardown(struct wpa_ctrl *ctrl, int argc,
 }
 
 
+static int wpa_cli_cmd_tdls_link_status(struct wpa_ctrl *ctrl, int argc,
+					char *argv[])
+{
+	return wpa_cli_cmd(ctrl, "TDLS_LINK_STATUS", 1, argc, argv);
+}
+
+
 static int wpa_cli_cmd_wmm_ac_addts(struct wpa_ctrl *ctrl, int argc,
 				    char *argv[])
 {
@@ -2570,6 +2816,13 @@ static int wpa_cli_cmd_mac_rand_scan(struct wpa_ctrl *ctrl, int argc,
 }
 
 
+static int wpa_cli_cmd_get_pref_freq_list(struct wpa_ctrl *ctrl, int argc,
+					  char *argv[])
+{
+	return wpa_cli_cmd(ctrl, "GET_PREF_FREQ_LIST", 1, argc, argv);
+}
+
+
 enum wpa_cli_cmd_flags {
 	cli_cmd_flag_none		= 0x00,
 	cli_cmd_flag_sensitive		= 0x01
@@ -2583,7 +2836,7 @@ struct wpa_cli_cmd {
 	const char *usage;
 };
 
-static struct wpa_cli_cmd wpa_cli_commands[] = {
+static const struct wpa_cli_cmd wpa_cli_commands[] = {
 	{ "status", wpa_cli_cmd_status, NULL,
 	  cli_cmd_flag_none,
 	  "[verbose] = get current WPA/EAPOL/EAP status" },
@@ -2624,7 +2877,7 @@ static struct wpa_cli_cmd wpa_cli_commands[] = {
 	{ "dump", wpa_cli_cmd_dump, NULL,
 	  cli_cmd_flag_none,
 	  "= dump config variables" },
-	{ "get", wpa_cli_cmd_get, NULL,
+	{ "get", wpa_cli_cmd_get, wpa_cli_complete_get,
 	  cli_cmd_flag_none,
 	  "<name> = get information" },
 	{ "logon", wpa_cli_cmd_logon, NULL,
@@ -2686,29 +2939,33 @@ static struct wpa_cli_cmd wpa_cli_commands[] = {
 	{ "list_networks", wpa_cli_cmd_list_networks, NULL,
 	  cli_cmd_flag_none,
 	  "= list configured networks" },
-	{ "select_network", wpa_cli_cmd_select_network, NULL,
+	{ "select_network", wpa_cli_cmd_select_network,
+	  wpa_cli_complete_network_id,
 	  cli_cmd_flag_none,
 	  "<network id> = select a network (disable others)" },
-	{ "enable_network", wpa_cli_cmd_enable_network, NULL,
+	{ "enable_network", wpa_cli_cmd_enable_network,
+	  wpa_cli_complete_network_id,
 	  cli_cmd_flag_none,
 	  "<network id> = enable a network" },
-	{ "disable_network", wpa_cli_cmd_disable_network, NULL,
+	{ "disable_network", wpa_cli_cmd_disable_network,
+	  wpa_cli_complete_network_id,
 	  cli_cmd_flag_none,
 	  "<network id> = disable a network" },
 	{ "add_network", wpa_cli_cmd_add_network, NULL,
 	  cli_cmd_flag_none,
 	  "= add a network" },
-	{ "remove_network", wpa_cli_cmd_remove_network, NULL,
+	{ "remove_network", wpa_cli_cmd_remove_network,
+	  wpa_cli_complete_network_id,
 	  cli_cmd_flag_none,
 	  "<network id> = remove a network" },
-	{ "set_network", wpa_cli_cmd_set_network, NULL,
+	{ "set_network", wpa_cli_cmd_set_network, wpa_cli_complete_network,
 	  cli_cmd_flag_sensitive,
 	  "<network id> <variable> <value> = set network variables (shows\n"
 	  "  list of variables when run without arguments)" },
-	{ "get_network", wpa_cli_cmd_get_network, NULL,
+	{ "get_network", wpa_cli_cmd_get_network, wpa_cli_complete_network,
 	  cli_cmd_flag_none,
 	  "<network id> <variable> = get network variables" },
-	{ "dup_network", wpa_cli_cmd_dup_network, NULL,
+	{ "dup_network", wpa_cli_cmd_dup_network, wpa_cli_complete_dup_network,
 	  cli_cmd_flag_none,
 	  "<src network id> <dst network id> <variable> = duplicate network variables"
 	},
@@ -2750,7 +3007,7 @@ static struct wpa_cli_cmd wpa_cli_commands[] = {
 	{ "get_capability", wpa_cli_cmd_get_capability, NULL,
 	  cli_cmd_flag_none,
 	  "<eap/pairwise/group/key_mgmt/proto/auth_alg/channels/freq/modes> "
-	  "= get capabilies" },
+	  "= get capabilities" },
 	{ "reconfigure", wpa_cli_cmd_reconfigure, NULL,
 	  cli_cmd_flag_none,
 	  "= force wpa_supplicant to re-read its configuration file" },
@@ -3054,6 +3311,9 @@ static struct wpa_cli_cmd wpa_cli_commands[] = {
 	{ "tdls_teardown", wpa_cli_cmd_tdls_teardown, NULL,
 	  cli_cmd_flag_none,
 	  "<addr> = tear down TDLS with <addr>" },
+	{ "tdls_link_status", wpa_cli_cmd_tdls_link_status, NULL,
+	  cli_cmd_flag_none,
+	  "<addr> = TDLS link status with <addr>" },
 	{ "wmm_ac_addts", wpa_cli_cmd_wmm_ac_addts, NULL,
 	  cli_cmd_flag_none,
 	  "<uplink/downlink/bidi> <tsid=0..7> <up=0..7> [nominal_msdu_size=#] "
@@ -3117,6 +3377,9 @@ static struct wpa_cli_cmd wpa_cli_commands[] = {
 	  "<scan|sched|pno|all> enable=<0/1> [addr=mac-address "
 	  "mask=mac-address-mask] = scan MAC randomization"
 	},
+	{ "get_pref_freq_list", wpa_cli_cmd_get_pref_freq_list, NULL,
+	  cli_cmd_flag_none,
+	  "<interface type> = retrieve preferred freq list for the specified interface type" },
 	{ NULL, NULL, NULL, cli_cmd_flag_none, NULL }
 };
 
@@ -3124,7 +3387,7 @@ static struct wpa_cli_cmd wpa_cli_commands[] = {
 /*
  * Prints command usage, lines are padded with the specified string.
  */
-static void print_cmd_help(struct wpa_cli_cmd *cmd, const char *pad)
+static void print_cmd_help(const struct wpa_cli_cmd *cmd, const char *pad)
 {
 	char c;
 	size_t n;
@@ -3262,7 +3525,7 @@ static char ** wpa_cli_edit_completion_cb(void *ctx, const char *str, int pos)
 
 static int wpa_request(struct wpa_ctrl *ctrl, int argc, char *argv[])
 {
-	struct wpa_cli_cmd *cmd, *match = NULL;
+	const struct wpa_cli_cmd *cmd, *match = NULL;
 	int count;
 	int ret = 0;
 
@@ -3347,6 +3610,9 @@ static void wpa_cli_action_process(const char *msg)
 	const char *ifname = ctrl_ifname;
 	char ifname_buf[100];
 
+	if (eloop_terminated())
+		return;
+
 	pos = msg;
 	if (os_strncmp(pos, "IFNAME=", 7) == 0) {
 		const char *end;
@@ -3524,7 +3790,7 @@ static void cli_event(const char *str)
 		s = os_strchr(start, ' ');
 		if (s == NULL)
 			return;
-		cli_txt_list_add_word(&p2p_groups, s + 1);
+		cli_txt_list_add_word(&p2p_groups, s + 1, ' ');
 		return;
 	}
 
@@ -3532,7 +3798,7 @@ static void cli_event(const char *str)
 		s = os_strchr(start, ' ');
 		if (s == NULL)
 			return;
-		cli_txt_list_del_word(&p2p_groups, s + 1);
+		cli_txt_list_del_word(&p2p_groups, s + 1, ' ');
 		return;
 	}
 #endif /* CONFIG_P2P */
@@ -3691,7 +3957,11 @@ static void start_edit(void)
 	ps = wpa_ctrl_get_remote_ifname(ctrl_conn);
 #endif /* CONFIG_CTRL_IFACE_UDP_REMOTE */
 
+#ifdef CONFIG_WPA_CLI_HISTORY_DIR
+	home = CONFIG_WPA_CLI_HISTORY_DIR;
+#else /* CONFIG_WPA_CLI_HISTORY_DIR */
 	home = getenv("HOME");
+#endif /* CONFIG_WPA_CLI_HISTORY_DIR */
 	if (home) {
 		const char *fname = ".wpa_cli_history";
 		int hfile_len = os_strlen(home) + 1 + os_strlen(fname) + 1;
@@ -3774,6 +4044,38 @@ static void update_ifnames(struct wpa_ctrl *ctrl)
 }
 
 
+static void update_networks(struct wpa_ctrl *ctrl)
+{
+	char buf[4096];
+	size_t len = sizeof(buf);
+	int ret;
+	char *cmd = "LIST_NETWORKS";
+	char *pos, *end;
+	int header = 1;
+
+	cli_txt_list_flush(&networks);
+
+	if (ctrl == NULL)
+		return;
+	ret = wpa_ctrl_request(ctrl, cmd, os_strlen(cmd), buf, &len, NULL);
+	if (ret < 0)
+		return;
+	buf[len] = '\0';
+
+	pos = buf;
+	while (pos) {
+		end = os_strchr(pos, '\n');
+		if (end == NULL)
+			break;
+		*end = '\0';
+		if (!header)
+			cli_txt_list_add_word(&networks, pos, '\t');
+		header = 0;
+		pos = end + 1;
+	}
+}
+
+
 static void try_connection(void *eloop_ctx, void *timeout_ctx)
 {
 	if (ctrl_conn)
@@ -3794,6 +4096,7 @@ static void try_connection(void *eloop_ctx, void *timeout_ctx)
 	}
 
 	update_bssid_list(ctrl_conn);
+	update_networks(ctrl_conn);
 
 	if (warning_displayed)
 		printf("Connection established.\n");
@@ -3815,6 +4118,7 @@ static void wpa_cli_interactive(void)
 	cli_txt_list_flush(&p2p_groups);
 	cli_txt_list_flush(&bsses);
 	cli_txt_list_flush(&ifnames);
+	cli_txt_list_flush(&networks);
 	if (edit_started)
 		edit_deinit(hfile, wpa_cli_edit_filter_history_cb);
 	os_free(hfile);
@@ -3823,45 +4127,49 @@ static void wpa_cli_interactive(void)
 }
 
 
+static void wpa_cli_action_ping(void *eloop_ctx, void *timeout_ctx)
+{
+	struct wpa_ctrl *ctrl = eloop_ctx;
+	char buf[256];
+	size_t len;
+
+	/* verify that connection is still working */
+	len = sizeof(buf) - 1;
+	if (wpa_ctrl_request(ctrl, "PING", 4, buf, &len,
+			     wpa_cli_action_cb) < 0 ||
+	    len < 4 || os_memcmp(buf, "PONG", 4) != 0) {
+		printf("wpa_supplicant did not reply to PING command - exiting\n");
+		eloop_terminate();
+		return;
+	}
+	eloop_register_timeout(ping_interval, 0, wpa_cli_action_ping,
+			       ctrl, NULL);
+}
+
+
+static void wpa_cli_action_receive(int sock, void *eloop_ctx, void *sock_ctx)
+{
+	struct wpa_ctrl *ctrl = eloop_ctx;
+
+	wpa_cli_recv_pending(ctrl, 1);
+}
+
+
 static void wpa_cli_action(struct wpa_ctrl *ctrl)
 {
 #ifdef CONFIG_ANSI_C_EXTRA
 	/* TODO: ANSI C version(?) */
 	printf("Action processing not supported in ANSI C build.\n");
 #else /* CONFIG_ANSI_C_EXTRA */
-	fd_set rfds;
-	int fd, res;
-	struct timeval tv;
-	char buf[256]; /* note: large enough to fit in unsolicited messages */
-	size_t len;
+	int fd;
 
 	fd = wpa_ctrl_get_fd(ctrl);
-
-	while (!wpa_cli_quit) {
-		FD_ZERO(&rfds);
-		FD_SET(fd, &rfds);
-		tv.tv_sec = ping_interval;
-		tv.tv_usec = 0;
-		res = select(fd + 1, &rfds, NULL, NULL, &tv);
-		if (res < 0 && errno != EINTR) {
-			perror("select");
-			break;
-		}
-
-		if (FD_ISSET(fd, &rfds))
-			wpa_cli_recv_pending(ctrl, 1);
-		else {
-			/* verify that connection is still working */
-			len = sizeof(buf) - 1;
-			if (wpa_ctrl_request(ctrl, "PING", 4, buf, &len,
-					     wpa_cli_action_cb) < 0 ||
-			    len < 4 || os_memcmp(buf, "PONG", 4) != 0) {
-				printf("wpa_supplicant did not reply to PING "
-				       "command - exiting\n");
-				break;
-			}
-		}
-	}
+	eloop_register_timeout(ping_interval, 0, wpa_cli_action_ping,
+			       ctrl, NULL);
+	eloop_register_read_sock(fd, wpa_cli_action_receive, ctrl, NULL);
+	eloop_run();
+	eloop_cancel_timeout(wpa_cli_action_ping, ctrl, NULL);
+	eloop_unregister_read_sock(fd);
 #endif /* CONFIG_ANSI_C_EXTRA */
 }
 
@@ -3886,18 +4194,17 @@ static char * wpa_cli_get_default_ifname(void)
 {
 	char *ifname = NULL;
 
+#ifdef ANDROID
+	char ifprop[PROPERTY_VALUE_MAX];
+	if (property_get("wifi.interface", ifprop, NULL) != 0) {
+		ifname = os_strdup(ifprop);
+		printf("Using interface '%s'\n", ifname ? ifname : "N/A");
+	}
+#else /* ANDROID */
 #ifdef CONFIG_CTRL_IFACE_UNIX
 	struct dirent *dent;
 	DIR *dir = opendir(ctrl_iface_dir);
 	if (!dir) {
-#ifdef ANDROID
-		char ifprop[PROPERTY_VALUE_MAX];
-		if (property_get("wifi.interface", ifprop, NULL) != 0) {
-			ifname = os_strdup(ifprop);
-			printf("Using interface '%s'\n", ifname);
-			return ifname;
-		}
-#endif /* ANDROID */
 		return NULL;
 	}
 	while ((dent = readdir(dir))) {
@@ -3941,6 +4248,7 @@ static char * wpa_cli_get_default_ifname(void)
 	}
 	wpa_ctrl_close(ctrl);
 #endif /* CONFIG_CTRL_IFACE_NAMED_PIPE */
+#endif /* ANDROID */
 
 	return ifname;
 }
@@ -3957,7 +4265,7 @@ int main(int argc, char *argv[])
 		return -1;
 
 	for (;;) {
-		c = getopt(argc, argv, "a:Bg:G:hi:p:P:v");
+		c = getopt(argc, argv, "a:Bg:G:hi:p:P:s:v");
 		if (c < 0)
 			break;
 		switch (c) {
@@ -3989,6 +4297,9 @@ int main(int argc, char *argv[])
 		case 'P':
 			pid_file = optarg;
 			break;
+		case 's':
+			client_socket_dir = optarg;
+			break;
 		default:
 			usage();
 			return -1;
diff --git a/wpa_supplicant/wpa_priv.c b/wpa_supplicant/wpa_priv.c
index ac38d69d5d1b..850ec405b42c 100644
--- a/wpa_supplicant/wpa_priv.c
+++ b/wpa_supplicant/wpa_priv.c
@@ -29,8 +29,9 @@ struct wpa_priv_interface {
 	char *sock_name;
 	int fd;
 
-	struct wpa_driver_ops *driver;
+	const struct wpa_driver_ops *driver;
 	void *drv_priv;
+	void *drv_global_priv;
 	struct sockaddr_un drv_addr;
 	int wpas_registered;
 
@@ -48,6 +49,10 @@ static void wpa_priv_cmd_register(struct wpa_priv_interface *iface,
 		if (iface->driver->deinit)
 			iface->driver->deinit(iface->drv_priv);
 		iface->drv_priv = NULL;
+		if (iface->drv_global_priv) {
+			iface->driver->global_deinit(iface->drv_global_priv);
+			iface->drv_global_priv = NULL;
+		}
 		iface->wpas_registered = 0;
 	}
 
@@ -58,10 +63,24 @@ static void wpa_priv_cmd_register(struct wpa_priv_interface *iface,
 		iface->l2 = NULL;
 	}
 
-	if (iface->driver->init == NULL)
+	if (iface->driver->init2) {
+		if (iface->driver->global_init) {
+			iface->drv_global_priv = iface->driver->global_init();
+			if (!iface->drv_global_priv) {
+				wpa_printf(MSG_INFO,
+					   "Failed to initialize driver global context");
+				return;
+			}
+		} else {
+			iface->drv_global_priv = NULL;
+		}
+		iface->drv_priv = iface->driver->init2(iface, iface->ifname,
+						       iface->drv_global_priv);
+	} else if (iface->driver->init) {
+		iface->drv_priv = iface->driver->init(iface, iface->ifname);
+	} else {
 		return;
-
-	iface->drv_priv = iface->driver->init(iface, iface->ifname);
+	}
 	if (iface->drv_priv == NULL) {
 		wpa_printf(MSG_DEBUG, "Failed to initialize driver wrapper");
 		return;
@@ -87,6 +106,10 @@ static void wpa_priv_cmd_unregister(struct wpa_priv_interface *iface,
 		if (iface->driver->deinit)
 			iface->driver->deinit(iface->drv_priv);
 		iface->drv_priv = NULL;
+		if (iface->drv_global_priv) {
+			iface->driver->global_deinit(iface->drv_global_priv);
+			iface->drv_global_priv = NULL;
+		}
 		iface->wpas_registered = 0;
 	}
 }
@@ -172,6 +195,58 @@ static void wpa_priv_cmd_get_scan_results(struct wpa_priv_interface *iface,
 }
 
 
+static void wpa_priv_cmd_authenticate(struct wpa_priv_interface *iface,
+				      void *buf, size_t len)
+{
+	struct wpa_driver_auth_params params;
+	struct privsep_cmd_authenticate *auth;
+	int res, i;
+
+	if (iface->drv_priv == NULL || iface->driver->authenticate == NULL)
+		return;
+
+	if (len < sizeof(*auth)) {
+		wpa_printf(MSG_DEBUG, "Invalid authentication request");
+		return;
+	}
+
+	auth = buf;
+	if (sizeof(*auth) + auth->ie_len + auth->sae_data_len > len) {
+		wpa_printf(MSG_DEBUG, "Authentication request overflow");
+		return;
+	}
+
+	os_memset(&params, 0, sizeof(params));
+	params.freq = auth->freq;
+	params.bssid = auth->bssid;
+	params.ssid = auth->ssid;
+	if (auth->ssid_len > SSID_MAX_LEN)
+		return;
+	params.ssid_len = auth->ssid_len;
+	params.auth_alg = auth->auth_alg;
+	for (i = 0; i < 4; i++) {
+		if (auth->wep_key_len[i]) {
+			params.wep_key[i] = auth->wep_key[i];
+			params.wep_key_len[i] = auth->wep_key_len[i];
+		}
+	}
+	params.wep_tx_keyidx = auth->wep_tx_keyidx;
+	params.local_state_change = auth->local_state_change;
+	params.p2p = auth->p2p;
+	if (auth->ie_len) {
+		params.ie = (u8 *) (auth + 1);
+		params.ie_len = auth->ie_len;
+	}
+	if (auth->sae_data_len) {
+		params.sae_data = ((u8 *) (auth + 1)) + auth->ie_len;
+		params.sae_data_len = auth->sae_data_len;
+	}
+
+	res = iface->driver->authenticate(iface->drv_priv, &params);
+	wpa_printf(MSG_DEBUG, "drv->authenticate: res=%d", res);
+}
+
+
 static void wpa_priv_cmd_associate(struct wpa_priv_interface *iface,
 				   void *buf, size_t len)
 {
@@ -199,7 +274,7 @@ static void wpa_priv_cmd_associate(struct wpa_priv_interface *iface,
 	if (bssid[0] | bssid[1] | bssid[2] | bssid[3] | bssid[4] | bssid[5])
 		params.bssid = bssid;
 	params.ssid = assoc->ssid;
-	if (assoc->ssid_len > 32)
+	if (assoc->ssid_len > SSID_MAX_LEN)
 		return;
 	params.ssid_len = assoc->ssid_len;
 	params.freq.mode = assoc->hwmode;
@@ -244,7 +319,7 @@ fail:
 static void wpa_priv_cmd_get_ssid(struct wpa_priv_interface *iface,
 				  struct sockaddr_un *from)
 {
-	u8 ssid[sizeof(int) + 32];
+	u8 ssid[sizeof(int) + SSID_MAX_LEN];
 	int res;
 
 	if (iface->drv_priv == NULL)
@@ -254,7 +329,7 @@ static void wpa_priv_cmd_get_ssid(struct wpa_priv_interface *iface,
 		goto fail;
 
 	res = iface->driver->get_ssid(iface->drv_priv, &ssid[sizeof(int)]);
-	if (res < 0 || res > 32)
+	if (res < 0 || res > SSID_MAX_LEN)
 		goto fail;
 	os_memcpy(ssid, &res, sizeof(int));
 
@@ -307,6 +382,10 @@ static void wpa_priv_cmd_get_capa(struct wpa_priv_interface *iface,
 	    iface->driver->get_capa(iface->drv_priv, &capa) < 0)
 		goto fail;
 
+	/* For now, no support for passing extended_capa pointers */
+	capa.extended_capa = NULL;
+	capa.extended_capa_mask = NULL;
+	capa.extended_capa_len = 0;
 	sendto(iface->fd, &capa, sizeof(capa), 0, (struct sockaddr *) from,
 	       sizeof(*from));
 	return;
@@ -356,7 +435,8 @@ static void wpa_priv_cmd_l2_register(struct wpa_priv_interface *iface,
 	}
 
 	proto = reg_cmd[0];
-	if (proto != ETH_P_EAPOL && proto != ETH_P_RSN_PREAUTH) {
+	if (proto != ETH_P_EAPOL && proto != ETH_P_RSN_PREAUTH &&
+	    proto != ETH_P_80211_ENCAP) {
 		wpa_printf(MSG_DEBUG, "Refused l2_packet connection for "
 			   "ethertype 0x%x", proto);
 		return;
@@ -529,6 +609,9 @@ static void wpa_priv_receive(int sock, void *eloop_ctx, void *sock_ctx)
 		pos[cmd_len] = '\0';
 		wpa_priv_cmd_set_country(iface, pos);
 		break;
+	case PRIVSEP_CMD_AUTHENTICATE:
+		wpa_priv_cmd_authenticate(iface, cmd_buf, cmd_len);
+		break;
 	}
 }
 
@@ -698,6 +781,36 @@ static int wpa_priv_send_event(struct wpa_priv_interface *iface, int event,
 }
 
 
+static void wpa_priv_send_auth(struct wpa_priv_interface *iface,
+			       union wpa_event_data *data)
+{
+	size_t buflen = sizeof(struct privsep_event_auth) + data->auth.ies_len;
+	struct privsep_event_auth *auth;
+	u8 *buf, *pos;
+
+	buf = os_malloc(buflen);
+	if (buf == NULL)
+		return;
+
+	auth = (struct privsep_event_auth *) buf;
+	pos = (u8 *) (auth + 1);
+
+	os_memcpy(auth->peer, data->auth.peer, ETH_ALEN);
+	os_memcpy(auth->bssid, data->auth.bssid, ETH_ALEN);
+	auth->auth_type = data->auth.auth_type;
+	auth->auth_transaction = data->auth.auth_transaction;
+	auth->status_code = data->auth.status_code;
+	if (data->auth.ies) {
+		os_memcpy(pos, data->auth.ies, data->auth.ies_len);
+		auth->ies_len = data->auth.ies_len;
+	}
+
+	wpa_priv_send_event(iface, PRIVSEP_EVENT_AUTH, buf, buflen);
+
+	os_free(buf);
+}
+
+
 static void wpa_priv_send_assoc(struct wpa_priv_interface *iface, int event,
 				union wpa_event_data *data)
 {
@@ -851,6 +964,10 @@ void wpa_supplicant_event(void *ctx, enum wpa_event_type event,
 				    &data->michael_mic_failure.unicast,
 				    sizeof(int));
 		break;
+	case EVENT_SCAN_STARTED:
+		wpa_priv_send_event(iface, PRIVSEP_EVENT_SCAN_STARTED, NULL,
+				    0);
+		break;
 	case EVENT_SCAN_RESULTS:
 		wpa_priv_send_event(iface, PRIVSEP_EVENT_SCAN_RESULTS, NULL,
 				    0);
@@ -874,9 +991,12 @@ void wpa_supplicant_event(void *ctx, enum wpa_event_type event,
 	case EVENT_FT_RESPONSE:
 		wpa_priv_send_ft_response(iface, data);
 		break;
+	case EVENT_AUTH:
+		wpa_priv_send_auth(iface, data);
+		break;
 	default:
-		wpa_printf(MSG_DEBUG, "Unsupported driver event %d - TODO",
-			   event);
+		wpa_printf(MSG_DEBUG, "Unsupported driver event %d (%s) - TODO",
+			   event, event_to_string(event));
 		break;
 	}
 }
@@ -944,8 +1064,9 @@ static void usage(void)
 	       "contributors\n"
 	       "\n"
 	       "usage:\n"
-	       "  wpa_priv [-Bdd] [-P<pid file>] <driver:ifname> "
-	       "[driver:ifname ...]\n");
+	       "  wpa_priv [-Bdd] [-c<ctrl dir>] [-P<pid file>] "
+	       "<driver:ifname> \\\n"
+	       "           [driver:ifname ...]\n");
 }
 
 
@@ -982,20 +1103,20 @@ int main(int argc, char *argv[])
 			break;
 		default:
 			usage();
-			goto out;
+			goto out2;
 		}
 	}
 
 	if (optind >= argc) {
 		usage();
-		goto out;
+		goto out2;
 	}
 
 	wpa_printf(MSG_DEBUG, "wpa_priv control directory: '%s'", ctrl_dir);
 
 	if (eloop_init()) {
 		wpa_printf(MSG_ERROR, "Failed to initialize event loop");
-		goto out;
+		goto out2;
 	}
 
 	for (i = optind; i < argc; i++) {
@@ -1025,7 +1146,9 @@ out:
 
 	eloop_destroy();
 
-	os_daemonize_terminate(pid_file);
+out2:
+	if (daemonize)
+		os_daemonize_terminate(pid_file);
 	os_free(pid_file);
 	os_program_deinit();
 
diff --git a/wpa_supplicant/wpa_supplicant.c b/wpa_supplicant/wpa_supplicant.c
index 19fb8900bd75..ef55fdcf79c0 100644
--- a/wpa_supplicant/wpa_supplicant.c
+++ b/wpa_supplicant/wpa_supplicant.c
@@ -35,6 +35,7 @@
 #include "common/ieee802_11_defs.h"
 #include "common/hw_features_common.h"
 #include "p2p/p2p.h"
+#include "fst/fst.h"
 #include "blacklist.h"
 #include "wpas_glue.h"
 #include "wps_supplicant.h"
@@ -55,11 +56,11 @@
 #include "wpas_kay.h"
 #include "mesh.h"
 
-const char *wpa_supplicant_version =
+const char *const wpa_supplicant_version =
 "wpa_supplicant v" VERSION_STR "\n"
 "Copyright (c) 2003-2015, Jouni Malinen <j@w1.fi> and contributors";
 
-const char *wpa_supplicant_license =
+const char *const wpa_supplicant_license =
 "This software may be distributed under the terms of the BSD license.\n"
 "See README for more details.\n"
 #ifdef EAP_TLS_OPENSSL
@@ -70,16 +71,16 @@ const char *wpa_supplicant_license =
 
 #ifndef CONFIG_NO_STDOUT_DEBUG
 /* Long text divided into parts in order to fit in C89 strings size limits. */
-const char *wpa_supplicant_full_license1 =
+const char *const wpa_supplicant_full_license1 =
 "";
-const char *wpa_supplicant_full_license2 =
+const char *const wpa_supplicant_full_license2 =
 "This software may be distributed under the terms of the BSD license.\n"
 "\n"
 "Redistribution and use in source and binary forms, with or without\n"
 "modification, are permitted provided that the following conditions are\n"
 "met:\n"
 "\n";
-const char *wpa_supplicant_full_license3 =
+const char *const wpa_supplicant_full_license3 =
 "1. Redistributions of source code must retain the above copyright\n"
 "   notice, this list of conditions and the following disclaimer.\n"
 "\n"
@@ -87,7 +88,7 @@ const char *wpa_supplicant_full_license3 =
 "   notice, this list of conditions and the following disclaimer in the\n"
 "   documentation and/or other materials provided with the distribution.\n"
 "\n";
-const char *wpa_supplicant_full_license4 =
+const char *const wpa_supplicant_full_license4 =
 "3. Neither the name(s) of the above-listed copyright holder(s) nor the\n"
 "   names of its contributors may be used to endorse or promote products\n"
 "   derived from this software without specific prior written permission.\n"
@@ -96,7 +97,7 @@ const char *wpa_supplicant_full_license4 =
 "\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n"
 "LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n"
 "A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n";
-const char *wpa_supplicant_full_license5 =
+const char *const wpa_supplicant_full_license5 =
 "OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n"
 "SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n"
 "LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n"
@@ -456,6 +457,8 @@ static void wpa_supplicant_cleanup(struct wpa_supplicant *wpa_s)
 			     wpa_s, NULL);
 #endif /* CONFIG_DELAYED_MIC_ERROR_REPORT */
 
+	eloop_cancel_timeout(wpas_network_reenabled, wpa_s, NULL);
+
 	wpas_wps_deinit(wpa_s);
 
 	wpabuf_free(wpa_s->pending_eapol_rx);
@@ -491,6 +494,16 @@ static void wpa_supplicant_cleanup(struct wpa_supplicant *wpa_s)
 
 	wpas_mac_addr_rand_scan_clear(wpa_s, MAC_ADDR_RAND_ALL);
 
+	/*
+	 * Need to remove any pending gas-query radio work before the
+	 * gas_query_deinit() call because gas_query::work has not yet been set
+	 * for works that have not been started. gas_query_free() will be unable
+	 * to cancel such pending radio works and once the pending gas-query
+	 * radio work eventually gets removed, the deinit notification call to
+	 * gas_query_start_cb() would result in dereferencing freed memory.
+	 */
+	if (wpa_s->radio)
+		radio_remove_works(wpa_s, "gas-query", 0);
 	gas_query_deinit(wpa_s->gas);
 	wpa_s->gas = NULL;
 
@@ -716,6 +729,30 @@ void wpa_supplicant_set_state(struct wpa_supplicant *wpa_s,
 		wpa_s->normal_scans = 0;
 	}
 
+#ifdef CONFIG_P2P
+	/*
+	 * P2PS client has to reply to Probe Request frames received on the
+	 * group operating channel. Enable Probe Request frame reporting for
+	 * P2P connected client in case p2p_cli_probe configuration property is
+	 * set to 1.
+	 */
+	if (wpa_s->conf->p2p_cli_probe && wpa_s->current_ssid &&
+	    wpa_s->current_ssid->mode == WPAS_MODE_INFRA &&
+	    wpa_s->current_ssid->p2p_group) {
+		if (state == WPA_COMPLETED && !wpa_s->p2p_cli_probe) {
+			wpa_dbg(wpa_s, MSG_DEBUG,
+				"P2P: Enable CLI Probe Request RX reporting");
+			wpa_s->p2p_cli_probe =
+				wpa_drv_probe_req_report(wpa_s, 1) >= 0;
+		} else if (state != WPA_COMPLETED && wpa_s->p2p_cli_probe) {
+			wpa_dbg(wpa_s, MSG_DEBUG,
+				"P2P: Disable CLI Probe Request RX reporting");
+			wpa_s->p2p_cli_probe = 0;
+			wpa_drv_probe_req_report(wpa_s, 0);
+		}
+	}
+#endif /* CONFIG_P2P */
+
 	if (state != WPA_SCANNING)
 		wpa_supplicant_notify_scanning(wpa_s, 0);
 
@@ -729,6 +766,7 @@ void wpa_supplicant_set_state(struct wpa_supplicant *wpa_s,
 			ssid && ssid->id_str ? ssid->id_str : "");
 #endif /* CONFIG_CTRL_IFACE || !CONFIG_NO_STDOUT_DEBUG */
 		wpas_clear_temp_disabled(wpa_s, ssid, 1);
+		wpa_blacklist_clear(wpa_s);
 		wpa_s->extra_blacklist_count = 0;
 		wpa_s->new_connection = 0;
 		wpa_drv_set_operstate(wpa_s, 1);
@@ -870,7 +908,8 @@ int wpa_supplicant_reload_configuration(struct wpa_supplicant *wpa_s)
 
 	eapol_sm_invalidate_cached_session(wpa_s->eapol);
 	if (wpa_s->current_ssid) {
-		wpa_s->own_disconnect_req = 1;
+		if (wpa_s->wpa_state >= WPA_AUTHENTICATING)
+			wpa_s->own_disconnect_req = 1;
 		wpa_supplicant_deauthenticate(wpa_s,
 					      WLAN_REASON_DEAUTH_LEAVING);
 	}
@@ -1237,7 +1276,12 @@ int wpa_supplicant_set_suites(struct wpa_supplicant *wpa_s,
 	}
 
 	if (wpa_key_mgmt_wpa_psk(ssid->key_mgmt)) {
-		wpa_sm_set_pmk(wpa_s->wpa, ssid->psk, PMK_LEN, NULL);
+		int psk_set = 0;
+
+		if (ssid->psk_set) {
+			wpa_sm_set_pmk(wpa_s->wpa, ssid->psk, PMK_LEN, NULL);
+			psk_set = 1;
+		}
 #ifndef CONFIG_NO_PBKDF2
 		if (bss && ssid->bssid_set && ssid->ssid_len == 0 &&
 		    ssid->passphrase) {
@@ -1247,6 +1291,7 @@ int wpa_supplicant_set_suites(struct wpa_supplicant *wpa_s,
 		        wpa_hexdump_key(MSG_MSGDUMP, "PSK (from passphrase)",
 					psk, PMK_LEN);
 			wpa_sm_set_pmk(wpa_s->wpa, psk, PMK_LEN, NULL);
+			psk_set = 1;
 			os_memset(psk, 0, sizeof(psk));
 		}
 #endif /* CONFIG_NO_PBKDF2 */
@@ -1284,6 +1329,7 @@ int wpa_supplicant_set_suites(struct wpa_supplicant *wpa_s,
 						"external passphrase)",
 						psk, PMK_LEN);
 				wpa_sm_set_pmk(wpa_s->wpa, psk, PMK_LEN, NULL);
+				psk_set = 1;
 				os_memset(psk, 0, sizeof(psk));
 			} else
 #endif /* CONFIG_NO_PBKDF2 */
@@ -1296,6 +1342,7 @@ int wpa_supplicant_set_suites(struct wpa_supplicant *wpa_s,
 					return -1;
 				}
 				wpa_sm_set_pmk(wpa_s->wpa, psk, PMK_LEN, NULL);
+				psk_set = 1;
 				os_memset(psk, 0, sizeof(psk));
 			} else {
 				wpa_msg(wpa_s, MSG_INFO, "EXT PW: No suitable "
@@ -1309,6 +1356,12 @@ int wpa_supplicant_set_suites(struct wpa_supplicant *wpa_s,
 			ext_password_free(pw);
 		}
 #endif /* CONFIG_EXT_PASSWORD */
+
+		if (!psk_set) {
+			wpa_msg(wpa_s, MSG_INFO,
+				"No PSK available for association");
+			return -1;
+		}
 	} else
 		wpa_sm_set_pmk_from_pmksa(wpa_s->wpa);
 
@@ -1914,7 +1967,8 @@ static void wpas_start_assoc_cb(struct wpa_radio_work *work, int deinit)
 
 	wpa_s->connect_work = work;
 
-	if (cwork->bss_removed || !wpas_valid_bss_ssid(wpa_s, bss, ssid)) {
+	if (cwork->bss_removed || !wpas_valid_bss_ssid(wpa_s, bss, ssid) ||
+	    wpas_network_disabled(wpa_s, ssid)) {
 		wpa_dbg(wpa_s, MSG_DEBUG, "BSS/SSID entry for association not valid anymore - drop connection attempt");
 		wpas_connect_work_done(wpa_s);
 		return;
@@ -1963,7 +2017,9 @@ static void wpas_start_assoc_cb(struct wpa_radio_work *work, int deinit)
 			wpa_ssid_txt(ssid->ssid, ssid->ssid_len));
 		os_memset(wpa_s->pending_bssid, 0, ETH_ALEN);
 	}
-	wpa_supplicant_cancel_sched_scan(wpa_s);
+	if (!wpa_s->pno)
+		wpa_supplicant_cancel_sched_scan(wpa_s);
+
 	wpa_supplicant_cancel_scan(wpa_s);
 
 	/* Starting new association, so clear the possibly used WPA IE from the
@@ -2136,6 +2192,18 @@ static void wpas_start_assoc_cb(struct wpa_radio_work *work, int deinit)
 		}
 	}
 
+#ifdef CONFIG_FST
+	if (wpa_s->fst_ies) {
+		int fst_ies_len = wpabuf_len(wpa_s->fst_ies);
+
+		if (wpa_ie_len + fst_ies_len <= sizeof(wpa_ie)) {
+			os_memcpy(wpa_ie + wpa_ie_len,
+				  wpabuf_head(wpa_s->fst_ies), fst_ies_len);
+			wpa_ie_len += fst_ies_len;
+		}
+	}
+#endif /* CONFIG_FST */
+
 	wpa_clear_keys(wpa_s, bss ? bss->bssid : NULL);
 	use_crypt = 1;
 	cipher_pairwise = wpa_s->pairwise_cipher;
@@ -2383,7 +2451,8 @@ static void wpas_start_assoc_cb(struct wpa_radio_work *work, int deinit)
 	}
 	old_ssid = wpa_s->current_ssid;
 	wpa_s->current_ssid = ssid;
-	wpa_s->current_bss = bss;
+	if (!wpas_driver_bss_selection(wpa_s) || ssid->bssid_set)
+		wpa_s->current_bss = bss;
 	wpa_supplicant_rsn_supp_set_config(wpa_s, wpa_s->current_ssid);
 	wpa_supplicant_initiate_eapol(wpa_s);
 	if (old_ssid != wpa_s->current_ssid)
@@ -2505,15 +2574,20 @@ void wpa_supplicant_enable_network(struct wpa_supplicant *wpa_s,
 	} else
 		wpa_supplicant_enable_one_network(wpa_s, ssid);
 
-	if (wpa_s->reassociate && !wpa_s->disconnected) {
+	if (wpa_s->reassociate && !wpa_s->disconnected &&
+	    (!wpa_s->current_ssid ||
+	     wpa_s->wpa_state == WPA_DISCONNECTED ||
+	     wpa_s->wpa_state == WPA_SCANNING)) {
 		if (wpa_s->sched_scanning) {
 			wpa_printf(MSG_DEBUG, "Stop ongoing sched_scan to add "
 				   "new network to scan filters");
 			wpa_supplicant_cancel_sched_scan(wpa_s);
 		}
 
-		if (wpa_supplicant_fast_associate(wpa_s) != 1)
+		if (wpa_supplicant_fast_associate(wpa_s) != 1) {
+			wpa_s->scan_req = NORMAL_SCAN_REQ;
 			wpa_supplicant_req_scan(wpa_s, 0, 0);
+		}
 	}
 }
 
@@ -2586,7 +2660,8 @@ void wpa_supplicant_select_network(struct wpa_supplicant *wpa_s,
 	int disconnected = 0;
 
 	if (ssid && ssid != wpa_s->current_ssid && wpa_s->current_ssid) {
-		wpa_s->own_disconnect_req = 1;
+		if (wpa_s->wpa_state >= WPA_AUTHENTICATING)
+			wpa_s->own_disconnect_req = 1;
 		wpa_supplicant_deauthenticate(
 			wpa_s, WLAN_REASON_DEAUTH_LEAVING);
 		disconnected = 1;
@@ -2625,6 +2700,13 @@ void wpa_supplicant_select_network(struct wpa_supplicant *wpa_s,
 		eapol_sm_notify_config(wpa_s->eapol, NULL, NULL);
 		wpa_s->connect_without_scan =
 			(ssid->mode == WPAS_MODE_MESH) ? ssid : NULL;
+
+		/*
+		 * Don't optimize next scan freqs since a new ESS has been
+		 * selected.
+		 */
+		os_free(wpa_s->next_scan_freqs);
+		wpa_s->next_scan_freqs = NULL;
 	} else {
 		wpa_s->connect_without_scan = NULL;
 	}
@@ -2633,8 +2715,10 @@ void wpa_supplicant_select_network(struct wpa_supplicant *wpa_s,
 	wpa_s->reassociate = 1;
 
 	if (wpa_s->connect_without_scan ||
-	    wpa_supplicant_fast_associate(wpa_s) != 1)
+	    wpa_supplicant_fast_associate(wpa_s) != 1) {
+		wpa_s->scan_req = NORMAL_SCAN_REQ;
 		wpa_supplicant_req_scan(wpa_s, 0, disconnected ? 100000 : 0);
+	}
 
 	if (ssid)
 		wpas_notify_network_selected(wpa_s, ssid);
@@ -2709,6 +2793,11 @@ int wpa_supplicant_set_ap_scan(struct wpa_supplicant *wpa_s, int ap_scan)
 	if (ap_scan < 0 || ap_scan > 2)
 		return -1;
 
+	if (ap_scan == 2 && os_strcmp(wpa_s->driver->name, "nl80211") == 0) {
+		wpa_printf(MSG_INFO,
+			   "Note: nl80211 driver interface is not designed to be used with ap_scan=2; this can result in connection failures");
+	}
+
 #ifdef ANDROID
 	if (ap_scan == 2 && ap_scan != wpa_s->conf->ap_scan &&
 	    wpa_s->wpa_state >= WPA_ASSOCIATING &&
@@ -2848,7 +2937,7 @@ int wpa_supplicant_set_debug_params(struct wpa_global *global, int debug_level,
 struct wpa_ssid * wpa_supplicant_get_ssid(struct wpa_supplicant *wpa_s)
 {
 	struct wpa_ssid *entry;
-	u8 ssid[MAX_SSID_LEN];
+	u8 ssid[SSID_MAX_LEN];
 	int res;
 	size_t ssid_len;
 	u8 bssid[ETH_ALEN];
@@ -3053,12 +3142,36 @@ void wpa_supplicant_rx_eapol(void *ctx, const u8 *src_addr,
 	    (wpa_s->current_ssid == NULL ||
 	     wpa_s->current_ssid->mode != IEEE80211_MODE_IBSS)) {
 		/* Timeout for completing IEEE 802.1X and WPA authentication */
-		wpa_supplicant_req_auth_timeout(
-			wpa_s,
-			(wpa_key_mgmt_wpa_ieee8021x(wpa_s->key_mgmt) ||
-			 wpa_s->key_mgmt == WPA_KEY_MGMT_IEEE8021X_NO_WPA ||
-			 wpa_s->key_mgmt == WPA_KEY_MGMT_WPS) ?
-			70 : 10, 0);
+		int timeout = 10;
+
+		if (wpa_key_mgmt_wpa_ieee8021x(wpa_s->key_mgmt) ||
+		    wpa_s->key_mgmt == WPA_KEY_MGMT_IEEE8021X_NO_WPA ||
+		    wpa_s->key_mgmt == WPA_KEY_MGMT_WPS) {
+			/* Use longer timeout for IEEE 802.1X/EAP */
+			timeout = 70;
+		}
+
+#ifdef CONFIG_WPS
+		if (wpa_s->current_ssid && wpa_s->current_bss &&
+		    (wpa_s->current_ssid->key_mgmt & WPA_KEY_MGMT_WPS) &&
+		    eap_is_wps_pin_enrollee(&wpa_s->current_ssid->eap)) {
+			/*
+			 * Use shorter timeout if going through WPS AP iteration
+			 * for PIN config method with an AP that does not
+			 * advertise Selected Registrar.
+			 */
+			struct wpabuf *wps_ie;
+
+			wps_ie = wpa_bss_get_vendor_ie_multi(
+				wpa_s->current_bss, WPS_IE_VENDOR_TYPE);
+			if (wps_ie &&
+			    !wps_is_addr_authorized(wps_ie, wpa_s->own_addr, 1))
+				timeout = 10;
+			wpabuf_free(wps_ie);
+		}
+#endif /* CONFIG_WPS */
+
+		wpa_supplicant_req_auth_timeout(wpa_s, timeout, 0);
 	}
 	wpa_s->eapol_received++;
 
@@ -3190,6 +3303,12 @@ int wpa_supplicant_driver_init(struct wpa_supplicant *wpa_s)
 		}
 	}
 
+	if (wpa_s->conf->ap_scan == 2 &&
+	    os_strcmp(wpa_s->driver->name, "nl80211") == 0) {
+		wpa_printf(MSG_INFO,
+			   "Note: nl80211 driver interface is not designed to be used with ap_scan=2; this can result in connection failures");
+	}
+
 	wpa_clear_keys(wpa_s, NULL);
 
 	/* Make sure that TKIP countermeasures are not left enabled (could
@@ -3615,6 +3734,124 @@ int wpas_init_ext_pw(struct wpa_supplicant *wpa_s)
 }
 
 
+#ifdef CONFIG_FST
+
+static const u8 * wpas_fst_get_bssid_cb(void *ctx)
+{
+	struct wpa_supplicant *wpa_s = ctx;
+
+	return (is_zero_ether_addr(wpa_s->bssid) ||
+		wpa_s->wpa_state != WPA_COMPLETED) ? NULL : wpa_s->bssid;
+}
+
+
+static void wpas_fst_get_channel_info_cb(void *ctx,
+					 enum hostapd_hw_mode *hw_mode,
+					 u8 *channel)
+{
+	struct wpa_supplicant *wpa_s = ctx;
+
+	if (wpa_s->current_bss) {
+		*hw_mode = ieee80211_freq_to_chan(wpa_s->current_bss->freq,
+						  channel);
+	} else if (wpa_s->hw.num_modes) {
+		*hw_mode = wpa_s->hw.modes[0].mode;
+	} else {
+		WPA_ASSERT(0);
+		*hw_mode = 0;
+	}
+}
+
+
+static int wpas_fst_get_hw_modes(void *ctx, struct hostapd_hw_modes **modes)
+{
+	struct wpa_supplicant *wpa_s = ctx;
+
+	*modes = wpa_s->hw.modes;
+	return wpa_s->hw.num_modes;
+}
+
+
+static void wpas_fst_set_ies_cb(void *ctx, const struct wpabuf *fst_ies)
+{
+	struct wpa_supplicant *wpa_s = ctx;
+
+	wpa_hexdump_buf(MSG_DEBUG, "FST: Set IEs", fst_ies);
+	wpa_s->fst_ies = fst_ies;
+}
+
+
+static int wpas_fst_send_action_cb(void *ctx, const u8 *da, struct wpabuf *data)
+{
+	struct wpa_supplicant *wpa_s = ctx;
+
+	WPA_ASSERT(os_memcmp(wpa_s->bssid, da, ETH_ALEN) == 0);
+	return wpa_drv_send_action(wpa_s, wpa_s->assoc_freq, 0, wpa_s->bssid,
+					  wpa_s->own_addr, wpa_s->bssid,
+					  wpabuf_head(data), wpabuf_len(data),
+				   0);
+}
+
+
+static const struct wpabuf * wpas_fst_get_mb_ie_cb(void *ctx, const u8 *addr)
+{
+	struct wpa_supplicant *wpa_s = ctx;
+
+	WPA_ASSERT(os_memcmp(wpa_s->bssid, addr, ETH_ALEN) == 0);
+	return wpa_s->received_mb_ies;
+}
+
+
+static void wpas_fst_update_mb_ie_cb(void *ctx, const u8 *addr,
+				     const u8 *buf, size_t size)
+{
+	struct wpa_supplicant *wpa_s = ctx;
+	struct mb_ies_info info;
+
+	WPA_ASSERT(os_memcmp(wpa_s->bssid, addr, ETH_ALEN) == 0);
+
+	if (!mb_ies_info_by_ies(&info, buf, size)) {
+		wpabuf_free(wpa_s->received_mb_ies);
+		wpa_s->received_mb_ies = mb_ies_by_info(&info);
+	}
+}
+
+
+const u8 * wpas_fst_get_peer_first(void *ctx, struct fst_get_peer_ctx **get_ctx,
+				   Boolean mb_only)
+{
+	struct wpa_supplicant *wpa_s = ctx;
+
+	*get_ctx = NULL;
+	if (!is_zero_ether_addr(wpa_s->bssid))
+		return (wpa_s->received_mb_ies || !mb_only) ?
+			wpa_s->bssid : NULL;
+	return NULL;
+}
+
+
+const u8 * wpas_fst_get_peer_next(void *ctx, struct fst_get_peer_ctx **get_ctx,
+				  Boolean mb_only)
+{
+	return NULL;
+}
+
+void fst_wpa_supplicant_fill_iface_obj(struct wpa_supplicant *wpa_s,
+				       struct fst_wpa_obj *iface_obj)
+{
+	iface_obj->ctx              = wpa_s;
+	iface_obj->get_bssid        = wpas_fst_get_bssid_cb;
+	iface_obj->get_channel_info = wpas_fst_get_channel_info_cb;
+	iface_obj->get_hw_modes     = wpas_fst_get_hw_modes;
+	iface_obj->set_ies          = wpas_fst_set_ies_cb;
+	iface_obj->send_action      = wpas_fst_send_action_cb;
+	iface_obj->get_mb_ie        = wpas_fst_get_mb_ie_cb;
+	iface_obj->update_mb_ie     = wpas_fst_update_mb_ie_cb;
+	iface_obj->get_peer_first   = wpas_fst_get_peer_first;
+	iface_obj->get_peer_next    = wpas_fst_get_peer_next;
+}
+#endif /* CONFIG_FST */
+
 static int wpas_set_wowlan_triggers(struct wpa_supplicant *wpa_s,
 				    const struct wpa_driver_capa *capa)
 {
@@ -4153,6 +4390,28 @@ static int wpa_supplicant_init_iface(struct wpa_supplicant *wpa_s,
 		return -1;
 	}
 
+#ifdef CONFIG_FST
+	if (wpa_s->conf->fst_group_id) {
+		struct fst_iface_cfg cfg;
+		struct fst_wpa_obj iface_obj;
+
+		fst_wpa_supplicant_fill_iface_obj(wpa_s, &iface_obj);
+		os_strlcpy(cfg.group_id, wpa_s->conf->fst_group_id,
+			   sizeof(cfg.group_id));
+		cfg.priority = wpa_s->conf->fst_priority;
+		cfg.llt = wpa_s->conf->fst_llt;
+
+		wpa_s->fst = fst_attach(wpa_s->ifname, wpa_s->own_addr,
+					&iface_obj, &cfg);
+		if (!wpa_s->fst) {
+			wpa_msg(wpa_s, MSG_ERROR,
+				"FST: Cannot attach iface %s to group %s",
+				wpa_s->ifname, cfg.group_id);
+			return -1;
+		}
+	}
+#endif /* CONFIG_FST */
+
 	if (wpas_wps_init(wpa_s))
 		return -1;
 
@@ -4261,6 +4520,17 @@ static void wpa_supplicant_deinit_iface(struct wpa_supplicant *wpa_s,
 	wpas_ctrl_radio_work_flush(wpa_s);
 	radio_remove_interface(wpa_s);
 
+#ifdef CONFIG_FST
+	if (wpa_s->fst) {
+		fst_detach(wpa_s->fst);
+		wpa_s->fst = NULL;
+	}
+	if (wpa_s->received_mb_ies) {
+		wpabuf_free(wpa_s->received_mb_ies);
+		wpa_s->received_mb_ies = NULL;
+	}
+#endif /* CONFIG_FST */
+
 	if (wpa_s->drv_priv)
 		wpa_drv_deinit(wpa_s);
 
@@ -4287,6 +4557,8 @@ static void wpa_supplicant_deinit_iface(struct wpa_supplicant *wpa_s,
 		wpa_s->conf = NULL;
 	}
 
+	os_free(wpa_s->ssids_from_scan_req);
+
 	os_free(wpa_s);
 }
 
@@ -4362,8 +4634,10 @@ struct wpa_supplicant * wpa_supplicant_add_iface(struct wpa_global *global,
 
 #ifdef CONFIG_P2P
 	if (wpa_s->global->p2p == NULL &&
+	    !wpa_s->global->p2p_disabled && !wpa_s->conf->p2p_disabled &&
 	    (wpa_s->drv_flags & WPA_DRIVER_FLAGS_DEDICATED_P2P_DEVICE) &&
-	    wpas_p2p_add_p2pdev_interface(wpa_s, iface->conf_p2p_dev) < 0) {
+	    wpas_p2p_add_p2pdev_interface(
+		    wpa_s, wpa_s->global->params.conf_p2p_dev) < 0) {
 		wpa_printf(MSG_INFO,
 			   "P2P: Failed to enable P2P Device interface");
 		/* Try to continue without. P2P will be disabled. */
@@ -4489,6 +4763,33 @@ static const char * wpa_supplicant_msg_ifname_cb(void *ctx)
 #endif /* CONFIG_NO_WPA_MSG */
 
 
+#ifndef WPA_SUPPLICANT_CLEANUP_INTERVAL
+#define WPA_SUPPLICANT_CLEANUP_INTERVAL 10
+#endif /* WPA_SUPPLICANT_CLEANUP_INTERVAL */
+
+/* Periodic cleanup tasks */
+static void wpas_periodic(void *eloop_ctx, void *timeout_ctx)
+{
+	struct wpa_global *global = eloop_ctx;
+	struct wpa_supplicant *wpa_s;
+
+	eloop_register_timeout(WPA_SUPPLICANT_CLEANUP_INTERVAL, 0,
+			       wpas_periodic, global, NULL);
+
+#ifdef CONFIG_P2P
+	if (global->p2p)
+		p2p_expire_peers(global->p2p);
+#endif /* CONFIG_P2P */
+
+	for (wpa_s = global->ifaces; wpa_s; wpa_s = wpa_s->next) {
+		wpa_bss_flush_by_age(wpa_s, wpa_s->conf->bss_expiration_age);
+#ifdef CONFIG_AP
+		ap_periodic(wpa_s);
+#endif /* CONFIG_AP */
+	}
+}
+
+
 /**
  * wpa_supplicant_init - Initialize %wpa_supplicant
  * @params: Parameters for %wpa_supplicant
@@ -4563,6 +4864,11 @@ struct wpa_global * wpa_supplicant_init(struct wpa_params *params)
 	if (params->override_ctrl_interface)
 		global->params.override_ctrl_interface =
 			os_strdup(params->override_ctrl_interface);
+#ifdef CONFIG_P2P
+	if (params->conf_p2p_dev)
+		global->params.conf_p2p_dev =
+			os_strdup(params->conf_p2p_dev);
+#endif /* CONFIG_P2P */
 	wpa_debug_level = global->params.wpa_debug_level =
 		params->wpa_debug_level;
 	wpa_debug_show_keys = global->params.wpa_debug_show_keys =
@@ -4612,6 +4918,9 @@ struct wpa_global * wpa_supplicant_init(struct wpa_params *params)
 	}
 #endif /* CONFIG_WIFI_DISPLAY */
 
+	eloop_register_timeout(WPA_SUPPLICANT_CLEANUP_INTERVAL, 0,
+			       wpas_periodic, global, NULL);
+
 	return global;
 }
 
@@ -4663,6 +4972,8 @@ void wpa_supplicant_deinit(struct wpa_global *global)
 	if (global == NULL)
 		return;
 
+	eloop_cancel_timeout(wpas_periodic, global, NULL);
+
 #ifdef CONFIG_WIFI_DISPLAY
 	wifi_display_deinit(global);
 #endif /* CONFIG_WIFI_DISPLAY */
@@ -4699,6 +5010,9 @@ void wpa_supplicant_deinit(struct wpa_global *global)
 	os_free(global->params.ctrl_interface_group);
 	os_free(global->params.override_driver);
 	os_free(global->params.override_ctrl_interface);
+#ifdef CONFIG_P2P
+	os_free(global->params.conf_p2p_dev);
+#endif /* CONFIG_P2P */
 
 	os_free(global->p2p_disallow_freq.range);
 	os_free(global->p2p_go_avoid_freq.range);
@@ -4958,6 +5272,15 @@ int wpa_supplicant_ctrl_iface_ctrl_rsp_handle(struct wpa_supplicant *wpa_s,
 		str_clear_free(eap->external_sim_resp);
 		eap->external_sim_resp = os_strdup(value);
 		break;
+	case WPA_CTRL_REQ_PSK_PASSPHRASE:
+		if (wpa_config_set(ssid, "psk", value, 0) < 0)
+			return -1;
+		ssid->mem_only_psk = 1;
+		if (ssid->passphrase)
+			wpa_config_update_psk(ssid);
+		if (wpa_s->wpa_state == WPA_SCANNING && !wpa_s->scanning)
+			wpa_supplicant_req_scan(wpa_s, 0, 0);
+		break;
 	default:
 		wpa_printf(MSG_DEBUG, "CTRL_IFACE: Unknown field '%s'", field);
 		return -1;
@@ -5005,7 +5328,8 @@ int wpas_network_disabled(struct wpa_supplicant *wpa_s, struct wpa_ssid *ssid)
 	}
 
 	if (wpa_key_mgmt_wpa_psk(ssid->key_mgmt) && !ssid->psk_set &&
-	    (!ssid->passphrase || ssid->ssid_len != 0) && !ssid->ext_psk)
+	    (!ssid->passphrase || ssid->ssid_len != 0) && !ssid->ext_psk &&
+	    !ssid->mem_only_psk)
 		return 1;
 
 	return 0;
@@ -5227,7 +5551,8 @@ int get_shared_radio_freqs_data(struct wpa_supplicant *wpa_s,
 			continue;
 
 		if (ifs->current_ssid->mode == WPAS_MODE_AP ||
-		    ifs->current_ssid->mode == WPAS_MODE_P2P_GO)
+		    ifs->current_ssid->mode == WPAS_MODE_P2P_GO ||
+		    ifs->current_ssid->mode == WPAS_MODE_MESH)
 			freq = ifs->current_ssid->frequency;
 		else if (wpa_drv_get_bssid(ifs, bssid) == 0)
 			freq = ifs->assoc_freq;
@@ -5243,7 +5568,7 @@ int get_shared_radio_freqs_data(struct wpa_supplicant *wpa_s,
 			freqs_data[idx++].freq = freq;
 
 		if (ifs->current_ssid->mode == WPAS_MODE_INFRA) {
-			freqs_data[i].flags = ifs->current_ssid->p2p_group ?
+			freqs_data[i].flags |= ifs->current_ssid->p2p_group ?
 				WPA_FREQ_USED_BY_P2P_CLIENT :
 				WPA_FREQ_USED_BY_INFRA_STATION;
 		}
diff --git a/wpa_supplicant/wpa_supplicant.conf b/wpa_supplicant/wpa_supplicant.conf
index 8964b3f720a2..6fe67e473c5e 100644
--- a/wpa_supplicant/wpa_supplicant.conf
+++ b/wpa_supplicant/wpa_supplicant.conf
@@ -108,6 +108,10 @@ eapol_version=1
 #    the driver reports successful association; each network block should have
 #    explicit security policy (i.e., only one option in the lists) for
 #    key_mgmt, pairwise, group, proto variables
+# Note: ap_scan=2 should not be used with the nl80211 driver interface (the
+# current Linux interface). ap_scan=1 is optimized work working with nl80211.
+# For finding networks using hidden SSID, scan_ssid=1 in the network block can
+# be used with nl80211.
 # When using IBSS or AP mode, ap_scan=2 mode can force the new network to be
 # created immediately regardless of scan results. ap_scan=1 mode will first try
 # to scan for existing networks and only if no matches with the enabled
@@ -268,6 +272,11 @@ fast_reauth=1
 #wps_nfc_dh_privkey: Hexdump of DH Private Key
 #wps_nfc_dev_pw: Hexdump of Device Password
 
+# Priority for the networks added through WPS
+# This priority value will be set to each network profile that is added
+# by executing the WPS protocol.
+#wps_priority=0
+
 # Maximum number of BSS entries to keep in memory
 # Default: 200
 # This can be used to limit memory use on the BSS entries (cached scan
@@ -297,6 +306,10 @@ fast_reauth=1
 # format: <backend name>[:<optional backend parameters>]
 #ext_password_backend=test:pw1=password|pw2=testing
 
+
+# Disable P2P functionality
+# p2p_disabled=1
+
 # Timeout in seconds to detect STA inactivity (default: 300 seconds)
 #
 # This timeout value is used in P2P GO mode to clean up
@@ -740,6 +753,11 @@ fast_reauth=1
 # startup and reconfiguration time can be optimized by generating the PSK only
 # only when the passphrase or SSID has actually changed.
 #
+# mem_only_psk: Whether to keep PSK/passphrase only in memory
+# 0 = allow psk/passphrase to be stored to the configuration file
+# 1 = do not store psk/passphrase to the configuration file
+#mem_only_psk=0
+#
 # eapol_flags: IEEE 802.1X/EAPOL options (bit field)
 # Dynamic WEP key required for non-WPA mode
 # bit0 (1): require dynamically generated unicast WEP key
@@ -969,9 +987,10 @@ fast_reauth=1
 # tls_disable_session_ticket=0 - allow TLS Session Ticket extension to be used
 #	Note: If not set, this is automatically set to 1 for EAP-TLS/PEAP/TTLS
 #	as a workaround for broken authentication server implementations unless
-#	EAP workarounds are disabled with eap_workarounds=0.
+#	EAP workarounds are disabled with eap_workaround=0.
 #	For EAP-FAST, this must be set to 0 (or left unconfigured for the
 #	default value to be used automatically).
+# tls_disable_tlsv1_0=1 - disable use of TLSv1.0
 # tls_disable_tlsv1_1=1 - disable use of TLSv1.1 (a workaround for AAA servers
 #	that have issues interoperating with updated TLS version)
 # tls_disable_tlsv1_2=1 - disable use of TLSv1.2 (a workaround for AAA servers
@@ -1122,6 +1141,32 @@ fast_reauth=1
 #  2: MCS 0-9
 #  3: not supported
 
+##### Fast Session Transfer (FST) support #####################################
+#
+# The options in this section are only available when the build configuration
+# option CONFIG_FST is set while compiling hostapd. They allow this interface
+# to be a part of FST setup.
+#
+# FST is the transfer of a session from a channel to another channel, in the
+# same or different frequency bands.
+#
+# For detals, see IEEE Std 802.11ad-2012.
+
+# Identifier of an FST Group  the interface belongs to.
+#fst_group_id=bond0
+
+# Interface priority within the FST Group.
+# Announcing a higher priority for an interface means declaring it more
+# preferable for FST switch.
+# fst_priority is in 1..255 range with 1 being the lowest priority.
+#fst_priority=100
+
+# Default LLT value for this interface in milliseconds. The value used in case
+# no value provided during session setup. Default is 50 msec.
+# fst_llt is in 1..4294967 range (due to spec limitation, see 10.32.2.2
+# Transitioning between states).
+#fst_llt=100
+
 # Example blocks:
 
 # Simple case: WPA-PSK, PSK as an ASCII passphrase, allow all valid ciphers
diff --git a/wpa_supplicant/wpa_supplicant_i.h b/wpa_supplicant/wpa_supplicant_i.h
index 26ff216b02da..58df48c548ea 100644
--- a/wpa_supplicant/wpa_supplicant_i.h
+++ b/wpa_supplicant/wpa_supplicant_i.h
@@ -17,14 +17,14 @@
 #include "config_ssid.h"
 #include "wmm_ac.h"
 
-extern const char *wpa_supplicant_version;
-extern const char *wpa_supplicant_license;
+extern const char *const wpa_supplicant_version;
+extern const char *const wpa_supplicant_license;
 #ifndef CONFIG_NO_STDOUT_DEBUG
-extern const char *wpa_supplicant_full_license1;
-extern const char *wpa_supplicant_full_license2;
-extern const char *wpa_supplicant_full_license3;
-extern const char *wpa_supplicant_full_license4;
-extern const char *wpa_supplicant_full_license5;
+extern const char *const wpa_supplicant_full_license1;
+extern const char *const wpa_supplicant_full_license2;
+extern const char *const wpa_supplicant_full_license3;
+extern const char *const wpa_supplicant_full_license4;
+extern const char *const wpa_supplicant_full_license5;
 #endif /* CONFIG_NO_STDOUT_DEBUG */
 
 struct wpa_sm;
@@ -66,17 +66,6 @@ struct wpa_interface {
 	 */
 	const char *confanother;
 
-#ifdef CONFIG_P2P
-	/**
-	 * conf_p2p_dev - Configuration file used to hold the
-	 * P2P Device configuration parameters.
-	 *
-	 * This can also be %NULL. In such a case, if a P2P Device dedicated
-	 * interfaces is created, the main configuration file will be used.
-	 */
-	const char *conf_p2p_dev;
-#endif /* CONFIG_P2P */
-
 	/**
 	 * ctrl_interface - Control interface parameter
 	 *
@@ -227,6 +216,18 @@ struct wpa_params {
 	 * its internal entropy store over restarts.
 	 */
 	char *entropy_file;
+
+#ifdef CONFIG_P2P
+	/**
+	 * conf_p2p_dev - Configuration file used to hold the
+	 * P2P Device configuration parameters.
+	 *
+	 * This can also be %NULL. In such a case, if a P2P Device dedicated
+	 * interfaces is created, the main configuration file will be used.
+	 */
+	char *conf_p2p_dev;
+#endif /* CONFIG_P2P */
+
 };
 
 struct p2p_srv_bonjour {
@@ -366,10 +367,12 @@ struct wps_ap_info {
 	} type;
 	unsigned int tries;
 	struct os_reltime last_attempt;
+	unsigned int pbc_active;
+	u8 uuid[WPS_UUID_LEN];
 };
 
 struct wpa_ssid_value {
-	u8 ssid[32];
+	u8 ssid[SSID_MAX_LEN];
 	size_t ssid_len;
 };
 
@@ -479,7 +482,7 @@ struct wpa_supplicant {
 	struct wpa_ssid_value *disallow_aps_ssid;
 	size_t disallow_aps_ssid_count;
 
-	enum { WPA_SETBAND_AUTO, WPA_SETBAND_5G, WPA_SETBAND_2G } setband;
+	enum set_band setband;
 
 	/* Preferred network for the next connection attempt */
 	struct wpa_ssid *next_ssid;
@@ -518,7 +521,7 @@ struct wpa_supplicant {
 	unsigned int last_scan_res_size;
 	struct os_reltime last_scan;
 
-	struct wpa_driver_ops *driver;
+	const struct wpa_driver_ops *driver;
 	int interface_removed; /* whether the network interface has been
 				* removed */
 	struct wpa_sm *wpa;
@@ -590,6 +593,8 @@ struct wpa_supplicant {
 	} scan_req, last_scan_req;
 	enum wpa_states scan_prev_wpa_state;
 	struct os_reltime scan_trigger_time, scan_start_time;
+	/* Minimum freshness requirement for connection purposes */
+	struct os_reltime scan_min_time;
 	int scan_runs; /* number of scan runs since WPS was started */
 	int *next_scan_freqs;
 	int *manual_scan_freqs;
@@ -609,6 +614,9 @@ struct wpa_supplicant {
 	int scan_id[MAX_SCAN_ID];
 	unsigned int scan_id_count;
 
+	struct wpa_ssid_value *ssids_from_scan_req;
+	unsigned int num_ssids_from_scan_req;
+
 	u64 drv_flags;
 	unsigned int drv_enc;
 	unsigned int drv_smps_modes;
@@ -639,6 +647,7 @@ struct wpa_supplicant {
 	int wps_success; /* WPS success event received */
 	struct wps_er *wps_er;
 	unsigned int wps_run;
+	struct os_reltime wps_pin_start_time;
 	int blacklist_cleared;
 
 	struct wpabuf *pending_eapol_rx;
@@ -648,6 +657,7 @@ struct wpa_supplicant {
 	unsigned int eap_expected_failure:1;
 	unsigned int reattach:1; /* reassociation to the same BSS requested */
 	unsigned int mac_addr_changed:1;
+	unsigned int added_vif:1;
 
 	struct os_reltime last_mac_addr_change;
 	int last_mac_addr_style;
@@ -661,7 +671,7 @@ struct wpa_supplicant {
 
 #ifdef CONFIG_SME
 	struct {
-		u8 ssid[32];
+		u8 ssid[SSID_MAX_LEN];
 		size_t ssid_len;
 		int freq;
 		u8 assoc_req_ie[200];
@@ -736,7 +746,6 @@ struct wpa_supplicant {
 	int p2p_mgmt;
 
 #ifdef CONFIG_P2P
-	struct wpa_supplicant *p2p_dev;
 	struct p2p_go_neg_results *go_params;
 	int create_p2p_iface;
 	u8 pending_interface_addr[ETH_ALEN];
@@ -767,7 +776,7 @@ struct wpa_supplicant {
 	u8 pending_join_iface_addr[ETH_ALEN];
 	u8 pending_join_dev_addr[ETH_ALEN];
 	int pending_join_wps_method;
-	u8 p2p_join_ssid[32];
+	u8 p2p_join_ssid[SSID_MAX_LEN];
 	size_t p2p_join_ssid_len;
 	int p2p_join_scan_count;
 	int auto_pd_scan_retry;
@@ -813,7 +822,8 @@ struct wpa_supplicant {
 	unsigned int p2p_nfc_tag_enabled:1;
 	unsigned int p2p_peer_oob_pk_hash_known:1;
 	unsigned int p2p_disable_ip_addr_req:1;
-	unsigned int p2ps_join_addr_valid:1;
+	unsigned int p2ps_method_config_any:1;
+	unsigned int p2p_cli_probe:1;
 	int p2p_persistent_go_freq;
 	int p2p_persistent_id;
 	int p2p_go_intent;
@@ -969,6 +979,12 @@ struct wpa_supplicant {
 	u8 last_tspecs_count;
 
 	struct rrm_data rrm;
+
+#ifdef CONFIG_FST
+	struct fst_iface *fst;
+	const struct wpabuf *fst_ies;
+	struct wpabuf *received_mb_ies;
+#endif /* CONFIG_FST */
 };
 
 
@@ -1116,13 +1132,13 @@ struct wpa_bss * wpa_supplicant_pick_network(struct wpa_supplicant *wpa_s,
 int eap_register_methods(void);
 
 /**
- * Utility method to tell if a given network is a persistent group
+ * Utility method to tell if a given network is for persistent group storage
  * @ssid: Network object
  * Returns: 1 if network is a persistent group, 0 otherwise
  */
 static inline int network_is_persistent_group(struct wpa_ssid *ssid)
 {
-	return ((ssid->disabled == 2) || ssid->p2p_persistent_group);
+	return ssid->disabled == 2 && ssid->p2p_persistent_group;
 }
 
 int wpas_network_disabled(struct wpa_supplicant *wpa_s, struct wpa_ssid *ssid);
@@ -1140,4 +1156,15 @@ int get_shared_radio_freqs_data(struct wpa_supplicant *wpa_s,
 int get_shared_radio_freqs(struct wpa_supplicant *wpa_s,
 			   int *freq_array, unsigned int len);
 
+void wpas_network_reenabled(void *eloop_ctx, void *timeout_ctx);
+
+#ifdef CONFIG_FST
+
+struct fst_wpa_obj;
+
+void fst_wpa_supplicant_fill_iface_obj(struct wpa_supplicant *wpa_s,
+				       struct fst_wpa_obj *iface_obj);
+
+#endif /* CONFIG_FST */
+
 #endif /* WPA_SUPPLICANT_I_H */
diff --git a/wpa_supplicant/wpas_glue.c b/wpa_supplicant/wpas_glue.c
index 1bb82ba71696..29c22ba2c967 100644
--- a/wpa_supplicant/wpas_glue.c
+++ b/wpa_supplicant/wpas_glue.c
@@ -737,6 +737,8 @@ enum wpa_ctrl_req_type wpa_supplicant_ctrl_req_from_string(const char *field)
 		return WPA_CTRL_REQ_EAP_PASSPHRASE;
 	else if (os_strcmp(field, "SIM") == 0)
 		return WPA_CTRL_REQ_SIM;
+	else if (os_strcmp(field, "PSK_PASSPHRASE") == 0)
+		return WPA_CTRL_REQ_PSK_PASSPHRASE;
 	return WPA_CTRL_REQ_UNKNOWN;
 }
 
@@ -776,6 +778,10 @@ const char * wpa_supplicant_ctrl_req_to_string(enum wpa_ctrl_req_type field,
 	case WPA_CTRL_REQ_SIM:
 		ret = "SIM";
 		break;
+	case WPA_CTRL_REQ_PSK_PASSPHRASE:
+		*txt = "PSK or passphrase";
+		ret = "PSK_PASSPHRASE";
+		break;
 	default:
 		break;
 	}
@@ -789,6 +795,35 @@ const char * wpa_supplicant_ctrl_req_to_string(enum wpa_ctrl_req_type field,
 	return ret;
 }
 
+
+void wpas_send_ctrl_req(struct wpa_supplicant *wpa_s, struct wpa_ssid *ssid,
+			const char *field_name, const char *txt)
+{
+	char *buf;
+	size_t buflen;
+	int len;
+
+	buflen = 100 + os_strlen(txt) + ssid->ssid_len;
+	buf = os_malloc(buflen);
+	if (buf == NULL)
+		return;
+	len = os_snprintf(buf, buflen, "%s-%d:%s needed for SSID ",
+			  field_name, ssid->id, txt);
+	if (os_snprintf_error(buflen, len)) {
+		os_free(buf);
+		return;
+	}
+	if (ssid->ssid && buflen > len + ssid->ssid_len) {
+		os_memcpy(buf + len, ssid->ssid, ssid->ssid_len);
+		len += ssid->ssid_len;
+		buf[len] = '\0';
+	}
+	buf[buflen - 1] = '\0';
+	wpa_msg(wpa_s, MSG_INFO, WPA_CTRL_REQ "%s", buf);
+	os_free(buf);
+}
+
+
 #ifdef IEEE8021X_EAPOL
 #if defined(CONFIG_CTRL_IFACE) || !defined(CONFIG_NO_STDOUT_DEBUG)
 static void wpa_supplicant_eap_param_needed(void *ctx,
@@ -798,9 +833,6 @@ static void wpa_supplicant_eap_param_needed(void *ctx,
 	struct wpa_supplicant *wpa_s = ctx;
 	struct wpa_ssid *ssid = wpa_s->current_ssid;
 	const char *field_name, *txt = NULL;
-	char *buf;
-	size_t buflen;
-	int len;
 
 	if (ssid == NULL)
 		return;
@@ -817,25 +849,7 @@ static void wpa_supplicant_eap_param_needed(void *ctx,
 
 	wpas_notify_eap_status(wpa_s, "eap parameter needed", field_name);
 
-	buflen = 100 + os_strlen(txt) + ssid->ssid_len;
-	buf = os_malloc(buflen);
-	if (buf == NULL)
-		return;
-	len = os_snprintf(buf, buflen,
-			  WPA_CTRL_REQ "%s-%d:%s needed for SSID ",
-			  field_name, ssid->id, txt);
-	if (os_snprintf_error(buflen, len)) {
-		os_free(buf);
-		return;
-	}
-	if (ssid->ssid && buflen > len + ssid->ssid_len) {
-		os_memcpy(buf + len, ssid->ssid, ssid->ssid_len);
-		len += ssid->ssid_len;
-		buf[len] = '\0';
-	}
-	buf[buflen - 1] = '\0';
-	wpa_msg(wpa_s, MSG_INFO, "%s", buf);
-	os_free(buf);
+	wpas_send_ctrl_req(wpa_s, ssid, field_name, txt);
 }
 #else /* CONFIG_CTRL_IFACE || !CONFIG_NO_STDOUT_DEBUG */
 #define wpa_supplicant_eap_param_needed NULL
@@ -1007,7 +1021,8 @@ static int wpa_supplicant_key_mgmt_set_pmk(void *ctx, const u8 *pmk,
 {
 	struct wpa_supplicant *wpa_s = ctx;
 
-	if (wpa_s->conf->key_mgmt_offload)
+	if (wpa_s->conf->key_mgmt_offload &&
+	    (wpa_s->drv_flags & WPA_DRIVER_FLAGS_KEY_MGMT_OFFLOAD))
 		return wpa_drv_set_key(wpa_s, WPA_ALG_PMK, NULL, 0, 0,
 				       NULL, 0, pmk, pmk_len);
 	else
diff --git a/wpa_supplicant/wpas_glue.h b/wpa_supplicant/wpas_glue.h
index 9808c2265f99..5585e5615a65 100644
--- a/wpa_supplicant/wpas_glue.h
+++ b/wpa_supplicant/wpas_glue.h
@@ -22,4 +22,7 @@ const char * wpa_supplicant_ctrl_req_to_string(enum wpa_ctrl_req_type field,
 
 enum wpa_ctrl_req_type wpa_supplicant_ctrl_req_from_string(const char *field);
 
+void wpas_send_ctrl_req(struct wpa_supplicant *wpa_s, struct wpa_ssid *ssid,
+			const char *field_name, const char *txt);
+
 #endif /* WPAS_GLUE_H */
diff --git a/wpa_supplicant/wps_supplicant.c b/wpa_supplicant/wps_supplicant.c
index eabe986541c3..60f761c81b80 100644
--- a/wpa_supplicant/wps_supplicant.c
+++ b/wpa_supplicant/wps_supplicant.c
@@ -39,6 +39,14 @@
 #define WPS_PIN_SCAN_IGNORE_SEL_REG 3
 #endif /* WPS_PIN_SCAN_IGNORE_SEL_REG */
 
+/*
+ * The minimum time in seconds before trying to associate to a WPS PIN AP that
+ * does not have Selected Registrar TRUE.
+ */
+#ifndef WPS_PIN_TIME_IGNORE_SEL_REG
+#define WPS_PIN_TIME_IGNORE_SEL_REG 5
+#endif /* WPS_PIN_TIME_IGNORE_SEL_REG */
+
 static void wpas_wps_timeout(void *eloop_ctx, void *timeout_ctx);
 static void wpas_clear_wps(struct wpa_supplicant *wpa_s);
 
@@ -539,6 +547,7 @@ static int wpa_supplicant_wps_cred(void *ctx,
 			return -1;
 		}
 	}
+	ssid->priority = wpa_s->conf->wps_priority;
 
 	wpas_wps_security_workaround(wpa_s, ssid, cred);
 
@@ -552,6 +561,9 @@ static int wpa_supplicant_wps_cred(void *ctx,
 	}
 #endif /* CONFIG_NO_CONFIG_WRITE */
 
+	if (ssid->priority)
+		wpa_config_update_prio_list(wpa_s->conf);
+
 	/*
 	 * Optimize the post-WPS scan based on the channel used during
 	 * the provisioning in case EAP-Failure is not received.
@@ -880,7 +892,8 @@ static int wpa_supplicant_wps_rf_band(void *ctx)
 	if (!wpa_s->current_ssid || !wpa_s->assoc_freq)
 		return 0;
 
-	return (wpa_s->assoc_freq > 2484) ? WPS_RF_50GHZ : WPS_RF_24GHZ;
+	return (wpa_s->assoc_freq > 50000) ? WPS_RF_60GHZ :
+		(wpa_s->assoc_freq > 2484) ? WPS_RF_50GHZ : WPS_RF_24GHZ;
 }
 
 
@@ -942,8 +955,20 @@ static void wpas_clear_wps(struct wpa_supplicant *wpa_s)
 static void wpas_wps_timeout(void *eloop_ctx, void *timeout_ctx)
 {
 	struct wpa_supplicant *wpa_s = eloop_ctx;
+	union wps_event_data data;
+
 	wpa_msg(wpa_s, MSG_INFO, WPS_EVENT_TIMEOUT "Requested operation timed "
 		"out");
+	os_memset(&data, 0, sizeof(data));
+	data.fail.config_error = WPS_CFG_MSG_TIMEOUT;
+	data.fail.error_indication = WPS_EI_NO_ERROR;
+	/*
+	 * Call wpas_notify_wps_event_fail() directly instead of through
+	 * wpa_supplicant_wps_event() which would end up registering unnecessary
+	 * timeouts (those are only for the case where the failure happens
+	 * during an EAP-WSC exchange).
+	 */
+	wpas_notify_wps_event_fail(wpa_s, &data.fail);
 	wpas_clear_wps(wpa_s);
 }
 
@@ -1178,6 +1203,7 @@ static int wpas_wps_start_dev_pw(struct wpa_supplicant *wpa_s,
 	}
 #ifdef CONFIG_P2P
 	if (p2p_group && wpa_s->go_params && wpa_s->go_params->ssid_len) {
+		os_free(ssid->ssid);
 		ssid->ssid = os_zalloc(wpa_s->go_params->ssid_len + 1);
 		if (ssid->ssid) {
 			ssid->ssid_len = wpa_s->go_params->ssid_len;
@@ -1216,11 +1242,28 @@ static int wpas_wps_start_dev_pw(struct wpa_supplicant *wpa_s,
 int wpas_wps_start_pin(struct wpa_supplicant *wpa_s, const u8 *bssid,
 		       const char *pin, int p2p_group, u16 dev_pw_id)
 {
+	os_get_reltime(&wpa_s->wps_pin_start_time);
 	return wpas_wps_start_dev_pw(wpa_s, NULL, bssid, pin, p2p_group,
 				     dev_pw_id, NULL, NULL, 0, 0);
 }
 
 
+void wpas_wps_pbc_overlap(struct wpa_supplicant *wpa_s)
+{
+	union wps_event_data data;
+
+	os_memset(&data, 0, sizeof(data));
+	data.fail.config_error = WPS_CFG_MULTIPLE_PBC_DETECTED;
+	data.fail.error_indication = WPS_EI_NO_ERROR;
+	/*
+	 * Call wpas_notify_wps_event_fail() directly instead of through
+	 * wpa_supplicant_wps_event() which would end up registering unnecessary
+	 * timeouts (those are only for the case where the failure happens
+	 * during an EAP-WSC exchange).
+	 */
+	wpas_notify_wps_event_fail(wpa_s, &data.fail);
+}
+
 /* Cancel the wps pbc/pin requests */
 int wpas_wps_cancel(struct wpa_supplicant *wpa_s)
 {
@@ -1487,6 +1530,8 @@ int wpas_wps_init(struct wpa_supplicant *wpa_s)
 				wps->dev.rf_bands |= WPS_RF_24GHZ;
 			else if (modes[m].mode == HOSTAPD_MODE_IEEE80211A)
 				wps->dev.rf_bands |= WPS_RF_50GHZ;
+			else if (modes[m].mode == HOSTAPD_MODE_IEEE80211AD)
+				wps->dev.rf_bands |= WPS_RF_60GHZ;
 		}
 	}
 	if (wps->dev.rf_bands == 0) {
@@ -1609,9 +1654,15 @@ int wpas_wps_ssid_bss_match(struct wpa_supplicant *wpa_s,
 		 * external Registrar.
 		 */
 		if (!wps_is_addr_authorized(wps_ie, wpa_s->own_addr, 1)) {
-			if (wpa_s->scan_runs < WPS_PIN_SCAN_IGNORE_SEL_REG) {
-				wpa_printf(MSG_DEBUG, "   skip - WPS AP "
-					   "without active PIN Registrar");
+			struct os_reltime age;
+
+			os_reltime_age(&wpa_s->wps_pin_start_time, &age);
+
+			if (wpa_s->scan_runs < WPS_PIN_SCAN_IGNORE_SEL_REG ||
+			    age.sec < WPS_PIN_TIME_IGNORE_SEL_REG) {
+				wpa_printf(MSG_DEBUG,
+					   "   skip - WPS AP without active PIN Registrar (scan_runs=%d age=%d)",
+					   wpa_s->scan_runs, (int) age.sec);
 				wpabuf_free(wps_ie);
 				return 0;
 			}
@@ -1694,10 +1745,10 @@ int wpas_wps_ssid_wildcard_ok(struct wpa_supplicant *wpa_s,
 int wpas_wps_scan_pbc_overlap(struct wpa_supplicant *wpa_s,
 			      struct wpa_bss *selected, struct wpa_ssid *ssid)
 {
-	const u8 *sel_uuid, *uuid;
+	const u8 *sel_uuid;
 	struct wpabuf *wps_ie;
 	int ret = 0;
-	struct wpa_bss *bss;
+	size_t i;
 
 	if (!eap_is_wps_pbc_enrollee(&ssid->eap))
 		return 0;
@@ -1718,40 +1769,28 @@ int wpas_wps_scan_pbc_overlap(struct wpa_supplicant *wpa_s,
 		sel_uuid = NULL;
 	}
 
-	dl_list_for_each(bss, &wpa_s->bss, struct wpa_bss, list) {
-		struct wpabuf *ie;
-		if (bss == selected)
+	for (i = 0; i < wpa_s->num_wps_ap; i++) {
+		struct wps_ap_info *ap = &wpa_s->wps_ap[i];
+
+		if (!ap->pbc_active ||
+		    os_memcmp(selected->bssid, ap->bssid, ETH_ALEN) == 0)
 			continue;
-		ie = wpa_bss_get_vendor_ie_multi(bss, WPS_IE_VENDOR_TYPE);
-		if (!ie)
-			continue;
-		if (!wps_is_selected_pbc_registrar(ie)) {
-			wpabuf_free(ie);
-			continue;
-		}
+
 		wpa_printf(MSG_DEBUG, "WPS: Another BSS in active PBC mode: "
-			   MACSTR, MAC2STR(bss->bssid));
-		uuid = wps_get_uuid_e(ie);
+			   MACSTR, MAC2STR(ap->bssid));
 		wpa_hexdump(MSG_DEBUG, "WPS: UUID of the other BSS",
-			    uuid, UUID_LEN);
-		if (os_memcmp(selected->bssid, bss->bssid, ETH_ALEN) == 0) {
-			wpabuf_free(ie);
-			continue;
-		}
-		if (sel_uuid == NULL || uuid == NULL ||
-		    os_memcmp(sel_uuid, uuid, UUID_LEN) != 0) {
+			    ap->uuid, UUID_LEN);
+		if (sel_uuid == NULL ||
+		    os_memcmp(sel_uuid, ap->uuid, UUID_LEN) != 0) {
 			ret = 1; /* PBC overlap */
 			wpa_msg(wpa_s, MSG_INFO, "WPS: PBC overlap detected: "
 				MACSTR " and " MACSTR,
 				MAC2STR(selected->bssid),
-				MAC2STR(bss->bssid));
-			wpabuf_free(ie);
+				MAC2STR(ap->bssid));
 			break;
 		}
 
 		/* TODO: verify that this is reasonable dual-band situation */
-
-		wpabuf_free(ie);
 	}
 
 	wpabuf_free(wps_ie);
@@ -1910,7 +1949,7 @@ static int wpas_wps_network_to_cred(struct wpa_ssid *ssid,
 				    struct wps_credential *cred)
 {
 	os_memset(cred, 0, sizeof(*cred));
-	if (ssid->ssid_len > 32)
+	if (ssid->ssid_len > SSID_MAX_LEN)
 		return -1;
 	os_memcpy(cred->ssid, ssid->ssid, ssid->ssid_len);
 	cred->ssid_len = ssid->ssid_len;
@@ -2582,6 +2621,10 @@ static int wpas_wps_nfc_rx_handover_sel(struct wpa_supplicant *wpa_s,
 			 (attr.rf_bands == NULL ||
 			  *attr.rf_bands & WPS_RF_50GHZ))
 			freq = 5000 + 5 * chan;
+		else if (chan >= 1 && chan <= 4 &&
+			 (attr.rf_bands == NULL ||
+			  *attr.rf_bands & WPS_RF_60GHZ))
+			freq = 56160 + 2160 * chan;
 
 		if (freq) {
 			wpa_printf(MSG_DEBUG,
@@ -2771,7 +2814,8 @@ static void wpas_wps_update_ap_info_bss(struct wpa_supplicant *wpa_s,
 	struct wpabuf *wps;
 	enum wps_ap_info_type type;
 	struct wps_ap_info *ap;
-	int r;
+	int r, pbc_active;
+	const u8 *uuid;
 
 	if (wpa_scan_get_vendor_ie(res, WPS_IE_VENDOR_TYPE) == NULL)
 		return;
@@ -2788,7 +2832,8 @@ static void wpas_wps_update_ap_info_bss(struct wpa_supplicant *wpa_s,
 	else
 		type = WPS_AP_NOT_SEL_REG;
 
-	wpabuf_free(wps);
+	uuid = wps_get_uuid_e(wps);
+	pbc_active = wps_is_selected_pbc_registrar(wps);
 
 	ap = wpas_wps_get_ap_info(wpa_s, res->bssid);
 	if (ap) {
@@ -2800,13 +2845,16 @@ static void wpas_wps_update_ap_info_bss(struct wpa_supplicant *wpa_s,
 			if (type != WPS_AP_NOT_SEL_REG)
 				wpa_blacklist_del(wpa_s, ap->bssid);
 		}
-		return;
+		ap->pbc_active = pbc_active;
+		if (uuid)
+			os_memcpy(ap->uuid, uuid, WPS_UUID_LEN);
+		goto out;
 	}
 
 	ap = os_realloc_array(wpa_s->wps_ap, wpa_s->num_wps_ap + 1,
 			      sizeof(struct wps_ap_info));
 	if (ap == NULL)
-		return;
+		goto out;
 
 	wpa_s->wps_ap = ap;
 	ap = &wpa_s->wps_ap[wpa_s->num_wps_ap];
@@ -2815,8 +2863,14 @@ static void wpas_wps_update_ap_info_bss(struct wpa_supplicant *wpa_s,
 	os_memset(ap, 0, sizeof(*ap));
 	os_memcpy(ap->bssid, res->bssid, ETH_ALEN);
 	ap->type = type;
+	ap->pbc_active = pbc_active;
+	if (uuid)
+		os_memcpy(ap->uuid, uuid, WPS_UUID_LEN);
 	wpa_printf(MSG_DEBUG, "WPS: AP " MACSTR " type %d added",
 		   MAC2STR(ap->bssid), ap->type);
+
+out:
+	wpabuf_free(wps);
 }
 
 
diff --git a/wpa_supplicant/wps_supplicant.h b/wpa_supplicant/wps_supplicant.h
index 683bd50e4bab..3c25ca86dc65 100644
--- a/wpa_supplicant/wps_supplicant.h
+++ b/wpa_supplicant/wps_supplicant.h
@@ -33,6 +33,7 @@ int wpas_wps_start_pbc(struct wpa_supplicant *wpa_s, const u8 *bssid,
 		       int p2p_group);
 int wpas_wps_start_pin(struct wpa_supplicant *wpa_s, const u8 *bssid,
 		       const char *pin, int p2p_group, u16 dev_pw_id);
+void wpas_wps_pbc_overlap(struct wpa_supplicant *wpa_s);
 int wpas_wps_cancel(struct wpa_supplicant *wpa_s);
 int wpas_wps_start_reg(struct wpa_supplicant *wpa_s, const u8 *bssid,
 		       const char *pin, struct wps_new_ap_settings *settings);

From 3451a7cd5bc9b8f33d720b0d2b575d9093c4680f Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Wed, 14 Oct 2015 04:42:05 +0000
Subject: [PATCH 187/380] Follow-up r288218 by ensuring common objects are
 built before recursing.

Some example where this is a problem:
  lib/atf/libatf-c++/tests/Makefile:SRCS.${_T}=   ${_T}.cpp test_helpers.cpp
  lib/atf/libatf-c++/tests/detail/Makefile:SRCS.${_T}=    ${_T}.cpp test_helpers.cpp
  lib/atf/libatf-c/tests/Makefile:SRCS.${_T}=     ${_T}.c test_helpers.c
  lib/atf/libatf-c/tests/detail/Makefile:SRCS.${_T}=      ${_T}.c test_helpers.c
  lib/libpam/libpam/tests/Makefile:SRCS.${test} = ${test}.c ${COMMONSRC}

A similar change may be needed for FILES, SCRIPTS, or INCS, but for now stay
with just SRCS.

Reported by:	rodrigc
MFC after:	3 weeks
X-MFC-With:	r288218
Sponsored by:	EMC / Isilon Storage Division
---
 share/mk/bsd.progs.mk | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/share/mk/bsd.progs.mk b/share/mk/bsd.progs.mk
index ab527089c807..3efa691f45a8 100644
--- a/share/mk/bsd.progs.mk
+++ b/share/mk/bsd.progs.mk
@@ -84,6 +84,25 @@ $v =
 # tell progs.mk we might want to install things
 PROGS_TARGETS+= checkdpadd clean cleandepend cleandir depend install
 
+# Find common sources among the PROGS and depend on them before building
+# anything.  This allows parallelization without them each fighting over
+# the same objects.
+_PROGS_COMMON_SRCS=
+_PROGS_ALL_SRCS=
+.for p in ${PROGS}
+.for s in ${SRCS.${p}}
+.if ${_PROGS_ALL_SRCS:M${s}} && !${_PROGS_COMMON_SRCS:M${s}}
+_PROGS_COMMON_SRCS+=	${s}
+.else
+_PROGS_ALL_SRCS+=	${s}
+.endif
+.endfor
+.endfor
+.if !empty(_PROGS_COMMON_SRCS)
+_PROGS_COMMON_OBJS=	${_PROGS_COMMON_SRCS:N*.h:R:S/$/.o/g}
+${PROGS}: ${_PROGS_COMMON_OBJS}
+.endif
+
 .for p in ${PROGS}
 .if defined(PROGS_CXX) && !empty(PROGS_CXX:M$p)
 # bsd.prog.mk may need to know this

From b79a4ad7442e3124557f37a4a8eefee57f99cb61 Mon Sep 17 00:00:00 2001
From: adrian <adrian@FreeBSD.org>
Date: Wed, 14 Oct 2015 05:16:56 +0000
Subject: [PATCH 188/380] Fix date.

Noticed by:	bdrewery
---
 share/man/man4/rsu.4 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/share/man/man4/rsu.4 b/share/man/man4/rsu.4
index cfaf43e602d9..dceb55d0f11c 100644
--- a/share/man/man4/rsu.4
+++ b/share/man/man4/rsu.4
@@ -15,7 +15,7 @@
 .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 .\"
-.Dd November 19, 2014
+.Dd October 13, 2015
 .Dt RSU 4
 .Os
 .Sh NAME

From c7b6fda55e23a071077905546a1d87c5ddf8fa93 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Wed, 14 Oct 2015 05:50:16 +0000
Subject: [PATCH 189/380] Fix support for building a PROG_CXX, and PROG,
 directly.

For example in lib/atf/libatf-c++/tests/detail it is now possible to
run 'make application_test'.  This was intended to worked for PROGS,
but lacked support for PROGS_CXX.

Also fix redefining the main PROG target to recurse.  This isn't needed
since the main process is setting PROG/PROG_CXX to handle it directly
via bsd.prog.mk.

MFC after:	3 weeks
Sponsored by:	EMC / Isilon Storage Division
---
 share/mk/bsd.progs.mk | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/share/mk/bsd.progs.mk b/share/mk/bsd.progs.mk
index 3efa691f45a8..2da30c70ebb5 100644
--- a/share/mk/bsd.progs.mk
+++ b/share/mk/bsd.progs.mk
@@ -31,6 +31,9 @@ UPDATE_DEPENDFILE_PROG = ${PROGS:[1]}
 # They may have asked us to build just one
 .for t in ${PROGS}
 .if make($t)
+.if ${PROGS_CXX:M${t}}
+PROG_CXX ?= $t
+.endif
 PROG ?= $t
 .endif
 .endfor
@@ -61,7 +64,7 @@ UPDATE_DEPENDFILE ?= yes
 UPDATE_DEPENDFILE ?= NO
 
 # prog.mk will do the rest
-.else
+.else # !defined(PROG)
 all: ${PROGS}
 
 # We cannot capture dependencies for meta mode here
@@ -80,7 +83,7 @@ $v =
 # handle being called [bsd.]progs.mk
 .include <bsd.prog.mk>
 
-.if !empty(PROGS) && !defined(_RECURSING_PROGS)
+.if !empty(PROGS) && !defined(_RECURSING_PROGS) && !defined(PROG)
 # tell progs.mk we might want to install things
 PROGS_TARGETS+= checkdpadd clean cleandepend cleandir depend install
 
@@ -130,4 +133,4 @@ $p.$t: .PHONY .MAKE
 .for t in ${PROGS_TARGETS:O:u}
 $t: ${PROGS:%=%.$t}
 .endfor
-.endif
+.endif	# !empty(PROGS) && !defined(_RECURSING_PROGS) && !defined(PROG)

From c9534bc93d8fce7dde17877987c9760dbdd52202 Mon Sep 17 00:00:00 2001
From: hiren <hiren@FreeBSD.org>
Date: Wed, 14 Oct 2015 06:57:28 +0000
Subject: [PATCH 190/380] Fix an unnecessarily aggressive behavior where mtu
 clamping begins on first retransmission timeout (rto) when blackhole
 detection is enabled.  Make sure it only happens when the second attempt to
 send the same segment also fails with rto.

Also make sure that each mtu probing stage (usually 1448 -> 1188 -> 524) follows
the same pattern and gets 2 chances (rto) before further clamping down.

Note: RFC4821 doesn't specify implementation details on how this situation
should be handled.

Differential Revision:	https://reviews.freebsd.org/D3434
Reviewed by:	sbruno, gnn (previous version)
MFC after:	2 weeks
Sponsored by:	Limelight Networks
---
 sys/netinet/tcp_timer.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/sys/netinet/tcp_timer.c b/sys/netinet/tcp_timer.c
index 60b882f869f7..d39339674970 100644
--- a/sys/netinet/tcp_timer.c
+++ b/sys/netinet/tcp_timer.c
@@ -664,9 +664,15 @@ tcp_timer_rexmt(void * xtp)
 		int isipv6;
 #endif
 
+		/*
+		 * Idea here is that at each stage of mtu probe (usually, 1448
+		 * -> 1188 -> 524) should be given 2 chances to recover before
+		 *  further clamping down. 'tp->t_rxtshift % 2 == 0' should
+		 *  take care of that.
+		 */
 		if (((tp->t_flags2 & (TF2_PLPMTU_PMTUD|TF2_PLPMTU_MAXSEGSNT)) ==
 		    (TF2_PLPMTU_PMTUD|TF2_PLPMTU_MAXSEGSNT)) &&
-		    (tp->t_rxtshift <= 2)) {
+		    (tp->t_rxtshift >= 2 && tp->t_rxtshift % 2 == 0)) {
 			/*
 			 * Enter Path MTU Black-hole Detection mechanism:
 			 * - Disable Path MTU Discovery (IP "DF" bit).
@@ -734,9 +740,11 @@ tcp_timer_rexmt(void * xtp)
 			 * with a lowered MTU, maybe this isn't a blackhole and
 			 * we restore the previous MSS and blackhole detection
 			 * flags.
+			 * The limit '6' is determined by giving each probe
+			 * stage (1448, 1188, 524) 2 chances to recover.
 			 */
 			if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) &&
-			    (tp->t_rxtshift > 4)) {
+			    (tp->t_rxtshift > 6)) {
 				tp->t_flags2 |= TF2_PLPMTU_PMTUD;
 				tp->t_flags2 &= ~TF2_PLPMTU_BLACKHOLE;
 				optlen = tp->t_maxopd - tp->t_maxseg;

From 05e3e0c72286e8a94d36f6cff4bdfedac570693a Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Wed, 14 Oct 2015 07:28:36 +0000
Subject: [PATCH 191/380] 5219 l2arc_write_buffers() may write beyond target_sz

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Saso Kiselkov <skiselkov@gmail.com>
Reviewed by: George Wilson <george@delphix.com>
Reviewed by: Steven Hartland <steven.hartland@multiplay.co.uk>
Reviewed by: Justin Gibbs <gibbs@FreeBSD.org>
Approved by: Matthew Ahrens <mahrens@delphix.com>
Author: Andriy Gapon <avg@freebsd.org>

illumos/illumos-gate@d7d9a6d919f92d74ea0510a53f8441396048e800
---
 uts/common/fs/zfs/arc.c | 44 ++++++++++++++++++++++++++++++-----------
 1 file changed, 33 insertions(+), 11 deletions(-)

diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index 92e089e4b3d4..c3d449321514 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -5902,7 +5902,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
     boolean_t *headroom_boost)
 {
 	arc_buf_hdr_t *hdr, *hdr_prev, *head;
-	uint64_t write_asize, write_psize, write_sz, headroom,
+	uint64_t write_asize, write_sz, headroom,
 	    buf_compress_minsz;
 	void *buf_data;
 	boolean_t full;
@@ -5917,7 +5917,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 	*headroom_boost = B_FALSE;
 
 	pio = NULL;
-	write_sz = write_asize = write_psize = 0;
+	write_sz = write_asize = 0;
 	full = B_FALSE;
 	head = kmem_cache_alloc(hdr_l2only_cache, KM_PUSHPAGE);
 	head->b_flags |= ARC_FLAG_L2_WRITE_HEAD;
@@ -5954,6 +5954,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 		for (; hdr; hdr = hdr_prev) {
 			kmutex_t *hash_lock;
 			uint64_t buf_sz;
+			uint64_t buf_a_sz;
 
 			if (arc_warm == B_FALSE)
 				hdr_prev = multilist_sublist_next(mls, hdr);
@@ -5982,7 +5983,15 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 				continue;
 			}
 
-			if ((write_sz + hdr->b_size) > target_sz) {
+			/*
+			 * Assume that the buffer is not going to be compressed
+			 * and could take more space on disk because of a larger
+			 * disk block size.
+			 */
+			buf_sz = hdr->b_size;
+			buf_a_sz = vdev_psize_to_asize(dev->l2ad_vdev, buf_sz);
+
+			if ((write_asize + buf_a_sz) > target_sz) {
 				full = B_TRUE;
 				mutex_exit(hash_lock);
 				break;
@@ -6046,7 +6055,6 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 			 */
 			hdr->b_l2hdr.b_daddr = L2ARC_ADDR_UNSET;
 
-			buf_sz = hdr->b_size;
 			hdr->b_flags |= ARC_FLAG_HAS_L2HDR;
 
 			mutex_enter(&dev->l2ad_mtx);
@@ -6063,6 +6071,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 			mutex_exit(hash_lock);
 
 			write_sz += buf_sz;
+			write_asize += buf_a_sz;
 		}
 
 		multilist_sublist_unlock(mls);
@@ -6081,6 +6090,19 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 
 	mutex_enter(&dev->l2ad_mtx);
 
+	/*
+	 * Note that elsewhere in this file arcstat_l2_asize
+	 * and the used space on l2ad_vdev are updated using b_asize,
+	 * which is not necessarily rounded up to the device block size.
+	 * Too keep accounting consistent we do the same here as well:
+	 * stats_size accumulates the sum of b_asize of the written buffers,
+	 * while write_asize accumulates the sum of b_asize rounded up
+	 * to the device block size.
+	 * The latter sum is used only to validate the corectness of the code.
+	 */
+	uint64_t stats_size = 0;
+	write_asize = 0;
+
 	/*
 	 * Now start writing the buffers. We're starting at the write head
 	 * and work backwards, retracing the course of the buffer selector
@@ -6134,7 +6156,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 
 		/* Compression may have squashed the buffer to zero length. */
 		if (buf_sz != 0) {
-			uint64_t buf_p_sz;
+			uint64_t buf_a_sz;
 
 			wzio = zio_write_phys(pio, dev->l2ad_vdev,
 			    dev->l2ad_hand, buf_sz, buf_data, ZIO_CHECKSUM_OFF,
@@ -6145,14 +6167,14 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 			    zio_t *, wzio);
 			(void) zio_nowait(wzio);
 
-			write_asize += buf_sz;
+			stats_size += buf_sz;
 
 			/*
 			 * Keep the clock hand suitably device-aligned.
 			 */
-			buf_p_sz = vdev_psize_to_asize(dev->l2ad_vdev, buf_sz);
-			write_psize += buf_p_sz;
-			dev->l2ad_hand += buf_p_sz;
+			buf_a_sz = vdev_psize_to_asize(dev->l2ad_vdev, buf_sz);
+			write_asize += buf_a_sz;
+			dev->l2ad_hand += buf_a_sz;
 		}
 	}
 
@@ -6162,8 +6184,8 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 	ARCSTAT_BUMP(arcstat_l2_writes_sent);
 	ARCSTAT_INCR(arcstat_l2_write_bytes, write_asize);
 	ARCSTAT_INCR(arcstat_l2_size, write_sz);
-	ARCSTAT_INCR(arcstat_l2_asize, write_asize);
-	vdev_space_update(dev->l2ad_vdev, write_asize, 0, 0);
+	ARCSTAT_INCR(arcstat_l2_asize, stats_size);
+	vdev_space_update(dev->l2ad_vdev, stats_size, 0, 0);
 
 	/*
 	 * Bump device hand to the device start if it is approaching the end.

From afbdb89044aa9bcf99a858ac03118bab8d159319 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Wed, 14 Oct 2015 07:37:02 +0000
Subject: [PATCH 192/380] MFV r289294: 5219 l2arc_write_buffers() may write
 beyond target_sz

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Saso Kiselkov <skiselkov@gmail.com>
Reviewed by: George Wilson <george@delphix.com>
Reviewed by: Steven Hartland <steven.hartland@multiplay.co.uk>
Reviewed by: Justin Gibbs <gibbs@FreeBSD.org>
Approved by: Matthew Ahrens <mahrens@delphix.com>
Author: Andriy Gapon <avg@freebsd.org>

illumos/illumos-gate@d7d9a6d919f92d74ea0510a53f8441396048e800
---
 sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c
index 42c001a7cda3..08d4f310c345 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c
@@ -6243,7 +6243,8 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
     boolean_t *headroom_boost)
 {
 	arc_buf_hdr_t *hdr, *hdr_prev, *head;
-	uint64_t write_asize, write_sz, headroom, buf_compress_minsz;
+	uint64_t write_asize, write_sz, headroom,
+	    buf_compress_minsz;
 	void *buf_data;
 	boolean_t full;
 	l2arc_write_callback_t *cb;
@@ -6405,6 +6406,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 			 * using it to denote the header's state change.
 			 */
 			hdr->b_l2hdr.b_daddr = L2ARC_ADDR_UNSET;
+
 			hdr->b_flags |= ARC_FLAG_HAS_L2HDR;
 
 			mutex_enter(&dev->l2ad_mtx);

From c50d6675518a4e4210ca89c2e61b455017910f14 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Wed, 14 Oct 2015 07:44:19 +0000
Subject: [PATCH 193/380] 6288 dmu_buf_will_dirty could be faster

Reviewed by: George Wilson <george.wilson@delphix.com>
Reviewed by: Paul Dagnelie <pcd@delphix.com>
Reviewed by: Justin Gibbs <gibbs@scsiguy.com>
Reviewed by: Richard Elling <Richard.Elling@RichardElling.com>
Approved by: Robert Mustacchi <rm@joyent.com>
Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@0f2e7d03b8f588387cb8dd8dd500cbe5ff4484e0
---
 uts/common/fs/zfs/dbuf.c | 61 +++++++++++++++++++++++++++++++++-------
 1 file changed, 51 insertions(+), 10 deletions(-)

diff --git a/uts/common/fs/zfs/dbuf.c b/uts/common/fs/zfs/dbuf.c
index e75bac094ecb..514aaa2365ef 100644
--- a/uts/common/fs/zfs/dbuf.c
+++ b/uts/common/fs/zfs/dbuf.c
@@ -1115,6 +1115,32 @@ dbuf_release_bp(dmu_buf_impl_t *db)
 	(void) arc_release(db->db_buf, db);
 }
 
+/*
+ * We already have a dirty record for this TXG, and we are being
+ * dirtied again.
+ */
+static void
+dbuf_redirty(dbuf_dirty_record_t *dr)
+{
+	dmu_buf_impl_t *db = dr->dr_dbuf;
+
+	ASSERT(MUTEX_HELD(&db->db_mtx));
+
+	if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) {
+		/*
+		 * If this buffer has already been written out,
+		 * we now need to reset its state.
+		 */
+		dbuf_unoverride(dr);
+		if (db->db.db_object != DMU_META_DNODE_OBJECT &&
+		    db->db_state != DB_NOFILL) {
+			/* Already released on initial dirty, so just thaw. */
+			ASSERT(arc_released(db->db_buf));
+			arc_buf_thaw(db->db_buf);
+		}
+	}
+}
+
 dbuf_dirty_record_t *
 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
 {
@@ -1187,16 +1213,7 @@ dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
 	if (dr && dr->dr_txg == tx->tx_txg) {
 		DB_DNODE_EXIT(db);
 
-		if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) {
-			/*
-			 * If this buffer has already been written out,
-			 * we now need to reset its state.
-			 */
-			dbuf_unoverride(dr);
-			if (db->db.db_object != DMU_META_DNODE_OBJECT &&
-			    db->db_state != DB_NOFILL)
-				arc_buf_thaw(db->db_buf);
-		}
+		dbuf_redirty(dr);
 		mutex_exit(&db->db_mtx);
 		return (dr);
 	}
@@ -1500,6 +1517,30 @@ dmu_buf_will_dirty(dmu_buf_t *db_fake, dmu_tx_t *tx)
 	ASSERT(tx->tx_txg != 0);
 	ASSERT(!refcount_is_zero(&db->db_holds));
 
+	/*
+	 * Quick check for dirtyness.  For already dirty blocks, this
+	 * reduces runtime of this function by >90%, and overall performance
+	 * by 50% for some workloads (e.g. file deletion with indirect blocks
+	 * cached).
+	 */
+	mutex_enter(&db->db_mtx);
+	dbuf_dirty_record_t *dr;
+	for (dr = db->db_last_dirty;
+	    dr != NULL && dr->dr_txg >= tx->tx_txg; dr = dr->dr_next) {
+		/*
+		 * It's possible that it is already dirty but not cached,
+		 * because there are some calls to dbuf_dirty() that don't
+		 * go through dmu_buf_will_dirty().
+		 */
+		if (dr->dr_txg == tx->tx_txg && db->db_state == DB_CACHED) {
+			/* This dbuf is already dirty and cached. */
+			dbuf_redirty(dr);
+			mutex_exit(&db->db_mtx);
+			return;
+		}
+	}
+	mutex_exit(&db->db_mtx);
+
 	DB_DNODE_ENTER(db);
 	if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock))
 		rf |= DB_RF_HAVESTRUCT;

From 4e77c724b399db44ae9227197594f6dab36e6386 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Wed, 14 Oct 2015 07:45:44 +0000
Subject: [PATCH 194/380] MFV r289296: 6288 dmu_buf_will_dirty could be faster

Reviewed by: George Wilson <george.wilson@delphix.com>
Reviewed by: Paul Dagnelie <pcd@delphix.com>
Reviewed by: Justin Gibbs <gibbs@scsiguy.com>
Reviewed by: Richard Elling <Richard.Elling@RichardElling.com>
Approved by: Robert Mustacchi <rm@joyent.com>
Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@0f2e7d03b8f588387cb8dd8dd500cbe5ff4484e0
---
 .../opensolaris/uts/common/fs/zfs/dbuf.c      | 61 ++++++++++++++++---
 1 file changed, 51 insertions(+), 10 deletions(-)

diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c
index 4ca13568dbac..981de6340238 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c
@@ -1115,6 +1115,32 @@ dbuf_release_bp(dmu_buf_impl_t *db)
 	(void) arc_release(db->db_buf, db);
 }
 
+/*
+ * We already have a dirty record for this TXG, and we are being
+ * dirtied again.
+ */
+static void
+dbuf_redirty(dbuf_dirty_record_t *dr)
+{
+	dmu_buf_impl_t *db = dr->dr_dbuf;
+
+	ASSERT(MUTEX_HELD(&db->db_mtx));
+
+	if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) {
+		/*
+		 * If this buffer has already been written out,
+		 * we now need to reset its state.
+		 */
+		dbuf_unoverride(dr);
+		if (db->db.db_object != DMU_META_DNODE_OBJECT &&
+		    db->db_state != DB_NOFILL) {
+			/* Already released on initial dirty, so just thaw. */
+			ASSERT(arc_released(db->db_buf));
+			arc_buf_thaw(db->db_buf);
+		}
+	}
+}
+
 dbuf_dirty_record_t *
 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
 {
@@ -1187,16 +1213,7 @@ dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
 	if (dr && dr->dr_txg == tx->tx_txg) {
 		DB_DNODE_EXIT(db);
 
-		if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) {
-			/*
-			 * If this buffer has already been written out,
-			 * we now need to reset its state.
-			 */
-			dbuf_unoverride(dr);
-			if (db->db.db_object != DMU_META_DNODE_OBJECT &&
-			    db->db_state != DB_NOFILL)
-				arc_buf_thaw(db->db_buf);
-		}
+		dbuf_redirty(dr);
 		mutex_exit(&db->db_mtx);
 		return (dr);
 	}
@@ -1500,6 +1517,30 @@ dmu_buf_will_dirty(dmu_buf_t *db_fake, dmu_tx_t *tx)
 	ASSERT(tx->tx_txg != 0);
 	ASSERT(!refcount_is_zero(&db->db_holds));
 
+	/*
+	 * Quick check for dirtyness.  For already dirty blocks, this
+	 * reduces runtime of this function by >90%, and overall performance
+	 * by 50% for some workloads (e.g. file deletion with indirect blocks
+	 * cached).
+	 */
+	mutex_enter(&db->db_mtx);
+	dbuf_dirty_record_t *dr;
+	for (dr = db->db_last_dirty;
+	    dr != NULL && dr->dr_txg >= tx->tx_txg; dr = dr->dr_next) {
+		/*
+		 * It's possible that it is already dirty but not cached,
+		 * because there are some calls to dbuf_dirty() that don't
+		 * go through dmu_buf_will_dirty().
+		 */
+		if (dr->dr_txg == tx->tx_txg && db->db_state == DB_CACHED) {
+			/* This dbuf is already dirty and cached. */
+			dbuf_redirty(dr);
+			mutex_exit(&db->db_mtx);
+			return;
+		}
+	}
+	mutex_exit(&db->db_mtx);
+
 	DB_DNODE_ENTER(db);
 	if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock))
 		rf |= DB_RF_HAVESTRUCT;

From 3a4a55e0a9fc201ad6c731915673fa1cc060f490 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Wed, 14 Oct 2015 07:48:39 +0000
Subject: [PATCH 195/380] 6286 ZFS internal error when set large block on
 bootfs

Reviewed by: Paul Dagnelie <pcd@delphix.com>
Reviewed by: George Wilson <george.wilson@delphix.com>
Reviewed by: Andriy Gapon <avg@FreeBSD.org>
Approved by: Robert Mustacchi <rm@joyent.com>
Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@6de9bb5603e65b16816b7ab29e39bac820e2da2b
---
 uts/common/fs/zfs/zfs_ioctl.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/uts/common/fs/zfs/zfs_ioctl.c b/uts/common/fs/zfs/zfs_ioctl.c
index 5a56021c8cae..4552cbb3e3d1 100644
--- a/uts/common/fs/zfs/zfs_ioctl.c
+++ b/uts/common/fs/zfs/zfs_ioctl.c
@@ -24,7 +24,7 @@
  * Portions Copyright 2011 Martin Matuska
  * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2014, Joyent, Inc. All rights reserved.
- * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
  */
@@ -3835,7 +3835,7 @@ zfs_check_settable(const char *dsname, nvpair_t *pair, cred_t *cr)
 			 */
 			if (zfs_is_bootfs(dsname) &&
 			    intval > SPA_OLD_MAXBLOCKSIZE) {
-				return (SET_ERROR(EDOM));
+				return (SET_ERROR(ERANGE));
 			}
 
 			/*
@@ -3844,7 +3844,7 @@ zfs_check_settable(const char *dsname, nvpair_t *pair, cred_t *cr)
 			 */
 			if (intval > zfs_max_recordsize ||
 			    intval > SPA_MAXBLOCKSIZE)
-				return (SET_ERROR(EDOM));
+				return (SET_ERROR(ERANGE));
 
 			if ((err = spa_open(dsname, &spa, FTAG)) != 0)
 				return (err);

From f3d984a1de37056cd53d15248244687c6bc6dae9 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Wed, 14 Oct 2015 07:50:08 +0000
Subject: [PATCH 196/380] MFV r289298: 6286 ZFS internal error when set large
 block on bootfs

Reviewed by: Paul Dagnelie <pcd@delphix.com>
Reviewed by: George Wilson <george.wilson@delphix.com>
Reviewed by: Andriy Gapon <avg@FreeBSD.org>
Approved by: Robert Mustacchi <rm@joyent.com>
Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@6de9bb5603e65b16816b7ab29e39bac820e2da2b
---
 sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c
index 2e5191672fd2..9ead0b71fc02 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c
@@ -27,7 +27,7 @@
  * Copyright 2014 Xin Li <delphij@FreeBSD.org>. All rights reserved.
  * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2014, Joyent, Inc. All rights reserved.
- * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
  */
@@ -3921,7 +3921,7 @@ zfs_check_settable(const char *dsname, nvpair_t *pair, cred_t *cr)
 			 */
 			if (zfs_is_bootfs(dsname) &&
 			    intval > SPA_OLD_MAXBLOCKSIZE) {
-				return (SET_ERROR(EDOM));
+				return (SET_ERROR(ERANGE));
 			}
 
 			/*
@@ -3930,7 +3930,7 @@ zfs_check_settable(const char *dsname, nvpair_t *pair, cred_t *cr)
 			 */
 			if (intval > zfs_max_recordsize ||
 			    intval > SPA_MAXBLOCKSIZE)
-				return (SET_ERROR(EDOM));
+				return (SET_ERROR(ERANGE));
 
 			if ((err = spa_open(dsname, &spa, FTAG)) != 0)
 				return (err);

From 176234430e93ca5a44579b9c7234e756f07ee271 Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Wed, 14 Oct 2015 09:03:32 +0000
Subject: [PATCH 197/380] Invalid pages should not appear on the inactive
 queue.  Change the check into an assertion.

Reviewed by:	alc
Tested by:	pho
Sponsored by:	The FreeBSD Foundation
---
 sys/vm/vm_pageout.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c
index e0d00f5172f3..5611ae66694a 100644
--- a/sys/vm/vm_pageout.c
+++ b/sys/vm/vm_pageout.c
@@ -1174,11 +1174,12 @@ unlock_page:
 		queues_locked = FALSE;
 
 		/*
-		 * Invalid pages can be easily freed. They cannot be
-		 * mapped, vm_page_free() asserts this.
+		 * Invalid pages cannot appear on a queue.  If
+		 * vm_pageout_fallback_object_lock() allowed a window
+		 * where the page could be invalidated, it should
+		 * detect this.
 		 */
-		if (m->valid == 0)
-			goto free_page;
+		KASSERT(m->valid != 0, ("Invalid page %p on inact queue", m));
 
 		/*
 		 * If the page has been referenced and the object is not dead,
@@ -1231,7 +1232,6 @@ unlock_page:
 			/*
 			 * Clean pages can be freed.
 			 */
-free_page:
 			vm_page_free(m);
 			PCPU_INC(cnt.v_dfree);
 			--page_shortage;

From c90bce1ee130340a907c067f64d81097bb803678 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Wed, 14 Oct 2015 10:21:21 +0000
Subject: [PATCH 198/380] 6293 ztest failure: error == 28 (0xc == 0x1c) in
 ztest_tx_assign()

Reviewed by: George Wilson <george.wilson@delphix.com>
Reviewed by: Prakash Surya <prakash.surya@delphix.com>
Reviewed by: Richard Elling <Richard.Elling@RichardElling.com>
Approved by: Richard Lowe <richlowe@richlowe.net>
Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@8fe00bfb8790ad51653f67b01d5ac14256cbb404
---
 uts/common/fs/zfs/arc.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index c3d449321514..80d86e1b8d0e 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -5020,6 +5020,16 @@ arc_init(void)
 		arc_c_max = arc_c_min;
 	arc_c_max = MAX(allmem * 3 / 4, arc_c_max);
 
+	/*
+	 * In userland, there's only the memory pressure that we artificially
+	 * create (see arc_available_memory()).  Don't let arc_c get too
+	 * small, because it can cause transactions to be larger than
+	 * arc_c, causing arc_tempreserve_space() to fail.
+	 */
+#ifndef _KERNEL
+	arc_c_min = arc_c_max / 2;
+#endif
+
 	/*
 	 * Allow the tunables to override our calculations if they are
 	 * reasonable (ie. over 64MB)

From 89a4cb886e13bdfef3d3ee5f241359e0f534bbc9 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Wed, 14 Oct 2015 10:28:29 +0000
Subject: [PATCH 199/380] MFV r289304: 6293 ztest failure: error == 28 (0xc ==
 0x1c) in ztest_tx_assign()

Reviewed by: George Wilson <george.wilson@delphix.com>
Reviewed by: Prakash Surya <prakash.surya@delphix.com>
Reviewed by: Richard Elling <Richard.Elling@RichardElling.com>
Approved by: Richard Lowe <richlowe@richlowe.net>
Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@8fe00bfb8790ad51653f67b01d5ac14256cbb404
---
 sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c
index 08d4f310c345..5269359193aa 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c
@@ -5295,6 +5295,16 @@ arc_init(void)
 		arc_c_max = arc_c_min;
 	arc_c_max = MAX(arc_c * 5, arc_c_max);
 
+	/*
+	 * In userland, there's only the memory pressure that we artificially
+	 * create (see arc_available_memory()).  Don't let arc_c get too
+	 * small, because it can cause transactions to be larger than
+	 * arc_c, causing arc_tempreserve_space() to fail.
+	 */
+#ifndef _KERNEL
+	arc_c_min = arc_c_max / 2;
+#endif
+
 #ifdef _KERNEL
 	/*
 	 * Allow the tunables to override our calculations if they are

From 930ff2b90c3913ae568b1215949634e562c4431c Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Wed, 14 Oct 2015 10:30:24 +0000
Subject: [PATCH 200/380] 6295 metaslab_condense's dbgmsg should include vdev
 id

Reviewed by: George Wilson <george.wilson@delphix.com>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Andriy Gapon <avg@freebsd.org>
Reviewed by: Xin Li <delphij@freebsd.org>
Reviewed by: Justin Gibbs <gibbs@scsiguy.com>
Approved by: Richard Lowe <richlowe@richlowe.net>
Author: Joe Stein <joe.stein@delphix.com>

illumos/illumos-gate@daec38ecb4fb5e73e4ca9e99be84f6b8c50c02fa
---
 uts/common/fs/zfs/metaslab.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/uts/common/fs/zfs/metaslab.c b/uts/common/fs/zfs/metaslab.c
index 9ba9fd384141..852534eff8bd 100644
--- a/uts/common/fs/zfs/metaslab.c
+++ b/uts/common/fs/zfs/metaslab.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  */
 
@@ -1705,10 +1705,11 @@ metaslab_condense(metaslab_t *msp, uint64_t txg, dmu_tx_t *tx)
 	ASSERT(msp->ms_loaded);
 
 
-	spa_dbgmsg(spa, "condensing: txg %llu, msp[%llu] %p, "
-	    "smp size %llu, segments %lu, forcing condense=%s", txg,
-	    msp->ms_id, msp, space_map_length(msp->ms_sm),
-	    avl_numnodes(&msp->ms_tree->rt_root),
+	spa_dbgmsg(spa, "condensing: txg %llu, msp[%llu] %p, vdev id %llu, "
+	    "spa %s, smp size %llu, segments %lu, forcing condense=%s", txg,
+	    msp->ms_id, msp, msp->ms_group->mg_vd->vdev_id,
+	    msp->ms_group->mg_vd->vdev_spa->spa_name,
+	    space_map_length(msp->ms_sm), avl_numnodes(&msp->ms_tree->rt_root),
 	    msp->ms_condense_wanted ? "TRUE" : "FALSE");
 
 	msp->ms_condense_wanted = B_FALSE;

From 984f7bcd4ca17d983f6cf97fb5ce6bf455a06163 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Wed, 14 Oct 2015 10:31:50 +0000
Subject: [PATCH 201/380] MFV r289306: 6295 metaslab_condense's dbgmsg should
 include vdev id

Reviewed by: George Wilson <george.wilson@delphix.com>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Andriy Gapon <avg@freebsd.org>
Reviewed by: Xin Li <delphij@freebsd.org>
Reviewed by: Justin Gibbs <gibbs@scsiguy.com>
Approved by: Richard Lowe <richlowe@richlowe.net>
Author: Joe Stein <joe.stein@delphix.com>

illumos/illumos-gate@daec38ecb4fb5e73e4ca9e99be84f6b8c50c02fa
---
 .../contrib/opensolaris/uts/common/fs/zfs/metaslab.c  | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c
index c565e8b8fbd5..6a20d25667de 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  */
 
@@ -1794,10 +1794,11 @@ metaslab_condense(metaslab_t *msp, uint64_t txg, dmu_tx_t *tx)
 	ASSERT(msp->ms_loaded);
 
 
-	spa_dbgmsg(spa, "condensing: txg %llu, msp[%llu] %p, "
-	    "smp size %llu, segments %lu, forcing condense=%s", txg,
-	    msp->ms_id, msp, space_map_length(msp->ms_sm),
-	    avl_numnodes(&msp->ms_tree->rt_root),
+	spa_dbgmsg(spa, "condensing: txg %llu, msp[%llu] %p, vdev id %llu, "
+	    "spa %s, smp size %llu, segments %lu, forcing condense=%s", txg,
+	    msp->ms_id, msp, msp->ms_group->mg_vd->vdev_id,
+	    msp->ms_group->mg_vd->vdev_spa->spa_name,
+	    space_map_length(msp->ms_sm), avl_numnodes(&msp->ms_tree->rt_root),
 	    msp->ms_condense_wanted ? "TRUE" : "FALSE");
 
 	msp->ms_condense_wanted = B_FALSE;

From 44b92e6b372337009a1a679f24372d2c804e8994 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Wed, 14 Oct 2015 10:36:43 +0000
Subject: [PATCH 202/380] 6267 dn_bonus evicted too early

Reviewed by: Richard Yao <ryao@gentoo.org>
Reviewed by: Xin LI <delphij@freebsd.org>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Approved by: Richard Lowe <richlowe@richlowe.net>
Author: Justin T. Gibbs <gibbs@FreeBSD.org>

illumos/illumos-gate@d2058105c61ec61df3a2dd3f839fed8c3fe7bfd6
---
 uts/common/fs/zfs/dbuf.c           | 47 ++++++++----------------------
 uts/common/fs/zfs/dmu_objset.c     |  1 -
 uts/common/fs/zfs/dnode_sync.c     | 14 +++++----
 uts/common/fs/zfs/sys/dbuf.h       | 18 +++++++++++-
 uts/common/fs/zfs/sys/dmu_objset.h |  1 -
 5 files changed, 38 insertions(+), 43 deletions(-)

diff --git a/uts/common/fs/zfs/dbuf.c b/uts/common/fs/zfs/dbuf.c
index 514aaa2365ef..8b0c71db5c86 100644
--- a/uts/common/fs/zfs/dbuf.c
+++ b/uts/common/fs/zfs/dbuf.c
@@ -272,7 +272,7 @@ dbuf_verify_user(dmu_buf_impl_t *db, dbvu_verify_type_t verify_type)
 		 */
 		ASSERT3U(holds, >=, db->db_dirtycnt);
 	} else {
-		if (db->db_immediate_evict == TRUE)
+		if (db->db_user_immediate_evict == TRUE)
 			ASSERT3U(holds, >=, db->db_dirtycnt);
 		else
 			ASSERT3U(holds, >, 0);
@@ -1875,8 +1875,9 @@ dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid,
 	db->db_blkptr = blkptr;
 
 	db->db_user = NULL;
-	db->db_immediate_evict = 0;
-	db->db_freed_in_flight = 0;
+	db->db_user_immediate_evict = FALSE;
+	db->db_freed_in_flight = FALSE;
+	db->db_pending_evict = FALSE;
 
 	if (blkid == DMU_BONUS_BLKID) {
 		ASSERT3P(parent, ==, dn->dn_dbuf);
@@ -2432,12 +2433,13 @@ dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag)
 		arc_buf_freeze(db->db_buf);
 
 	if (holds == db->db_dirtycnt &&
-	    db->db_level == 0 && db->db_immediate_evict)
+	    db->db_level == 0 && db->db_user_immediate_evict)
 		dbuf_evict_user(db);
 
 	if (holds == 0) {
 		if (db->db_blkid == DMU_BONUS_BLKID) {
 			dnode_t *dn;
+			boolean_t evict_dbuf = db->db_pending_evict;
 
 			/*
 			 * If the dnode moves here, we cannot cross this
@@ -2452,7 +2454,7 @@ dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag)
 			 * Decrementing the dbuf count means that the bonus
 			 * buffer's dnode hold is no longer discounted in
 			 * dnode_move(). The dnode cannot move until after
-			 * the dnode_rele_and_unlock() below.
+			 * the dnode_rele() below.
 			 */
 			DB_DNODE_EXIT(db);
 
@@ -2462,35 +2464,10 @@ dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag)
 			 */
 			mutex_exit(&db->db_mtx);
 
-			/*
-			 * If the dnode has been freed, evict the bonus
-			 * buffer immediately.	The data in the bonus
-			 * buffer is no longer relevant and this prevents
-			 * a stale bonus buffer from being associated
-			 * with this dnode_t should the dnode_t be reused
-			 * prior to being destroyed.
-			 */
-			mutex_enter(&dn->dn_mtx);
-			if (dn->dn_type == DMU_OT_NONE ||
-			    dn->dn_free_txg != 0) {
-				/*
-				 * Drop dn_mtx.  It is a leaf lock and
-				 * cannot be held when dnode_evict_bonus()
-				 * acquires other locks in order to
-				 * perform the eviction.
-				 *
-				 * Freed dnodes cannot be reused until the
-				 * last hold is released.  Since this bonus
-				 * buffer has a hold, the dnode will remain
-				 * in the free state, even without dn_mtx
-				 * held, until the dnode_rele_and_unlock()
-				 * below.
-				 */
-				mutex_exit(&dn->dn_mtx);
+			if (evict_dbuf)
 				dnode_evict_bonus(dn);
-				mutex_enter(&dn->dn_mtx);
-			}
-			dnode_rele_and_unlock(dn, db);
+
+			dnode_rele(dn, db);
 		} else if (db->db_buf == NULL) {
 			/*
 			 * This is a special case: we never associated this
@@ -2537,7 +2514,7 @@ dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag)
 				} else {
 					dbuf_clear(db);
 				}
-			} else if (db->db_objset->os_evicting ||
+			} else if (db->db_pending_evict ||
 			    arc_buf_eviction_needed(db->db_buf)) {
 				dbuf_clear(db);
 			} else {
@@ -2585,7 +2562,7 @@ dmu_buf_set_user_ie(dmu_buf_t *db_fake, dmu_buf_user_t *user)
 {
 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
 
-	db->db_immediate_evict = TRUE;
+	db->db_user_immediate_evict = TRUE;
 	return (dmu_buf_set_user(db_fake, user));
 }
 
diff --git a/uts/common/fs/zfs/dmu_objset.c b/uts/common/fs/zfs/dmu_objset.c
index f84ff378c947..6846572f8d5c 100644
--- a/uts/common/fs/zfs/dmu_objset.c
+++ b/uts/common/fs/zfs/dmu_objset.c
@@ -686,7 +686,6 @@ dmu_objset_evict(objset_t *os)
 	if (os->os_sa)
 		sa_tear_down(os);
 
-	os->os_evicting = B_TRUE;
 	dmu_objset_evict_dbufs(os);
 
 	mutex_enter(&os->os_lock);
diff --git a/uts/common/fs/zfs/dnode_sync.c b/uts/common/fs/zfs/dnode_sync.c
index 0787885229d4..9aee51381872 100644
--- a/uts/common/fs/zfs/dnode_sync.c
+++ b/uts/common/fs/zfs/dnode_sync.c
@@ -424,6 +424,7 @@ dnode_evict_dbufs(dnode_t *dn)
 			db_next = AVL_NEXT(&dn->dn_dbufs, &db_marker);
 			avl_remove(&dn->dn_dbufs, &db_marker);
 		} else {
+			db->db_pending_evict = TRUE;
 			mutex_exit(&db->db_mtx);
 			db_next = AVL_NEXT(&dn->dn_dbufs, db);
 		}
@@ -437,10 +438,14 @@ void
 dnode_evict_bonus(dnode_t *dn)
 {
 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
-	if (dn->dn_bonus && refcount_is_zero(&dn->dn_bonus->db_holds)) {
-		mutex_enter(&dn->dn_bonus->db_mtx);
-		dbuf_evict(dn->dn_bonus);
-		dn->dn_bonus = NULL;
+	if (dn->dn_bonus != NULL) {
+		if (refcount_is_zero(&dn->dn_bonus->db_holds)) {
+			mutex_enter(&dn->dn_bonus->db_mtx);
+			dbuf_evict(dn->dn_bonus);
+			dn->dn_bonus = NULL;
+		} else {
+			dn->dn_bonus->db_pending_evict = TRUE;
+		}
 	}
 	rw_exit(&dn->dn_struct_rwlock);
 }
@@ -492,7 +497,6 @@ dnode_sync_free(dnode_t *dn, dmu_tx_t *tx)
 
 	dnode_undirty_dbufs(&dn->dn_dirty_records[txgoff]);
 	dnode_evict_dbufs(dn);
-	ASSERT(avl_is_empty(&dn->dn_dbufs));
 
 	/*
 	 * XXX - It would be nice to assert this, but we may still
diff --git a/uts/common/fs/zfs/sys/dbuf.h b/uts/common/fs/zfs/sys/dbuf.h
index 482ccb01dacb..233d541342ad 100644
--- a/uts/common/fs/zfs/sys/dbuf.h
+++ b/uts/common/fs/zfs/sys/dbuf.h
@@ -230,9 +230,25 @@ typedef struct dmu_buf_impl {
 	/* User callback information. */
 	dmu_buf_user_t *db_user;
 
-	uint8_t db_immediate_evict;
+	/*
+	 * Evict user data as soon as the dirty and reference
+	 * counts are equal.
+	 */
+	uint8_t db_user_immediate_evict;
+
+	/*
+	 * This block was freed while a read or write was
+	 * active.
+	 */
 	uint8_t db_freed_in_flight;
 
+	/*
+	 * dnode_evict_dbufs() or dnode_evict_bonus() tried to
+	 * evict this dbuf, but couldn't due to outstanding
+	 * references.  Evict once the refcount drops to 0.
+	 */
+	uint8_t db_pending_evict;
+
 	uint8_t db_dirtycnt;
 } dmu_buf_impl_t;
 
diff --git a/uts/common/fs/zfs/sys/dmu_objset.h b/uts/common/fs/zfs/sys/dmu_objset.h
index 9e98350f001f..8a263a39e6b3 100644
--- a/uts/common/fs/zfs/sys/dmu_objset.h
+++ b/uts/common/fs/zfs/sys/dmu_objset.h
@@ -93,7 +93,6 @@ struct objset {
 	uint8_t os_copies;
 	enum zio_checksum os_dedup_checksum;
 	boolean_t os_dedup_verify;
-	boolean_t os_evicting;
 	zfs_logbias_op_t os_logbias;
 	zfs_cache_type_t os_primary_cache;
 	zfs_cache_type_t os_secondary_cache;

From 297adc95512ebc2d14e1e9aaf999b3d9272312b3 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Wed, 14 Oct 2015 10:38:05 +0000
Subject: [PATCH 203/380] MFV r289308: 6267 dn_bonus evicted too early

Reviewed by: Richard Yao <ryao@gentoo.org>
Reviewed by: Xin LI <delphij@freebsd.org>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Approved by: Richard Lowe <richlowe@richlowe.net>
Author: Justin T. Gibbs <gibbs@FreeBSD.org>

illumos/illumos-gate@d2058105c61ec61df3a2dd3f839fed8c3fe7bfd6
---
 .../opensolaris/uts/common/fs/zfs/dbuf.c      | 47 +++++--------------
 .../uts/common/fs/zfs/dmu_objset.c            |  1 -
 .../uts/common/fs/zfs/dnode_sync.c            | 14 ++++--
 .../opensolaris/uts/common/fs/zfs/sys/dbuf.h  | 18 ++++++-
 .../uts/common/fs/zfs/sys/dmu_objset.h        |  1 -
 5 files changed, 38 insertions(+), 43 deletions(-)

diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c
index 981de6340238..ac7ef3dc742d 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c
@@ -272,7 +272,7 @@ dbuf_verify_user(dmu_buf_impl_t *db, dbvu_verify_type_t verify_type)
 		 */
 		ASSERT3U(holds, >=, db->db_dirtycnt);
 	} else {
-		if (db->db_immediate_evict == TRUE)
+		if (db->db_user_immediate_evict == TRUE)
 			ASSERT3U(holds, >=, db->db_dirtycnt);
 		else
 			ASSERT3U(holds, >, 0);
@@ -1875,8 +1875,9 @@ dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid,
 	db->db_blkptr = blkptr;
 
 	db->db_user = NULL;
-	db->db_immediate_evict = 0;
-	db->db_freed_in_flight = 0;
+	db->db_user_immediate_evict = FALSE;
+	db->db_freed_in_flight = FALSE;
+	db->db_pending_evict = FALSE;
 
 	if (blkid == DMU_BONUS_BLKID) {
 		ASSERT3P(parent, ==, dn->dn_dbuf);
@@ -2432,12 +2433,13 @@ dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag)
 		arc_buf_freeze(db->db_buf);
 
 	if (holds == db->db_dirtycnt &&
-	    db->db_level == 0 && db->db_immediate_evict)
+	    db->db_level == 0 && db->db_user_immediate_evict)
 		dbuf_evict_user(db);
 
 	if (holds == 0) {
 		if (db->db_blkid == DMU_BONUS_BLKID) {
 			dnode_t *dn;
+			boolean_t evict_dbuf = db->db_pending_evict;
 
 			/*
 			 * If the dnode moves here, we cannot cross this
@@ -2452,7 +2454,7 @@ dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag)
 			 * Decrementing the dbuf count means that the bonus
 			 * buffer's dnode hold is no longer discounted in
 			 * dnode_move(). The dnode cannot move until after
-			 * the dnode_rele_and_unlock() below.
+			 * the dnode_rele() below.
 			 */
 			DB_DNODE_EXIT(db);
 
@@ -2462,35 +2464,10 @@ dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag)
 			 */
 			mutex_exit(&db->db_mtx);
 
-			/*
-			 * If the dnode has been freed, evict the bonus
-			 * buffer immediately.	The data in the bonus
-			 * buffer is no longer relevant and this prevents
-			 * a stale bonus buffer from being associated
-			 * with this dnode_t should the dnode_t be reused
-			 * prior to being destroyed.
-			 */
-			mutex_enter(&dn->dn_mtx);
-			if (dn->dn_type == DMU_OT_NONE ||
-			    dn->dn_free_txg != 0) {
-				/*
-				 * Drop dn_mtx.  It is a leaf lock and
-				 * cannot be held when dnode_evict_bonus()
-				 * acquires other locks in order to
-				 * perform the eviction.
-				 *
-				 * Freed dnodes cannot be reused until the
-				 * last hold is released.  Since this bonus
-				 * buffer has a hold, the dnode will remain
-				 * in the free state, even without dn_mtx
-				 * held, until the dnode_rele_and_unlock()
-				 * below.
-				 */
-				mutex_exit(&dn->dn_mtx);
+			if (evict_dbuf)
 				dnode_evict_bonus(dn);
-				mutex_enter(&dn->dn_mtx);
-			}
-			dnode_rele_and_unlock(dn, db);
+
+			dnode_rele(dn, db);
 		} else if (db->db_buf == NULL) {
 			/*
 			 * This is a special case: we never associated this
@@ -2537,7 +2514,7 @@ dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag)
 				} else {
 					dbuf_clear(db);
 				}
-			} else if (db->db_objset->os_evicting ||
+			} else if (db->db_pending_evict ||
 			    arc_buf_eviction_needed(db->db_buf)) {
 				dbuf_clear(db);
 			} else {
@@ -2585,7 +2562,7 @@ dmu_buf_set_user_ie(dmu_buf_t *db_fake, dmu_buf_user_t *user)
 {
 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
 
-	db->db_immediate_evict = TRUE;
+	db->db_user_immediate_evict = TRUE;
 	return (dmu_buf_set_user(db_fake, user));
 }
 
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_objset.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_objset.c
index f84ff378c947..6846572f8d5c 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_objset.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_objset.c
@@ -686,7 +686,6 @@ dmu_objset_evict(objset_t *os)
 	if (os->os_sa)
 		sa_tear_down(os);
 
-	os->os_evicting = B_TRUE;
 	dmu_objset_evict_dbufs(os);
 
 	mutex_enter(&os->os_lock);
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dnode_sync.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dnode_sync.c
index 0787885229d4..9aee51381872 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dnode_sync.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dnode_sync.c
@@ -424,6 +424,7 @@ dnode_evict_dbufs(dnode_t *dn)
 			db_next = AVL_NEXT(&dn->dn_dbufs, &db_marker);
 			avl_remove(&dn->dn_dbufs, &db_marker);
 		} else {
+			db->db_pending_evict = TRUE;
 			mutex_exit(&db->db_mtx);
 			db_next = AVL_NEXT(&dn->dn_dbufs, db);
 		}
@@ -437,10 +438,14 @@ void
 dnode_evict_bonus(dnode_t *dn)
 {
 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
-	if (dn->dn_bonus && refcount_is_zero(&dn->dn_bonus->db_holds)) {
-		mutex_enter(&dn->dn_bonus->db_mtx);
-		dbuf_evict(dn->dn_bonus);
-		dn->dn_bonus = NULL;
+	if (dn->dn_bonus != NULL) {
+		if (refcount_is_zero(&dn->dn_bonus->db_holds)) {
+			mutex_enter(&dn->dn_bonus->db_mtx);
+			dbuf_evict(dn->dn_bonus);
+			dn->dn_bonus = NULL;
+		} else {
+			dn->dn_bonus->db_pending_evict = TRUE;
+		}
 	}
 	rw_exit(&dn->dn_struct_rwlock);
 }
@@ -492,7 +497,6 @@ dnode_sync_free(dnode_t *dn, dmu_tx_t *tx)
 
 	dnode_undirty_dbufs(&dn->dn_dirty_records[txgoff]);
 	dnode_evict_dbufs(dn);
-	ASSERT(avl_is_empty(&dn->dn_dbufs));
 
 	/*
 	 * XXX - It would be nice to assert this, but we may still
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dbuf.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dbuf.h
index 482ccb01dacb..233d541342ad 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dbuf.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dbuf.h
@@ -230,9 +230,25 @@ typedef struct dmu_buf_impl {
 	/* User callback information. */
 	dmu_buf_user_t *db_user;
 
-	uint8_t db_immediate_evict;
+	/*
+	 * Evict user data as soon as the dirty and reference
+	 * counts are equal.
+	 */
+	uint8_t db_user_immediate_evict;
+
+	/*
+	 * This block was freed while a read or write was
+	 * active.
+	 */
 	uint8_t db_freed_in_flight;
 
+	/*
+	 * dnode_evict_dbufs() or dnode_evict_bonus() tried to
+	 * evict this dbuf, but couldn't due to outstanding
+	 * references.  Evict once the refcount drops to 0.
+	 */
+	uint8_t db_pending_evict;
+
 	uint8_t db_dirtycnt;
 } dmu_buf_impl_t;
 
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dmu_objset.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dmu_objset.h
index 9e98350f001f..8a263a39e6b3 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dmu_objset.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dmu_objset.h
@@ -93,7 +93,6 @@ struct objset {
 	uint8_t os_copies;
 	enum zio_checksum os_dedup_checksum;
 	boolean_t os_dedup_verify;
-	boolean_t os_evicting;
 	zfs_logbias_op_t os_logbias;
 	zfs_cache_type_t os_primary_cache;
 	zfs_cache_type_t os_secondary_cache;

From d000743c4fef0f3f266ef337cba5460d3d27b485 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Wed, 14 Oct 2015 11:12:47 +0000
Subject: [PATCH 204/380] 4185 add new cryptographic checksums to ZFS: SHA-512,
 Skein, Edon-R

Reviewed by: George Wilson <george.wilson@delphix.com>
Reviewed by: Prakash Surya <prakash.surya@delphix.com>
Reviewed by: Saso Kiselkov <saso.kiselkov@nexenta.com>
Reviewed by: Richard Lowe <richlowe@richlowe.net>
Approved by: Garrett D'Amore <garrett@damore.org>
Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@45818ee124adeaaf947698996b4f4c722afc6d1f
---
 common/zfs/zfeature_common.c         |  13 +
 common/zfs/zfeature_common.h         |   3 +
 common/zfs/zfs_fletcher.c            |  19 +-
 common/zfs/zfs_fletcher.h            |  17 +-
 common/zfs/zfs_prop.c                |  19 +-
 uts/common/Makefile.files            |   6 +
 uts/common/crypto/io/edonr_mod.c     |  63 ++
 uts/common/crypto/io/skein_mod.c     | 830 +++++++++++++++++++++++++++
 uts/common/fs/zfs/arc.c              |   6 +-
 uts/common/fs/zfs/ddt.c              |   5 +-
 uts/common/fs/zfs/dmu.c              |  24 +-
 uts/common/fs/zfs/dmu_send.c         |   3 +-
 uts/common/fs/zfs/dsl_dataset.c      |   8 +
 uts/common/fs/zfs/edonr_zfs.c        | 102 ++++
 uts/common/fs/zfs/sha256.c           |  33 +-
 uts/common/fs/zfs/skein_zfs.c        |  91 +++
 uts/common/fs/zfs/spa.c              |  34 ++
 uts/common/fs/zfs/spa_misc.c         |   7 +-
 uts/common/fs/zfs/sys/dmu.h          |   2 +
 uts/common/fs/zfs/sys/spa.h          |   9 +
 uts/common/fs/zfs/sys/spa_impl.h     |   5 +
 uts/common/fs/zfs/sys/zio.h          |   3 +
 uts/common/fs/zfs/sys/zio_checksum.h |  51 +-
 uts/common/fs/zfs/zfs_ioctl.c        |  45 +-
 uts/common/fs/zfs/zio.c              |  37 +-
 uts/common/fs/zfs/zio_checksum.c     | 161 +++++-
 uts/common/sys/debug.h               |   9 +
 uts/common/sys/edonr.h               |  93 +++
 uts/common/sys/skein.h               | 178 ++++++
 29 files changed, 1799 insertions(+), 77 deletions(-)
 create mode 100644 uts/common/crypto/io/edonr_mod.c
 create mode 100644 uts/common/crypto/io/skein_mod.c
 create mode 100644 uts/common/fs/zfs/edonr_zfs.c
 create mode 100644 uts/common/fs/zfs/skein_zfs.c
 create mode 100644 uts/common/sys/edonr.h
 create mode 100644 uts/common/sys/skein.h

diff --git a/common/zfs/zfeature_common.c b/common/zfs/zfeature_common.c
index 025c227e677f..f75894b44dee 100644
--- a/common/zfs/zfeature_common.c
+++ b/common/zfs/zfeature_common.c
@@ -231,4 +231,17 @@ zpool_feature_init(void)
 	    "org.open-zfs:large_blocks", "large_blocks",
 	    "Support for blocks larger than 128KB.",
 	    ZFEATURE_FLAG_PER_DATASET, large_blocks_deps);
+
+	zfeature_register(SPA_FEATURE_SHA512,
+	    "org.illumos:sha512", "sha512",
+	    "SHA-512/256 hash algorithm.",
+	    ZFEATURE_FLAG_PER_DATASET, NULL);
+	zfeature_register(SPA_FEATURE_SKEIN,
+	    "org.illumos:skein", "skein",
+	    "Skein hash algorithm.",
+	    ZFEATURE_FLAG_PER_DATASET, NULL);
+	zfeature_register(SPA_FEATURE_EDONR,
+	    "org.illumos:edonr", "edonr",
+	    "Edon-R hash algorithm.",
+	    ZFEATURE_FLAG_PER_DATASET, NULL);
 }
diff --git a/common/zfs/zfeature_common.h b/common/zfs/zfeature_common.h
index 794290bfe0c5..6b229aebc4b5 100644
--- a/common/zfs/zfeature_common.h
+++ b/common/zfs/zfeature_common.h
@@ -52,6 +52,9 @@ typedef enum spa_feature {
 	SPA_FEATURE_BOOKMARKS,
 	SPA_FEATURE_FS_SS_LIMIT,
 	SPA_FEATURE_LARGE_BLOCKS,
+	SPA_FEATURE_SHA512,
+	SPA_FEATURE_SKEIN,
+	SPA_FEATURE_EDONR,
 	SPA_FEATURES
 } spa_feature_t;
 
diff --git a/common/zfs/zfs_fletcher.c b/common/zfs/zfs_fletcher.c
index fa43ce6bdb5d..a58fa14b7c02 100644
--- a/common/zfs/zfs_fletcher.c
+++ b/common/zfs/zfs_fletcher.c
@@ -22,6 +22,9 @@
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright 2013 Saso Kiselkov. All rights reserved.
+ */
 
 /*
  * Fletcher Checksums
@@ -131,8 +134,10 @@
 #include <sys/zio.h>
 #include <sys/spa.h>
 
+/*ARGSUSED*/
 void
-fletcher_2_native(const void *buf, uint64_t size, zio_cksum_t *zcp)
+fletcher_2_native(const void *buf, uint64_t size,
+    const void *ctx_template, zio_cksum_t *zcp)
 {
 	const uint64_t *ip = buf;
 	const uint64_t *ipend = ip + (size / sizeof (uint64_t));
@@ -148,8 +153,10 @@ fletcher_2_native(const void *buf, uint64_t size, zio_cksum_t *zcp)
 	ZIO_SET_CHECKSUM(zcp, a0, a1, b0, b1);
 }
 
+/*ARGSUSED*/
 void
-fletcher_2_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp)
+fletcher_2_byteswap(const void *buf, uint64_t size,
+    const void *ctx_template, zio_cksum_t *zcp)
 {
 	const uint64_t *ip = buf;
 	const uint64_t *ipend = ip + (size / sizeof (uint64_t));
@@ -165,8 +172,10 @@ fletcher_2_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp)
 	ZIO_SET_CHECKSUM(zcp, a0, a1, b0, b1);
 }
 
+/*ARGSUSED*/
 void
-fletcher_4_native(const void *buf, uint64_t size, zio_cksum_t *zcp)
+fletcher_4_native(const void *buf, uint64_t size,
+    const void *ctx_template, zio_cksum_t *zcp)
 {
 	const uint32_t *ip = buf;
 	const uint32_t *ipend = ip + (size / sizeof (uint32_t));
@@ -182,8 +191,10 @@ fletcher_4_native(const void *buf, uint64_t size, zio_cksum_t *zcp)
 	ZIO_SET_CHECKSUM(zcp, a, b, c, d);
 }
 
+/*ARGSUSED*/
 void
-fletcher_4_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp)
+fletcher_4_byteswap(const void *buf, uint64_t size,
+    const void *ctx_template, zio_cksum_t *zcp)
 {
 	const uint32_t *ip = buf;
 	const uint32_t *ipend = ip + (size / sizeof (uint32_t));
diff --git a/common/zfs/zfs_fletcher.h b/common/zfs/zfs_fletcher.h
index b49df0cf4f0f..a920cc816d45 100644
--- a/common/zfs/zfs_fletcher.h
+++ b/common/zfs/zfs_fletcher.h
@@ -22,6 +22,9 @@
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright 2013 Saso Kiselkov. All rights reserved.
+ */
 
 #ifndef	_ZFS_FLETCHER_H
 #define	_ZFS_FLETCHER_H
@@ -37,14 +40,12 @@ extern "C" {
  * fletcher checksum functions
  */
 
-void fletcher_2_native(const void *, uint64_t, zio_cksum_t *);
-void fletcher_2_byteswap(const void *, uint64_t, zio_cksum_t *);
-void fletcher_4_native(const void *, uint64_t, zio_cksum_t *);
-void fletcher_4_byteswap(const void *, uint64_t, zio_cksum_t *);
-void fletcher_4_incremental_native(const void *, uint64_t,
-    zio_cksum_t *);
-void fletcher_4_incremental_byteswap(const void *, uint64_t,
-    zio_cksum_t *);
+void fletcher_2_native(const void *, uint64_t, const void *, zio_cksum_t *);
+void fletcher_2_byteswap(const void *, uint64_t, const void *, zio_cksum_t *);
+void fletcher_4_native(const void *, uint64_t, const void *, zio_cksum_t *);
+void fletcher_4_byteswap(const void *, uint64_t, const void *, zio_cksum_t *);
+void fletcher_4_incremental_native(const void *, uint64_t, zio_cksum_t *);
+void fletcher_4_incremental_byteswap(const void *, uint64_t, zio_cksum_t *);
 
 #ifdef	__cplusplus
 }
diff --git a/common/zfs/zfs_prop.c b/common/zfs/zfs_prop.c
index e145b1c866e3..11abc0d46fa8 100644
--- a/common/zfs/zfs_prop.c
+++ b/common/zfs/zfs_prop.c
@@ -71,6 +71,9 @@ zfs_prop_init(void)
 		{ "fletcher4",	ZIO_CHECKSUM_FLETCHER_4 },
 		{ "sha256",	ZIO_CHECKSUM_SHA256 },
 		{ "noparity",	ZIO_CHECKSUM_NOPARITY },
+		{ "sha512",	ZIO_CHECKSUM_SHA512 },
+		{ "skein",	ZIO_CHECKSUM_SKEIN },
+		{ "edonr",	ZIO_CHECKSUM_EDONR },
 		{ NULL }
 	};
 
@@ -81,6 +84,14 @@ zfs_prop_init(void)
 		{ "sha256",	ZIO_CHECKSUM_SHA256 },
 		{ "sha256,verify",
 				ZIO_CHECKSUM_SHA256 | ZIO_CHECKSUM_VERIFY },
+		{ "sha512",	ZIO_CHECKSUM_SHA512 },
+		{ "sha512,verify",
+				ZIO_CHECKSUM_SHA512 | ZIO_CHECKSUM_VERIFY },
+		{ "skein",	ZIO_CHECKSUM_SKEIN },
+		{ "skein,verify",
+				ZIO_CHECKSUM_SKEIN | ZIO_CHECKSUM_VERIFY },
+		{ "edonr,verify",
+				ZIO_CHECKSUM_EDONR | ZIO_CHECKSUM_VERIFY },
 		{ NULL }
 	};
 
@@ -217,12 +228,12 @@ zfs_prop_init(void)
 	zprop_register_index(ZFS_PROP_CHECKSUM, "checksum",
 	    ZIO_CHECKSUM_DEFAULT, PROP_INHERIT, ZFS_TYPE_FILESYSTEM |
 	    ZFS_TYPE_VOLUME,
-	    "on | off | fletcher2 | fletcher4 | sha256", "CHECKSUM",
-	    checksum_table);
+	    "on | off | fletcher2 | fletcher4 | sha256 | sha512 | "
+	    "skein | edonr", "CHECKSUM", checksum_table);
 	zprop_register_index(ZFS_PROP_DEDUP, "dedup", ZIO_CHECKSUM_OFF,
 	    PROP_INHERIT, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME,
-	    "on | off | verify | sha256[,verify]", "DEDUP",
-	    dedup_table);
+	    "on | off | verify | sha256[,verify], sha512[,verify], "
+	    "skein[,verify], edonr,verify", "DEDUP", dedup_table);
 	zprop_register_index(ZFS_PROP_COMPRESSION, "compression",
 	    ZIO_COMPRESS_DEFAULT, PROP_INHERIT,
 	    ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME,
diff --git a/uts/common/Makefile.files b/uts/common/Makefile.files
index cc8b16a70787..f33cd43b86ff 100644
--- a/uts/common/Makefile.files
+++ b/uts/common/Makefile.files
@@ -509,6 +509,10 @@ SHA1_OBJS +=	sha1.o sha1_mod.o
 
 SHA2_OBJS +=	sha2.o sha2_mod.o
 
+SKEIN_OBJS +=	skein.o skein_block.o skein_iv.o skein_mod.o
+
+EDONR_OBJS +=	edonr.o edonr_mod.o
+
 IPGPC_OBJS +=   classifierddi.o classifier.o filters.o trie.o table.o \
 		ba_table.o
 
@@ -1370,6 +1374,8 @@ ZFS_COMMON_OBJS +=		\
 	rrwlock.o		\
 	sa.o			\
 	sha256.o		\
+	edonr_zfs.o		\
+	skein_zfs.o		\
 	spa.o			\
 	spa_config.o		\
 	spa_errlog.o		\
diff --git a/uts/common/crypto/io/edonr_mod.c b/uts/common/crypto/io/edonr_mod.c
new file mode 100644
index 000000000000..d17bbe990060
--- /dev/null
+++ b/uts/common/crypto/io/edonr_mod.c
@@ -0,0 +1,63 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://opensource.org/licenses/CDDL-1.0.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2013 Saso Kiselkov. All rights reserved.
+ */
+
+#include <sys/modctl.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/spi.h>
+#include <sys/sysmacros.h>
+#include <sys/systm.h>
+#include <sys/edonr.h>
+
+/*
+ * Unlike sha2 or skein, we won't expose edonr via the Kernel Cryptographic
+ * Framework (KCF), because Edon-R is *NOT* suitable for general-purpose
+ * cryptographic use. Users of Edon-R must interface directly to this module.
+ */
+
+static struct modlmisc modlmisc = {
+	&mod_miscops,
+	"Edon-R Message-Digest Algorithm"
+};
+
+static struct modlinkage modlinkage = {
+	MODREV_1, &modlmisc, NULL
+};
+
+int
+_init(void)
+{
+	int error;
+
+	if ((error = mod_install(&modlinkage)) != 0)
+		return (error);
+
+	return (0);
+}
+
+int
+_info(struct modinfo *modinfop)
+{
+	return (mod_info(&modlinkage, modinfop));
+}
diff --git a/uts/common/crypto/io/skein_mod.c b/uts/common/crypto/io/skein_mod.c
new file mode 100644
index 000000000000..aca7582dcbac
--- /dev/null
+++ b/uts/common/crypto/io/skein_mod.c
@@ -0,0 +1,830 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://opensource.org/licenses/CDDL-1.0.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2013 Saso Kiselkov. All rights reserved.
+ */
+
+#include <sys/modctl.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/spi.h>
+#include <sys/strsun.h>
+#include <sys/sysmacros.h>
+#include <sys/systm.h>
+#define	SKEIN_MODULE_IMPL
+#include <sys/skein.h>
+
+/*
+ * Like the sha2 module, we create the skein module with two modlinkages:
+ * - modlmisc to allow direct calls to Skein_* API functions.
+ * - modlcrypto to integrate well into the Kernel Crypto Framework (KCF).
+ */
+static struct modlmisc modlmisc = {
+	&mod_miscops,
+	"Skein Message-Digest Algorithm"
+};
+
+static struct modlcrypto modlcrypto = {
+	&mod_cryptoops,
+	"Skein Kernel SW Provider"
+};
+
+static struct modlinkage modlinkage = {
+	MODREV_1, &modlmisc, &modlcrypto, NULL
+};
+
+static crypto_mech_info_t skein_mech_info_tab[] = {
+	{CKM_SKEIN_256, SKEIN_256_MECH_INFO_TYPE,
+	    CRYPTO_FG_DIGEST | CRYPTO_FG_DIGEST_ATOMIC,
+	    0, 0, CRYPTO_KEYSIZE_UNIT_IN_BITS},
+	{CKM_SKEIN_256_MAC, SKEIN_256_MAC_MECH_INFO_TYPE,
+	    CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC, 1, INT_MAX,
+	    CRYPTO_KEYSIZE_UNIT_IN_BYTES},
+	{CKM_SKEIN_512, SKEIN_512_MECH_INFO_TYPE,
+	    CRYPTO_FG_DIGEST | CRYPTO_FG_DIGEST_ATOMIC,
+	    0, 0, CRYPTO_KEYSIZE_UNIT_IN_BITS},
+	{CKM_SKEIN_512_MAC, SKEIN_512_MAC_MECH_INFO_TYPE,
+	    CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC, 1, INT_MAX,
+	    CRYPTO_KEYSIZE_UNIT_IN_BYTES},
+	{CKM_SKEIN1024, SKEIN1024_MECH_INFO_TYPE,
+	    CRYPTO_FG_DIGEST | CRYPTO_FG_DIGEST_ATOMIC,
+	    0, 0, CRYPTO_KEYSIZE_UNIT_IN_BITS},
+	{CKM_SKEIN1024_MAC, SKEIN1024_MAC_MECH_INFO_TYPE,
+	    CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC, 1, INT_MAX,
+	    CRYPTO_KEYSIZE_UNIT_IN_BYTES}
+};
+
+static void skein_provider_status(crypto_provider_handle_t, uint_t *);
+
+static crypto_control_ops_t skein_control_ops = {
+	skein_provider_status
+};
+
+static int skein_digest_init(crypto_ctx_t *, crypto_mechanism_t *,
+    crypto_req_handle_t);
+static int skein_digest(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
+    crypto_req_handle_t);
+static int skein_update(crypto_ctx_t *, crypto_data_t *, crypto_req_handle_t);
+static int skein_final(crypto_ctx_t *, crypto_data_t *, crypto_req_handle_t);
+static int skein_digest_atomic(crypto_provider_handle_t, crypto_session_id_t,
+    crypto_mechanism_t *, crypto_data_t *, crypto_data_t *,
+    crypto_req_handle_t);
+
+static crypto_digest_ops_t skein_digest_ops = {
+	skein_digest_init,
+	skein_digest,
+	skein_update,
+	NULL,
+	skein_final,
+	skein_digest_atomic
+};
+
+static int skein_mac_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *,
+    crypto_spi_ctx_template_t, crypto_req_handle_t);
+static int skein_mac_atomic(crypto_provider_handle_t, crypto_session_id_t,
+    crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *,
+    crypto_spi_ctx_template_t, crypto_req_handle_t);
+
+static crypto_mac_ops_t skein_mac_ops = {
+	skein_mac_init,
+	NULL,
+	skein_update,	/* using regular digest update is OK here */
+	skein_final,	/* using regular digest final is OK here */
+	skein_mac_atomic,
+	NULL
+};
+
+static int skein_create_ctx_template(crypto_provider_handle_t,
+    crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t *,
+    size_t *, crypto_req_handle_t);
+static int skein_free_context(crypto_ctx_t *);
+
+static crypto_ctx_ops_t skein_ctx_ops = {
+	skein_create_ctx_template,
+	skein_free_context
+};
+
+static crypto_ops_t skein_crypto_ops = {
+	&skein_control_ops,
+	&skein_digest_ops,
+	NULL,
+	&skein_mac_ops,
+	NULL,
+	NULL,
+	NULL,
+	NULL,
+	NULL,
+	NULL,
+	NULL,
+	NULL,
+	NULL,
+	&skein_ctx_ops,
+	NULL,
+	NULL,
+	NULL
+};
+
+static crypto_provider_info_t skein_prov_info = {
+	CRYPTO_SPI_VERSION_4,
+	"Skein Software Provider",
+	CRYPTO_SW_PROVIDER,
+	{&modlinkage},
+	NULL,
+	&skein_crypto_ops,
+	sizeof (skein_mech_info_tab) / sizeof (crypto_mech_info_t),
+	skein_mech_info_tab
+};
+
+static crypto_kcf_provider_handle_t skein_prov_handle = NULL;
+
+typedef struct skein_ctx {
+	skein_mech_type_t		sc_mech_type;
+	size_t				sc_digest_bitlen;
+	/*LINTED(E_ANONYMOUS_UNION_DECL)*/
+	union {
+		Skein_256_Ctxt_t	sc_256;
+		Skein_512_Ctxt_t	sc_512;
+		Skein1024_Ctxt_t	sc_1024;
+	};
+} skein_ctx_t;
+#define	SKEIN_CTX(_ctx_)	((skein_ctx_t *)((_ctx_)->cc_provider_private))
+#define	SKEIN_CTX_LVALUE(_ctx_)	(_ctx_)->cc_provider_private
+#define	SKEIN_OP(_skein_ctx, _op, ...)					\
+	do {								\
+		skein_ctx_t	*sc = (_skein_ctx);			\
+		switch (sc->sc_mech_type) {				\
+		case SKEIN_256_MECH_INFO_TYPE:				\
+		case SKEIN_256_MAC_MECH_INFO_TYPE:			\
+			(void) Skein_256_ ## _op(&sc->sc_256, __VA_ARGS__);\
+			break;						\
+		case SKEIN_512_MECH_INFO_TYPE:				\
+		case SKEIN_512_MAC_MECH_INFO_TYPE:			\
+			(void) Skein_512_ ## _op(&sc->sc_512, __VA_ARGS__);\
+			break;						\
+		case SKEIN1024_MECH_INFO_TYPE:				\
+		case SKEIN1024_MAC_MECH_INFO_TYPE:			\
+			(void) Skein1024_ ## _op(&sc->sc_1024, __VA_ARGS__);\
+			break;						\
+		}							\
+		_NOTE(CONSTCOND)					\
+	} while (0)
+
+static int
+skein_get_digest_bitlen(const crypto_mechanism_t *mechanism, size_t *result)
+{
+	if (mechanism->cm_param != NULL) {
+		/*LINTED(E_BAD_PTR_CAST_ALIGN)*/
+		skein_param_t	*param = (skein_param_t *)mechanism->cm_param;
+
+		if (mechanism->cm_param_len != sizeof (*param) ||
+		    param->sp_digest_bitlen == 0) {
+			return (CRYPTO_MECHANISM_PARAM_INVALID);
+		}
+		*result = param->sp_digest_bitlen;
+	} else {
+		switch (mechanism->cm_type) {
+		case SKEIN_256_MECH_INFO_TYPE:
+			*result = 256;
+			break;
+		case SKEIN_512_MECH_INFO_TYPE:
+			*result = 512;
+			break;
+		case SKEIN1024_MECH_INFO_TYPE:
+			*result = 1024;
+			break;
+		default:
+			return (CRYPTO_MECHANISM_INVALID);
+		}
+	}
+	return (CRYPTO_SUCCESS);
+}
+
+int
+_init(void)
+{
+	int error;
+
+	if ((error = mod_install(&modlinkage)) != 0)
+		return (error);
+
+	/*
+	 * Try to register with KCF - failure shouldn't unload us, since we
+	 * still may want to continue providing misc/skein functionality.
+	 */
+	(void) crypto_register_provider(&skein_prov_info, &skein_prov_handle);
+
+	return (0);
+}
+
+int
+_info(struct modinfo *modinfop)
+{
+	return (mod_info(&modlinkage, modinfop));
+}
+
+/*
+ * KCF software provider control entry points.
+ */
+/* ARGSUSED */
+static void
+skein_provider_status(crypto_provider_handle_t provider, uint_t *status)
+{
+	*status = CRYPTO_PROVIDER_READY;
+}
+
+/*
+ * General Skein hashing helper functions.
+ */
+
+/*
+ * Performs an Update on a context with uio input data.
+ */
+static int
+skein_digest_update_uio(skein_ctx_t *ctx, const crypto_data_t *data)
+{
+	off_t		offset = data->cd_offset;
+	size_t		length = data->cd_length;
+	uint_t		vec_idx;
+	size_t		cur_len;
+	const uio_t	*uio = data->cd_uio;
+
+	/* we support only kernel buffer */
+	if (uio->uio_segflg != UIO_SYSSPACE)
+		return (CRYPTO_ARGUMENTS_BAD);
+
+	/*
+	 * Jump to the first iovec containing data to be
+	 * digested.
+	 */
+	for (vec_idx = 0; vec_idx < uio->uio_iovcnt &&
+	    offset >= uio->uio_iov[vec_idx].iov_len;
+	    offset -= uio->uio_iov[vec_idx++].iov_len)
+		;
+	if (vec_idx == uio->uio_iovcnt) {
+		/*
+		 * The caller specified an offset that is larger than the
+		 * total size of the buffers it provided.
+		 */
+		return (CRYPTO_DATA_LEN_RANGE);
+	}
+
+	/*
+	 * Now do the digesting on the iovecs.
+	 */
+	while (vec_idx < uio->uio_iovcnt && length > 0) {
+		cur_len = MIN(uio->uio_iov[vec_idx].iov_len - offset, length);
+		SKEIN_OP(ctx, Update, (uint8_t *)uio->uio_iov[vec_idx].iov_base
+		    + offset, cur_len);
+		length -= cur_len;
+		vec_idx++;
+		offset = 0;
+	}
+
+	if (vec_idx == uio->uio_iovcnt && length > 0) {
+		/*
+		 * The end of the specified iovec's was reached but
+		 * the length requested could not be processed, i.e.
+		 * The caller requested to digest more data than it provided.
+		 */
+		return (CRYPTO_DATA_LEN_RANGE);
+	}
+
+	return (CRYPTO_SUCCESS);
+}
+
+/*
+ * Performs a Final on a context and writes to a uio digest output.
+ */
+static int
+skein_digest_final_uio(skein_ctx_t *ctx, crypto_data_t *digest,
+    crypto_req_handle_t req)
+{
+	off_t	offset = digest->cd_offset;
+	uint_t	vec_idx;
+	uio_t	*uio = digest->cd_uio;
+
+	/* we support only kernel buffer */
+	if (uio->uio_segflg != UIO_SYSSPACE)
+		return (CRYPTO_ARGUMENTS_BAD);
+
+	/*
+	 * Jump to the first iovec containing ptr to the digest to be returned.
+	 */
+	for (vec_idx = 0; offset >= uio->uio_iov[vec_idx].iov_len &&
+	    vec_idx < uio->uio_iovcnt;
+	    offset -= uio->uio_iov[vec_idx++].iov_len)
+		;
+	if (vec_idx == uio->uio_iovcnt) {
+		/*
+		 * The caller specified an offset that is larger than the
+		 * total size of the buffers it provided.
+		 */
+		return (CRYPTO_DATA_LEN_RANGE);
+	}
+	if (offset + CRYPTO_BITS2BYTES(ctx->sc_digest_bitlen) <=
+	    uio->uio_iov[vec_idx].iov_len) {
+		/* The computed digest will fit in the current iovec. */
+		SKEIN_OP(ctx, Final,
+		    (uchar_t *)uio->uio_iov[vec_idx].iov_base + offset);
+	} else {
+		uint8_t *digest_tmp;
+		off_t scratch_offset = 0;
+		size_t length = CRYPTO_BITS2BYTES(ctx->sc_digest_bitlen);
+		size_t cur_len;
+
+		digest_tmp = kmem_alloc(CRYPTO_BITS2BYTES(
+		    ctx->sc_digest_bitlen), crypto_kmflag(req));
+		if (digest_tmp == NULL)
+			return (CRYPTO_HOST_MEMORY);
+		SKEIN_OP(ctx, Final, digest_tmp);
+		while (vec_idx < uio->uio_iovcnt && length > 0) {
+			cur_len = MIN(uio->uio_iov[vec_idx].iov_len - offset,
+			    length);
+			bcopy(digest_tmp + scratch_offset,
+			    uio->uio_iov[vec_idx].iov_base + offset, cur_len);
+
+			length -= cur_len;
+			vec_idx++;
+			scratch_offset += cur_len;
+			offset = 0;
+		}
+		kmem_free(digest_tmp, CRYPTO_BITS2BYTES(ctx->sc_digest_bitlen));
+
+		if (vec_idx == uio->uio_iovcnt && length > 0) {
+			/*
+			 * The end of the specified iovec's was reached but
+			 * the length requested could not be processed, i.e.
+			 * The caller requested to digest more data than it
+			 * provided.
+			 */
+			return (CRYPTO_DATA_LEN_RANGE);
+		}
+	}
+
+	return (CRYPTO_SUCCESS);
+}
+
+/*
+ * Performs an Update on a context with mblk input data.
+ */
+static int
+skein_digest_update_mblk(skein_ctx_t *ctx, crypto_data_t *data)
+{
+	off_t offset = data->cd_offset;
+	size_t length = data->cd_length;
+	mblk_t *mp;
+	size_t cur_len;
+
+	/* Jump to the first mblk_t containing data to be digested. */
+	for (mp = data->cd_mp; mp != NULL && offset >= MBLKL(mp);
+	    offset -= MBLKL(mp), mp = mp->b_cont)
+		;
+	if (mp == NULL) {
+		/*
+		 * The caller specified an offset that is larger than the
+		 * total size of the buffers it provided.
+		 */
+		return (CRYPTO_DATA_LEN_RANGE);
+	}
+
+	/* Now do the digesting on the mblk chain. */
+	while (mp != NULL && length > 0) {
+		cur_len = MIN(MBLKL(mp) - offset, length);
+		SKEIN_OP(ctx, Update, mp->b_rptr + offset, cur_len);
+		length -= cur_len;
+		offset = 0;
+		mp = mp->b_cont;
+	}
+
+	if (mp == NULL && length > 0) {
+		/*
+		 * The end of the mblk was reached but the length requested
+		 * could not be processed, i.e. The caller requested
+		 * to digest more data than it provided.
+		 */
+		return (CRYPTO_DATA_LEN_RANGE);
+	}
+
+	return (CRYPTO_SUCCESS);
+}
+
+/*
+ * Performs a Final on a context and writes to an mblk digest output.
+ */
+static int
+skein_digest_final_mblk(skein_ctx_t *ctx, crypto_data_t *digest,
+    crypto_req_handle_t req)
+{
+	off_t	offset = digest->cd_offset;
+	mblk_t	*mp;
+
+	/* Jump to the first mblk_t that will be used to store the digest. */
+	for (mp = digest->cd_mp; mp != NULL && offset >= MBLKL(mp);
+	    offset -= MBLKL(mp), mp = mp->b_cont)
+		;
+	if (mp == NULL) {
+		/* caller specified offset is too large */
+		return (CRYPTO_DATA_LEN_RANGE);
+	}
+
+	if (offset + CRYPTO_BITS2BYTES(ctx->sc_digest_bitlen) <= MBLKL(mp)) {
+		/* The digest will fit in the current mblk. */
+		SKEIN_OP(ctx, Final, mp->b_rptr + offset);
+	} else {
+		/* Split the digest up between the individual buffers. */
+		uint8_t *digest_tmp;
+		off_t scratch_offset = 0;
+		size_t length = CRYPTO_BITS2BYTES(ctx->sc_digest_bitlen);
+		size_t cur_len;
+
+		digest_tmp = kmem_alloc(CRYPTO_BITS2BYTES(
+		    ctx->sc_digest_bitlen), crypto_kmflag(req));
+		if (digest_tmp == NULL)
+			return (CRYPTO_HOST_MEMORY);
+		SKEIN_OP(ctx, Final, digest_tmp);
+		while (mp != NULL && length > 0) {
+			cur_len = MIN(MBLKL(mp) - offset, length);
+			bcopy(digest_tmp + scratch_offset,
+			    mp->b_rptr + offset, cur_len);
+			length -= cur_len;
+			mp = mp->b_cont;
+			scratch_offset += cur_len;
+			offset = 0;
+		}
+		kmem_free(digest_tmp, CRYPTO_BITS2BYTES(ctx->sc_digest_bitlen));
+		if (mp == NULL && length > 0) {
+			/* digest too long to fit in the mblk buffers */
+			return (CRYPTO_DATA_LEN_RANGE);
+		}
+	}
+
+	return (CRYPTO_SUCCESS);
+}
+
+/*
+ * KCF software provider digest entry points.
+ */
+
+/*
+ * Initializes a skein digest context to the configuration in `mechanism'.
+ * The mechanism cm_type must be one of SKEIN_*_MECH_INFO_TYPE. The cm_param
+ * field may contain a skein_param_t structure indicating the length of the
+ * digest the algorithm should produce. Otherwise the default output lengths
+ * are applied (32 bytes for Skein-256, 64 bytes for Skein-512 and 128 bytes
+ * for Skein-1024).
+ */
+static int
+skein_digest_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
+    crypto_req_handle_t req)
+{
+	int	error = CRYPTO_SUCCESS;
+
+	if (!VALID_SKEIN_DIGEST_MECH(mechanism->cm_type))
+		return (CRYPTO_MECHANISM_INVALID);
+
+	SKEIN_CTX_LVALUE(ctx) = kmem_alloc(sizeof (*SKEIN_CTX(ctx)),
+	    crypto_kmflag(req));
+	if (SKEIN_CTX(ctx) == NULL)
+		return (CRYPTO_HOST_MEMORY);
+
+	SKEIN_CTX(ctx)->sc_mech_type = mechanism->cm_type;
+	error = skein_get_digest_bitlen(mechanism,
+	    &SKEIN_CTX(ctx)->sc_digest_bitlen);
+	if (error != CRYPTO_SUCCESS)
+		goto errout;
+	SKEIN_OP(SKEIN_CTX(ctx), Init, SKEIN_CTX(ctx)->sc_digest_bitlen);
+
+	return (CRYPTO_SUCCESS);
+errout:
+	bzero(SKEIN_CTX(ctx), sizeof (*SKEIN_CTX(ctx)));
+	kmem_free(SKEIN_CTX(ctx), sizeof (*SKEIN_CTX(ctx)));
+	SKEIN_CTX_LVALUE(ctx) = NULL;
+	return (error);
+}
+
+/*
+ * Executes a skein_update and skein_digest on a pre-initialized crypto
+ * context in a single step. See the documentation to these functions to
+ * see what to pass here.
+ */
+static int
+skein_digest(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *digest,
+    crypto_req_handle_t req)
+{
+	int error = CRYPTO_SUCCESS;
+
+	ASSERT(SKEIN_CTX(ctx) != NULL);
+
+	if (digest->cd_length <
+	    CRYPTO_BITS2BYTES(SKEIN_CTX(ctx)->sc_digest_bitlen)) {
+		digest->cd_length =
+		    CRYPTO_BITS2BYTES(SKEIN_CTX(ctx)->sc_digest_bitlen);
+		return (CRYPTO_BUFFER_TOO_SMALL);
+	}
+
+	error = skein_update(ctx, data, req);
+	if (error != CRYPTO_SUCCESS) {
+		bzero(SKEIN_CTX(ctx), sizeof (*SKEIN_CTX(ctx)));
+		kmem_free(SKEIN_CTX(ctx), sizeof (*SKEIN_CTX(ctx)));
+		SKEIN_CTX_LVALUE(ctx) = NULL;
+		digest->cd_length = 0;
+		return (error);
+	}
+	error = skein_final(ctx, digest, req);
+
+	return (error);
+}
+
+/*
+ * Performs a skein Update with the input message in `data' (successive calls
+ * can push more data). This is used both for digest and MAC operation.
+ * Supported input data formats are raw, uio and mblk.
+ */
+/*ARGSUSED*/
+static int
+skein_update(crypto_ctx_t *ctx, crypto_data_t *data, crypto_req_handle_t req)
+{
+	int error = CRYPTO_SUCCESS;
+
+	ASSERT(SKEIN_CTX(ctx) != NULL);
+
+	switch (data->cd_format) {
+	case CRYPTO_DATA_RAW:
+		SKEIN_OP(SKEIN_CTX(ctx), Update,
+		    (uint8_t *)data->cd_raw.iov_base + data->cd_offset,
+		    data->cd_length);
+		break;
+	case CRYPTO_DATA_UIO:
+		error = skein_digest_update_uio(SKEIN_CTX(ctx), data);
+		break;
+	case CRYPTO_DATA_MBLK:
+		error = skein_digest_update_mblk(SKEIN_CTX(ctx), data);
+		break;
+	default:
+		error = CRYPTO_ARGUMENTS_BAD;
+	}
+
+	return (error);
+}
+
+/*
+ * Performs a skein Final, writing the output to `digest'. This is used both
+ * for digest and MAC operation.
+ * Supported output digest formats are raw, uio and mblk.
+ */
+/*ARGSUSED*/
+static int
+skein_final(crypto_ctx_t *ctx, crypto_data_t *digest, crypto_req_handle_t req)
+{
+	int error = CRYPTO_SUCCESS;
+
+	ASSERT(SKEIN_CTX(ctx) != NULL);
+
+	if (digest->cd_length <
+	    CRYPTO_BITS2BYTES(SKEIN_CTX(ctx)->sc_digest_bitlen)) {
+		digest->cd_length =
+		    CRYPTO_BITS2BYTES(SKEIN_CTX(ctx)->sc_digest_bitlen);
+		return (CRYPTO_BUFFER_TOO_SMALL);
+	}
+
+	switch (digest->cd_format) {
+	case CRYPTO_DATA_RAW:
+		SKEIN_OP(SKEIN_CTX(ctx), Final,
+		    (uint8_t *)digest->cd_raw.iov_base + digest->cd_offset);
+		break;
+	case CRYPTO_DATA_UIO:
+		error = skein_digest_final_uio(SKEIN_CTX(ctx), digest, req);
+		break;
+	case CRYPTO_DATA_MBLK:
+		error = skein_digest_final_mblk(SKEIN_CTX(ctx), digest, req);
+		break;
+	default:
+		error = CRYPTO_ARGUMENTS_BAD;
+	}
+
+	if (error == CRYPTO_SUCCESS)
+		digest->cd_length =
+		    CRYPTO_BITS2BYTES(SKEIN_CTX(ctx)->sc_digest_bitlen);
+	else
+		digest->cd_length = 0;
+
+	bzero(SKEIN_CTX(ctx), sizeof (*SKEIN_CTX(ctx)));
+	kmem_free(SKEIN_CTX(ctx), sizeof (*(SKEIN_CTX(ctx))));
+	SKEIN_CTX_LVALUE(ctx) = NULL;
+
+	return (error);
+}
+
+/*
+ * Performs a full skein digest computation in a single call, configuring the
+ * algorithm according to `mechanism', reading the input to be digested from
+ * `data' and writing the output to `digest'.
+ * Supported input/output formats are raw, uio and mblk.
+ */
+/*ARGSUSED*/
+static int
+skein_digest_atomic(crypto_provider_handle_t provider,
+    crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
+    crypto_data_t *data, crypto_data_t *digest, crypto_req_handle_t req)
+{
+	int		error;
+	skein_ctx_t	skein_ctx;
+	crypto_ctx_t	ctx;
+	SKEIN_CTX_LVALUE(&ctx) = &skein_ctx;
+
+	/* Init */
+	if (!VALID_SKEIN_DIGEST_MECH(mechanism->cm_type))
+		return (CRYPTO_MECHANISM_INVALID);
+	skein_ctx.sc_mech_type = mechanism->cm_type;
+	error = skein_get_digest_bitlen(mechanism, &skein_ctx.sc_digest_bitlen);
+	if (error != CRYPTO_SUCCESS)
+		goto out;
+	SKEIN_OP(&skein_ctx, Init, skein_ctx.sc_digest_bitlen);
+
+	if ((error = skein_update(&ctx, data, digest)) != CRYPTO_SUCCESS)
+		goto out;
+	if ((error = skein_final(&ctx, data, digest)) != CRYPTO_SUCCESS)
+		goto out;
+
+out:
+	if (error == CRYPTO_SUCCESS)
+		digest->cd_length =
+		    CRYPTO_BITS2BYTES(skein_ctx.sc_digest_bitlen);
+	else
+		digest->cd_length = 0;
+	bzero(&skein_ctx, sizeof (skein_ctx));
+
+	return (error);
+}
+
+/*
+ * Helper function that builds a Skein MAC context from the provided
+ * mechanism and key.
+ */
+static int
+skein_mac_ctx_build(skein_ctx_t *ctx, crypto_mechanism_t *mechanism,
+    crypto_key_t *key)
+{
+	int error;
+
+	if (!VALID_SKEIN_MAC_MECH(mechanism->cm_type))
+		return (CRYPTO_MECHANISM_INVALID);
+	if (key->ck_format != CRYPTO_KEY_RAW)
+		return (CRYPTO_ARGUMENTS_BAD);
+	ctx->sc_mech_type = mechanism->cm_type;
+	error = skein_get_digest_bitlen(mechanism, &ctx->sc_digest_bitlen);
+	if (error != CRYPTO_SUCCESS)
+		return (error);
+	SKEIN_OP(ctx, InitExt, ctx->sc_digest_bitlen, 0, key->ck_data,
+	    CRYPTO_BITS2BYTES(key->ck_length));
+
+	return (CRYPTO_SUCCESS);
+}
+
+/*
+ * KCF software provide mac entry points.
+ */
+/*
+ * Initializes a skein MAC context. You may pass a ctx_template, in which
+ * case the template will be reused to make initialization more efficient.
+ * Otherwise a new context will be constructed. The mechanism cm_type must
+ * be one of SKEIN_*_MAC_MECH_INFO_TYPE. Same as in skein_digest_init, you
+ * may pass a skein_param_t in cm_param to configure the length of the
+ * digest. The key must be in raw format.
+ */
+static int
+skein_mac_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
+    crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
+    crypto_req_handle_t req)
+{
+	int	error;
+
+	SKEIN_CTX_LVALUE(ctx) = kmem_alloc(sizeof (*SKEIN_CTX(ctx)),
+	    crypto_kmflag(req));
+	if (SKEIN_CTX(ctx) == NULL)
+		return (CRYPTO_HOST_MEMORY);
+
+	if (ctx_template != NULL) {
+		bcopy(ctx_template, SKEIN_CTX(ctx),
+		    sizeof (*SKEIN_CTX(ctx)));
+	} else {
+		error = skein_mac_ctx_build(SKEIN_CTX(ctx), mechanism, key);
+		if (error != CRYPTO_SUCCESS)
+			goto errout;
+	}
+
+	return (CRYPTO_SUCCESS);
+errout:
+	bzero(SKEIN_CTX(ctx), sizeof (*SKEIN_CTX(ctx)));
+	kmem_free(SKEIN_CTX(ctx), sizeof (*SKEIN_CTX(ctx)));
+	return (error);
+}
+
+/*
+ * The MAC update and final calls are reused from the regular digest code.
+ */
+
+/*ARGSUSED*/
+/*
+ * Same as skein_digest_atomic, performs an atomic Skein MAC operation in
+ * one step. All the same properties apply to the arguments of this
+ * function as to those of the partial operations above.
+ */
+static int
+skein_mac_atomic(crypto_provider_handle_t provider,
+    crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
+    crypto_key_t *key, crypto_data_t *data, crypto_data_t *mac,
+    crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
+{
+	/* faux crypto context just for skein_digest_{update,final} */
+	int		error;
+	crypto_ctx_t	ctx;
+	skein_ctx_t	skein_ctx;
+	SKEIN_CTX_LVALUE(&ctx) = &skein_ctx;
+
+	if (ctx_template != NULL) {
+		bcopy(ctx_template, &skein_ctx, sizeof (skein_ctx));
+	} else {
+		error = skein_mac_ctx_build(&skein_ctx, mechanism, key);
+		if (error != CRYPTO_SUCCESS)
+			goto errout;
+	}
+
+	if ((error = skein_update(&ctx, data, req)) != CRYPTO_SUCCESS)
+		goto errout;
+	if ((error = skein_final(&ctx, mac, req)) != CRYPTO_SUCCESS)
+		goto errout;
+
+	return (CRYPTO_SUCCESS);
+errout:
+	bzero(&skein_ctx, sizeof (skein_ctx));
+	return (error);
+}
+
+/*
+ * KCF software provider context management entry points.
+ */
+
+/*
+ * Constructs a context template for the Skein MAC algorithm. The same
+ * properties apply to the arguments of this function as to those of
+ * skein_mac_init.
+ */
+/*ARGSUSED*/
+static int
+skein_create_ctx_template(crypto_provider_handle_t provider,
+    crypto_mechanism_t *mechanism, crypto_key_t *key,
+    crypto_spi_ctx_template_t *ctx_template, size_t *ctx_template_size,
+    crypto_req_handle_t req)
+{
+	int		error;
+	skein_ctx_t	*ctx_tmpl;
+
+	ctx_tmpl = kmem_alloc(sizeof (*ctx_tmpl), crypto_kmflag(req));
+	if (ctx_tmpl == NULL)
+		return (CRYPTO_HOST_MEMORY);
+	error = skein_mac_ctx_build(ctx_tmpl, mechanism, key);
+	if (error != CRYPTO_SUCCESS)
+		goto errout;
+	*ctx_template = ctx_tmpl;
+	*ctx_template_size = sizeof (*ctx_tmpl);
+
+	return (CRYPTO_SUCCESS);
+errout:
+	bzero(ctx_tmpl, sizeof (*ctx_tmpl));
+	kmem_free(ctx_tmpl, sizeof (*ctx_tmpl));
+	return (error);
+}
+
+/*
+ * Frees a skein context in a parent crypto context.
+ */
+static int
+skein_free_context(crypto_ctx_t *ctx)
+{
+	if (SKEIN_CTX(ctx) != NULL) {
+		bzero(SKEIN_CTX(ctx), sizeof (*SKEIN_CTX(ctx)));
+		kmem_free(SKEIN_CTX(ctx), sizeof (*SKEIN_CTX(ctx)));
+		SKEIN_CTX_LVALUE(ctx) = NULL;
+	}
+
+	return (CRYPTO_SUCCESS);
+}
diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index 80d86e1b8d0e..52f582be1633 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -1372,7 +1372,7 @@ arc_cksum_verify(arc_buf_t *buf)
 		mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
 		return;
 	}
-	fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
+	fletcher_2_native(buf->b_data, buf->b_hdr->b_size, NULL, &zc);
 	if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc))
 		panic("buffer modified while frozen!");
 	mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
@@ -1385,7 +1385,7 @@ arc_cksum_equal(arc_buf_t *buf)
 	int equal;
 
 	mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
-	fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
+	fletcher_2_native(buf->b_data, buf->b_hdr->b_size, NULL, &zc);
 	equal = ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc);
 	mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
 
@@ -1405,7 +1405,7 @@ arc_cksum_compute(arc_buf_t *buf, boolean_t force)
 	}
 	buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP);
 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size,
-	    buf->b_hdr->b_freeze_cksum);
+	    NULL, buf->b_hdr->b_freeze_cksum);
 	mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
 	arc_buf_watch(buf);
 }
diff --git a/uts/common/fs/zfs/ddt.c b/uts/common/fs/zfs/ddt.c
index 7d938965722a..9955f89e7759 100644
--- a/uts/common/fs/zfs/ddt.c
+++ b/uts/common/fs/zfs/ddt.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -59,7 +59,8 @@ ddt_object_create(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
 	spa_t *spa = ddt->ddt_spa;
 	objset_t *os = ddt->ddt_os;
 	uint64_t *objectp = &ddt->ddt_object[type][class];
-	boolean_t prehash = zio_checksum_table[ddt->ddt_checksum].ci_dedup;
+	boolean_t prehash = zio_checksum_table[ddt->ddt_checksum].ci_flags &
+	    ZCHECKSUM_FLAG_DEDUP;
 	char name[DDT_NAMELEN];
 
 	ddt_object_name(ddt, type, class, name);
diff --git a/uts/common/fs/zfs/dmu.c b/uts/common/fs/zfs/dmu.c
index 6ebd95cd3ef2..5b24268af1a8 100644
--- a/uts/common/fs/zfs/dmu.c
+++ b/uts/common/fs/zfs/dmu.c
@@ -1424,7 +1424,8 @@ dmu_sync_done(zio_t *zio, arc_buf_t *buf, void *varg)
 
 			ASSERT(BP_EQUAL(bp, bp_orig));
 			ASSERT(zio->io_prop.zp_compress != ZIO_COMPRESS_OFF);
-			ASSERT(zio_checksum_table[chksum].ci_dedup);
+			ASSERT(zio_checksum_table[chksum].ci_flags &
+			    ZCHECKSUM_FLAG_NOPWRITE);
 		}
 		dr->dt.dl.dr_overridden_by = *zio->io_bp;
 		dr->dt.dl.dr_override_state = DR_OVERRIDDEN;
@@ -1769,8 +1770,10 @@ dmu_write_policy(objset_t *os, dnode_t *dn, int level, int wp, zio_prop_t *zp)
 		 * as well.  Otherwise, the metadata checksum defaults
 		 * to fletcher4.
 		 */
-		if (zio_checksum_table[checksum].ci_correctable < 1 ||
-		    zio_checksum_table[checksum].ci_eck)
+		if (!(zio_checksum_table[checksum].ci_flags &
+		    ZCHECKSUM_FLAG_METADATA) ||
+		    (zio_checksum_table[checksum].ci_flags &
+		    ZCHECKSUM_FLAG_EMBEDDED))
 			checksum = ZIO_CHECKSUM_FLETCHER_4;
 
 		if (os->os_redundant_metadata == ZFS_REDUNDANT_METADATA_ALL ||
@@ -1809,17 +1812,20 @@ dmu_write_policy(objset_t *os, dnode_t *dn, int level, int wp, zio_prop_t *zp)
 		 */
 		if (dedup_checksum != ZIO_CHECKSUM_OFF) {
 			dedup = (wp & WP_DMU_SYNC) ? B_FALSE : B_TRUE;
-			if (!zio_checksum_table[checksum].ci_dedup)
+			if (!(zio_checksum_table[checksum].ci_flags &
+			    ZCHECKSUM_FLAG_DEDUP))
 				dedup_verify = B_TRUE;
 		}
 
 		/*
-		 * Enable nopwrite if we have a cryptographically secure
-		 * checksum that has no known collisions (i.e. SHA-256)
-		 * and compression is enabled.  We don't enable nopwrite if
-		 * dedup is enabled as the two features are mutually exclusive.
+		 * Enable nopwrite if we have secure enough checksum
+		 * algorithm (see comment in zio_nop_write) and
+		 * compression is enabled.  We don't enable nopwrite if
+		 * dedup is enabled as the two features are mutually
+		 * exclusive.
 		 */
-		nopwrite = (!dedup && zio_checksum_table[checksum].ci_dedup &&
+		nopwrite = (!dedup && (zio_checksum_table[checksum].ci_flags &
+		    ZCHECKSUM_FLAG_NOPWRITE) &&
 		    compress != ZIO_COMPRESS_OFF && zfs_nopwrite_enabled);
 	}
 
diff --git a/uts/common/fs/zfs/dmu_send.c b/uts/common/fs/zfs/dmu_send.c
index 0b57da6b8aaa..a0bf23349aa6 100644
--- a/uts/common/fs/zfs/dmu_send.c
+++ b/uts/common/fs/zfs/dmu_send.c
@@ -268,7 +268,8 @@ dump_write(dmu_sendarg_t *dsp, dmu_object_type_t type,
 		drrw->drr_checksumtype = ZIO_CHECKSUM_OFF;
 	} else {
 		drrw->drr_checksumtype = BP_GET_CHECKSUM(bp);
-		if (zio_checksum_table[drrw->drr_checksumtype].ci_dedup)
+		if (zio_checksum_table[drrw->drr_checksumtype].ci_flags &
+		    ZCHECKSUM_FLAG_DEDUP)
 			drrw->drr_checksumflags |= DRR_CHECKSUM_DEDUP;
 		DDK_SET_LSIZE(&drrw->drr_key, BP_GET_LSIZE(bp));
 		DDK_SET_PSIZE(&drrw->drr_key, BP_GET_PSIZE(bp));
diff --git a/uts/common/fs/zfs/dsl_dataset.c b/uts/common/fs/zfs/dsl_dataset.c
index 6606d89da27e..8f9e26bc1683 100644
--- a/uts/common/fs/zfs/dsl_dataset.c
+++ b/uts/common/fs/zfs/dsl_dataset.c
@@ -50,6 +50,8 @@
 #include <sys/dsl_destroy.h>
 #include <sys/dsl_userhold.h>
 #include <sys/dsl_bookmark.h>
+#include <sys/dmu_send.h>
+#include <sys/zio_checksum.h>
 
 /*
  * The SPA supports block sizes up to 16MB.  However, very large blocks
@@ -124,10 +126,16 @@ dsl_dataset_block_born(dsl_dataset_t *ds, const blkptr_t *bp, dmu_tx_t *tx)
 	dsl_dataset_phys(ds)->ds_compressed_bytes += compressed;
 	dsl_dataset_phys(ds)->ds_uncompressed_bytes += uncompressed;
 	dsl_dataset_phys(ds)->ds_unique_bytes += used;
+
 	if (BP_GET_LSIZE(bp) > SPA_OLD_MAXBLOCKSIZE) {
 		ds->ds_feature_activation_needed[SPA_FEATURE_LARGE_BLOCKS] =
 		    B_TRUE;
 	}
+
+	spa_feature_t f = zio_checksum_to_feature(BP_GET_CHECKSUM(bp));
+	if (f != SPA_FEATURE_NONE)
+		ds->ds_feature_activation_needed[f] = B_TRUE;
+
 	mutex_exit(&ds->ds_lock);
 	dsl_dir_diduse_space(ds->ds_dir, DD_USED_HEAD, delta,
 	    compressed, uncompressed, tx);
diff --git a/uts/common/fs/zfs/edonr_zfs.c b/uts/common/fs/zfs/edonr_zfs.c
new file mode 100644
index 000000000000..93f1221fd532
--- /dev/null
+++ b/uts/common/fs/zfs/edonr_zfs.c
@@ -0,0 +1,102 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://opensource.org/licenses/CDDL-1.0.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2013 Saso Kiselkov.  All rights reserved.
+ * Use is subject to license terms.
+ */
+#include <sys/zfs_context.h>
+#include <sys/zio.h>
+#include <sys/edonr.h>
+
+#define	EDONR_MODE		512
+#define	EDONR_BLOCK_SIZE	EdonR512_BLOCK_SIZE
+
+/*
+ * Native zio_checksum interface for the Edon-R hash function.
+ */
+/*ARGSUSED*/
+void
+zio_checksum_edonr_native(const void *buf, uint64_t size,
+    const void *ctx_template, zio_cksum_t *zcp)
+{
+	uint8_t		digest[EDONR_MODE / 8];
+	EdonRState	ctx;
+
+	ASSERT(ctx_template != NULL);
+	bcopy(ctx_template, &ctx, sizeof (ctx));
+	EdonRUpdate(&ctx, buf, size * 8);
+	EdonRFinal(&ctx, digest);
+	bcopy(digest, zcp->zc_word, sizeof (zcp->zc_word));
+}
+
+/*
+ * Byteswapped zio_checksum interface for the Edon-R hash function.
+ */
+void
+zio_checksum_edonr_byteswap(const void *buf, uint64_t size,
+    const void *ctx_template, zio_cksum_t *zcp)
+{
+	zio_cksum_t	tmp;
+
+	zio_checksum_edonr_native(buf, size, ctx_template, &tmp);
+	zcp->zc_word[0] = BSWAP_64(zcp->zc_word[0]);
+	zcp->zc_word[1] = BSWAP_64(zcp->zc_word[1]);
+	zcp->zc_word[2] = BSWAP_64(zcp->zc_word[2]);
+	zcp->zc_word[3] = BSWAP_64(zcp->zc_word[3]);
+}
+
+void *
+zio_checksum_edonr_tmpl_init(const zio_cksum_salt_t *salt)
+{
+	EdonRState	*ctx;
+	uint8_t		salt_block[EDONR_BLOCK_SIZE];
+
+	/*
+	 * Edon-R needs all but the last hash invocation to be on full-size
+	 * blocks, but the salt is too small. Rather than simply padding it
+	 * with zeros, we expand the salt into a new salt block of proper
+	 * size by double-hashing it (the new salt block will be composed of
+	 * H(salt) || H(H(salt))).
+	 */
+	CTASSERT(EDONR_BLOCK_SIZE == 2 * (EDONR_MODE / 8));
+	EdonRHash(EDONR_MODE, salt->zcs_bytes, sizeof (salt->zcs_bytes) * 8,
+	    salt_block);
+	EdonRHash(EDONR_MODE, salt_block, EDONR_MODE, salt_block +
+	    EDONR_MODE / 8);
+
+	/*
+	 * Feed the new salt block into the hash function - this will serve
+	 * as our MAC key.
+	 */
+	ctx = kmem_zalloc(sizeof (*ctx), KM_SLEEP);
+	EdonRInit(ctx, EDONR_MODE);
+	EdonRUpdate(ctx, salt_block, sizeof (salt_block) * 8);
+	return (ctx);
+}
+
+void
+zio_checksum_edonr_tmpl_free(void *ctx_template)
+{
+	EdonRState	*ctx = ctx_template;
+
+	bzero(ctx, sizeof (*ctx));
+	kmem_free(ctx, sizeof (*ctx));
+}
diff --git a/uts/common/fs/zfs/sha256.c b/uts/common/fs/zfs/sha256.c
index f515be6bb304..81a7f6b1c28a 100644
--- a/uts/common/fs/zfs/sha256.c
+++ b/uts/common/fs/zfs/sha256.c
@@ -22,12 +22,17 @@
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright 2013 Saso Kiselkov. All rights reserved.
+ */
 #include <sys/zfs_context.h>
 #include <sys/zio.h>
 #include <sys/sha2.h>
 
+/*ARGSUSED*/
 void
-zio_checksum_SHA256(const void *buf, uint64_t size, zio_cksum_t *zcp)
+zio_checksum_SHA256(const void *buf, uint64_t size,
+    const void *ctx_template, zio_cksum_t *zcp)
 {
 	SHA2_CTX ctx;
 	zio_cksum_t tmp;
@@ -48,3 +53,29 @@ zio_checksum_SHA256(const void *buf, uint64_t size, zio_cksum_t *zcp)
 	zcp->zc_word[2] = BE_64(tmp.zc_word[2]);
 	zcp->zc_word[3] = BE_64(tmp.zc_word[3]);
 }
+
+/*ARGSUSED*/
+void
+zio_checksum_SHA512_native(const void *buf, uint64_t size,
+    const void *ctx_template, zio_cksum_t *zcp)
+{
+	SHA2_CTX	ctx;
+
+	SHA2Init(SHA512_256, &ctx);
+	SHA2Update(&ctx, buf, size);
+	SHA2Final(zcp, &ctx);
+}
+
+/*ARGSUSED*/
+void
+zio_checksum_SHA512_byteswap(const void *buf, uint64_t size,
+    const void *ctx_template, zio_cksum_t *zcp)
+{
+	zio_cksum_t	tmp;
+
+	zio_checksum_SHA512_native(buf, size, ctx_template, &tmp);
+	zcp->zc_word[0] = BSWAP_64(tmp.zc_word[0]);
+	zcp->zc_word[1] = BSWAP_64(tmp.zc_word[1]);
+	zcp->zc_word[2] = BSWAP_64(tmp.zc_word[2]);
+	zcp->zc_word[3] = BSWAP_64(tmp.zc_word[3]);
+}
diff --git a/uts/common/fs/zfs/skein_zfs.c b/uts/common/fs/zfs/skein_zfs.c
new file mode 100644
index 000000000000..65923403968d
--- /dev/null
+++ b/uts/common/fs/zfs/skein_zfs.c
@@ -0,0 +1,91 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://opensource.org/licenses/CDDL-1.0.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2013 Saso Kiselkov.  All rights reserved.
+ */
+#include <sys/zfs_context.h>
+#include <sys/zio.h>
+#include <sys/skein.h>
+
+/*
+ * Computes a native 256-bit skein MAC checksum. Please note that this
+ * function requires the presence of a ctx_template that should be allocated
+ * using zio_checksum_skein_tmpl_init.
+ */
+/*ARGSUSED*/
+void
+zio_checksum_skein_native(const void *buf, uint64_t size,
+    const void *ctx_template, zio_cksum_t *zcp)
+{
+	Skein_512_Ctxt_t	ctx;
+
+	ASSERT(ctx_template != NULL);
+	bcopy(ctx_template, &ctx, sizeof (ctx));
+	(void) Skein_512_Update(&ctx, buf, size);
+	(void) Skein_512_Final(&ctx, (uint8_t *)zcp);
+	bzero(&ctx, sizeof (ctx));
+}
+
+/*
+ * Byteswapped version of zio_checksum_skein_native. This just invokes
+ * the native checksum function and byteswaps the resulting checksum (since
+ * skein is internally endian-insensitive).
+ */
+void
+zio_checksum_skein_byteswap(const void *buf, uint64_t size,
+    const void *ctx_template, zio_cksum_t *zcp)
+{
+	zio_cksum_t	tmp;
+
+	zio_checksum_skein_native(buf, size, ctx_template, &tmp);
+	zcp->zc_word[0] = BSWAP_64(tmp.zc_word[0]);
+	zcp->zc_word[1] = BSWAP_64(tmp.zc_word[1]);
+	zcp->zc_word[2] = BSWAP_64(tmp.zc_word[2]);
+	zcp->zc_word[3] = BSWAP_64(tmp.zc_word[3]);
+}
+
+/*
+ * Allocates a skein MAC template suitable for using in skein MAC checksum
+ * computations and returns a pointer to it.
+ */
+void *
+zio_checksum_skein_tmpl_init(const zio_cksum_salt_t *salt)
+{
+	Skein_512_Ctxt_t	*ctx;
+
+	ctx = kmem_zalloc(sizeof (*ctx), KM_SLEEP);
+	(void) Skein_512_InitExt(ctx, sizeof (zio_cksum_t) * 8, 0,
+	    salt->zcs_bytes, sizeof (salt->zcs_bytes));
+	return (ctx);
+}
+
+/*
+ * Frees a skein context template previously allocated using
+ * zio_checksum_skein_tmpl_init.
+ */
+void
+zio_checksum_skein_tmpl_free(void *ctx_template)
+{
+	Skein_512_Ctxt_t	*ctx = ctx_template;
+
+	bzero(ctx, sizeof (*ctx));
+	kmem_free(ctx, sizeof (*ctx));
+}
diff --git a/uts/common/fs/zfs/spa.c b/uts/common/fs/zfs/spa.c
index 537c0945d15a..95a6b0fae776 100644
--- a/uts/common/fs/zfs/spa.c
+++ b/uts/common/fs/zfs/spa.c
@@ -24,6 +24,7 @@
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2015, Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
+ * Copyright 2013 Saso Kiselkov. All rights reserved.
  */
 
 /*
@@ -2512,6 +2513,19 @@ spa_load_impl(spa_t *spa, uint64_t pool_guid, nvlist_t *config,
 		return (spa_load(spa, state, SPA_IMPORT_EXISTING, B_TRUE));
 	}
 
+	/* Grab the secret checksum salt from the MOS. */
+	error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
+	    DMU_POOL_CHECKSUM_SALT, 1,
+	    sizeof (spa->spa_cksum_salt.zcs_bytes),
+	    spa->spa_cksum_salt.zcs_bytes);
+	if (error == ENOENT) {
+		/* Generate a new salt for subsequent use */
+		(void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes,
+		    sizeof (spa->spa_cksum_salt.zcs_bytes));
+	} else if (error != 0) {
+		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
+	}
+
 	if (spa_dir_prop(spa, DMU_POOL_SYNC_BPOBJ, &obj) != 0)
 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
 	error = bpobj_open(&spa->spa_deferred_bpobj, spa->spa_meta_objset, obj);
@@ -3664,6 +3678,12 @@ spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props,
 	if (version >= SPA_VERSION_ZPOOL_HISTORY)
 		spa_history_create_obj(spa, tx);
 
+	/*
+	 * Generate some random noise for salted checksums to operate on.
+	 */
+	(void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes,
+	    sizeof (spa->spa_cksum_salt.zcs_bytes));
+
 	/*
 	 * Set pool properties.
 	 */
@@ -6215,6 +6235,20 @@ spa_sync_upgrades(spa_t *spa, dmu_tx_t *tx)
 		if (lz4_en && !lz4_ac)
 			spa_feature_incr(spa, SPA_FEATURE_LZ4_COMPRESS, tx);
 	}
+
+	/*
+	 * If we haven't written the salt, do so now.  Note that the
+	 * feature may not be activated yet, but that's fine since
+	 * the presence of this ZAP entry is backwards compatible.
+	 */
+	if (zap_contains(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
+	    DMU_POOL_CHECKSUM_SALT) == ENOENT) {
+		VERIFY0(zap_add(spa->spa_meta_objset,
+		    DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CHECKSUM_SALT, 1,
+		    sizeof (spa->spa_cksum_salt.zcs_bytes),
+		    spa->spa_cksum_salt.zcs_bytes, tx));
+	}
+
 	rrw_exit(&dp->dp_config_rwlock, FTAG);
 }
 
diff --git a/uts/common/fs/zfs/spa_misc.c b/uts/common/fs/zfs/spa_misc.c
index c5af055a8608..9148a0343316 100644
--- a/uts/common/fs/zfs/spa_misc.c
+++ b/uts/common/fs/zfs/spa_misc.c
@@ -23,6 +23,7 @@
  * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
+ * Copyright 2013 Saso Kiselkov. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -50,7 +51,7 @@
 #include <sys/arc.h>
 #include <sys/ddt.h>
 #include "zfs_prop.h"
-#include "zfeature_common.h"
+#include <sys/zfeature.h>
 
 /*
  * SPA locking
@@ -556,6 +557,7 @@ spa_add(const char *name, nvlist_t *config, const char *altroot)
 	mutex_init(&spa->spa_history_lock, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&spa->spa_proc_lock, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&spa->spa_props_lock, NULL, MUTEX_DEFAULT, NULL);
+	mutex_init(&spa->spa_cksum_tmpls_lock, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&spa->spa_scrub_lock, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&spa->spa_suspend_lock, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&spa->spa_vdev_top_lock, NULL, MUTEX_DEFAULT, NULL);
@@ -716,6 +718,8 @@ spa_remove(spa_t *spa)
 	for (int t = 0; t < TXG_SIZE; t++)
 		bplist_destroy(&spa->spa_free_bplist[t]);
 
+	zio_checksum_templates_free(spa);
+
 	cv_destroy(&spa->spa_async_cv);
 	cv_destroy(&spa->spa_evicting_os_cv);
 	cv_destroy(&spa->spa_proc_cv);
@@ -729,6 +733,7 @@ spa_remove(spa_t *spa)
 	mutex_destroy(&spa->spa_history_lock);
 	mutex_destroy(&spa->spa_proc_lock);
 	mutex_destroy(&spa->spa_props_lock);
+	mutex_destroy(&spa->spa_cksum_tmpls_lock);
 	mutex_destroy(&spa->spa_scrub_lock);
 	mutex_destroy(&spa->spa_suspend_lock);
 	mutex_destroy(&spa->spa_vdev_top_lock);
diff --git a/uts/common/fs/zfs/sys/dmu.h b/uts/common/fs/zfs/sys/dmu.h
index 149cb112f96a..c10806989d46 100644
--- a/uts/common/fs/zfs/sys/dmu.h
+++ b/uts/common/fs/zfs/sys/dmu.h
@@ -27,6 +27,7 @@
  * Copyright 2013 DEY Storage Systems, Inc.
  * Copyright 2014 HybridCluster. All rights reserved.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
+ * Copyright 2013 Saso Kiselkov. All rights reserved.
  */
 
 /* Portions Copyright 2010 Robert Milkowski */
@@ -318,6 +319,7 @@ typedef struct dmu_buf {
 #define	DMU_POOL_FREE_BPOBJ		"free_bpobj"
 #define	DMU_POOL_BPTREE_OBJ		"bptree_obj"
 #define	DMU_POOL_EMPTY_BPOBJ		"empty_bpobj"
+#define	DMU_POOL_CHECKSUM_SALT		"org.illumos:checksum_salt"
 
 /*
  * Allocate an object from this objset.  The range of object numbers
diff --git a/uts/common/fs/zfs/sys/spa.h b/uts/common/fs/zfs/sys/spa.h
index 2c4887b6ca50..7ac78390338a 100644
--- a/uts/common/fs/zfs/sys/spa.h
+++ b/uts/common/fs/zfs/sys/spa.h
@@ -23,6 +23,7 @@
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
+ * Copyright 2013 Saso Kiselkov. All rights reserved.
  */
 
 #ifndef _SYS_SPA_H
@@ -146,6 +147,14 @@ typedef struct zio_cksum {
 	uint64_t	zc_word[4];
 } zio_cksum_t;
 
+/*
+ * Some checksums/hashes need a 256-bit initialization salt. This salt is kept
+ * secret and is suitable for use in MAC algorithms as the key.
+ */
+typedef struct zio_cksum_salt {
+	uint8_t		zcs_bytes[32];
+} zio_cksum_salt_t;
+
 /*
  * Each block is described by its DVAs, time of birth, checksum, etc.
  * The word-by-word, bit-by-bit layout of the blkptr is as follows:
diff --git a/uts/common/fs/zfs/sys/spa_impl.h b/uts/common/fs/zfs/sys/spa_impl.h
index 56e6adb713d2..441800198215 100644
--- a/uts/common/fs/zfs/sys/spa_impl.h
+++ b/uts/common/fs/zfs/sys/spa_impl.h
@@ -23,6 +23,7 @@
  * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
+ * Copyright 2013 Saso Kiselkov. All rights reserved.
  */
 
 #ifndef _SYS_SPA_IMPL_H
@@ -165,6 +166,10 @@ struct spa {
 	uint64_t	spa_syncing_txg;	/* txg currently syncing */
 	bpobj_t		spa_deferred_bpobj;	/* deferred-free bplist */
 	bplist_t	spa_free_bplist[TXG_SIZE]; /* bplist of stuff to free */
+	zio_cksum_salt_t spa_cksum_salt;	/* secret salt for cksum */
+	/* checksum context templates */
+	kmutex_t	spa_cksum_tmpls_lock;
+	void		*spa_cksum_tmpls[ZIO_CHECKSUM_FUNCTIONS];
 	uberblock_t	spa_ubsync;		/* last synced uberblock */
 	uberblock_t	spa_uberblock;		/* current uberblock */
 	boolean_t	spa_extreme_rewind;	/* rewind past deferred frees */
diff --git a/uts/common/fs/zfs/sys/zio.h b/uts/common/fs/zfs/sys/zio.h
index d160c9468659..877e2839bf80 100644
--- a/uts/common/fs/zfs/sys/zio.h
+++ b/uts/common/fs/zfs/sys/zio.h
@@ -82,6 +82,9 @@ enum zio_checksum {
 	ZIO_CHECKSUM_SHA256,
 	ZIO_CHECKSUM_ZILOG2,
 	ZIO_CHECKSUM_NOPARITY,
+	ZIO_CHECKSUM_SHA512,
+	ZIO_CHECKSUM_SKEIN,
+	ZIO_CHECKSUM_EDONR,
 	ZIO_CHECKSUM_FUNCTIONS
 };
 
diff --git a/uts/common/fs/zfs/sys/zio_checksum.h b/uts/common/fs/zfs/sys/zio_checksum.h
index 0c293ab20e2c..572b29d3cb1f 100644
--- a/uts/common/fs/zfs/sys/zio_checksum.h
+++ b/uts/common/fs/zfs/sys/zio_checksum.h
@@ -20,13 +20,15 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2014, 2015 by Delphix. All rights reserved.
+ * Copyright Saso Kiselkov 2013, All rights reserved.
  */
 
 #ifndef _SYS_ZIO_CHECKSUM_H
 #define	_SYS_ZIO_CHECKSUM_H
 
 #include <sys/zio.h>
+#include <zfeature_common.h>
 
 #ifdef	__cplusplus
 extern "C" {
@@ -35,17 +37,34 @@ extern "C" {
 /*
  * Signature for checksum functions.
  */
-typedef void zio_checksum_func_t(const void *, uint64_t, zio_cksum_t *);
+typedef void zio_checksum_t(const void *data, uint64_t size,
+    const void *ctx_template, zio_cksum_t *zcp);
+typedef void *zio_checksum_tmpl_init_t(const zio_cksum_salt_t *salt);
+typedef void zio_checksum_tmpl_free_t(void *ctx_template);
+
+typedef enum zio_checksum_flags {
+	/* Strong enough for metadata? */
+	ZCHECKSUM_FLAG_METADATA = (1 << 1),
+	/* ZIO embedded checksum */
+	ZCHECKSUM_FLAG_EMBEDDED = (1 << 2),
+	/* Strong enough for dedup (without verification)? */
+	ZCHECKSUM_FLAG_DEDUP = (1 << 3),
+	/* Uses salt value */
+	ZCHECKSUM_FLAG_SALTED = (1 << 4),
+	/* Strong enough for nopwrite? */
+	ZCHECKSUM_FLAG_NOPWRITE = (1 << 5)
+} zio_checksum_flags_t;
 
 /*
  * Information about each checksum function.
  */
 typedef struct zio_checksum_info {
-	zio_checksum_func_t *ci_func[2]; /* checksum function per byteorder */
-	int		ci_correctable;	/* number of correctable bits	*/
-	int		ci_eck;		/* uses zio embedded checksum? */
-	boolean_t	ci_dedup;	/* strong enough for dedup? */
-	char		*ci_name;	/* descriptive name */
+	/* checksum function for each byteorder */
+	zio_checksum_t			*ci_func[2];
+	zio_checksum_tmpl_init_t	*ci_tmpl_init;
+	zio_checksum_tmpl_free_t	*ci_tmpl_free;
+	zio_checksum_flags_t		ci_flags;
+	char				*ci_name;	/* descriptive name */
 } zio_checksum_info_t;
 
 typedef struct zio_bad_cksum {
@@ -62,12 +81,28 @@ extern zio_checksum_info_t zio_checksum_table[ZIO_CHECKSUM_FUNCTIONS];
 /*
  * Checksum routines.
  */
-extern zio_checksum_func_t zio_checksum_SHA256;
+extern zio_checksum_t zio_checksum_SHA256;
+extern zio_checksum_t zio_checksum_SHA512_native;
+extern zio_checksum_t zio_checksum_SHA512_byteswap;
+
+/* Skein */
+extern zio_checksum_t zio_checksum_skein_native;
+extern zio_checksum_t zio_checksum_skein_byteswap;
+extern zio_checksum_tmpl_init_t zio_checksum_skein_tmpl_init;
+extern zio_checksum_tmpl_free_t zio_checksum_skein_tmpl_free;
+
+/* Edon-R */
+extern zio_checksum_t zio_checksum_edonr_native;
+extern zio_checksum_t zio_checksum_edonr_byteswap;
+extern zio_checksum_tmpl_init_t zio_checksum_edonr_tmpl_init;
+extern zio_checksum_tmpl_free_t zio_checksum_edonr_tmpl_free;
 
 extern void zio_checksum_compute(zio_t *zio, enum zio_checksum checksum,
     void *data, uint64_t size);
 extern int zio_checksum_error(zio_t *zio, zio_bad_cksum_t *out);
 extern enum zio_checksum spa_dedup_checksum(spa_t *spa);
+extern void zio_checksum_templates_free(spa_t *spa);
+extern spa_feature_t zio_checksum_to_feature(enum zio_checksum cksum);
 
 #ifdef	__cplusplus
 }
diff --git a/uts/common/fs/zfs/zfs_ioctl.c b/uts/common/fs/zfs/zfs_ioctl.c
index 4552cbb3e3d1..0388ca25f95b 100644
--- a/uts/common/fs/zfs/zfs_ioctl.c
+++ b/uts/common/fs/zfs/zfs_ioctl.c
@@ -180,6 +180,7 @@
 #include <sys/dsl_bookmark.h>
 #include <sys/dsl_userhold.h>
 #include <sys/zfeature.h>
+#include <sys/zio_checksum.h>
 
 #include "zfs_namecheck.h"
 #include "zfs_prop.h"
@@ -3817,11 +3818,6 @@ zfs_check_settable(const char *dsname, nvpair_t *pair, cred_t *cr)
 			return (SET_ERROR(ENOTSUP));
 		break;
 
-	case ZFS_PROP_DEDUP:
-		if (zfs_earlier_version(dsname, SPA_VERSION_DEDUP))
-			return (SET_ERROR(ENOTSUP));
-		break;
-
 	case ZFS_PROP_RECORDSIZE:
 		/* Record sizes above 128k need the feature to be enabled */
 		if (nvpair_value_uint64(pair, &intval) == 0 &&
@@ -3872,6 +3868,45 @@ zfs_check_settable(const char *dsname, nvpair_t *pair, cred_t *cr)
 				return (SET_ERROR(ENOTSUP));
 		}
 		break;
+
+	case ZFS_PROP_CHECKSUM:
+	case ZFS_PROP_DEDUP:
+	{
+		spa_feature_t feature;
+		spa_t *spa;
+
+		/* dedup feature version checks */
+		if (prop == ZFS_PROP_DEDUP &&
+		    zfs_earlier_version(dsname, SPA_VERSION_DEDUP))
+			return (SET_ERROR(ENOTSUP));
+
+		if (nvpair_value_uint64(pair, &intval) != 0)
+			return (SET_ERROR(EINVAL));
+
+		/* check prop value is enabled in features */
+		feature = zio_checksum_to_feature(intval);
+		if (feature == SPA_FEATURE_NONE)
+			break;
+
+		if ((err = spa_open(dsname, &spa, FTAG)) != 0)
+			return (err);
+		/*
+		 * Salted checksums are not supported on root pools.
+		 */
+		if (spa_bootfs(spa) != 0 &&
+		    intval < ZIO_CHECKSUM_FUNCTIONS &&
+		    (zio_checksum_table[intval].ci_flags &
+		    ZCHECKSUM_FLAG_SALTED)) {
+			spa_close(spa, FTAG);
+			return (SET_ERROR(ERANGE));
+		}
+		if (!spa_feature_is_enabled(spa, feature)) {
+			spa_close(spa, FTAG);
+			return (SET_ERROR(ENOTSUP));
+		}
+		spa_close(spa, FTAG);
+		break;
+	}
 	}
 
 	return (zfs_secpolicy_setprop(dsname, prop, pair, CRED()));
diff --git a/uts/common/fs/zfs/zio.c b/uts/common/fs/zfs/zio.c
index f92ecb4739b2..7fa795ea8cad 100644
--- a/uts/common/fs/zfs/zio.c
+++ b/uts/common/fs/zfs/zio.c
@@ -929,7 +929,7 @@ zio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size,
 
 	zio->io_prop.zp_checksum = checksum;
 
-	if (zio_checksum_table[checksum].ci_eck) {
+	if (zio_checksum_table[checksum].ci_flags & ZCHECKSUM_FLAG_EMBEDDED) {
 		/*
 		 * zec checksums are necessarily destructive -- they modify
 		 * the end of the write buffer to hold the verifier/checksum.
@@ -1125,8 +1125,8 @@ zio_write_bp_init(zio_t *zio)
 		if (BP_IS_HOLE(bp) || !zp->zp_dedup)
 			return (ZIO_PIPELINE_CONTINUE);
 
-		ASSERT(zio_checksum_table[zp->zp_checksum].ci_dedup ||
-		    zp->zp_dedup_verify);
+		ASSERT((zio_checksum_table[zp->zp_checksum].ci_flags &
+		    ZCHECKSUM_FLAG_DEDUP) || zp->zp_dedup_verify);
 
 		if (BP_GET_CHECKSUM(bp) == zp->zp_checksum) {
 			BP_SET_DEDUP(bp, 1);
@@ -1981,12 +1981,22 @@ zio_write_gang_block(zio_t *pio)
 }
 
 /*
- * The zio_nop_write stage in the pipeline determines if allocating
- * a new bp is necessary.  By leveraging a cryptographically secure checksum,
- * such as SHA256, we can compare the checksums of the new data and the old
- * to determine if allocating a new block is required.  The nopwrite
- * feature can handle writes in either syncing or open context (i.e. zil
- * writes) and as a result is mutually exclusive with dedup.
+ * The zio_nop_write stage in the pipeline determines if allocating a
+ * new bp is necessary.  The nopwrite feature can handle writes in
+ * either syncing or open context (i.e. zil writes) and as a result is
+ * mutually exclusive with dedup.
+ *
+ * By leveraging a cryptographically secure checksum, such as SHA256, we
+ * can compare the checksums of the new data and the old to determine if
+ * allocating a new block is required.  Note that our requirements for
+ * cryptographic strength are fairly weak: there can't be any accidental
+ * hash collisions, but we don't need to be secure against intentional
+ * (malicious) collisions.  To trigger a nopwrite, you have to be able
+ * to write the file to begin with, and triggering an incorrect (hash
+ * collision) nopwrite is no worse than simply writing to the file.
+ * That said, there are no known attacks against the checksum algorithms
+ * used for nopwrite, assuming that the salt and the checksums
+ * themselves remain secret.
  */
 static int
 zio_nop_write(zio_t *zio)
@@ -2009,7 +2019,8 @@ zio_nop_write(zio_t *zio)
 	 * allocate a new bp.
 	 */
 	if (BP_IS_HOLE(bp_orig) ||
-	    !zio_checksum_table[BP_GET_CHECKSUM(bp)].ci_dedup ||
+	    !(zio_checksum_table[BP_GET_CHECKSUM(bp)].ci_flags &
+	    ZCHECKSUM_FLAG_NOPWRITE) ||
 	    BP_GET_CHECKSUM(bp) != BP_GET_CHECKSUM(bp_orig) ||
 	    BP_GET_COMPRESS(bp) != BP_GET_COMPRESS(bp_orig) ||
 	    BP_GET_DEDUP(bp) != BP_GET_DEDUP(bp_orig) ||
@@ -2021,7 +2032,8 @@ zio_nop_write(zio_t *zio)
 	 * avoid allocating a new bp and issuing any I/O.
 	 */
 	if (ZIO_CHECKSUM_EQUAL(bp->blk_cksum, bp_orig->blk_cksum)) {
-		ASSERT(zio_checksum_table[zp->zp_checksum].ci_dedup);
+		ASSERT(zio_checksum_table[zp->zp_checksum].ci_flags &
+		    ZCHECKSUM_FLAG_NOPWRITE);
 		ASSERT3U(BP_GET_PSIZE(bp), ==, BP_GET_PSIZE(bp_orig));
 		ASSERT3U(BP_GET_LSIZE(bp), ==, BP_GET_LSIZE(bp_orig));
 		ASSERT(zp->zp_compress != ZIO_COMPRESS_OFF);
@@ -2302,7 +2314,8 @@ zio_ddt_write(zio_t *zio)
 		 * we can't resolve it, so just convert to an ordinary write.
 		 * (And automatically e-mail a paper to Nature?)
 		 */
-		if (!zio_checksum_table[zp->zp_checksum].ci_dedup) {
+		if (!(zio_checksum_table[zp->zp_checksum].ci_flags &
+		    ZCHECKSUM_FLAG_DEDUP)) {
 			zp->zp_checksum = spa_dedup_checksum(spa);
 			zio_pop_transforms(zio);
 			zio->io_stage = ZIO_STAGE_OPEN;
diff --git a/uts/common/fs/zfs/zio_checksum.c b/uts/common/fs/zfs/zio_checksum.c
index d1c60c3ffaba..b471ad904749 100644
--- a/uts/common/fs/zfs/zio_checksum.c
+++ b/uts/common/fs/zfs/zio_checksum.c
@@ -22,10 +22,12 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2013 by Delphix. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
+ * Copyright 2013 Saso Kiselkov. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
 #include <sys/spa.h>
+#include <sys/spa_impl.h>
 #include <sys/zio.h>
 #include <sys/zio_checksum.h>
 #include <sys/zil.h>
@@ -59,29 +61,95 @@
  * checksum function of the appropriate strength.  When reading a block,
  * we compare the expected checksum against the actual checksum, which we
  * compute via the checksum function specified by BP_GET_CHECKSUM(bp).
+ *
+ * SALTED CHECKSUMS
+ *
+ * To enable the use of less secure hash algorithms with dedup, we
+ * introduce the notion of salted checksums (MACs, really).  A salted
+ * checksum is fed both a random 256-bit value (the salt) and the data
+ * to be checksummed.  This salt is kept secret (stored on the pool, but
+ * never shown to the user).  Thus even if an attacker knew of collision
+ * weaknesses in the hash algorithm, they won't be able to mount a known
+ * plaintext attack on the DDT, since the actual hash value cannot be
+ * known ahead of time.  How the salt is used is algorithm-specific
+ * (some might simply prefix it to the data block, others might need to
+ * utilize a full-blown HMAC).  On disk the salt is stored in a ZAP
+ * object in the MOS (DMU_POOL_CHECKSUM_SALT).
+ *
+ * CONTEXT TEMPLATES
+ *
+ * Some hashing algorithms need to perform a substantial amount of
+ * initialization work (e.g. salted checksums above may need to pre-hash
+ * the salt) before being able to process data.  Performing this
+ * redundant work for each block would be wasteful, so we instead allow
+ * a checksum algorithm to do the work once (the first time it's used)
+ * and then keep this pre-initialized context as a template inside the
+ * spa_t (spa_cksum_tmpls).  If the zio_checksum_info_t contains
+ * non-NULL ci_tmpl_init and ci_tmpl_free callbacks, they are used to
+ * construct and destruct the pre-initialized checksum context.  The
+ * pre-initialized context is then reused during each checksum
+ * invocation and passed to the checksum function.
  */
 
 /*ARGSUSED*/
 static void
-zio_checksum_off(const void *buf, uint64_t size, zio_cksum_t *zcp)
+zio_checksum_off(const void *buf, uint64_t size,
+    const void *ctx_template, zio_cksum_t *zcp)
 {
 	ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0);
 }
 
 zio_checksum_info_t zio_checksum_table[ZIO_CHECKSUM_FUNCTIONS] = {
-	{{NULL,			NULL},			0, 0, 0, "inherit"},
-	{{NULL,			NULL},			0, 0, 0, "on"},
-	{{zio_checksum_off,	zio_checksum_off},	0, 0, 0, "off"},
-	{{zio_checksum_SHA256,	zio_checksum_SHA256},	1, 1, 0, "label"},
-	{{zio_checksum_SHA256,	zio_checksum_SHA256},	1, 1, 0, "gang_header"},
-	{{fletcher_2_native,	fletcher_2_byteswap},	0, 1, 0, "zilog"},
-	{{fletcher_2_native,	fletcher_2_byteswap},	0, 0, 0, "fletcher2"},
-	{{fletcher_4_native,	fletcher_4_byteswap},	1, 0, 0, "fletcher4"},
-	{{zio_checksum_SHA256,	zio_checksum_SHA256},	1, 0, 1, "sha256"},
-	{{fletcher_4_native,	fletcher_4_byteswap},	0, 1, 0, "zilog2"},
-	{{zio_checksum_off,	zio_checksum_off},	0, 0, 0, "noparity"},
+	{{NULL, NULL}, NULL, NULL, 0, "inherit"},
+	{{NULL, NULL}, NULL, NULL, 0, "on"},
+	{{zio_checksum_off,		zio_checksum_off},
+	    NULL, NULL, 0, "off"},
+	{{zio_checksum_SHA256,		zio_checksum_SHA256},
+	    NULL, NULL, ZCHECKSUM_FLAG_METADATA | ZCHECKSUM_FLAG_EMBEDDED,
+	    "label"},
+	{{zio_checksum_SHA256,		zio_checksum_SHA256},
+	    NULL, NULL, ZCHECKSUM_FLAG_METADATA | ZCHECKSUM_FLAG_EMBEDDED,
+	    "gang_header"},
+	{{fletcher_2_native,		fletcher_2_byteswap},
+	    NULL, NULL, ZCHECKSUM_FLAG_EMBEDDED, "zilog"},
+	{{fletcher_2_native,		fletcher_2_byteswap},
+	    NULL, NULL, 0, "fletcher2"},
+	{{fletcher_4_native,		fletcher_4_byteswap},
+	    NULL, NULL, ZCHECKSUM_FLAG_METADATA, "fletcher4"},
+	{{zio_checksum_SHA256,		zio_checksum_SHA256},
+	    NULL, NULL, ZCHECKSUM_FLAG_METADATA | ZCHECKSUM_FLAG_DEDUP |
+	    ZCHECKSUM_FLAG_NOPWRITE, "sha256"},
+	{{fletcher_4_native,		fletcher_4_byteswap},
+	    NULL, NULL, ZCHECKSUM_FLAG_EMBEDDED, "zilog2"},
+	{{zio_checksum_off,		zio_checksum_off},
+	    NULL, NULL, 0, "noparity"},
+	{{zio_checksum_SHA512_native,	zio_checksum_SHA512_byteswap},
+	    NULL, NULL, ZCHECKSUM_FLAG_METADATA | ZCHECKSUM_FLAG_DEDUP |
+	    ZCHECKSUM_FLAG_NOPWRITE, "sha512"},
+	{{zio_checksum_skein_native,	zio_checksum_skein_byteswap},
+	    zio_checksum_skein_tmpl_init, zio_checksum_skein_tmpl_free,
+	    ZCHECKSUM_FLAG_METADATA | ZCHECKSUM_FLAG_DEDUP |
+	    ZCHECKSUM_FLAG_SALTED | ZCHECKSUM_FLAG_NOPWRITE, "skein"},
+	{{zio_checksum_edonr_native,	zio_checksum_edonr_byteswap},
+	    zio_checksum_edonr_tmpl_init, zio_checksum_edonr_tmpl_free,
+	    ZCHECKSUM_FLAG_METADATA | ZCHECKSUM_FLAG_SALTED |
+	    ZCHECKSUM_FLAG_NOPWRITE, "edonr"},
 };
 
+spa_feature_t
+zio_checksum_to_feature(enum zio_checksum cksum)
+{
+	switch (cksum) {
+	case ZIO_CHECKSUM_SHA512:
+		return (SPA_FEATURE_SHA512);
+	case ZIO_CHECKSUM_SKEIN:
+		return (SPA_FEATURE_SKEIN);
+	case ZIO_CHECKSUM_EDONR:
+		return (SPA_FEATURE_EDONR);
+	}
+	return (SPA_FEATURE_NONE);
+}
+
 enum zio_checksum
 zio_checksum_select(enum zio_checksum child, enum zio_checksum parent)
 {
@@ -115,7 +183,8 @@ zio_checksum_dedup_select(spa_t *spa, enum zio_checksum child,
 	if (child == (ZIO_CHECKSUM_ON | ZIO_CHECKSUM_VERIFY))
 		return (spa_dedup_checksum(spa) | ZIO_CHECKSUM_VERIFY);
 
-	ASSERT(zio_checksum_table[child & ZIO_CHECKSUM_MASK].ci_dedup ||
+	ASSERT((zio_checksum_table[child & ZIO_CHECKSUM_MASK].ci_flags &
+	    ZCHECKSUM_FLAG_DEDUP) ||
 	    (child & ZIO_CHECKSUM_VERIFY) || child == ZIO_CHECKSUM_OFF);
 
 	return (child);
@@ -147,6 +216,30 @@ zio_checksum_label_verifier(zio_cksum_t *zcp, uint64_t offset)
 	ZIO_SET_CHECKSUM(zcp, offset, 0, 0, 0);
 }
 
+/*
+ * Calls the template init function of a checksum which supports context
+ * templates and installs the template into the spa_t.
+ */
+static void
+zio_checksum_template_init(enum zio_checksum checksum, spa_t *spa)
+{
+	zio_checksum_info_t *ci = &zio_checksum_table[checksum];
+
+	if (ci->ci_tmpl_init == NULL)
+		return;
+	if (spa->spa_cksum_tmpls[checksum] != NULL)
+		return;
+
+	VERIFY(ci->ci_tmpl_free != NULL);
+	mutex_enter(&spa->spa_cksum_tmpls_lock);
+	if (spa->spa_cksum_tmpls[checksum] == NULL) {
+		spa->spa_cksum_tmpls[checksum] =
+		    ci->ci_tmpl_init(&spa->spa_cksum_salt);
+		VERIFY(spa->spa_cksum_tmpls[checksum] != NULL);
+	}
+	mutex_exit(&spa->spa_cksum_tmpls_lock);
+}
+
 /*
  * Generate the checksum.
  */
@@ -158,11 +251,14 @@ zio_checksum_compute(zio_t *zio, enum zio_checksum checksum,
 	uint64_t offset = zio->io_offset;
 	zio_checksum_info_t *ci = &zio_checksum_table[checksum];
 	zio_cksum_t cksum;
+	spa_t *spa = zio->io_spa;
 
 	ASSERT((uint_t)checksum < ZIO_CHECKSUM_FUNCTIONS);
 	ASSERT(ci->ci_func[0] != NULL);
 
-	if (ci->ci_eck) {
+	zio_checksum_template_init(checksum, spa);
+
+	if (ci->ci_flags & ZCHECKSUM_FLAG_EMBEDDED) {
 		zio_eck_t *eck;
 
 		if (checksum == ZIO_CHECKSUM_ZILOG2) {
@@ -181,10 +277,12 @@ zio_checksum_compute(zio_t *zio, enum zio_checksum checksum,
 		else
 			bp->blk_cksum = eck->zec_cksum;
 		eck->zec_magic = ZEC_MAGIC;
-		ci->ci_func[0](data, size, &cksum);
+		ci->ci_func[0](data, size, spa->spa_cksum_tmpls[checksum],
+		    &cksum);
 		eck->zec_cksum = cksum;
 	} else {
-		ci->ci_func[0](data, size, &bp->blk_cksum);
+		ci->ci_func[0](data, size, spa->spa_cksum_tmpls[checksum],
+		    &bp->blk_cksum);
 	}
 }
 
@@ -202,11 +300,14 @@ zio_checksum_error(zio_t *zio, zio_bad_cksum_t *info)
 	void *data = zio->io_data;
 	zio_checksum_info_t *ci = &zio_checksum_table[checksum];
 	zio_cksum_t actual_cksum, expected_cksum, verifier;
+	spa_t *spa = zio->io_spa;
 
 	if (checksum >= ZIO_CHECKSUM_FUNCTIONS || ci->ci_func[0] == NULL)
 		return (SET_ERROR(EINVAL));
 
-	if (ci->ci_eck) {
+	zio_checksum_template_init(checksum, spa);
+
+	if (ci->ci_flags & ZCHECKSUM_FLAG_EMBEDDED) {
 		zio_eck_t *eck;
 
 		if (checksum == ZIO_CHECKSUM_ZILOG2) {
@@ -243,7 +344,8 @@ zio_checksum_error(zio_t *zio, zio_bad_cksum_t *info)
 
 		expected_cksum = eck->zec_cksum;
 		eck->zec_cksum = verifier;
-		ci->ci_func[byteswap](data, size, &actual_cksum);
+		ci->ci_func[byteswap](data, size,
+		    spa->spa_cksum_tmpls[checksum], &actual_cksum);
 		eck->zec_cksum = expected_cksum;
 
 		if (byteswap)
@@ -253,7 +355,8 @@ zio_checksum_error(zio_t *zio, zio_bad_cksum_t *info)
 		ASSERT(!BP_IS_GANG(bp));
 		byteswap = BP_SHOULD_BYTESWAP(bp);
 		expected_cksum = bp->blk_cksum;
-		ci->ci_func[byteswap](data, size, &actual_cksum);
+		ci->ci_func[byteswap](data, size,
+		    spa->spa_cksum_tmpls[checksum], &actual_cksum);
 	}
 
 	info->zbc_expected = expected_cksum;
@@ -275,3 +378,23 @@ zio_checksum_error(zio_t *zio, zio_bad_cksum_t *info)
 
 	return (0);
 }
+
+/*
+ * Called by a spa_t that's about to be deallocated. This steps through
+ * all of the checksum context templates and deallocates any that were
+ * initialized using the algorithm-specific template init function.
+ */
+void
+zio_checksum_templates_free(spa_t *spa)
+{
+	for (enum zio_checksum checksum = 0;
+	    checksum < ZIO_CHECKSUM_FUNCTIONS; checksum++) {
+		if (spa->spa_cksum_tmpls[checksum] != NULL) {
+			zio_checksum_info_t *ci = &zio_checksum_table[checksum];
+
+			VERIFY(ci->ci_tmpl_free != NULL);
+			ci->ci_tmpl_free(spa->spa_cksum_tmpls[checksum]);
+			spa->spa_cksum_tmpls[checksum] = NULL;
+		}
+	}
+}
diff --git a/uts/common/sys/debug.h b/uts/common/sys/debug.h
index 27b84beb8807..8efc8956b675 100644
--- a/uts/common/sys/debug.h
+++ b/uts/common/sys/debug.h
@@ -27,6 +27,7 @@
 
 /*
  * Copyright (c) 2012 by Delphix. All rights reserved.
+ * Copyright 2013 Saso Kiselkov. All rights reserved.
  */
 
 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
@@ -124,6 +125,14 @@ _NOTE(CONSTCOND) } while (0)
 #define	ASSERT0(x)		((void)0)
 #endif
 
+/*
+ * Compile-time assertion. The condition 'x' must be constant.
+ */
+#define	CTASSERT(x)		_CTASSERT(x, __LINE__)
+#define	_CTASSERT(x, y)		__CTASSERT(x, y)
+#define	__CTASSERT(x, y) \
+	typedef char __compile_time_assertion__ ## y [(x) ? 1 : -1]
+
 #ifdef	_KERNEL
 
 extern void abort_sequence_enter(char *);
diff --git a/uts/common/sys/edonr.h b/uts/common/sys/edonr.h
new file mode 100644
index 000000000000..e65118dc1f83
--- /dev/null
+++ b/uts/common/sys/edonr.h
@@ -0,0 +1,93 @@
+/*
+ * IDI,NTNU
+ *
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://opensource.org/licenses/CDDL-1.0.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ * Copyright (C) 2009, 2010, Jorn Amundsen <jorn.amundsen@ntnu.no>
+ *
+ * Tweaked Edon-R implementation for SUPERCOP, based on NIST API.
+ *
+ * $Id: edonr.h 517 2013-02-17 20:34:39Z joern $
+ */
+/*
+ * Portions copyright (c) 2013, Saso Kiselkov, All rights reserved
+ */
+
+#ifndef	_SYS_EDONR_H_
+#define	_SYS_EDONR_H_
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#include <sys/types.h>
+
+/*
+ * EdonR allows to call EdonRUpdate() consecutively only if the total length
+ * of stored unprocessed data and the new supplied data is less than or equal
+ * to the BLOCK_SIZE on which the compression functions operates.
+ * Otherwise an assertion failure is invoked.
+ */
+
+/* Specific algorithm definitions */
+#define	EdonR224_DIGEST_SIZE	28
+#define	EdonR224_BLOCK_SIZE	64
+#define	EdonR256_DIGEST_SIZE	32
+#define	EdonR256_BLOCK_SIZE	64
+#define	EdonR384_DIGEST_SIZE	48
+#define	EdonR384_BLOCK_SIZE	128
+#define	EdonR512_DIGEST_SIZE	64
+#define	EdonR512_BLOCK_SIZE	128
+
+#define	EdonR256_BLOCK_BITSIZE	512
+#define	EdonR512_BLOCK_BITSIZE	1024
+
+typedef struct {
+	uint32_t DoublePipe[16];
+	uint8_t LastPart[EdonR256_BLOCK_SIZE * 2];
+} EdonRData256;
+typedef struct {
+	uint64_t DoublePipe[16];
+	uint8_t LastPart[EdonR512_BLOCK_SIZE * 2];
+} EdonRData512;
+
+typedef struct {
+	size_t hashbitlen;
+
+	/* + algorithm specific parameters */
+	int unprocessed_bits;
+	uint64_t bits_processed;
+	union {
+		EdonRData256 p256[1];
+		EdonRData512 p512[1];
+	} pipe[1];
+} EdonRState;
+
+void EdonRInit(EdonRState *state, size_t hashbitlen);
+void EdonRUpdate(EdonRState *state, const uint8_t *data, size_t databitlen);
+void EdonRFinal(EdonRState *state, uint8_t *hashval);
+void EdonRHash(size_t hashbitlen, const uint8_t *data, size_t databitlen,
+    uint8_t *hashval);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _SYS_EDONR_H_ */
diff --git a/uts/common/sys/skein.h b/uts/common/sys/skein.h
new file mode 100644
index 000000000000..423e1dc34eb9
--- /dev/null
+++ b/uts/common/sys/skein.h
@@ -0,0 +1,178 @@
+/*
+ * Interface declarations for Skein hashing.
+ * Source code author: Doug Whiting, 2008.
+ * This algorithm and source code is released to the public domain.
+ *
+ * The following compile-time switches may be defined to control some
+ * tradeoffs between speed, code size, error checking, and security.
+ *
+ * The "default" note explains what happens when the switch is not defined.
+ *
+ *  SKEIN_DEBUG            -- make callouts from inside Skein code
+ *                            to examine/display intermediate values.
+ *                            [default: no callouts (no overhead)]
+ *
+ *  SKEIN_ERR_CHECK        -- how error checking is handled inside Skein
+ *                            code. If not defined, most error checking
+ *                            is disabled (for performance). Otherwise,
+ *                            the switch value is interpreted as:
+ *                                0: use assert()      to flag errors
+ *                                1: return SKEIN_FAIL to flag errors
+ */
+/* Copyright 2013 Doug Whiting. This code is released to the public domain. */
+#ifndef	_SYS_SKEIN_H_
+#define	_SYS_SKEIN_H_
+
+#include <sys/types.h>		/* get size_t definition */
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+enum {
+	SKEIN_SUCCESS = 0,	/* return codes from Skein calls */
+	SKEIN_FAIL = 1,
+	SKEIN_BAD_HASHLEN = 2
+};
+
+#define	SKEIN_MODIFIER_WORDS	(2)	/* number of modifier (tweak) words */
+
+#define	SKEIN_256_STATE_WORDS	(4)
+#define	SKEIN_512_STATE_WORDS	(8)
+#define	SKEIN1024_STATE_WORDS	(16)
+#define	SKEIN_MAX_STATE_WORDS	(16)
+
+#define	SKEIN_256_STATE_BYTES	(8 * SKEIN_256_STATE_WORDS)
+#define	SKEIN_512_STATE_BYTES	(8 * SKEIN_512_STATE_WORDS)
+#define	SKEIN1024_STATE_BYTES	(8 * SKEIN1024_STATE_WORDS)
+
+#define	SKEIN_256_STATE_BITS	(64 * SKEIN_256_STATE_WORDS)
+#define	SKEIN_512_STATE_BITS	(64 * SKEIN_512_STATE_WORDS)
+#define	SKEIN1024_STATE_BITS	(64 * SKEIN1024_STATE_WORDS)
+
+#define	SKEIN_256_BLOCK_BYTES	(8 * SKEIN_256_STATE_WORDS)
+#define	SKEIN_512_BLOCK_BYTES	(8 * SKEIN_512_STATE_WORDS)
+#define	SKEIN1024_BLOCK_BYTES	(8 * SKEIN1024_STATE_WORDS)
+
+typedef struct {
+	size_t hashBitLen;	/* size of hash result, in bits */
+	size_t bCnt;		/* current byte count in buffer b[] */
+	/* tweak words: T[0]=byte cnt, T[1]=flags */
+	uint64_t T[SKEIN_MODIFIER_WORDS];
+} Skein_Ctxt_Hdr_t;
+
+typedef struct {		/*  256-bit Skein hash context structure */
+	Skein_Ctxt_Hdr_t h;	/* common header context variables */
+	uint64_t X[SKEIN_256_STATE_WORDS];	/* chaining variables */
+	/* partial block buffer (8-byte aligned) */
+	uint8_t b[SKEIN_256_BLOCK_BYTES];
+} Skein_256_Ctxt_t;
+
+typedef struct {		/*  512-bit Skein hash context structure */
+	Skein_Ctxt_Hdr_t h;	/* common header context variables */
+	uint64_t X[SKEIN_512_STATE_WORDS];	/* chaining variables */
+	/* partial block buffer (8-byte aligned) */
+	uint8_t b[SKEIN_512_BLOCK_BYTES];
+} Skein_512_Ctxt_t;
+
+typedef struct {		/* 1024-bit Skein hash context structure */
+	Skein_Ctxt_Hdr_t h;	/* common header context variables */
+	uint64_t X[SKEIN1024_STATE_WORDS];	/* chaining variables */
+	/* partial block buffer (8-byte aligned) */
+	uint8_t b[SKEIN1024_BLOCK_BYTES];
+} Skein1024_Ctxt_t;
+
+/*   Skein APIs for (incremental) "straight hashing" */
+int Skein_256_Init(Skein_256_Ctxt_t *ctx, size_t hashBitLen);
+int Skein_512_Init(Skein_512_Ctxt_t *ctx, size_t hashBitLen);
+int Skein1024_Init(Skein1024_Ctxt_t *ctx, size_t hashBitLen);
+
+int Skein_256_Update(Skein_256_Ctxt_t *ctx, const uint8_t *msg,
+    size_t msgByteCnt);
+int Skein_512_Update(Skein_512_Ctxt_t *ctx, const uint8_t *msg,
+    size_t msgByteCnt);
+int Skein1024_Update(Skein1024_Ctxt_t *ctx, const uint8_t *msg,
+    size_t msgByteCnt);
+
+int Skein_256_Final(Skein_256_Ctxt_t *ctx, uint8_t *hashVal);
+int Skein_512_Final(Skein_512_Ctxt_t *ctx, uint8_t *hashVal);
+int Skein1024_Final(Skein1024_Ctxt_t *ctx, uint8_t *hashVal);
+
+/*
+ * Skein APIs for "extended" initialization: MAC keys, tree hashing.
+ * After an InitExt() call, just use Update/Final calls as with Init().
+ *
+ * Notes: Same parameters as _Init() calls, plus treeInfo/key/keyBytes.
+ *          When keyBytes == 0 and treeInfo == SKEIN_SEQUENTIAL,
+ *              the results of InitExt() are identical to calling Init().
+ *          The function Init() may be called once to "precompute" the IV for
+ *              a given hashBitLen value, then by saving a copy of the context
+ *              the IV computation may be avoided in later calls.
+ *          Similarly, the function InitExt() may be called once per MAC key
+ *              to precompute the MAC IV, then a copy of the context saved and
+ *              reused for each new MAC computation.
+ */
+int Skein_256_InitExt(Skein_256_Ctxt_t *ctx, size_t hashBitLen,
+    uint64_t treeInfo, const uint8_t *key, size_t keyBytes);
+int Skein_512_InitExt(Skein_512_Ctxt_t *ctx, size_t hashBitLen,
+    uint64_t treeInfo, const uint8_t *key, size_t keyBytes);
+int Skein1024_InitExt(Skein1024_Ctxt_t *ctx, size_t hashBitLen,
+    uint64_t treeInfo, const uint8_t *key, size_t keyBytes);
+
+/*
+ * Skein APIs for MAC and tree hash:
+ *	Final_Pad: pad, do final block, but no OUTPUT type
+ *	Output:    do just the output stage
+ */
+int Skein_256_Final_Pad(Skein_256_Ctxt_t *ctx, uint8_t *hashVal);
+int Skein_512_Final_Pad(Skein_512_Ctxt_t *ctx, uint8_t *hashVal);
+int Skein1024_Final_Pad(Skein1024_Ctxt_t *ctx, uint8_t *hashVal);
+
+#ifndef	SKEIN_TREE_HASH
+#define	SKEIN_TREE_HASH (1)
+#endif
+#if	SKEIN_TREE_HASH
+int Skein_256_Output(Skein_256_Ctxt_t *ctx, uint8_t *hashVal);
+int Skein_512_Output(Skein_512_Ctxt_t *ctx, uint8_t *hashVal);
+int Skein1024_Output(Skein1024_Ctxt_t *ctx, uint8_t *hashVal);
+#endif
+
+/*
+ * When you initialize a Skein KCF hashing method you can pass this param
+ * structure in cm_param to fine-tune the algorithm's defaults.
+ */
+typedef struct skein_param {
+	size_t	sp_digest_bitlen;		/* length of digest in bits */
+} skein_param_t;
+
+/* Module definitions */
+#ifdef	SKEIN_MODULE_IMPL
+#define	CKM_SKEIN_256				"CKM_SKEIN_256"
+#define	CKM_SKEIN_512				"CKM_SKEIN_512"
+#define	CKM_SKEIN1024				"CKM_SKEIN1024"
+#define	CKM_SKEIN_256_MAC			"CKM_SKEIN_256_MAC"
+#define	CKM_SKEIN_512_MAC			"CKM_SKEIN_512_MAC"
+#define	CKM_SKEIN1024_MAC			"CKM_SKEIN1024_MAC"
+
+typedef enum skein_mech_type {
+	SKEIN_256_MECH_INFO_TYPE,
+	SKEIN_512_MECH_INFO_TYPE,
+	SKEIN1024_MECH_INFO_TYPE,
+	SKEIN_256_MAC_MECH_INFO_TYPE,
+	SKEIN_512_MAC_MECH_INFO_TYPE,
+	SKEIN1024_MAC_MECH_INFO_TYPE
+} skein_mech_type_t;
+
+#define	VALID_SKEIN_DIGEST_MECH(__mech)				\
+	((int)(__mech) >= SKEIN_256_MECH_INFO_TYPE &&		\
+	(__mech) <= SKEIN1024_MECH_INFO_TYPE)
+#define	VALID_SKEIN_MAC_MECH(__mech)				\
+	((int)(__mech) >= SKEIN_256_MAC_MECH_INFO_TYPE &&	\
+	(__mech) <= SKEIN1024_MAC_MECH_INFO_TYPE)
+#endif	/* SKEIN_MODULE_IMPL */
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _SYS_SKEIN_H_ */

From b800f004107ceb768773500aa53149cd9cc29f77 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Wed, 14 Oct 2015 11:52:58 +0000
Subject: [PATCH 205/380] MFV r289311: 5764 "zfs send -nv" directs output to
 stderr

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Paul Dagnelie <paul.dagnelie@delphix.com>
Reviewed by: Basil Crow <basil.crow@delphix.com>
Reviewed by: Steven Hartland <killing@multiplay.co.uk>
Reviewed by: Bayard Bell <buffer.g.overflow@gmail.com>
Approved by: Dan McDonald <danmcd@omniti.com>
Author: Manoj Joseph <manoj.joseph@delphix.com>

illumos/illumos-gate@dc5f28a3c341db7c241bba77ddc109c141072f27
---
 cddl/contrib/opensolaris/cmd/zfs/zfs.8        |  3 +++
 .../lib/libzfs/common/libzfs_sendrecv.c       | 24 ++++++++++++-------
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/cddl/contrib/opensolaris/cmd/zfs/zfs.8 b/cddl/contrib/opensolaris/cmd/zfs/zfs.8
index e534ae9a02ad..b9dd26bda106 100644
--- a/cddl/contrib/opensolaris/cmd/zfs/zfs.8
+++ b/cddl/contrib/opensolaris/cmd/zfs/zfs.8
@@ -2630,6 +2630,9 @@ useful in conjunction with the
 or
 .Fl P
 flags to determine what data will be sent.
+In this case, the verbose output will be written to
+standard output (contrast with a non-dry-run, where the stream is written
+to standard output and the verbose output goes to standard error).
 .It Fl P
 Print machine-parsable verbose information about the stream package generated.
 .It Fl v
diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_sendrecv.c b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_sendrecv.c
index f6efa97569c4..51388814bd07 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_sendrecv.c
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_sendrecv.c
@@ -818,7 +818,8 @@ typedef struct send_dump_data {
 	char prevsnap[ZFS_MAXNAMELEN];
 	uint64_t prevsnap_obj;
 	boolean_t seenfrom, seento, replicate, doall, fromorigin;
-	boolean_t verbose, dryrun, parsable, progress, embed_data, large_block;
+	boolean_t verbose, dryrun, parsable, progress, embed_data, std_out;
+	boolean_t large_block;
 	int outfd;
 	boolean_t err;
 	nvlist_t *fss;
@@ -1047,6 +1048,7 @@ dump_snapshot(zfs_handle_t *zhp, void *arg)
 	int err;
 	boolean_t isfromsnap, istosnap, fromorigin;
 	boolean_t exclude = B_FALSE;
+	FILE *fout = sdd->std_out ? stdout : stderr;
 
 	err = 0;
 	thissnap = strchr(zhp->zfs_name, '@') + 1;
@@ -1121,30 +1123,30 @@ dump_snapshot(zfs_handle_t *zhp, void *arg)
 
 		if (sdd->parsable) {
 			if (sdd->prevsnap[0] != '\0') {
-				(void) fprintf(stderr, "incremental\t%s\t%s",
+				(void) fprintf(fout, "incremental\t%s\t%s",
 				    sdd->prevsnap, zhp->zfs_name);
 			} else {
-				(void) fprintf(stderr, "full\t%s",
+				(void) fprintf(fout, "full\t%s",
 				    zhp->zfs_name);
 			}
 		} else {
-			(void) fprintf(stderr, dgettext(TEXT_DOMAIN,
+			(void) fprintf(fout, dgettext(TEXT_DOMAIN,
 			    "send from @%s to %s"),
 			    sdd->prevsnap, zhp->zfs_name);
 		}
 		if (err == 0) {
 			if (sdd->parsable) {
-				(void) fprintf(stderr, "\t%llu\n",
+				(void) fprintf(fout, "\t%llu\n",
 				    (longlong_t)size);
 			} else {
 				char buf[16];
 				zfs_nicenum(size, buf, sizeof (buf));
-				(void) fprintf(stderr, dgettext(TEXT_DOMAIN,
+				(void) fprintf(fout, dgettext(TEXT_DOMAIN,
 				    " estimated size is %s\n"), buf);
 			}
 			sdd->size += size;
 		} else {
-			(void) fprintf(stderr, "\n");
+			(void) fprintf(fout, "\n");
 		}
 	}
 
@@ -1388,6 +1390,7 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
 	int pipefd[2];
 	dedup_arg_t dda = { 0 };
 	int featureflags = 0;
+	FILE *fout;
 
 	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
 	    "cannot send '%s'"), zhp->zfs_name);
@@ -1515,6 +1518,9 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
 	sdd.filter_cb_arg = cb_arg;
 	if (debugnvp)
 		sdd.debugnv = *debugnvp;
+	if (sdd.verbose && sdd.dryrun)
+		sdd.std_out = B_TRUE;
+	fout = sdd.std_out ? stdout : stderr;
 
 	/*
 	 * Some flags require that we place user holds on the datasets that are
@@ -1554,12 +1560,12 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
 
 		if (flags->verbose) {
 			if (flags->parsable) {
-				(void) fprintf(stderr, "size\t%llu\n",
+				(void) fprintf(fout, "size\t%llu\n",
 				    (longlong_t)sdd.size);
 			} else {
 				char buf[16];
 				zfs_nicenum(sdd.size, buf, sizeof (buf));
-				(void) fprintf(stderr, dgettext(TEXT_DOMAIN,
+				(void) fprintf(fout, dgettext(TEXT_DOMAIN,
 				    "total estimated size is %s\n"), buf);
 			}
 		}

From 60458e70da25850f61b9877b8eae858c8ebe875d Mon Sep 17 00:00:00 2001
From: vangyzen <vangyzen@FreeBSD.org>
Date: Wed, 14 Oct 2015 14:26:44 +0000
Subject: [PATCH 206/380] resolver: automatically reload /etc/resolv.conf

On each resolver query, use stat(2) to see if the modification time
of /etc/resolv.conf has changed.  If so, reload the file and reinitialize
the resolver library.  However, only call stat(2) if at least two seconds
have passed since the last call to stat(2), since calling it on every
query could kill performance.

This new behavior is enabled by default.  Add a "reload-period" option
to disable it or change the period of the test.

Document this behavior and option in resolv.conf(5).

Polish the man page just enough to appease igor.

https://lists.freebsd.org/pipermail/freebsd-arch/2015-October/017342.html

Reviewed by:	kp, wblock
Discussed with:	jilles, imp, alfred
MFC after:	1 month
Relnotes:	yes
Sponsored by:	Dell Inc.
Differential Revision:	https://reviews.freebsd.org/D3867
---
 include/resolv.h            |  5 ++++-
 lib/libc/resolv/res_init.c  | 22 ++++++++++++++++++++++
 lib/libc/resolv/res_state.c | 31 +++++++++++++++++++++++++++++--
 share/man/man5/resolver.5   | 18 +++++++++++++++---
 4 files changed, 70 insertions(+), 6 deletions(-)

diff --git a/include/resolv.h b/include/resolv.h
index e3d4fd12a097..78da3748a20e 100644
--- a/include/resolv.h
+++ b/include/resolv.h
@@ -176,7 +176,8 @@ struct __res_state {
 	int	res_h_errno;		/*%< last one set for this context */
 	int	_vcsock;		/*%< PRIVATE: for res_send VC i/o */
 	u_int	_flags;			/*%< PRIVATE: see below */
-	u_int	_pad;			/*%< make _u 64 bit aligned */
+	u_short	reload_period;		/*%< seconds between stat(resolv.conf)*/
+	u_short	_pad;			/*%< make _u 64 bit aligned */
 	union {
 		/* On an 32-bit arch this means 512b total. */
 		char	pad[72 - 4*sizeof (int) - 3*sizeof (void *)];
@@ -188,6 +189,8 @@ struct __res_state {
 		} _ext;
 	} _u;
 	u_char	*_rnd;			/*%< PRIVATE: random state */
+	struct timespec	conf_mtim;	/*%< mod time of loaded resolv.conf */
+	time_t		conf_stat;	/*%< time of last stat(resolv.conf) */
 };
 
 typedef struct __res_state *res_state;
diff --git a/lib/libc/resolv/res_init.c b/lib/libc/resolv/res_init.c
index 4dbd6d1caca2..087e6a862568 100644
--- a/lib/libc/resolv/res_init.c
+++ b/lib/libc/resolv/res_init.c
@@ -78,6 +78,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/types.h>
 #include <sys/param.h>
 #include <sys/socket.h>
+#include <sys/stat.h>
 #include <sys/time.h>
 
 #include <netinet/in.h>
@@ -227,6 +228,7 @@ __res_vinit(res_state statp, int preinit) {
 	statp->pfcode = 0;
 	statp->_vcsock = -1;
 	statp->_flags = 0;
+	statp->reload_period = 2;
 	statp->qhook = NULL;
 	statp->rhook = NULL;
 	statp->_u._ext.nscount = 0;
@@ -321,6 +323,22 @@ __res_vinit(res_state statp, int preinit) {
 
 	nserv = 0;
 	if ((fp = fopen(_PATH_RESCONF, "re")) != NULL) {
+	    struct stat sb;
+	    struct timespec now;
+
+	    if (_fstat(fileno(fp), &sb) == 0) {
+		statp->conf_mtim = sb.st_mtim;
+		if (clock_gettime(CLOCK_MONOTONIC_FAST, &now) == 0) {
+		    statp->conf_stat = now.tv_sec;
+		} else {
+		    statp->conf_stat = 0;
+		}
+	    } else {
+		statp->conf_mtim.tv_sec = 0;
+		statp->conf_mtim.tv_nsec = 0;
+		statp->conf_stat = 0;
+	    }
+
 	    /* read the config file */
 	    while (fgets(buf, sizeof(buf), fp) != NULL) {
 		/* skip comments */
@@ -666,6 +684,10 @@ res_setoptions(res_state statp, const char *options, const char *source)
 		} else if (!strncmp(cp, "no-check-names",
 				    sizeof("no-check-names") - 1)) {
 			statp->options |= RES_NOCHECKNAME;
+		} else if (!strncmp(cp, "reload-period:",
+				    sizeof("reload-period:") - 1)) {
+			statp->reload_period = (u_short)
+				atoi(cp + sizeof("reload-period:") - 1);
 		}
 #ifdef RES_USE_EDNS0
 		else if (!strncmp(cp, "edns0", sizeof("edns0") - 1)) {
diff --git a/lib/libc/resolv/res_state.c b/lib/libc/resolv/res_state.c
index 59c943060757..87643422ebb2 100644
--- a/lib/libc/resolv/res_state.c
+++ b/lib/libc/resolv/res_state.c
@@ -26,6 +26,8 @@
  */
 
 #include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/time.h>
 #include <netinet/in.h>
 #include <arpa/nameser.h>
 #include <resolv.h>
@@ -59,13 +61,38 @@ res_keycreate(void)
 	res_thr_keycreated = thr_keycreate(&res_key, free_res) == 0;
 }
 
+static res_state
+res_check_reload(res_state statp)
+{
+	struct timespec now;
+	struct stat sb;
+
+	if ((statp->options & RES_INIT) == 0 || statp->reload_period == 0) {
+		return (statp);
+	}
+
+	if (clock_gettime(CLOCK_MONOTONIC_FAST, &now) != 0 ||
+	    (now.tv_sec - statp->conf_stat) < statp->reload_period) {
+		return (statp);
+	}
+
+	statp->conf_stat = now.tv_sec;
+	if (stat(_PATH_RESCONF, &sb) == 0 &&
+	    (sb.st_mtim.tv_sec  != statp->conf_mtim.tv_sec ||
+	     sb.st_mtim.tv_nsec != statp->conf_mtim.tv_nsec)) {
+		statp->options &= ~RES_INIT;
+	}
+
+	return (statp);
+}
+
 res_state
 __res_state(void)
 {
 	res_state statp;
 
 	if (thr_main() != 0)
-		return (&_res);
+		return res_check_reload(&_res);
 
 	if (thr_once(&res_init_once, res_keycreate) != 0 ||
 	    !res_thr_keycreated)
@@ -73,7 +100,7 @@ __res_state(void)
 
 	statp = thr_getspecific(res_key);
 	if (statp != NULL)
-		return (statp);
+		return res_check_reload(statp);
 	statp = calloc(1, sizeof(*statp));
 	if (statp == NULL)
 		return (&_res);
diff --git a/share/man/man5/resolver.5 b/share/man/man5/resolver.5
index 4116f23c9cdc..a3c5e05cf7bd 100644
--- a/share/man/man5/resolver.5
+++ b/share/man/man5/resolver.5
@@ -28,7 +28,7 @@
 .\"     @(#)resolver.5	8.1 (Berkeley) 6/5/93
 .\" $FreeBSD$
 .\"
-.Dd December 25, 2013
+.Dd October 12, 2015
 .Dt RESOLVER 5
 .Os
 .Sh NAME
@@ -175,6 +175,19 @@ the resolver from obeying the standard
 and
 .Sy search
 rules with the given name.
+.It Sy reload-period: Ns Ar n
+The resolver checks the modification time of
+.Pa /etc/resolv.conf
+every
+.Ar n
+seconds.
+If
+.Pa /etc/resolv.conf
+has changed, it is automatically reloaded.
+The default for
+.Ar n
+is two seconds.
+Setting it to zero disables the file check.
 .El
 .Pp
 Options may also be specified as a space or tab separated list using the
@@ -191,8 +204,7 @@ If more than one instance of these keywords is present,
 the last instance will override.
 .Pp
 The keyword and value must appear on a single line, and the keyword
-(e.g.\&
-.Sy nameserver )
+.Pq for example, Sy nameserver
 must start the line.
 The value follows the keyword, separated by white space.
 .Sh FILES

From 40bca2754d65948b01b3e0a63dda32938c1bd837 Mon Sep 17 00:00:00 2001
From: kp <kp@FreeBSD.org>
Date: Wed, 14 Oct 2015 16:21:41 +0000
Subject: [PATCH 207/380] pf: Fix TSO issues

In certain configurations (mostly but not exclusively as a VM on Xen) pf
produced packets with an invalid TCP checksum.

The problem was that pf could only handle packets with a full checksum. The
FreeBSD IP stack produces TCP packets with a pseudo-header checksum (only
addresses, length and protocol).
Certain network interfaces expect to see the pseudo-header checksum, so they
end up producing packets with invalid checksums.

To fix this stop calculating the full checksum and teach pf to only update TCP
checksums if TSO is disabled or the change affects the pseudo-header checksum.

PR:		154428, 193579, 198868
Reviewed by:	sbruno
MFC after:	1 week
Relnotes:	yes
Sponsored by:	RootBSD
Differential Revision:	https://reviews.freebsd.org/D3779
---
 sys/net/pfvar.h           |   5 ++
 sys/netpfil/pf/pf.c       | 112 +++++++++++++++++++++++++++-----------
 sys/netpfil/pf/pf_ioctl.c |  13 -----
 sys/netpfil/pf/pf_norm.c  |  13 +++--
 4 files changed, 93 insertions(+), 50 deletions(-)

diff --git a/sys/net/pfvar.h b/sys/net/pfvar.h
index ea90dc889141..747da412abdc 100644
--- a/sys/net/pfvar.h
+++ b/sys/net/pfvar.h
@@ -1552,6 +1552,8 @@ extern void			 pf_print_state(struct pf_state *);
 extern void			 pf_print_flags(u_int8_t);
 extern u_int16_t		 pf_cksum_fixup(u_int16_t, u_int16_t, u_int16_t,
 				    u_int8_t);
+extern u_int16_t		 pf_proto_cksum_fixup(struct mbuf *, u_int16_t,
+				    u_int16_t, u_int16_t, u_int8_t);
 
 VNET_DECLARE(struct ifnet *,		 sync_ifp);
 #define	V_sync_ifp		 	 VNET(sync_ifp);
@@ -1581,6 +1583,9 @@ u_int32_t	pf_new_isn(struct pf_state *);
 void   *pf_pull_hdr(struct mbuf *, int, void *, int, u_short *, u_short *,
 	    sa_family_t);
 void	pf_change_a(void *, u_int16_t *, u_int32_t, u_int8_t);
+void	pf_change_proto_a(struct mbuf *, void *, u_int16_t *, u_int32_t,
+	    u_int8_t);
+void	pf_change_tcp_a(struct mbuf *, void *, u_int16_t *, u_int32_t);
 void	pf_send_deferred_syn(struct pf_state *);
 int	pf_match_addr(u_int8_t, struct pf_addr *, struct pf_addr *,
 	    struct pf_addr *, sa_family_t);
diff --git a/sys/netpfil/pf/pf.c b/sys/netpfil/pf/pf.c
index 1f6d5a241658..812131447966 100644
--- a/sys/netpfil/pf/pf.c
+++ b/sys/netpfil/pf/pf.c
@@ -202,7 +202,7 @@ static void		 pf_init_threshold(struct pf_threshold *, u_int32_t,
 static void		 pf_add_threshold(struct pf_threshold *);
 static int		 pf_check_threshold(struct pf_threshold *);
 
-static void		 pf_change_ap(struct pf_addr *, u_int16_t *,
+static void		 pf_change_ap(struct mbuf *, struct pf_addr *, u_int16_t *,
 			    u_int16_t *, u_int16_t *, struct pf_addr *,
 			    u_int16_t, u_int8_t, sa_family_t);
 static int		 pf_modulate_sack(struct mbuf *, int, struct pf_pdesc *,
@@ -1991,6 +1991,22 @@ pf_addr_wrap_neq(struct pf_addr_wrap *aw1, struct pf_addr_wrap *aw2)
 	}
 }
 
+/**
+ * Checksum updates are a little complicated because the checksum in the TCP/UDP
+ * header isn't always a full checksum. In some cases (i.e. output) it's a
+ * pseudo-header checksum, which is a partial checksum over src/dst IP
+ * addresses, protocol number and length.
+ *
+ * That means we have the following cases:
+ *  * Input or forwarding: we don't have TSO, the checksum fields are full
+ *  	checksums, we need to update the checksum whenever we change anything.
+ *  * Output (i.e. the checksum is a pseudo-header checksum):
+ *  	x The field being updated is src/dst address or affects the length of
+ *  	the packet. We need to update the pseudo-header checksum (note that this
+ *  	checksum is not ones' complement).
+ *  	x Some other field is being modified (e.g. src/dst port numbers): We
+ *  	don't have to update anything.
+ **/
 u_int16_t
 pf_cksum_fixup(u_int16_t cksum, u_int16_t old, u_int16_t new, u_int8_t udp)
 {
@@ -2006,9 +2022,20 @@ pf_cksum_fixup(u_int16_t cksum, u_int16_t old, u_int16_t new, u_int8_t udp)
 	return (l);
 }
 
+u_int16_t
+pf_proto_cksum_fixup(struct mbuf *m, u_int16_t cksum, u_int16_t old,
+        u_int16_t new, u_int8_t udp)
+{
+	if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6))
+		return (cksum);
+
+	return (pf_cksum_fixup(cksum, old, new, udp));
+}
+
 static void
-pf_change_ap(struct pf_addr *a, u_int16_t *p, u_int16_t *ic, u_int16_t *pc,
-    struct pf_addr *an, u_int16_t pn, u_int8_t u, sa_family_t af)
+pf_change_ap(struct mbuf *m, struct pf_addr *a, u_int16_t *p, u_int16_t *ic,
+        u_int16_t *pc, struct pf_addr *an, u_int16_t pn, u_int8_t u,
+        sa_family_t af)
 {
 	struct pf_addr	ao;
 	u_int16_t	po = *p;
@@ -2016,6 +2043,9 @@ pf_change_ap(struct pf_addr *a, u_int16_t *p, u_int16_t *ic, u_int16_t *pc,
 	PF_ACPY(&ao, a, af);
 	PF_ACPY(a, an, af);
 
+	if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6))
+		*pc = ~*pc;
+
 	*p = pn;
 
 	switch (af) {
@@ -2025,17 +2055,19 @@ pf_change_ap(struct pf_addr *a, u_int16_t *p, u_int16_t *ic, u_int16_t *pc,
 		    ao.addr16[0], an->addr16[0], 0),
 		    ao.addr16[1], an->addr16[1], 0);
 		*p = pn;
-		*pc = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(*pc,
+
+		*pc = pf_cksum_fixup(pf_cksum_fixup(*pc,
 		    ao.addr16[0], an->addr16[0], u),
-		    ao.addr16[1], an->addr16[1], u),
-		    po, pn, u);
+		    ao.addr16[1], an->addr16[1], u);
+
+		*pc = pf_proto_cksum_fixup(m, *pc, po, pn, u);
 		break;
 #endif /* INET */
 #ifdef INET6
 	case AF_INET6:
 		*pc = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
 		    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
-		    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(*pc,
+		    pf_cksum_fixup(pf_cksum_fixup(*pc,
 		    ao.addr16[0], an->addr16[0], u),
 		    ao.addr16[1], an->addr16[1], u),
 		    ao.addr16[2], an->addr16[2], u),
@@ -2043,13 +2075,20 @@ pf_change_ap(struct pf_addr *a, u_int16_t *p, u_int16_t *ic, u_int16_t *pc,
 		    ao.addr16[4], an->addr16[4], u),
 		    ao.addr16[5], an->addr16[5], u),
 		    ao.addr16[6], an->addr16[6], u),
-		    ao.addr16[7], an->addr16[7], u),
-		    po, pn, u);
+		    ao.addr16[7], an->addr16[7], u);
+
+		*pc = pf_proto_cksum_fixup(m, *pc, po, pn, u);
 		break;
 #endif /* INET6 */
 	}
-}
 
+	if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | 
+	    CSUM_DELAY_DATA_IPV6)) {
+		*pc = ~*pc;
+		if (! *pc)
+			*pc = 0xffff;
+	}
+}
 
 /* Changes a u_int32_t.  Uses a void * so there are no align restrictions */
 void
@@ -2063,6 +2102,19 @@ pf_change_a(void *a, u_int16_t *c, u_int32_t an, u_int8_t u)
 	    ao % 65536, an % 65536, u);
 }
 
+void
+pf_change_proto_a(struct mbuf *m, void *a, u_int16_t *c, u_int32_t an, u_int8_t udp)
+{
+	u_int32_t	ao;
+
+	memcpy(&ao, a, sizeof(ao));
+	memcpy(a, &an, sizeof(u_int32_t));
+
+	*c = pf_proto_cksum_fixup(m,
+	    pf_proto_cksum_fixup(m, *c, ao / 65536, an / 65536, udp),
+	    ao % 65536, an % 65536, udp);
+}
+
 #ifdef INET6
 static void
 pf_change_a6(struct pf_addr *a, u_int16_t *c, struct pf_addr *an, u_int8_t u)
@@ -2208,12 +2260,10 @@ pf_modulate_sack(struct mbuf *m, int off, struct pf_pdesc *pd,
 				for (i = 2; i + TCPOLEN_SACK <= olen;
 				    i += TCPOLEN_SACK) {
 					memcpy(&sack, &opt[i], sizeof(sack));
-					pf_change_a(&sack.start, &th->th_sum,
-					    htonl(ntohl(sack.start) -
-					    dst->seqdiff), 0);
-					pf_change_a(&sack.end, &th->th_sum,
-					    htonl(ntohl(sack.end) -
-					    dst->seqdiff), 0);
+					pf_change_proto_a(m, &sack.start, &th->th_sum,
+					    htonl(ntohl(sack.start) - dst->seqdiff), 0);
+					pf_change_proto_a(m, &sack.end, &th->th_sum,
+					    htonl(ntohl(sack.end) - dst->seqdiff), 0);
 					memcpy(&opt[i], &sack, sizeof(sack));
 				}
 				copyback = 1;
@@ -3117,7 +3167,7 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction,
 
 			if (PF_ANEQ(saddr, &nk->addr[pd->sidx], af) ||
 			    nk->port[pd->sidx] != sport) {
-				pf_change_ap(saddr, &th->th_sport, pd->ip_sum,
+				pf_change_ap(m, saddr, &th->th_sport, pd->ip_sum,
 				    &th->th_sum, &nk->addr[pd->sidx],
 				    nk->port[pd->sidx], 0, af);
 				pd->sport = &th->th_sport;
@@ -3126,7 +3176,7 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction,
 
 			if (PF_ANEQ(daddr, &nk->addr[pd->didx], af) ||
 			    nk->port[pd->didx] != dport) {
-				pf_change_ap(daddr, &th->th_dport, pd->ip_sum,
+				pf_change_ap(m, daddr, &th->th_dport, pd->ip_sum,
 				    &th->th_sum, &nk->addr[pd->didx],
 				    nk->port[pd->didx], 0, af);
 				dport = th->th_dport;
@@ -3140,7 +3190,7 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction,
 
 			if (PF_ANEQ(saddr, &nk->addr[pd->sidx], af) ||
 			    nk->port[pd->sidx] != sport) {
-				pf_change_ap(saddr, &pd->hdr.udp->uh_sport,
+				pf_change_ap(m, saddr, &pd->hdr.udp->uh_sport,
 				    pd->ip_sum, &pd->hdr.udp->uh_sum,
 				    &nk->addr[pd->sidx],
 				    nk->port[pd->sidx], 1, af);
@@ -3150,7 +3200,7 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction,
 
 			if (PF_ANEQ(daddr, &nk->addr[pd->didx], af) ||
 			    nk->port[pd->didx] != dport) {
-				pf_change_ap(daddr, &pd->hdr.udp->uh_dport,
+				pf_change_ap(m, daddr, &pd->hdr.udp->uh_dport,
 				    pd->ip_sum, &pd->hdr.udp->uh_sum,
 				    &nk->addr[pd->didx],
 				    nk->port[pd->didx], 1, af);
@@ -3502,7 +3552,7 @@ pf_create_state(struct pf_rule *r, struct pf_rule *nr, struct pf_rule *a,
 			if ((s->src.seqdiff = pf_tcp_iss(pd) - s->src.seqlo) ==
 			    0)
 				s->src.seqdiff = 1;
-			pf_change_a(&th->th_seq, &th->th_sum,
+			pf_change_proto_a(m, &th->th_seq, &th->th_sum,
 			    htonl(s->src.seqlo + s->src.seqdiff), 0);
 			*rewrite = 1;
 		} else
@@ -3826,9 +3876,9 @@ pf_tcp_track_full(struct pf_state_peer *src, struct pf_state_peer *dst,
 			while ((src->seqdiff = arc4random() - seq) == 0)
 				;
 			ack = ntohl(th->th_ack) - dst->seqdiff;
-			pf_change_a(&th->th_seq, &th->th_sum, htonl(seq +
+			pf_change_proto_a(m, &th->th_seq, &th->th_sum, htonl(seq +
 			    src->seqdiff), 0);
-			pf_change_a(&th->th_ack, &th->th_sum, htonl(ack), 0);
+			pf_change_proto_a(m, &th->th_ack, &th->th_sum, htonl(ack), 0);
 			*copyback = 1;
 		} else {
 			ack = ntohl(th->th_ack);
@@ -3878,9 +3928,9 @@ pf_tcp_track_full(struct pf_state_peer *src, struct pf_state_peer *dst,
 		ack = ntohl(th->th_ack) - dst->seqdiff;
 		if (src->seqdiff) {
 			/* Modulate sequence numbers */
-			pf_change_a(&th->th_seq, &th->th_sum, htonl(seq +
+			pf_change_proto_a(m, &th->th_seq, &th->th_sum, htonl(seq +
 			    src->seqdiff), 0);
-			pf_change_a(&th->th_ack, &th->th_sum, htonl(ack), 0);
+			pf_change_proto_a(m, &th->th_ack, &th->th_sum, htonl(ack), 0);
 			*copyback = 1;
 		}
 		end = seq + pd->p_len;
@@ -4334,14 +4384,14 @@ pf_test_state_tcp(struct pf_state **state, int direction, struct pfi_kif *kif,
 
 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af) ||
 		    nk->port[pd->sidx] != th->th_sport)
-			pf_change_ap(pd->src, &th->th_sport, pd->ip_sum,
-			    &th->th_sum, &nk->addr[pd->sidx],
+			pf_change_ap(m, pd->src, &th->th_sport,
+			    pd->ip_sum, &th->th_sum, &nk->addr[pd->sidx],
 			    nk->port[pd->sidx], 0, pd->af);
 
 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af) ||
 		    nk->port[pd->didx] != th->th_dport)
-			pf_change_ap(pd->dst, &th->th_dport, pd->ip_sum,
-			    &th->th_sum, &nk->addr[pd->didx],
+			pf_change_ap(m, pd->dst, &th->th_dport,
+			    pd->ip_sum, &th->th_sum, &nk->addr[pd->didx],
 			    nk->port[pd->didx], 0, pd->af);
 		copyback = 1;
 	}
@@ -4405,13 +4455,13 @@ pf_test_state_udp(struct pf_state **state, int direction, struct pfi_kif *kif,
 
 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af) ||
 		    nk->port[pd->sidx] != uh->uh_sport)
-			pf_change_ap(pd->src, &uh->uh_sport, pd->ip_sum,
+			pf_change_ap(m, pd->src, &uh->uh_sport, pd->ip_sum,
 			    &uh->uh_sum, &nk->addr[pd->sidx],
 			    nk->port[pd->sidx], 1, pd->af);
 
 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af) ||
 		    nk->port[pd->didx] != uh->uh_dport)
-			pf_change_ap(pd->dst, &uh->uh_dport, pd->ip_sum,
+			pf_change_ap(m, pd->dst, &uh->uh_dport, pd->ip_sum,
 			    &uh->uh_sum, &nk->addr[pd->didx],
 			    nk->port[pd->didx], 1, pd->af);
 		m_copyback(m, off, sizeof(*uh), (caddr_t)uh);
diff --git a/sys/netpfil/pf/pf_ioctl.c b/sys/netpfil/pf/pf_ioctl.c
index ba43de8d37b8..a5a516fd3e5a 100644
--- a/sys/netpfil/pf/pf_ioctl.c
+++ b/sys/netpfil/pf/pf_ioctl.c
@@ -3566,12 +3566,6 @@ pf_check_out(void *arg, struct mbuf **m, struct ifnet *ifp, int dir,
 {
 	int chk;
 
-	/* We need a proper CSUM befor we start (s. OpenBSD ip_output) */
-	if ((*m)->m_pkthdr.csum_flags & CSUM_DELAY_DATA) {
-		in_delayed_cksum(*m);
-		(*m)->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA;
-	}
-
 	chk = pf_test(PF_OUT, ifp, m, inp);
 	if (chk && *m) {
 		m_freem(*m);
@@ -3610,13 +3604,6 @@ pf_check6_out(void *arg, struct mbuf **m, struct ifnet *ifp, int dir,
 {
 	int chk;
 
-	/* We need a proper CSUM before we start (s. OpenBSD ip_output) */
-	if ((*m)->m_pkthdr.csum_flags & CSUM_DELAY_DATA_IPV6) {
-		in6_delayed_cksum(*m,
-		    (*m)->m_pkthdr.len - sizeof(struct ip6_hdr),
-		    sizeof(struct ip6_hdr));
-		(*m)->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA_IPV6;
-	}
 	CURVNET_SET(ifp->if_vnet);
 	chk = pf_test6(PF_OUT, ifp, m, inp);
 	CURVNET_RESTORE();
diff --git a/sys/netpfil/pf/pf_norm.c b/sys/netpfil/pf/pf_norm.c
index 8605554442ac..99cca637c302 100644
--- a/sys/netpfil/pf/pf_norm.c
+++ b/sys/netpfil/pf/pf_norm.c
@@ -1215,13 +1215,14 @@ pf_normalize_tcp(int dir, struct pfi_kif *kif, struct mbuf *m, int ipoff,
 		th->th_x2 = 0;
 		nv = *(u_int16_t *)(&th->th_ack + 1);
 
-		th->th_sum = pf_cksum_fixup(th->th_sum, ov, nv, 0);
+		th->th_sum = pf_proto_cksum_fixup(m, th->th_sum, ov, nv, 0);
 		rewrite = 1;
 	}
 
 	/* Remove urgent pointer, if TH_URG is not set */
 	if (!(flags & TH_URG) && th->th_urp) {
-		th->th_sum = pf_cksum_fixup(th->th_sum, th->th_urp, 0, 0);
+		th->th_sum = pf_proto_cksum_fixup(m, th->th_sum, th->th_urp,
+		    0, 0);
 		th->th_urp = 0;
 		rewrite = 1;
 	}
@@ -1422,7 +1423,7 @@ pf_normalize_tcp_stateful(struct mbuf *m, int off, struct pf_pdesc *pd,
 					    (src->scrub->pfss_flags &
 					    PFSS_TIMESTAMP)) {
 						tsval = ntohl(tsval);
-						pf_change_a(&opt[2],
+						pf_change_proto_a(m, &opt[2],
 						    &th->th_sum,
 						    htonl(tsval +
 						    src->scrub->pfss_ts_mod),
@@ -1438,7 +1439,7 @@ pf_normalize_tcp_stateful(struct mbuf *m, int off, struct pf_pdesc *pd,
 					    PFSS_TIMESTAMP)) {
 						tsecr = ntohl(tsecr)
 						    - dst->scrub->pfss_ts_mod;
-						pf_change_a(&opt[6],
+						pf_change_proto_a(m, &opt[6],
 						    &th->th_sum, htonl(tsecr),
 						    0);
 						copyback = 1;
@@ -1765,8 +1766,8 @@ pf_normalize_tcpopt(struct pf_rule *r, struct mbuf *m, struct tcphdr *th,
 		case TCPOPT_MAXSEG:
 			mss = (u_int16_t *)(optp + 2);
 			if ((ntohs(*mss)) > r->max_mss) {
-				th->th_sum = pf_cksum_fixup(th->th_sum,
-				    *mss, htons(r->max_mss), 0);
+				th->th_sum = pf_proto_cksum_fixup(m,
+				    th->th_sum, *mss, htons(r->max_mss), 0);
 				*mss = htons(r->max_mss);
 				rewrite = 1;
 			}

From fec51f584e7da5ddd90dc0109c6dd78ed72a2c33 Mon Sep 17 00:00:00 2001
From: bz <bz@FreeBSD.org>
Date: Wed, 14 Oct 2015 16:56:25 +0000
Subject: [PATCH 208/380] For the Cortex-A8 use the a8 and not the a9 events
 table.

MFC after:		2 weeks
Sponsored by:		DARPA/AFRL
Differential Revision:	https://reviews.freebsd.org/D3882
---
 lib/libpmc/libpmc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/libpmc/libpmc.c b/lib/libpmc/libpmc.c
index 93056183dadf..129c064b5c21 100644
--- a/lib/libpmc/libpmc.c
+++ b/lib/libpmc/libpmc.c
@@ -388,7 +388,7 @@ PMC_CLASS_TABLE_DESC(tsc, TSC, tsc, tsc);
 #if	defined(__XSCALE__)
 PMC_CLASS_TABLE_DESC(xscale, XSCALE, xscale, xscale);
 #endif
-PMC_CLASS_TABLE_DESC(cortex_a8, ARMV7, cortex_a9, armv7);
+PMC_CLASS_TABLE_DESC(cortex_a8, ARMV7, cortex_a8, armv7);
 PMC_CLASS_TABLE_DESC(cortex_a9, ARMV7, cortex_a9, armv7);
 #endif
 #if	defined(__aarch64__)

From c3c1d7f7d22cdb10b1a3f6643dc692593803f21f Mon Sep 17 00:00:00 2001
From: bz <bz@FreeBSD.org>
Date: Wed, 14 Oct 2015 17:07:24 +0000
Subject: [PATCH 209/380] HWPMC depends on pmu.c even if device pmu is not
 specified. Would be great if we could just automatically enabled "device pmu"
 if we try to compile in HWPMC.

MFC after:		2 weeks
Sponsored by:		DARPA/AFRL
Reviewed by:		andrew
Differential Revision:	https://reviews.freebsd.org/D3877
---
 sys/conf/files.arm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/conf/files.arm b/sys/conf/files.arm
index 85830bb6717c..98410828d591 100644
--- a/sys/conf/files.arm
+++ b/sys/conf/files.arm
@@ -56,7 +56,7 @@ arm/arm/platform_if.m		optional	platform
 arm/arm/pmap.c			optional	!armv6
 arm/arm/pmap-v6.c		optional	armv6 !arm_new_pmap
 arm/arm/pmap-v6-new.c		optional	armv6 arm_new_pmap
-arm/arm/pmu.c			optional	pmu
+arm/arm/pmu.c			optional	pmu | hwpmc
 arm/arm/sc_machdep.c		optional	sc
 arm/arm/setcpsr.S		standard
 arm/arm/setstack.s		standard

From aa0582a388b245787566c0ec51ae7cfa41a720fb Mon Sep 17 00:00:00 2001
From: bz <bz@FreeBSD.org>
Date: Wed, 14 Oct 2015 17:20:19 +0000
Subject: [PATCH 210/380] Now that we can detect the Cortex-A8 properly, fix
 the event list according to the Cortex-A8 TRM r3p2 section 3.2.49. The A8
 list differs from the "ARM-v7 common" list, given the A8 was an earlier
 model.

There is still more work to be done for other Cortex-Ax version as
andrew points out, but I am just trying to fix A8 for now for teaching.

MFC after:		2 weeks
Sponsored by:		DARPA/AFRL
Obtained from:		Cambridge/L41
Reviewed by:		andrew
Differential Revision:	https://reviews.freebsd.org/D3876
---
 sys/dev/hwpmc/pmc_events.h | 43 ++++++++++++++++++++++++++++++++++----
 1 file changed, 39 insertions(+), 4 deletions(-)

diff --git a/sys/dev/hwpmc/pmc_events.h b/sys/dev/hwpmc/pmc_events.h
index 96dc9003f78d..696eea88e6c7 100644
--- a/sys/dev/hwpmc/pmc_events.h
+++ b/sys/dev/hwpmc/pmc_events.h
@@ -5027,7 +5027,7 @@ __PMC_EV_ALIAS("IMPC_C0H_TRK_REQUEST.ALL", UCP_EVENT_84H_01H)
 #define	PMC_EV_ARMV7_FIRST	PMC_EV_ARMV7_EVENT_00H
 #define	PMC_EV_ARMV7_LAST	PMC_EV_ARMV7_EVENT_FFH
 
-#define	__PMC_EV_ALIAS_ARMV7_COMMON()					\
+#define	__PMC_EV_ALIAS_ARMV7_COMMON_A8()				\
 	__PMC_EV_ALIAS("PMNC_SW_INCR",		ARMV7_EVENT_00H)	\
 	__PMC_EV_ALIAS("L1_ICACHE_REFILL",	ARMV7_EVENT_01H)	\
 	__PMC_EV_ALIAS("ITLB_REFILL",		ARMV7_EVENT_02H)	\
@@ -5046,7 +5046,10 @@ __PMC_EV_ALIAS("IMPC_C0H_TRK_REQUEST.ALL", UCP_EVENT_84H_01H)
 	__PMC_EV_ALIAS("MEM_UNALIGNED_ACCESS",	ARMV7_EVENT_0FH)	\
 	__PMC_EV_ALIAS("PC_BRANCH_MIS_PRED",	ARMV7_EVENT_10H)	\
 	__PMC_EV_ALIAS("CLOCK_CYCLES",		ARMV7_EVENT_11H)	\
-	__PMC_EV_ALIAS("PC_BRANCH_PRED",	ARMV7_EVENT_12H)	\
+	__PMC_EV_ALIAS("PC_BRANCH_PRED",	ARMV7_EVENT_12H)
+
+#define	__PMC_EV_ALIAS_ARMV7_COMMON()					\
+	__PMC_EV_ALIAS_ARMV7_COMMON_A8()				\
 	__PMC_EV_ALIAS("MEM_ACCESS",		ARMV7_EVENT_13H)	\
 	__PMC_EV_ALIAS("L1_ICACHE_ACCESS",	ARMV7_EVENT_14H)	\
 	__PMC_EV_ALIAS("L1_DCACHE_WB",		ARMV7_EVENT_15H)	\
@@ -5060,8 +5063,40 @@ __PMC_EV_ALIAS("IMPC_C0H_TRK_REQUEST.ALL", UCP_EVENT_84H_01H)
 	__PMC_EV_ALIAS("BUS_CYCLES",		ARMV7_EVENT_1DH)	\
 	__PMC_EV_ALIAS("CPU_CYCLES",		ARMV7_EVENT_FFH)
 
-#define	__PMC_EV_ALIAS_ARMV7_CORTEX_A8()				\
-	__PMC_EV_ALIAS_ARMV7_COMMON()
+#define	__PMC_EV_ALIAS_ARMV7_CORTEX_A8()				 \
+	__PMC_EV_ALIAS_ARMV7_COMMON_A8()				 \
+	__PMC_EV_ALIAS("WRITE_BUF_FULL",		ARMV7_EVENT_40H) \
+	__PMC_EV_ALIAS("L2_STORE_MERGED",		ARMV7_EVENT_41H) \
+	__PMC_EV_ALIAS("L2_STORE_BUFFERABLE",		ARMV7_EVENT_42H) \
+	__PMC_EV_ALIAS("L2_ACCESS",			ARMV7_EVENT_43H) \
+	__PMC_EV_ALIAS("L2_CACHE_MISS",			ARMV7_EVENT_44H) \
+	__PMC_EV_ALIAS("AXI_READ",			ARMV7_EVENT_45H) \
+	__PMC_EV_ALIAS("AXI_WRITE",			ARMV7_EVENT_46H) \
+	__PMC_EV_ALIAS("MEM_REPLAY_EVT",		ARMV7_EVENT_47H) \
+	__PMC_EV_ALIAS("MEM_UNALIGNED_ACCESS_REPLAY",	ARMV7_EVENT_48H) \
+	__PMC_EV_ALIAS("L1_DCACHE_HASH_MISS",		ARMV7_EVENT_49H) \
+	__PMC_EV_ALIAS("L1_ICACHE_HASH_MISS",		ARMV7_EVENT_4AH) \
+	__PMC_EV_ALIAS("L1_CACHE_PAGECOL_ALIAS",	ARMV7_EVENT_4BH) \
+	__PMC_EV_ALIAS("L1_DCACHE_NEON_ACCESS",		ARMV7_EVENT_4CH) \
+	__PMC_EV_ALIAS("L1_DCACHE_NEON_CACHEABLE",	ARMV7_EVENT_4DH) \
+	__PMC_EV_ALIAS("L2_CACHE_NEON_MEM_ACCESS",	ARMV7_EVENT_4EH) \
+	__PMC_EV_ALIAS("L2_CACHE_NEON_HIT",		ARMV7_EVENT_4FH) \
+	__PMC_EV_ALIAS("L1_CACHE_ACCESS_NOCP15",	ARMV7_EVENT_50H) \
+	__PMC_EV_ALIAS("RET_STACK_MISPREDICT",		ARMV7_EVENT_51H) \
+	__PMC_EV_ALIAS("BRANCH_DIR_MISPREDICT",		ARMV7_EVENT_52H) \
+	__PMC_EV_ALIAS("PRED_BRANCH_PRED_TAKEN",	ARMV7_EVENT_53H) \
+	__PMC_EV_ALIAS("PRED_BRANCH_EXEC_TAKEN",	ARMV7_EVENT_54H) \
+	__PMC_EV_ALIAS("OPS_ISSUED",			ARMV7_EVENT_55H) \
+	__PMC_EV_ALIAS("CYCLES_NO_INSTRUCTION",		ARMV7_EVENT_56H) \
+	__PMC_EV_ALIAS("INSTRUCTIONS_ISSUED_CYCLE",	ARMV7_EVENT_57H) \
+	__PMC_EV_ALIAS("CYCLES_STALLED_NEON_MRC",	ARMV7_EVENT_58H) \
+	__PMC_EV_ALIAS("CYCLES_STALLED_NEON_FULLQ",	ARMV7_EVENT_59H) \
+	__PMC_EV_ALIAS("CYCLES_NONIDLE_NEON_INT",	ARMV7_EVENT_5AH) \
+	__PMC_EV_ALIAS("PMUEXTIN0_EVT",			ARMV7_EVENT_70H) \
+	__PMC_EV_ALIAS("PMUEXTIN1_EVT",			ARMV7_EVENT_71H) \
+	__PMC_EV_ALIAS("PMUEXTIN_EVT",			ARMV7_EVENT_72H)
+#define	PMC_EV_ARMV7_CORTEX_A8_FIRST	PMC_EV_ARMV7_PMNC_SW_INCR
+#define	PMC_EV_ARMV7_CORTEX_A8_LAST	PMC_EV_ARMV7_PMUEXTIN_EVT
 
 #define	__PMC_EV_ALIAS_ARMV7_CORTEX_A9()					\
 	__PMC_EV_ALIAS_ARMV7_COMMON()						\

From a3209916785bf4ad8f5ea842bad113cc8c9d2458 Mon Sep 17 00:00:00 2001
From: des <des@FreeBSD.org>
Date: Wed, 14 Oct 2015 18:08:38 +0000
Subject: [PATCH 211/380] Apply r3505 (s/SIGQUIT/SIGTERM/ in man page)

PR:		203580
---
 contrib/unbound/doc/unbound.conf.5.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/unbound/doc/unbound.conf.5.in b/contrib/unbound/doc/unbound.conf.5.in
index c497eeebf33f..42653a51ccf2 100644
--- a/contrib/unbound/doc/unbound.conf.5.in
+++ b/contrib/unbound/doc/unbound.conf.5.in
@@ -481,7 +481,7 @@ kill \-HUP `cat @UNBOUND_PIDFILE@`
 .fi
 triggers a reload,
 .nf
-kill \-QUIT `cat @UNBOUND_PIDFILE@` 
+kill \-TERM `cat @UNBOUND_PIDFILE@` 
 .fi
 gracefully terminates.
 .TP

From a6091af92361b7cb9a9c075590da1ba9c2badaa1 Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Wed, 14 Oct 2015 18:27:35 +0000
Subject: [PATCH 212/380] Allow PT_INTERP and PT_NOTES segments to be located
 anywhere in the executable image.  Keep one page (arbitrary) limit on the max
 allowed size of the PT_NOTES.

The ELF image activators still require that program headers of the
executable are fully contained in the first page of the image file.

Reviewed by:	emaste, jhb
Sponsored by:	The FreeBSD Foundation
MFC after:	2 weeks
Differential revision:	https://reviews.freebsd.org/D3871
---
 sys/kern/imgact_elf.c | 140 ++++++++++++++++++++++++++++++------------
 1 file changed, 100 insertions(+), 40 deletions(-)

diff --git a/sys/kern/imgact_elf.c b/sys/kern/imgact_elf.c
index 303e86528d3d..94b25b8fe3b9 100644
--- a/sys/kern/imgact_elf.c
+++ b/sys/kern/imgact_elf.c
@@ -721,21 +721,22 @@ fail:
 static int
 __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp)
 {
-	const Elf_Ehdr *hdr = (const Elf_Ehdr *)imgp->image_header;
+	struct thread *td;
+	const Elf_Ehdr *hdr;
 	const Elf_Phdr *phdr;
 	Elf_Auxargs *elf_auxargs;
 	struct vmspace *vmspace;
-	vm_prot_t prot;
-	u_long text_size = 0, data_size = 0, total_size = 0;
-	u_long text_addr = 0, data_addr = 0;
-	u_long seg_size, seg_addr;
-	u_long addr, baddr, et_dyn_addr, entry = 0, proghdr = 0;
-	int32_t osrel = 0;
-	int error = 0, i, n, interp_name_len = 0;
-	const char *err_str = NULL, *interp = NULL, *newinterp = NULL;
+	const char *err_str, *newinterp;
+	char *interp, *interp_buf, *path;
 	Elf_Brandinfo *brand_info;
-	char *path;
 	struct sysentvec *sv;
+	vm_prot_t prot;
+	u_long text_size, data_size, total_size, text_addr, data_addr;
+	u_long seg_size, seg_addr, addr, baddr, et_dyn_addr, entry, proghdr;
+	int32_t osrel;
+	int error, i, n, interp_name_len, have_interp;
+
+	hdr = (const Elf_Ehdr *)imgp->image_header;
 
 	/*
 	 * Do we have a valid ELF header ?
@@ -763,8 +764,17 @@ __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp)
 		uprintf("Unaligned program headers\n");
 		return (ENOEXEC);
 	}
-	n = 0;
+
+	n = error = 0;
 	baddr = 0;
+	osrel = 0;
+	text_size = data_size = total_size = text_addr = data_addr = 0;
+	entry = proghdr = 0;
+	interp_name_len = 0;
+	err_str = newinterp = NULL;
+	interp = interp_buf = NULL;
+	td = curthread;
+
 	for (i = 0; i < hdr->e_phnum; i++) {
 		switch (phdr[i].p_type) {
 		case PT_LOAD:
@@ -774,14 +784,32 @@ __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp)
 			break;
 		case PT_INTERP:
 			/* Path to interpreter */
-			if (phdr[i].p_filesz > MAXPATHLEN ||
-			    phdr[i].p_offset > PAGE_SIZE ||
-			    phdr[i].p_filesz > PAGE_SIZE - phdr[i].p_offset) {
+			if (phdr[i].p_filesz > MAXPATHLEN) {
 				uprintf("Invalid PT_INTERP\n");
-				return (ENOEXEC);
+				error = ENOEXEC;
+				goto ret;
 			}
-			interp = imgp->image_header + phdr[i].p_offset;
 			interp_name_len = phdr[i].p_filesz;
+			if (phdr[i].p_offset > PAGE_SIZE ||
+			    interp_name_len > PAGE_SIZE - phdr[i].p_offset) {
+				VOP_UNLOCK(imgp->vp, 0);
+				interp_buf = malloc(interp_name_len + 1, M_TEMP,
+				    M_WAITOK);
+				vn_lock(imgp->vp, LK_EXCLUSIVE | LK_RETRY);
+				error = vn_rdwr(UIO_READ, imgp->vp, interp_buf,
+				    interp_name_len, phdr[i].p_offset,
+				    UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred,
+				    NOCRED, NULL, td);
+				if (error != 0) {
+					uprintf("i/o error PT_INTERP\n");
+					goto ret;
+				}
+				interp_buf[interp_name_len] = '\0';
+				interp = interp_buf;
+			} else {
+				interp = __DECONST(char *, imgp->image_header) +
+				    phdr[i].p_offset;
+			}
 			break;
 		case PT_GNU_STACK:
 			if (__elfN(nxstack))
@@ -797,12 +825,14 @@ __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp)
 	if (brand_info == NULL) {
 		uprintf("ELF binary type \"%u\" not known.\n",
 		    hdr->e_ident[EI_OSABI]);
-		return (ENOEXEC);
+		error = ENOEXEC;
+		goto ret;
 	}
 	if (hdr->e_type == ET_DYN) {
 		if ((brand_info->flags & BI_CAN_EXEC_DYN) == 0) {
 			uprintf("Cannot execute shared object\n");
-			return (ENOEXEC);
+			error = ENOEXEC;
+			goto ret;
 		}
 		/*
 		 * Honour the base load address from the dso if it is
@@ -835,8 +865,8 @@ __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp)
 	imgp->proc->p_sysent = sv;
 
 	vn_lock(imgp->vp, LK_EXCLUSIVE | LK_RETRY);
-	if (error)
-		return (error);
+	if (error != 0)
+		goto ret;
 
 	for (i = 0; i < hdr->e_phnum; i++) {
 		switch (phdr[i].p_type) {
@@ -849,7 +879,7 @@ __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp)
 			    phdr[i].p_memsz, phdr[i].p_filesz, prot,
 			    sv->sv_pagesize);
 			if (error != 0)
-				return (error);
+				goto ret;
 
 			/*
 			 * If this segment contains the program headers,
@@ -921,7 +951,8 @@ __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp)
 	if (err_str != NULL) {
 		PROC_UNLOCK(imgp->proc);
 		uprintf("%s\n", err_str);
-		return (ENOMEM);
+		error = ENOMEM;
+		goto ret;
 	}
 
 	vmspace = imgp->proc->p_vmspace;
@@ -936,14 +967,14 @@ __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp)
 	 * calculation is that it leaves room for the heap to grow to
 	 * its maximum allowed size.
 	 */
-	addr = round_page((vm_offset_t)vmspace->vm_daddr + lim_max(curthread,
+	addr = round_page((vm_offset_t)vmspace->vm_daddr + lim_max(td,
 	    RLIMIT_DATA));
 	PROC_UNLOCK(imgp->proc);
 
 	imgp->entry_addr = entry;
 
 	if (interp != NULL) {
-		int have_interp = FALSE;
+		have_interp = FALSE;
 		VOP_UNLOCK(imgp->vp, 0);
 		if (brand_info->emul_path != NULL &&
 		    brand_info->emul_path[0] != '\0') {
@@ -969,7 +1000,7 @@ __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp)
 		vn_lock(imgp->vp, LK_EXCLUSIVE | LK_RETRY);
 		if (error != 0) {
 			uprintf("ELF interpreter %s not found\n", interp);
-			return (error);
+			goto ret;
 		}
 	} else
 		addr = et_dyn_addr;
@@ -993,6 +1024,8 @@ __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp)
 	imgp->reloc_base = addr;
 	imgp->proc->p_osrel = osrel;
 
+ ret:
+	free(interp_buf, M_TEMP);
 	return (error);
 }
 
@@ -2122,19 +2155,42 @@ __elfN(parse_notes)(struct image_params *imgp, Elf_Brandnote *checknote,
 {
 	const Elf_Note *note, *note0, *note_end;
 	const char *note_name;
-	int i;
+	char *buf;
+	int i, error;
+	boolean_t res;
 
-	if (pnote == NULL || pnote->p_offset > PAGE_SIZE ||
-	    pnote->p_filesz > PAGE_SIZE - pnote->p_offset)
+	/* We need some limit, might as well use PAGE_SIZE. */
+	if (pnote == NULL || pnote->p_filesz > PAGE_SIZE)
 		return (FALSE);
-
-	note = note0 = (const Elf_Note *)(imgp->image_header + pnote->p_offset);
-	note_end = (const Elf_Note *)(imgp->image_header +
-	    pnote->p_offset + pnote->p_filesz);
+	ASSERT_VOP_LOCKED(imgp->vp, "parse_notes");
+	if (pnote->p_offset > PAGE_SIZE ||
+	    pnote->p_filesz > PAGE_SIZE - pnote->p_offset) {
+		VOP_UNLOCK(imgp->vp, 0);
+		buf = malloc(pnote->p_filesz, M_TEMP, M_WAITOK);
+		vn_lock(imgp->vp, LK_EXCLUSIVE | LK_RETRY);
+		error = vn_rdwr(UIO_READ, imgp->vp, buf, pnote->p_filesz,
+		    pnote->p_offset, UIO_SYSSPACE, IO_NODELOCKED,
+		    curthread->td_ucred, NOCRED, NULL, curthread);
+		if (error != 0) {
+			uprintf("i/o error PT_NOTE\n");
+			res = FALSE;
+			goto ret;
+		}
+		note = note0 = (const Elf_Note *)buf;
+		note_end = (const Elf_Note *)(buf + pnote->p_filesz);
+	} else {
+		note = note0 = (const Elf_Note *)(imgp->image_header +
+		    pnote->p_offset);
+		note_end = (const Elf_Note *)(imgp->image_header +
+		    pnote->p_offset + pnote->p_filesz);
+		buf = NULL;
+	}
 	for (i = 0; i < 100 && note >= note0 && note < note_end; i++) {
 		if (!aligned(note, Elf32_Addr) || (const char *)note_end -
-		    (const char *)note < sizeof(Elf_Note))
-			return (FALSE);
+		    (const char *)note < sizeof(Elf_Note)) {
+			res = FALSE;
+			goto ret;
+		}
 		if (note->n_namesz != checknote->hdr.n_namesz ||
 		    note->n_descsz != checknote->hdr.n_descsz ||
 		    note->n_type != checknote->hdr.n_type)
@@ -2150,17 +2206,21 @@ __elfN(parse_notes)(struct image_params *imgp, Elf_Brandnote *checknote,
 		 * from the ELF OSABI-note if necessary.
 		 */
 		if ((checknote->flags & BN_TRANSLATE_OSREL) != 0 &&
-		    checknote->trans_osrel != NULL)
-			return (checknote->trans_osrel(note, osrel));
-		return (TRUE);
-
+		    checknote->trans_osrel != NULL) {
+			res = checknote->trans_osrel(note, osrel);
+			goto ret;
+		}
+		res = TRUE;
+		goto ret;
 nextnote:
 		note = (const Elf_Note *)((const char *)(note + 1) +
 		    roundup2(note->n_namesz, ELF_NOTE_ROUNDSIZE) +
 		    roundup2(note->n_descsz, ELF_NOTE_ROUNDSIZE));
 	}
-
-	return (FALSE);
+	res = FALSE;
+ret:
+	free(buf, M_TEMP);
+	return (res);
 }
 
 /*

From f516f6c2bc8b716d0307cfcee9fd9483d4bb5db3 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Wed, 14 Oct 2015 18:29:00 +0000
Subject: [PATCH 213/380] Redirect all stdout from run to stderr so failures
 can be debugged

---
 tests/sys/acl/00.sh | 2 +-
 tests/sys/acl/01.sh | 2 +-
 tests/sys/acl/02.sh | 6 +++---
 tests/sys/acl/03.sh | 2 +-
 tests/sys/acl/04.sh | 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/sys/acl/00.sh b/tests/sys/acl/00.sh
index 61e8115525ff..98b5e6fdf509 100644
--- a/tests/sys/acl/00.sh
+++ b/tests/sys/acl/00.sh
@@ -75,7 +75,7 @@ chmod 600 xxx
 rm xxx
 echo "ok 2"
 
-perl $TESTDIR/run $TESTDIR/tools-posix.test > /dev/null
+perl $TESTDIR/run $TESTDIR/tools-posix.test >&2
 
 if [ $? -eq 0 ]; then
 	echo "ok 3"
diff --git a/tests/sys/acl/01.sh b/tests/sys/acl/01.sh
index 93487a77263f..a456c07f2c64 100644
--- a/tests/sys/acl/01.sh
+++ b/tests/sys/acl/01.sh
@@ -76,7 +76,7 @@ chmod 600 xxx
 rm xxx
 echo "ok 2"
 
-perl $TESTDIR/run $TESTDIR/tools-nfs4-psarc.test > /dev/null
+perl $TESTDIR/run $TESTDIR/tools-nfs4-psarc.test >&2
 
 if [ $? -eq 0 ]; then
 	echo "ok 3"
diff --git a/tests/sys/acl/02.sh b/tests/sys/acl/02.sh
index 8fbb39de4d6a..d1bc75d8b078 100644
--- a/tests/sys/acl/02.sh
+++ b/tests/sys/acl/02.sh
@@ -76,10 +76,10 @@ rm xxx
 echo "ok 2"
 
 if [ `sysctl -n vfs.acl_nfs4_old_semantics` = 0 ]; then
-	perl $TESTDIR/run $TESTDIR/tools-nfs4-psarc.test > /dev/null
+	perl $TESTDIR/run $TESTDIR/tools-nfs4-psarc.test
 else
-	perl $TESTDIR/run $TESTDIR/tools-nfs4.test > /dev/null
-fi
+	perl $TESTDIR/run $TESTDIR/tools-nfs4.test
+fi >&2
 
 if [ $? -eq 0 ]; then
 	echo "ok 3"
diff --git a/tests/sys/acl/03.sh b/tests/sys/acl/03.sh
index 2b81b2662306..e6379a6d4497 100644
--- a/tests/sys/acl/03.sh
+++ b/tests/sys/acl/03.sh
@@ -89,7 +89,7 @@ echo "ok 3"
 
 cd $MNTROOT
 
-perl $TESTDIR/run $TESTDIR/tools-crossfs.test > /dev/null
+perl $TESTDIR/run $TESTDIR/tools-crossfs.test >&2
 
 if [ $? -eq 0 ]; then
 	echo "ok 4"
diff --git a/tests/sys/acl/04.sh b/tests/sys/acl/04.sh
index 5669c5a48d24..ff1d3cb080bb 100644
--- a/tests/sys/acl/04.sh
+++ b/tests/sys/acl/04.sh
@@ -57,7 +57,7 @@ echo "ok 1"
 
 cd $MNT
 
-perl $TESTDIR/run $TESTDIR/tools-nfs4-trivial.test > /dev/null
+perl $TESTDIR/run $TESTDIR/tools-nfs4-trivial.test >&2
 
 if [ $? -eq 0 ]; then
 	echo "ok 2"

From 93aac33bebffcc094c942e8ef9bf5934f3cab81b Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Wed, 14 Oct 2015 18:29:21 +0000
Subject: [PATCH 214/380] Allow PT_NOTES segments to be located anywhere in the
 executable image.

The dynamic linker still requires that program headers of the
executable or dso are mapped by a PT_LOAD segment.

Reviewed by:	emaste, jhb
Sponsored by:	The FreeBSD Foundation
MFC after:	2 weeks
Differential revision:	https://reviews.freebsd.org/D3871
---
 libexec/rtld-elf/map_object.c | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/libexec/rtld-elf/map_object.c b/libexec/rtld-elf/map_object.c
index 2e17fbf06b78..6012015bcc28 100644
--- a/libexec/rtld-elf/map_object.c
+++ b/libexec/rtld-elf/map_object.c
@@ -88,6 +88,8 @@ map_object(int fd, const char *path, const struct stat *sb)
     size_t relro_size;
     Elf_Addr note_start;
     Elf_Addr note_end;
+    char *note_map;
+    size_t note_map_len;
 
     hdr = get_elf_header(fd, path);
     if (hdr == NULL)
@@ -108,6 +110,7 @@ map_object(int fd, const char *path, const struct stat *sb)
     relro_size = 0;
     note_start = 0;
     note_end = 0;
+    note_map = NULL;
     segs = alloca(sizeof(segs[0]) * hdr->e_phnum);
     stack_flags = RTLD_DEFAULT_STACK_PF_EXEC | PF_R | PF_W;
     while (phdr < phlimit) {
@@ -150,9 +153,20 @@ map_object(int fd, const char *path, const struct stat *sb)
 
 	case PT_NOTE:
 	    if (phdr->p_offset > PAGE_SIZE ||
-	      phdr->p_offset + phdr->p_filesz > PAGE_SIZE)
-		break;
-	    note_start = (Elf_Addr)(char *)hdr + phdr->p_offset;
+	      phdr->p_offset + phdr->p_filesz > PAGE_SIZE) {
+		note_map_len = round_page(phdr->p_offset +
+		  phdr->p_filesz) - trunc_page(phdr->p_offset);
+		note_map = mmap(NULL, note_map_len, PROT_READ,
+		  MAP_PRIVATE, fd, trunc_page(phdr->p_offset));
+		if (note_map == MAP_FAILED) {
+		    _rtld_error("%s: error mapping PT_NOTE (%d)", path, errno);
+		    goto error;
+		}
+		note_start = (Elf_Addr)(note_map + phdr->p_offset -
+		  trunc_page(phdr->p_offset));
+	    } else {
+		note_start = (Elf_Addr)(char *)hdr + phdr->p_offset;
+	    }
 	    note_end = note_start + phdr->p_filesz;
 	    break;
 	}
@@ -295,12 +309,16 @@ map_object(int fd, const char *path, const struct stat *sb)
     obj->relro_size = round_page(relro_size);
     if (note_start < note_end)
 	digest_notes(obj, note_start, note_end);
+    if (note_map != NULL)
+	munmap(note_map, note_map_len);
     munmap(hdr, PAGE_SIZE);
     return (obj);
 
 error1:
     munmap(mapbase, mapsize);
 error:
+    if (note_map != NULL && note_map != MAP_FAILED)
+	munmap(note_map, note_map_len);
     munmap(hdr, PAGE_SIZE);
     return (NULL);
 }

From 4640526e7e284cecc9856f90c9b4e1728a8a38e2 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Wed, 14 Oct 2015 18:29:53 +0000
Subject: [PATCH 215/380] Add Makefile missed in r289236

---
 tests/sys/acl/Makefile | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 tests/sys/acl/Makefile

diff --git a/tests/sys/acl/Makefile b/tests/sys/acl/Makefile
new file mode 100644
index 000000000000..298b0a27c91d
--- /dev/null
+++ b/tests/sys/acl/Makefile
@@ -0,0 +1,29 @@
+# $FreeBSD$
+
+TESTSDIR=	${TESTSBASE}/sys/acl
+
+BINDIR=		${TESTSDIR}
+
+FILES+=		tools-crossfs.test
+FILES+=		tools-nfs4.test
+FILES+=		tools-nfs4-psarc.test
+FILES+=		tools-nfs4-trivial.test
+FILES+=		tools-posix.test
+
+SCRIPTS+=	run
+
+TAP_TESTS_SH+=	00
+TAP_TESTS_SH+=	01
+TAP_TESTS_SH+=	02
+TAP_TESTS_SH+=	03
+TAP_TESTS_SH+=	04
+
+.for t in ${TAP_TESTS_SH}
+TEST_METADATA.$t+=	required_user="root"
+.endfor
+
+.for t in 01 03 04
+TEST_METADATA.$t+=	required_programs="/sbin/zpool"
+.endfor
+
+.include <bsd.test.mk>

From 857b8f754e7f36463bd3bd5ef6eda0de02778e80 Mon Sep 17 00:00:00 2001
From: bz <bz@FreeBSD.org>
Date: Wed, 14 Oct 2015 18:30:04 +0000
Subject: [PATCH 216/380] Properly define functions withut argument and wrap
 for { for style purposes as followed in the rest of the file.  This will
 hopefully make gcc more happy.

---
 sys/netinet/tcp_pcap.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/sys/netinet/tcp_pcap.c b/sys/netinet/tcp_pcap.c
index 7a3514f9646d..c431f2444bd8 100644
--- a/sys/netinet/tcp_pcap.c
+++ b/sys/netinet/tcp_pcap.c
@@ -76,12 +76,16 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_pcap_packets, CTLFLAG_RW,
 
 /* Initialize the values. */
 static void
-tcp_pcap_max_set() {
+tcp_pcap_max_set(void)
+{
+
 	tcp_pcap_clusters_referenced_max = nmbclusters / 4;
 }
 
 void
-tcp_pcap_init() {
+tcp_pcap_init(void)
+{
+
 	tcp_pcap_max_set();
 	EVENTHANDLER_REGISTER(nmbclusters_change, tcp_pcap_max_set,
 		NULL, EVENTHANDLER_PRI_ANY);

From e2d907351f884aa4c384974429600841ad1f7414 Mon Sep 17 00:00:00 2001
From: bz <bz@FreeBSD.org>
Date: Wed, 14 Oct 2015 18:32:06 +0000
Subject: [PATCH 217/380] Fix the dependencies to be similar to TCP as without
 TCP, e.g., NOIP kernels this will otherwise fail.

---
 sys/conf/files | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/conf/files b/sys/conf/files
index 5e7bdf384bbe..f32f60c0174c 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -3682,7 +3682,7 @@ netinet/tcp_input.c		optional inet | inet6
 netinet/tcp_lro.c		optional inet | inet6
 netinet/tcp_output.c		optional inet | inet6
 netinet/tcp_offload.c		optional tcp_offload inet | tcp_offload inet6
-netinet/tcp_pcap.c		optional tcppcap
+netinet/tcp_pcap.c		optional inet tcppcap | inet6 tcppcap
 netinet/tcp_reass.c		optional inet | inet6
 netinet/tcp_sack.c		optional inet | inet6
 netinet/tcp_subr.c		optional inet | inet6

From a1c8ec00371288bfc12465e841eb4feb2c093ef5 Mon Sep 17 00:00:00 2001
From: bz <bz@FreeBSD.org>
Date: Wed, 14 Oct 2015 18:53:34 +0000
Subject: [PATCH 218/380] Revert r289319 as it seems some ARM kernels include
 HWPMC but no FDT. To me that seems broken as certain interrupts will never be
 handled properly.  I'll re-open D3877 and we can seek a better solution and
 try again.  For now go back to that state and avoid compile time errors.

---
 sys/conf/files.arm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/conf/files.arm b/sys/conf/files.arm
index 98410828d591..85830bb6717c 100644
--- a/sys/conf/files.arm
+++ b/sys/conf/files.arm
@@ -56,7 +56,7 @@ arm/arm/platform_if.m		optional	platform
 arm/arm/pmap.c			optional	!armv6
 arm/arm/pmap-v6.c		optional	armv6 !arm_new_pmap
 arm/arm/pmap-v6-new.c		optional	armv6 arm_new_pmap
-arm/arm/pmu.c			optional	pmu | hwpmc
+arm/arm/pmu.c			optional	pmu
 arm/arm/sc_machdep.c		optional	sc
 arm/arm/setcpsr.S		standard
 arm/arm/setstack.s		standard

From 19ad2d24469a9ff2fe2e641e01c7e44e1c089139 Mon Sep 17 00:00:00 2001
From: emaste <emaste@FreeBSD.org>
Date: Wed, 14 Oct 2015 19:14:05 +0000
Subject: [PATCH 219/380] Add mtree entry for casper .debug files

This was missed in r258838.

Sponsored by:	The FreeBSD Foundation
---
 etc/mtree/BSD.debug.dist | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/etc/mtree/BSD.debug.dist b/etc/mtree/BSD.debug.dist
index fe5bc48cd1db..9584e591eb47 100644
--- a/etc/mtree/BSD.debug.dist
+++ b/etc/mtree/BSD.debug.dist
@@ -17,6 +17,8 @@
             ..
         ..
         libexec
+            casper
+            ..
         ..
         sbin
         ..

From d70c586bb85ddc1f7dac37ec85544b169e6b4b6e Mon Sep 17 00:00:00 2001
From: emaste <emaste@FreeBSD.org>
Date: Wed, 14 Oct 2015 19:19:44 +0000
Subject: [PATCH 220/380] /libexec subdirs are part of the base system (for
 *.debug files)

Sponsored by:	The FreeBSD Foundation
---
 share/mk/bsd.prog.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/share/mk/bsd.prog.mk b/share/mk/bsd.prog.mk
index 966c17e8cffa..b487541bc018 100644
--- a/share/mk/bsd.prog.mk
+++ b/share/mk/bsd.prog.mk
@@ -56,7 +56,7 @@ PROG_FULL=${PROG}.full
 # Use ${DEBUGDIR} for base system debug files, else .debug subdirectory
 .if defined(BINDIR) && (\
     ${BINDIR} == "/bin" ||\
-    ${BINDIR} == "/libexec" ||\
+    ${BINDIR:C%/libexec(/.*)?%/libexec%} == "/libexec" ||\
     ${BINDIR} == "/sbin" ||\
     ${BINDIR:C%/usr/(bin|bsdinstall|libexec|lpr|sendmail|sm.bin|sbin)(/.*)?%/usr/bin%} == "/usr/bin"\
      )

From 2606e8ae5cfe7009393b115c804e527fdc834c66 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Wed, 14 Oct 2015 19:30:04 +0000
Subject: [PATCH 221/380] Revert r289282 for now as the interaction with a
 directory containing bsd.files.mk and bsd.subdir.mk is recursing too many
 times.

---
 share/mk/bsd.confs.mk  |  3 ---
 share/mk/bsd.files.mk  |  3 ---
 share/mk/bsd.incs.mk   |  3 ---
 share/mk/bsd.subdir.mk | 26 ++++++++++++++++++++------
 4 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/share/mk/bsd.confs.mk b/share/mk/bsd.confs.mk
index bcc36bc18e6f..0750a4e4fa57 100644
--- a/share/mk/bsd.confs.mk
+++ b/share/mk/bsd.confs.mk
@@ -84,7 +84,4 @@ STAGE_TARGETS+= stage_config
 .endif
 .endif
 
-config: buildconfig installconfig
-.ORDER: buildconfig installconfig
-
 .endif # ${MK_INCLUDES} != "no"
diff --git a/share/mk/bsd.files.mk b/share/mk/bsd.files.mk
index d1d3263c4444..b9379d92b59a 100644
--- a/share/mk/bsd.files.mk
+++ b/share/mk/bsd.files.mk
@@ -94,7 +94,4 @@ buildfiles: stage_as
 .endif
 .endif
 
-files: buildfiles installfiles
-.ORDER: buildfiles installfiles
-
 .endif # !target(__<bsd.files.mk>__)
diff --git a/share/mk/bsd.incs.mk b/share/mk/bsd.incs.mk
index c985668c7c4b..e1398c95dcd1 100644
--- a/share/mk/bsd.incs.mk
+++ b/share/mk/bsd.incs.mk
@@ -99,7 +99,4 @@ STAGE_SYMLINKS.INCS= ${INCSLINKS}
 .endif
 .endif
 
-includes: buildincludes installincludes
-.ORDER: buildincludes installincludes
-
 .endif # ${MK_INCLUDES} != "no"
diff --git a/share/mk/bsd.subdir.mk b/share/mk/bsd.subdir.mk
index e4c3e69d1c27..8afe40a693b0 100644
--- a/share/mk/bsd.subdir.mk
+++ b/share/mk/bsd.subdir.mk
@@ -32,12 +32,9 @@
 .if !target(__<bsd.subdir.mk>__)
 __<bsd.subdir.mk>__:
 
-ALL_SUBDIR_TARGETS= all all-man buildconfig buildfiles buildincludes \
-		    checkdpadd clean cleandepend cleandir cleanilinks \
-		    cleanobj config depend distribute files includes \
-		    installconfig installfiles installincludes lint \
-		    maninstall manlint obj objlink realinstall regress tags \
-		    ${SUBDIR_TARGETS}
+ALL_SUBDIR_TARGETS= all all-man checkdpadd clean cleandepend cleandir \
+	cleanilinks cleanobj depend distribute lint maninstall manlint obj \
+	objlink realinstall regress tags ${SUBDIR_TARGETS}
 
 .include <bsd.init.mk>
 
@@ -126,6 +123,23 @@ _sub.${__target}: _SUBDIR
 .endif
 .endfor
 
+# This is to support 'make includes' calling 'make buildincludes' and
+# 'make installincludes' in the proper order, and to support these
+# targets as SUBDIR_TARGETS.
+.for __target in files includes config
+.for __stage in build install
+${__stage}${__target}:
+.if make(${__stage}${__target})
+${__stage}${__target}: _sub.${__stage}${__target}
+_sub.${__stage}${__target}: _SUBDIR
+.endif
+.endfor
+.if !target(${__target})
+${__target}: .MAKE
+	${_+_}cd ${.CURDIR}; ${MAKE} build${__target}; ${MAKE} install${__target}
+.endif
+.endfor
+
 .endif
 
 .if !target(install)

From 7983774b1872ca0b37aa6c93c5a76a92fea21b2c Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Wed, 14 Oct 2015 20:22:12 +0000
Subject: [PATCH 222/380] Fix test-fenv:test_dfl_env when run on some amd64
 CPUs

Compare the fields that the AMD [1] and Intel [2] specs say will be
set once fnstenv returns.

Not all amd64 capable processors zero out the env.__x87.__other field
(example: AMD Opteron 6308). The AMD64/x64 specs aren't explicit on what the
env.__x87.__other field will contain after fnstenv is executed, so the values
in env.__x87.__other could be filled with arbitrary data depending on how the
CPU-specific implementation of fnstenv.

1. http://support.amd.com/TechDocs/26569_APM_v5.pdf
2. http://www.intel.com/Assets/en_US/PDF/manual/253666.pdf

Discussed with: kib, Anton Rang <anton.rang@isilon.com>
Reviewed by: Daniel O'Connor <darius@dons.net.au> (earlier patch; pre-generalization)
MFC after: 1 week
Sponsored by: EMC / Isilon Storage Division
Reported by: Bill Morchin <wmorchin@isilon.com>
---
 tools/regression/lib/msun/test-fenv.c | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/tools/regression/lib/msun/test-fenv.c b/tools/regression/lib/msun/test-fenv.c
index 71e8eedb3bc9..0ea6e42a7c8b 100644
--- a/tools/regression/lib/msun/test-fenv.c
+++ b/tools/regression/lib/msun/test-fenv.c
@@ -133,7 +133,34 @@ test_dfl_env(void)
 	fenv_t env;
 
 	fegetenv(&env);
+
+#ifdef __amd64__
+	/*
+	 * Compare the fields that the AMD [1] and Intel [2] specs say will be
+	 * set once fnstenv returns.
+	 *
+	 * Not all amd64 capable processors implement the fnstenv instruction
+	 * by zero'ing out the env.__x87.__other field (example: AMD Opteron
+	 * 6308). The AMD64/x64 specs aren't explicit on what the
+	 * env.__x87.__other field will contain after fnstenv is executed, so
+	 * the values in env.__x87.__other could be filled with arbitrary
+	 * data depending on how the CPU implements fnstenv.
+	 *
+	 * 1. http://support.amd.com/TechDocs/26569_APM_v5.pdf
+	 * 2. http://www.intel.com/Assets/en_US/PDF/manual/253666.pdf
+	 */
+	assert(memcmp(&env.__mxcsr, &FE_DFL_ENV->__mxcsr,
+	    sizeof(env.__mxcsr)) == 0);
+	assert(memcmp(&env.__x87.__control, &FE_DFL_ENV->__x87.__control,
+	    sizeof(env.__x87.__control)) == 0);
+	assert(memcmp(&env.__x87.__status, &FE_DFL_ENV->__x87.__status,
+	    sizeof(env.__x87.__status)) == 0);
+	assert(memcmp(&env.__x87.__tag, &FE_DFL_ENV->__x87.__tag,
+	    sizeof(env.__x87.__tag)) == 0);
+#else
 	assert(memcmp(&env, FE_DFL_ENV, sizeof(env)) == 0);
+#endif
+
 #endif
 	assert(fetestexcept(FE_ALL_EXCEPT) == 0);
 }

From 132e71dfe9682c5a0ebb1e03e54e1fc3ce535afa Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Wed, 14 Oct 2015 20:28:15 +0000
Subject: [PATCH 223/380] Re-indent the ALL_SUBDIR_TARGETS list

---
 share/mk/bsd.subdir.mk | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/share/mk/bsd.subdir.mk b/share/mk/bsd.subdir.mk
index 8afe40a693b0..5aba6a52c96a 100644
--- a/share/mk/bsd.subdir.mk
+++ b/share/mk/bsd.subdir.mk
@@ -33,8 +33,9 @@
 __<bsd.subdir.mk>__:
 
 ALL_SUBDIR_TARGETS= all all-man checkdpadd clean cleandepend cleandir \
-	cleanilinks cleanobj depend distribute lint maninstall manlint obj \
-	objlink realinstall regress tags ${SUBDIR_TARGETS}
+		    cleanilinks cleanobj depend distribute lint maninstall \
+		    manlint obj objlink realinstall regress tags \
+		    ${SUBDIR_TARGETS}
 
 .include <bsd.init.mk>
 

From 174c029e4d37ec1d4909afd8fda9d37c8225f7f4 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Wed, 14 Oct 2015 20:30:32 +0000
Subject: [PATCH 224/380] Recurse on 'buildconfig' and 'installconfig'.  Remove
 the 'config' pseudo target.

The 'config' target isn't really needed right now so just remove it to avoid
any clashes with config(8) building.  It's also likely misspelled and should
be 'configs' if we decide to add it back.  This was just a convenience
target recently added.

Sponsored by:	EMC / Isilon Storage Division
---
 share/mk/bsd.subdir.mk | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/share/mk/bsd.subdir.mk b/share/mk/bsd.subdir.mk
index 5aba6a52c96a..037a7fec37ab 100644
--- a/share/mk/bsd.subdir.mk
+++ b/share/mk/bsd.subdir.mk
@@ -32,9 +32,10 @@
 .if !target(__<bsd.subdir.mk>__)
 __<bsd.subdir.mk>__:
 
-ALL_SUBDIR_TARGETS= all all-man checkdpadd clean cleandepend cleandir \
-		    cleanilinks cleanobj depend distribute lint maninstall \
-		    manlint obj objlink realinstall regress tags \
+ALL_SUBDIR_TARGETS= all all-man buildconfig checkdpadd clean cleandepend \
+		    cleandir cleanilinks cleanobj depend distribute \
+		    installconfig lint maninstall manlint obj objlink \
+		    realinstall regress tags \
 		    ${SUBDIR_TARGETS}
 
 .include <bsd.init.mk>
@@ -127,7 +128,7 @@ _sub.${__target}: _SUBDIR
 # This is to support 'make includes' calling 'make buildincludes' and
 # 'make installincludes' in the proper order, and to support these
 # targets as SUBDIR_TARGETS.
-.for __target in files includes config
+.for __target in files includes
 .for __stage in build install
 ${__stage}${__target}:
 .if make(${__stage}${__target})

From cca4c2208254d663d97f949aaec527b707293388 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Wed, 14 Oct 2015 20:38:51 +0000
Subject: [PATCH 225/380] Add missing targets to PHONY_NOTMAIN.

- buildconfig, installconfig (missed in r289085)
- files (missed in r241298)

Sponsored by:	EMC / Isilon Storage Division
---
 share/mk/bsd.sys.mk | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/share/mk/bsd.sys.mk b/share/mk/bsd.sys.mk
index 044b668ebada..d5a469676d41 100644
--- a/share/mk/bsd.sys.mk
+++ b/share/mk/bsd.sys.mk
@@ -165,13 +165,13 @@ CXXFLAGS+=	 ${CXXFLAGS.${COMPILER_TYPE}}
 # Tell bmake not to mistake standard targets for things to be searched for
 # or expect to ever be up-to-date.
 PHONY_NOTMAIN = afterdepend afterinstall all beforedepend beforeinstall \
-		beforelinking build build-tools buildfiles buildincludes \
-		checkdpadd clean cleandepend cleandir cleanobj configure \
-		depend dependall distclean distribute exe \
-		html includes install installfiles installincludes lint \
-		obj objlink objs objwarn realall realdepend \
-		realinstall regress subdir-all subdir-depend subdir-install \
-		tags whereobj
+		beforelinking build build-tools buildconfig buildfiles \
+		buildincludes checkdpadd clean cleandepend cleandir cleanobj \
+		configure depend dependall distclean distribute exe \
+		files html includes install installconfig installfiles \
+		installincludes lint obj objlink objs objwarn realall \
+		realdepend realinstall regress subdir-all subdir-depend \
+		subdir-install tags whereobj
 
 # we don't want ${PROG} to be PHONY
 .PHONY: ${PHONY_NOTMAIN:N${PROG:U}}

From 2ef6d5560f94f44e2a6e18ebf2e3ba237529b19b Mon Sep 17 00:00:00 2001
From: emaste <emaste@FreeBSD.org>
Date: Wed, 14 Oct 2015 21:10:05 +0000
Subject: [PATCH 226/380] Add Ubiquiti EdgeRouter Lite (ERL) kernel config file

The ERL is a fairly cheap (~$100 USD) and readily available dual core
MIPS64 device so it makes a useful MIPS reference platform.

This is based in part on the kernel config generated by the mkerlimage
script from http://rtfm.net/FreeBSD/ERL/.

Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D3884
---
 sys/mips/conf/ERL | 212 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 212 insertions(+)
 create mode 100644 sys/mips/conf/ERL

diff --git a/sys/mips/conf/ERL b/sys/mips/conf/ERL
new file mode 100644
index 000000000000..2afd12116ca6
--- /dev/null
+++ b/sys/mips/conf/ERL
@@ -0,0 +1,212 @@
+#
+# ERL - EdgeRouter Lite kernel config
+# Based on configuration from http://rtfm.net/FreeBSD/ERL
+#
+# For more information on this file, please read the config(5) manual page,
+# and/or the handbook section on Kernel Configuration Files:
+#
+#    http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html
+#
+# The handbook is also available locally in /usr/share/doc/handbook
+# if you've installed the doc distribution, otherwise always see the
+# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the
+# latest information.
+#
+# An exhaustive list of options and more detailed explanations of the
+# device lines is also present in the ../../conf/NOTES and NOTES files.
+# If you are in doubt as to the purpose or necessity of a line, check first
+# in NOTES.
+#
+# $FreeBSD$
+
+ident		ERL
+
+makeoptions	ARCH_FLAGS="-march=octeon -mabi=64"
+makeoptions	LDSCRIPT_NAME=ldscript.mips.octeon1
+
+# Don't build any modules yet.
+makeoptions	MODULES_OVERRIDE=""
+makeoptions	KERNLOADADDR=0xffffffff80100000
+
+# We don't need to build a trampolined version of the kernel.
+makeoptions	WITHOUT_KERNEL_TRAMPOLINE=1
+
+include		"../cavium/std.octeon1"
+
+hints		"OCTEON1.hints"		#Default places to look for devices.
+
+makeoptions	DEBUG=-g		#Build kernel with gdb(1) debug symbols
+
+# Board-specific support that cannot be auto-detected at runtime.
+#options 	OCTEON_VENDOR_LANNER		# Support for Lanner boards.
+#options 	OCTEON_VENDOR_RADISYS		# Support for Radisys boards.
+options 	OCTEON_VENDOR_UBIQUITI		# Support for Ubiquiti boards.
+#options	OCTEON_VENDOR_GEFES		# Support for GE LANIC boards
+#options 	OCTEON_BOARD_CAPK_0100ND	# Support for CAPK-0100nd.
+
+# Compile for a specified Octeon model.  If not specified, support for
+# detection at runtime will be used instead, which may give inferior
+# performance.
+#
+# See sys/contrib/octeon-sdk/octeon-model.h for possible values.
+options 	OCTEON_MODEL=OCTEON_CN50XX_PASS1
+
+options 	SCHED_ULE		# ULE scheduler
+options 	PREEMPTION		# Enable kernel thread preemption
+options 	INET			# InterNETworking
+options 	INET6			# IPv6 communications protocols
+options 	SCTP			# Stream Control Transmission Protocol
+options 	FFS			# Berkeley Fast Filesystem
+options 	SOFTUPDATES		# Enable FFS soft updates support
+options 	UFS_ACL			# Support for access control lists
+options 	UFS_DIRHASH		# Improve performance on big directories
+options 	UFS_GJOURNAL		# Enable gjournal-based UFS journaling
+options 	MD_ROOT			# MD is a potential root device
+options 	NFSCL			# Network Filesystem Client
+options 	NFSD			# Network Filesystem Server
+options 	NFSLOCKD		# Network Lock Manager
+options 	NFS_ROOT		# NFS usable as /, requires NFSCL
+options 	MSDOSFS			# MSDOS Filesystem
+options 	CD9660			# ISO 9660 Filesystem
+options 	PROCFS			# Process filesystem (requires PSEUDOFS)
+options 	PSEUDOFS		# Pseudo-filesystem framework
+options 	GEOM_PART_GPT		# GUID Partition Tables.
+options 	GEOM_LABEL		# Provides labelization
+options 	COMPAT_FREEBSD32	# Compatible with o32 binaries
+options 	SCSI_DELAY=5000		# Delay (in ms) before probing SCSI
+options 	KTRACE			# ktrace(1) support
+options 	STACK			# stack(9) support
+options 	SYSVSHM			# SYSV-style shared memory
+options 	SYSVMSG			# SYSV-style message queues
+options 	SYSVSEM			# SYSV-style semaphores
+options 	_KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions
+options 	PRINTF_BUFR_SIZE=128	# Prevent printf output being interspersed.
+options 	HWPMC_HOOKS		# Necessary kernel hooks for hwpmc(4)
+options 	AUDIT			# Security event auditing
+options 	MAC			# TrustedBSD MAC Framework
+#options 	KDTRACE_FRAME		# Ensure frames are compiled in
+#options 	KDTRACE_HOOKS		# Kernel DTrace hooks
+options 	INCLUDE_CONFIG_FILE     # Include this file in kernel
+options 	NO_SWAPPING		# Disable support for paging
+options		TMPFS			# Temporary file system
+
+# Debugging for use in -current
+#options 	KDB			# Enable kernel debugger support.
+#options 	DDB			# Support DDB.
+#options 	GDB			# Support remote GDB.
+#options 	DEADLKRES		# Enable the deadlock resolver
+#options 	INVARIANTS		# Enable calls of extra sanity checking
+#options 	INVARIANT_SUPPORT	# Extra sanity checks of internal structures, required by INVARIANTS
+#options 	WITNESS			# Enable checks to detect deadlocks and cycles
+#options 	WITNESS_SKIPSPIN	# Don't run witness on spinlocks for speed
+#options 	MALLOC_DEBUG_MAXZONES=8	# Separate malloc(9) zones
+
+# Make an SMP-capable kernel by default
+options 	SMP			# Symmetric MultiProcessor Kernel
+
+options 	ROOTDEVNAME=\"ufs:da0s2a\"	# Default root filesystem.
+
+# ATA/SCSI peripherals
+device		scbus		# SCSI bus (required for ATA/SCSI)
+device		ch		# SCSI media changers
+device		da		# Direct Access (disks)
+device		sa		# Sequential Access (tape etc)
+device		cd		# CD
+device		pass		# Passthrough device (direct ATA/SCSI access)
+device		ses		# Enclosure Services (SES and SAF-TE)
+
+# Serial (COM) ports
+device		uart		# Generic UART driver
+
+# On-board Cavium Octeon Ethernet.
+# NOTE: Be sure to keep the 'device miibus' line in order to use these NICs!
+device		octe
+
+# Cavium Octeon management Ethernet.
+device		octm
+
+# Switch PHY support for the octe driver.  These currently present a VLAN per
+# physical port, but may eventually provide support for DSA or similar instead.
+#device		mv88e61xxphy	# Marvell 88E61XX
+
+# Wireless NIC cards
+device		wlan		# 802.11 support
+options 	IEEE80211_DEBUG	# enable debug msgs
+options 	IEEE80211_AMPDU_AGE # age frames in AMPDU reorder q's
+options 	IEEE80211_SUPPORT_MESH	# enable 802.11s draft support
+device		wlan_wep	# 802.11 WEP support
+device		wlan_ccmp	# 802.11 CCMP support
+device		wlan_tkip	# 802.11 TKIP support
+device		wlan_amrr	# AMRR transmit rate control algorithm
+#device		ath		# Atheros NIC's
+#device		ath_pci		# Atheros pci/cardbus glue
+#device		ath_hal		# pci/cardbus chip support
+#options 	AH_SUPPORT_AR5416	# enable AR5416 tx/rx descriptors
+#device		ath_rate_sample	# SampleRate tx rate control for ath
+
+# Pseudo devices.
+device		loop		# Network loopback
+device		random		# Entropy device
+device		ether		# Ethernet support
+device		vlan		# 802.1Q VLAN support
+device		tun		# Packet tunnel.
+device		md		# Memory "disks"
+device		gif		# IPv6 and IPv4 tunneling
+device		firmware	# firmware assist module
+
+# The `bpf' device enables the Berkeley Packet Filter.
+# Be aware of the administrative consequences of enabling this!
+# Note that 'bpf' is required for DHCP.
+device		bpf		# Berkeley packet filter
+
+# Hardware watchdog support.
+#device		octeon_wdog	# Octeon hardware watchdog
+
+# USB support
+options 	USB_DEBUG	# enable debug msgs
+device		octusb		# Cavium Octeon on-board USB interface (USB 2.0)
+device		uhci		# UHCI PCI->USB interface
+device		ohci		# OHCI PCI->USB interface
+device		ehci		# EHCI PCI->USB interface (USB 2.0)
+device		usb		# USB Bus (required)
+#device		udbp		# USB Double Bulk Pipe devices
+device		uhid		# "Human Interface Devices"
+device		ulpt		# Printer
+device		umass		# Disks/Mass storage - Requires scbus and da
+device		ums		# Mouse
+device		urio		# Diamond Rio 500 MP3 player
+# USB Serial devices
+device		u3g		# USB-based 3G modems (Option, Huawei, Sierra)
+device		uark		# Technologies ARK3116 based serial adapters
+device		ubsa		# Belkin F5U103 and compatible serial adapters
+device		uftdi		# For FTDI usb serial adapters
+device		uipaq		# Some WinCE based devices
+device		uplcom		# Prolific PL-2303 serial adapters
+device		uslcom		# SI Labs CP2101/CP2102 serial adapters
+device		uvisor		# Visor and Palm devices
+device		uvscom		# USB serial support for DDI pocket's PHS
+# USB Ethernet, requires miibus
+device		miibus		# MII bus support
+device		aue		# ADMtek USB Ethernet
+device		axe		# ASIX Electronics USB Ethernet
+device		cdce		# Generic USB over Ethernet
+device		cue		# CATC USB Ethernet
+device		kue		# Kawasaki LSI USB Ethernet
+device		rue		# RealTek RTL8150 USB Ethernet
+device		udav		# Davicom DM9601E USB
+# USB Wireless
+device		rum		# Ralink Technology RT2501USB wireless NICs
+device		uath		# Atheros AR5523 wireless NICs
+device		ural		# Ralink Technology RT2500USB wireless NICs
+device		zyd		# ZyDAS zd1211/zd1211b wireless NICs
+
+# crypto subsystem
+device		crypto		# core crypto support
+device		cryptodev	# /dev/crypto for access to h/w
+device		cryptocteon	# Octeon coprocessor 2 crypto offload
+
+# GPIO support
+#device		gpio
+
+# PMC support
+#device		hwpmc

From 13af28171f83033458369c19749f63f40aa9d83f Mon Sep 17 00:00:00 2001
From: gjb <gjb@FreeBSD.org>
Date: Wed, 14 Oct 2015 22:33:11 +0000
Subject: [PATCH 227/380] Deprecate MD5 checksum generation in favor of SHA512.

This was discussed during the 10.2-RELEASE cycle, however
since we were nearing the end of the cycle, we decided to
defer this change until after 10.2-RELEASE.

Reminded by:	so (delphij), jmg
MFC after:	5 days
Sponsored by:	The FreeBSD Foundation
---
 release/Makefile         | 2 +-
 release/Makefile.mirrors | 2 +-
 release/Makefile.vm      | 8 ++++----
 release/release.sh       | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/release/Makefile b/release/Makefile
index 1dbb5a776097..d9dbb4f80c20 100644
--- a/release/Makefile
+++ b/release/Makefile
@@ -300,7 +300,7 @@ release-install:
 	${XZ_CMD} -k ${DESTDIR}/${OSRELEASE}-${I}
 . endif
 .endfor
+	cd ${DESTDIR} && sha512 ${OSRELEASE}* > ${DESTDIR}/CHECKSUM.SHA512
 	cd ${DESTDIR} && sha256 ${OSRELEASE}* > ${DESTDIR}/CHECKSUM.SHA256
-	cd ${DESTDIR} && md5 ${OSRELEASE}* > ${DESTDIR}/CHECKSUM.MD5
 
 .include "${.CURDIR}/Makefile.vm"
diff --git a/release/Makefile.mirrors b/release/Makefile.mirrors
index d5e8949c3831..92e5edabff11 100644
--- a/release/Makefile.mirrors
+++ b/release/Makefile.mirrors
@@ -76,7 +76,7 @@ VM_DIR=			${TLD}/VM-IMAGES/${REVISION}-${BRANCH}/${TARGET_ARCH}
 .endif
 
 CLEANFILES+=		${STAGE_TARGETS}
-CHECKSUM_FILES?=	SHA256 MD5
+CHECKSUM_FILES?=	SHA512 SHA256
 SNAP_SUFFIX!=		echo ${_SNAP_SUFFIX:S,^-,,1} | tr -d ' '
 ISO_DIR=		${TLD}/${TARGET}/${TARGET_ARCH}/ISO-IMAGES/${REVISION}
 FTP_DIR=		${TLD}/${TARGET}/${TARGET_ARCH}/${REVISION}-${BRANCH}
diff --git a/release/Makefile.vm b/release/Makefile.vm
index b39d05520768..902442fdfdd8 100644
--- a/release/Makefile.vm
+++ b/release/Makefile.vm
@@ -68,10 +68,10 @@ cw${_CW:tl}-install:
 	mkdir -p ${DESTDIR}/${_CW:tl}
 	cp -p ${${_CW}IMAGE} \
 		${DESTDIR}/${_CW:tl}/${${_CW}_DISK}
+	cd ${DESTDIR}/${_CW:tl} && sha512 ${${_CW}_DISK}* > \
+		${DESTDIR}/${_CW:tl}/CHECKSUM.SHA512
 	cd ${DESTDIR}/${_CW:tl} && sha256 ${${_CW}_DISK}* > \
 		${DESTDIR}/${_CW:tl}/CHECKSUM.SHA256
-	cd ${DESTDIR}/${_CW:tl} && md5 ${${_CW}_DISK}* > \
-		${DESTDIR}/${_CW:tl}/CHECKSUM.MD5
 
 cw${_CW:tl}-package:
 	@# Special target to handle packaging cloud images in the formats
@@ -140,10 +140,10 @@ vm-install:
 	${XZ_CMD} ${DESTDIR}/vmimages/${OSRELEASE}.${FORMAT}
 .  endfor
 . endif
+	cd ${DESTDIR}/vmimages && sha512 ${OSRELEASE}* > \
+		${DESTDIR}/vmimages/CHECKSUM.SHA512
 	cd ${DESTDIR}/vmimages && sha256 ${OSRELEASE}* > \
 		${DESTDIR}/vmimages/CHECKSUM.SHA256
-	cd ${DESTDIR}/vmimages && md5 ${OSRELEASE}* > \
-		${DESTDIR}/vmimages/CHECKSUM.MD5
 .endif
 
 vm-release:
diff --git a/release/release.sh b/release/release.sh
index e16842204080..313a16c8bb5c 100755
--- a/release/release.sh
+++ b/release/release.sh
@@ -362,10 +362,10 @@ chroot_arm_armv6_build_release() {
 	chroot ${CHROOTDIR} cp -p ${OBJDIR}/${OSRELEASE}-${KERNEL}.img \
 		/R/${OSRELEASE}-${KERNEL}.img
 	chroot ${CHROOTDIR} xz -T ${XZ_THREADS} /R/${OSRELEASE}-${KERNEL}.img
+	cd ${CHROOTDIR}/R && sha512 ${OSRELEASE}* \
+		> CHECKSUM.SHA512
 	cd ${CHROOTDIR}/R && sha256 ${OSRELEASE}* \
 		> CHECKSUM.SHA256
-	cd ${CHROOTDIR}/R && md5 ${OSRELEASE}* \
-		> CHECKSUM.MD5
 
 	return 0
 } # chroot_arm_armv6_build_release()

From 192e942c78242f1806719be83a56b61bd3bee193 Mon Sep 17 00:00:00 2001
From: np <np@FreeBSD.org>
Date: Wed, 14 Oct 2015 23:29:19 +0000
Subject: [PATCH 228/380] iw_cxgbe: use correct RFC number.

---
 sys/dev/cxgbe/iw_cxgbe/cm.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/dev/cxgbe/iw_cxgbe/cm.c b/sys/dev/cxgbe/iw_cxgbe/cm.c
index 18d572dc5485..b5dede78befc 100644
--- a/sys/dev/cxgbe/iw_cxgbe/cm.c
+++ b/sys/dev/cxgbe/iw_cxgbe/cm.c
@@ -809,7 +809,7 @@ SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, ep_timeout_secs, CTLFLAG_RWTUN, &ep_timeout_s
 
 static int mpa_rev = 1;
 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, mpa_rev, CTLFLAG_RWTUN, &mpa_rev, 0,
-		"MPA Revision, 0 supports amso1100, 1 is RFC0544 spec compliant, 2 is IETF MPA Peer Connect Draft compliant (default = 1)");
+		"MPA Revision, 0 supports amso1100, 1 is RFC5044 spec compliant, 2 is IETF MPA Peer Connect Draft compliant (default = 1)");
 
 static int markers_enabled;
 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, markers_enabled, CTLFLAG_RWTUN, &markers_enabled, 0,

From cccff74e5f2871384130c3fa8741cc87164cf8de Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Wed, 14 Oct 2015 23:44:42 +0000
Subject: [PATCH 229/380] NTB: MFV 403c63cb: client event cleanup

Provide a better event interface between the client and transport.

Authored by:	Jon Mason
Obtained from:	Linux (Dual BSD/GPL driver)
Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/if_ntb/if_ntb.c | 22 ++++++++++++++++++----
 sys/dev/ntb/ntb_hw/ntb_hw.h |  7 +++++--
 2 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/sys/dev/ntb/if_ntb/if_ntb.c b/sys/dev/ntb/if_ntb/if_ntb.c
index 321460700011..5b716a0e57a2 100644
--- a/sys/dev/ntb/if_ntb/if_ntb.c
+++ b/sys/dev/ntb/if_ntb/if_ntb.c
@@ -137,7 +137,7 @@ struct ntb_transport_qp {
 	uint64_t		rx_max_entry;
 	uint64_t		rx_max_frame;
 
-	void (*event_handler) (void *data, int status);
+	void (*event_handler) (void *data, enum ntb_link_event status);
 	struct callout		link_work;
 	struct callout		queue_full;
 	struct callout		rx_full;
@@ -161,7 +161,7 @@ struct ntb_queue_handlers {
 	    void *data, int len);
 	void (*tx_handler) (struct ntb_transport_qp *qp, void *qp_data,
 	    void *data, int len);
-	void (*event_handler) (void *data, int status);
+	void (*event_handler) (void *data, enum ntb_link_event status);
 };
 
 
@@ -234,7 +234,7 @@ static void ntb_net_tx_handler(struct ntb_transport_qp *qp, void *qp_data,
     void *data, int len);
 static void ntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data,
     void *data, int len);
-static void ntb_net_event_handler(void *data, int status);
+static void ntb_net_event_handler(void *data, enum ntb_link_event status);
 static int ntb_transport_init(struct ntb_softc *ntb);
 static void ntb_transport_free(void *transport);
 static void ntb_transport_init_queue(struct ntb_netdev *nt,
@@ -465,9 +465,23 @@ ntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data, void *data,
 }
 
 static void
-ntb_net_event_handler(void *data, int status)
+ntb_net_event_handler(void *data, enum ntb_link_event status)
 {
+	struct ifnet *ifp;
 
+	ifp = data;
+	(void)ifp;
+
+	/* XXX The Linux driver munges with the carrier status here. */
+
+	switch (status) {
+	case NTB_LINK_DOWN:
+		break;
+	case NTB_LINK_UP:
+		break;
+	default:
+		panic("Bogus ntb_link_event %u\n", status);
+	}
 }
 
 /* Transport Init and teardown */
diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.h b/sys/dev/ntb/ntb_hw/ntb_hw.h
index 861d17863746..7abd10c54dbf 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.h
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.h
@@ -32,8 +32,11 @@
 struct ntb_softc;
 
 #define NTB_NUM_MW	2
-#define NTB_LINK_DOWN	0
-#define NTB_LINK_UP	1
+
+enum ntb_link_event {
+	NTB_LINK_DOWN = 0,
+	NTB_LINK_UP,
+};
 
 enum ntb_hw_event {
 	NTB_EVENT_SW_EVENT0 = 0,

From 0ea8fc84ed45f335637c780e289ea92ca4bddcaf Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Wed, 14 Oct 2015 23:45:35 +0000
Subject: [PATCH 230/380] if_ntb: Cleanup style

---
 sys/dev/ntb/if_ntb/if_ntb.c | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/sys/dev/ntb/if_ntb/if_ntb.c b/sys/dev/ntb/if_ntb/if_ntb.c
index 5b716a0e57a2..751d24d28f06 100644
--- a/sys/dev/ntb/if_ntb/if_ntb.c
+++ b/sys/dev/ntb/if_ntb/if_ntb.c
@@ -116,7 +116,7 @@ struct ntb_transport_qp {
 	struct ntb_rx_info	*rx_info;
 	struct ntb_rx_info	*remote_rx_info;
 
-	void (*tx_handler) (struct ntb_transport_qp *qp, void *qp_data,
+	void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data,
 	    void *data, int len);
 	struct ntb_queue_list	tx_free_q;
 	struct mtx		ntb_tx_free_q_lock;
@@ -125,7 +125,7 @@ struct ntb_transport_qp {
 	uint64_t		tx_max_entry;
 	uint64_t		tx_max_frame;
 
-	void (*rx_handler) (struct ntb_transport_qp *qp, void *qp_data,
+	void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data,
 	    void *data, int len);
 	struct ntb_queue_list	rx_pend_q;
 	struct ntb_queue_list	rx_free_q;
@@ -137,7 +137,7 @@ struct ntb_transport_qp {
 	uint64_t		rx_max_entry;
 	uint64_t		rx_max_frame;
 
-	void (*event_handler) (void *data, enum ntb_link_event status);
+	void (*event_handler)(void *data, enum ntb_link_event status);
 	struct callout		link_work;
 	struct callout		queue_full;
 	struct callout		rx_full;
@@ -157,11 +157,11 @@ struct ntb_transport_qp {
 };
 
 struct ntb_queue_handlers {
-	void (*rx_handler) (struct ntb_transport_qp *qp, void *qp_data,
+	void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data,
 	    void *data, int len);
-	void (*tx_handler) (struct ntb_transport_qp *qp, void *qp_data,
+	void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data,
 	    void *data, int len);
-	void (*event_handler) (void *data, enum ntb_link_event status);
+	void (*event_handler)(void *data, enum ntb_link_event status);
 };
 
 
@@ -825,7 +825,7 @@ ntb_tx_copy_task(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry,
 
 	ntb_ring_doorbell(qp->ntb, qp->qp_num);
 
-	/* 
+	/*
 	 * The entry length can only be zero if the packet is intended to be a
 	 * "link down" or similar.  Since no payload is being sent in these
 	 * cases, there is nothing to add to the completion queue.
@@ -868,7 +868,7 @@ ntb_transport_rxc_db(void *arg, int dummy __unused)
 	uint64_t i;
 	int rc;
 
-	/* 
+	/*
 	 * Limit the number of packets processed in a single interrupt to
 	 * provide fairness to others
 	 */
@@ -1276,7 +1276,7 @@ ntb_transport_link_cleanup(struct ntb_netdev *nt)
 	else
 		nt->transport_link = NTB_LINK_DOWN;
 
-	/* 
+	/*
 	 * The scratchpad registers keep the values if the remote side
 	 * goes down, blast them now to give them a sane value the next
 	 * time they are accessed

From fa6dd4da61c28a9397f350233cb6ed77a1f0e9aa Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Wed, 14 Oct 2015 23:46:15 +0000
Subject: [PATCH 231/380] NTB: MFV 53a788a7: Split ntb_setup_interrupts() into
 SOC, Xeon, and legacy routines

The names don't line up 100% with Linux.  Our routines are named
ntb_setup_interrupts, ntb_setup_xeon_msix, ntb_setup_soc_msix, and
ntb_setup_legacy_interrupt.  Linux SNB = FreeBSD Xeon; Linux BWD =
FreeBSD SOC.  Original Linux commit log:

This is an cleanup effort to make ntb_setup_msix() more readable - use
ntb_setup_bwd_msix() to init MSI-Xs on BWD hardware and
ntb_setup_snb_msix() - on SNB hardware.

Function ntb_setup_snb_msix() also initializes MSI-Xs the way it should
has been done - looping pci_enable_msix() until success or failure.

Authored by:	Alexander Gordeev
Obtained from:	Linux (Dual BSD/GPL driver)
Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/ntb_hw/ntb_hw.c | 177 +++++++++++++++++++++++-------------
 1 file changed, 112 insertions(+), 65 deletions(-)

diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c
index 599e06858394..0f280fcd407a 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.c
@@ -200,6 +200,9 @@ static int map_memory_window_bar(struct ntb_softc *ntb,
     struct ntb_pci_bar_info *bar);
 static void ntb_unmap_pci_bar(struct ntb_softc *ntb);
 static int ntb_setup_interrupts(struct ntb_softc *ntb);
+static int ntb_setup_legacy_interrupt(struct ntb_softc *ntb);
+static int ntb_setup_xeon_msix(struct ntb_softc *ntb, uint32_t num_vectors);
+static int ntb_setup_soc_msix(struct ntb_softc *ntb, uint32_t num_vectors);
 static void ntb_teardown_interrupts(struct ntb_softc *ntb);
 static void handle_soc_irq(void *arg);
 static void handle_xeon_irq(void *arg);
@@ -208,7 +211,7 @@ static void ntb_handle_legacy_interrupt(void *arg);
 static void ntb_irq_work(void *arg);
 static void mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx);
 static void unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx);
-static int ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors);
+static int ntb_create_callbacks(struct ntb_softc *ntb, uint32_t num_vectors);
 static void ntb_free_callbacks(struct ntb_softc *ntb);
 static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id);
 static int ntb_setup_xeon(struct ntb_softc *ntb);
@@ -473,13 +476,79 @@ ntb_unmap_pci_bar(struct ntb_softc *ntb)
 }
 
 static int
-ntb_setup_interrupts(struct ntb_softc *ntb)
+ntb_setup_xeon_msix(struct ntb_softc *ntb, uint32_t num_vectors)
 {
 	void (*interrupt_handler)(void *);
 	void *int_arg;
-	bool use_msix = false;
+	uint32_t i;
+	int rc;
+
+	if (num_vectors < 4)
+		return (ENOSPC);
+
+	for (i = 0; i < num_vectors; i++) {
+		ntb->int_info[i].rid = i + 1;
+		ntb->int_info[i].res = bus_alloc_resource_any(ntb->device,
+		    SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE);
+		if (ntb->int_info[i].res == NULL) {
+			device_printf(ntb->device,
+			    "bus_alloc_resource failed\n");
+			return (ENOMEM);
+		}
+		ntb->int_info[i].tag = NULL;
+		ntb->allocated_interrupts++;
+		if (i == num_vectors - 1) {
+			interrupt_handler = handle_xeon_event_irq;
+			int_arg = ntb;
+		} else {
+			interrupt_handler = handle_xeon_irq;
+			int_arg = &ntb->db_cb[i];
+		}
+		rc = bus_setup_intr(ntb->device, ntb->int_info[i].res,
+		    INTR_MPSAFE | INTR_TYPE_MISC, NULL, interrupt_handler,
+		    int_arg, &ntb->int_info[i].tag);
+		if (rc != 0) {
+			device_printf(ntb->device,
+			    "bus_setup_intr failed\n");
+			return (ENXIO);
+		}
+	}
+	return (0);
+}
+
+static int
+ntb_setup_soc_msix(struct ntb_softc *ntb, uint32_t num_vectors)
+{
+	uint32_t i;
+	int rc;
+
+	for (i = 0; i < num_vectors; i++) {
+		ntb->int_info[i].rid = i + 1;
+		ntb->int_info[i].res = bus_alloc_resource_any(ntb->device,
+		    SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE);
+		if (ntb->int_info[i].res == NULL) {
+			device_printf(ntb->device,
+			    "bus_alloc_resource failed\n");
+			return (ENOMEM);
+		}
+		ntb->int_info[i].tag = NULL;
+		ntb->allocated_interrupts++;
+		rc = bus_setup_intr(ntb->device, ntb->int_info[i].res,
+		    INTR_MPSAFE | INTR_TYPE_MISC, NULL, handle_soc_irq,
+		    &ntb->db_cb[i], &ntb->int_info[i].tag);
+		if (rc != 0) {
+			device_printf(ntb->device, "bus_setup_intr failed\n");
+			return (ENXIO);
+		}
+	}
+	return (0);
+}
+
+static int
+ntb_setup_interrupts(struct ntb_softc *ntb)
+{
 	uint32_t num_vectors;
-	int i;
+	int rc;
 
 	ntb->allocated_interrupts = 0;
 	/*
@@ -494,69 +563,47 @@ ntb_setup_interrupts(struct ntb_softc *ntb)
 
 	num_vectors = MIN(pci_msix_count(ntb->device),
 	    ntb->limits.max_db_bits);
-	if (num_vectors >= 1) {
+	if (num_vectors >= 1)
 		pci_alloc_msix(ntb->device, &num_vectors);
-		if (num_vectors >= 4)
-			use_msix = true;
-	}
 
 	ntb_create_callbacks(ntb, num_vectors);
-	if (use_msix == true) {
-		for (i = 0; i < num_vectors; i++) {
-			ntb->int_info[i].rid = i + 1;
-			ntb->int_info[i].res = bus_alloc_resource_any(
-			    ntb->device, SYS_RES_IRQ, &ntb->int_info[i].rid,
-			    RF_ACTIVE);
-			if (ntb->int_info[i].res == NULL) {
-				device_printf(ntb->device,
-				    "bus_alloc_resource failed\n");
-				return (ENOMEM);
-			}
-			ntb->int_info[i].tag = NULL;
-			ntb->allocated_interrupts++;
-			if (ntb->type == NTB_SOC) {
-				interrupt_handler = handle_soc_irq;
-				int_arg = &ntb->db_cb[i];
-			} else {
-				if (i == num_vectors - 1) {
-					interrupt_handler =
-					    handle_xeon_event_irq;
-					int_arg = ntb;
-				} else {
-					interrupt_handler =
-					    handle_xeon_irq;
-					int_arg = &ntb->db_cb[i];
-				}
-			}
-			if (bus_setup_intr(ntb->device, ntb->int_info[i].res,
-			    INTR_MPSAFE | INTR_TYPE_MISC, NULL,
-			    interrupt_handler, int_arg,
-			    &ntb->int_info[i].tag) != 0) {
-				device_printf(ntb->device,
-				    "bus_setup_intr failed\n");
-				return (ENXIO);
-			}
-		}
-	} else {
-		ntb->int_info[0].rid = 0;
-		ntb->int_info[0].res = bus_alloc_resource_any(ntb->device,
-		    SYS_RES_IRQ, &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE);
-		interrupt_handler = ntb_handle_legacy_interrupt;
-		if (ntb->int_info[0].res == NULL) {
-			device_printf(ntb->device,
-			    "bus_alloc_resource failed\n");
-			return (ENOMEM);
-		}
-		ntb->int_info[0].tag = NULL;
-		ntb->allocated_interrupts = 1;
 
-		if (bus_setup_intr(ntb->device, ntb->int_info[0].res,
-			INTR_MPSAFE | INTR_TYPE_MISC, NULL,
-			interrupt_handler, ntb, &ntb->int_info[0].tag) != 0) {
+	if (ntb->type == NTB_XEON)
+		rc = ntb_setup_xeon_msix(ntb, num_vectors);
+	else
+		rc = ntb_setup_soc_msix(ntb, num_vectors);
+	if (rc != 0)
+		device_printf(ntb->device,
+		    "Error allocating MSI-X interrupts: %d\n", rc);
 
-			device_printf(ntb->device, "bus_setup_intr failed\n");
-			return (ENXIO);
-		}
+	if (ntb->type == NTB_XEON && rc == ENOSPC)
+		rc = ntb_setup_legacy_interrupt(ntb);
+
+	return (rc);
+}
+
+static int
+ntb_setup_legacy_interrupt(struct ntb_softc *ntb)
+{
+	int rc;
+
+	ntb->int_info[0].rid = 0;
+	ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ,
+	    &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE);
+	if (ntb->int_info[0].res == NULL) {
+		device_printf(ntb->device, "bus_alloc_resource failed\n");
+		return (ENOMEM);
+	}
+
+	ntb->int_info[0].tag = NULL;
+	ntb->allocated_interrupts = 1;
+
+	rc = bus_setup_intr(ntb->device, ntb->int_info[0].res,
+	    INTR_MPSAFE | INTR_TYPE_MISC, NULL, ntb_handle_legacy_interrupt,
+	    ntb, &ntb->int_info[0].tag);
+	if (rc != 0) {
+		device_printf(ntb->device, "bus_setup_intr failed\n");
+		return (ENXIO);
 	}
 
 	return (0);
@@ -688,9 +735,9 @@ ntb_handle_legacy_interrupt(void *arg)
 }
 
 static int
-ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors)
+ntb_create_callbacks(struct ntb_softc *ntb, uint32_t num_vectors)
 {
-	int i;
+	uint32_t i;
 
 	ntb->db_cb = malloc(num_vectors * sizeof(*ntb->db_cb), M_NTB,
 	    M_ZERO | M_WAITOK);
@@ -705,7 +752,7 @@ ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors)
 static void
 ntb_free_callbacks(struct ntb_softc *ntb)
 {
-	int i;
+	uint8_t i;
 
 	for (i = 0; i < ntb->limits.max_db_bits; i++)
 		ntb_unregister_db_callback(ntb, i);

From ec70a91173acddfdea6bd3b861aac0492e1ec2fa Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Wed, 14 Oct 2015 23:47:08 +0000
Subject: [PATCH 232/380] NTB: Reserve link event doorbell callback on Xeon

Consumers that registered on this bit would never see a callback and it
is likely a mistake.

This does not affect if_ntb, which limits itself to a single doorbell
callback.
---
 sys/dev/ntb/ntb_hw/ntb_hw.c | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c
index 0f280fcd407a..4fafa07094ab 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.c
@@ -110,6 +110,7 @@ struct ntb_db_cb {
 	void			*data;
 	struct ntb_softc	*ntb;
 	struct callout		irq_work;
+	bool			reserved;
 };
 
 struct ntb_softc {
@@ -513,6 +514,12 @@ ntb_setup_xeon_msix(struct ntb_softc *ntb, uint32_t num_vectors)
 			return (ENXIO);
 		}
 	}
+
+	/*
+	 * Prevent consumers from registering callbacks on the link event irq
+	 * slot, from which they will never be called back.
+	 */
+	ntb->db_cb[num_vectors - 1].reserved = true;
 	return (0);
 }
 
@@ -1302,15 +1309,17 @@ int
 ntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data,
     ntb_db_callback func)
 {
+	struct ntb_db_cb *db_cb = &ntb->db_cb[idx];
 
-	if (idx >= ntb->allocated_interrupts || ntb->db_cb[idx].callback) {
+	if (idx >= ntb->allocated_interrupts || db_cb->callback ||
+	    db_cb->reserved) {
 		device_printf(ntb->device, "Invalid Index.\n");
 		return (EINVAL);
 	}
 
-	ntb->db_cb[idx].callback = func;
-	ntb->db_cb[idx].data = data;
-	callout_init(&ntb->db_cb[idx].irq_work, 1);
+	db_cb->callback = func;
+	db_cb->data = data;
+	callout_init(&db_cb->irq_work, 1);
 
 	unmask_ldb_interrupt(ntb, idx);
 

From 1a710218be689153eaf94fef6ee65092d5c2a914 Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Wed, 14 Oct 2015 23:47:23 +0000
Subject: [PATCH 233/380] NTB: Remap MSI-X messages over available slots

Remap MSI-X messages over available slots rather than falling back to
legacy INTx when fewer MSI-X slots are available than were requested.

N.B. the Linux driver does *not* do this.

To aid in testing, a tunable 'hw.ntb.force_remap_mode' has been added.
It defaults to off (0).  When the tunable is enabled and sufficient
slots were available, the driver restricts the number of slots by one
and remaps the MSI-X messages over the remaining slots.

In case this is actually not okay (as I don't yet have access to this
hardware to test), a tunable 'hw.ntb.prefer_intx_to_remap' has been
added.  It defaults to off (0).  When the tunable is enabled and fewer
slots are available than requested, fall back to legacy INTx mode rather
than attempting to remap MSI-X messages.

Suggested by:	jhb
Reviewed by:	jhb (earlier version)
Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/ntb_hw/ntb_hw.c | 71 ++++++++++++++++++++++++++++++++++---
 1 file changed, 67 insertions(+), 4 deletions(-)

diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c
index 4fafa07094ab..3cddaeacfe7d 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.c
@@ -200,6 +200,7 @@ static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar);
 static int map_memory_window_bar(struct ntb_softc *ntb,
     struct ntb_pci_bar_info *bar);
 static void ntb_unmap_pci_bar(struct ntb_softc *ntb);
+static int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail);
 static int ntb_setup_interrupts(struct ntb_softc *ntb);
 static int ntb_setup_legacy_interrupt(struct ntb_softc *ntb);
 static int ntb_setup_xeon_msix(struct ntb_softc *ntb, uint32_t num_vectors);
@@ -551,10 +552,55 @@ ntb_setup_soc_msix(struct ntb_softc *ntb, uint32_t num_vectors)
 	return (0);
 }
 
+/*
+ * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector
+ * cannot be allocated for each MSI-X message.  JHB seems to think remapping
+ * should be okay.  This tunable should enable us to test that hypothesis
+ * when someone gets their hands on some Xeon hardware.
+ */
+static int ntb_force_remap_mode;
+SYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN,
+    &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped"
+    " to a smaller number of ithreads, even if the desired number are "
+    "available");
+
+/*
+ * In case it is NOT ok, give consumers an abort button.
+ */
+static int ntb_prefer_intx;
+SYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN,
+    &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather "
+    "than remapping MSI-X messages over available slots (match Linux driver "
+    "behavior)");
+
+/*
+ * Remap the desired number of MSI-X messages to available ithreads in a simple
+ * round-robin fashion.
+ */
+static int
+ntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail)
+{
+	u_int *vectors;
+	uint32_t i;
+	int rc;
+
+	if (ntb_prefer_intx != 0)
+		return (ENXIO);
+
+	vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK);
+
+	for (i = 0; i < desired; i++)
+		vectors[i] = (i % avail) + 1;
+
+	rc = pci_remap_msix(dev, desired, vectors);
+	free(vectors, M_NTB);
+	return (rc);
+}
+
 static int
 ntb_setup_interrupts(struct ntb_softc *ntb)
 {
-	uint32_t num_vectors;
+	uint32_t desired_vectors, num_vectors;
 	int rc;
 
 	ntb->allocated_interrupts = 0;
@@ -568,10 +614,27 @@ ntb_setup_interrupts(struct ntb_softc *ntb)
 		ntb_reg_write(2, ntb->reg_ofs.ldb_mask,
 		    (uint16_t) ~(1 << XEON_LINK_DB));
 
-	num_vectors = MIN(pci_msix_count(ntb->device),
+	num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device),
 	    ntb->limits.max_db_bits);
-	if (num_vectors >= 1)
-		pci_alloc_msix(ntb->device, &num_vectors);
+	if (desired_vectors >= 1) {
+		rc = pci_alloc_msix(ntb->device, &num_vectors);
+
+		if (ntb_force_remap_mode != 0 && rc == 0 &&
+		    num_vectors == desired_vectors)
+			num_vectors--;
+
+		if (rc == 0 && num_vectors < desired_vectors) {
+			rc = ntb_remap_msix(ntb->device, desired_vectors,
+			    num_vectors);
+			if (rc == 0)
+				num_vectors = desired_vectors;
+			else
+				pci_release_msi(ntb->device);
+		}
+		if (rc != 0)
+			num_vectors = 1;
+	} else
+		num_vectors = 1;
 
 	ntb_create_callbacks(ntb, num_vectors);
 

From 1a9e30b488763c19fbcfbc886564272e25e6876f Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Wed, 14 Oct 2015 23:47:35 +0000
Subject: [PATCH 234/380] NTB: MFV a1413cfb: correct the spread of queues over
 mw's

The detection of an uneven number of queues on the given memory windows
was not correct.  The mw_num is zero based and the mod should be
division to spread them evenly over the mw's.

Authored by:	Jon Mason
Obtained from:	Linux (Dual BSD/GPL driver)
Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/if_ntb/if_ntb.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sys/dev/ntb/if_ntb/if_ntb.c b/sys/dev/ntb/if_ntb/if_ntb.c
index 751d24d28f06..13c77fab70d2 100644
--- a/sys/dev/ntb/if_ntb/if_ntb.c
+++ b/sys/dev/ntb/if_ntb/if_ntb.c
@@ -566,7 +566,7 @@ ntb_transport_init_queue(struct ntb_netdev *nt, unsigned int qp_num)
 	qp->client_ready = NTB_LINK_DOWN;
 	qp->event_handler = NULL;
 
-	if (nt->max_qps % NTB_NUM_MW && mw_num < nt->max_qps % NTB_NUM_MW)
+	if (nt->max_qps % NTB_NUM_MW && mw_num + 1 < nt->max_qps / NTB_NUM_MW)
 		num_qps_mw = nt->max_qps / NTB_NUM_MW + 1;
 	else
 		num_qps_mw = nt->max_qps / NTB_NUM_MW;
@@ -1197,7 +1197,7 @@ ntb_transport_setup_qp_mw(struct ntb_netdev *nt, unsigned int qp_num)
 	uint8_t mw_num = QP_TO_MW(qp_num);
 	unsigned int i;
 
-	if (nt->max_qps % NTB_NUM_MW && mw_num < nt->max_qps % NTB_NUM_MW)
+	if (nt->max_qps % NTB_NUM_MW && mw_num + 1 < nt->max_qps / NTB_NUM_MW)
 		num_qps_mw = nt->max_qps / NTB_NUM_MW + 1;
 	else
 		num_qps_mw = nt->max_qps / NTB_NUM_MW;

From 05fa2677c44644a759bbbd821133be2e20fd247d Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Wed, 14 Oct 2015 23:47:52 +0000
Subject: [PATCH 235/380] if_ntb: MFV 3cc5ba19: Add alignment check to meet
 hardware requirement

Original Linux commit log:

The NTB translate register must have the value to be BAR size aligned.
This alignment check make sure that the DMA memory allocated has the
proper alignment. Another requirement for NTB to function properly with
memory window BAR size greater or equal to 4M is to use the CMA feature
in 3.16 kernel with the appropriate CONFIG_CMA_ALIGNMENT and
CONFIG_CMA_SIZE_MBYTES set.

Authored by:	Dave Jiang
Obtained from:	Linux (Dual BSD/GPL driver)
Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/if_ntb/if_ntb.c | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/sys/dev/ntb/if_ntb/if_ntb.c b/sys/dev/ntb/if_ntb/if_ntb.c
index 13c77fab70d2..169296564d74 100644
--- a/sys/dev/ntb/if_ntb/if_ntb.c
+++ b/sys/dev/ntb/if_ntb/if_ntb.c
@@ -1163,13 +1163,26 @@ ntb_set_mw(struct ntb_netdev *nt, int num_mw, unsigned int size)
 	    BUS_SPACE_MAXADDR, mw->size, 0);
 	if (mw->virt_addr == NULL) {
 		mw->size = 0;
-		printf("ntb: Unable to allocate MW buffer of size %d\n",
-		    (int)mw->size);
+		printf("ntb: Unable to allocate MW buffer of size %zu\n",
+		    mw->size);
 		return (ENOMEM);
 	}
 	/* TODO: replace with bus_space_* functions */
 	mw->dma_addr = vtophys(mw->virt_addr);
 
+	/*
+	 * Ensure that the allocation from contigmalloc is aligned as
+	 * requested.  XXX: This may not be needed -- brought in for parity
+	 * with the Linux driver.
+	 */
+	if (mw->dma_addr % size != 0) {
+		device_printf(ntb_get_device(nt->ntb),
+		    "DMA memory 0x%jx not aligned to BAR size 0x%x\n",
+		    (uintmax_t)mw->dma_addr, size);
+		ntb_free_mw(nt, num_mw);
+		return (ENOMEM);
+	}
+
 	/* Notify HW the memory location of the receive buffer */
 	ntb_set_mw_addr(nt->ntb, num_mw, mw->dma_addr);
 

From 72e57e0f5f73fcdae19d4ab8afcb2b332a410593 Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Wed, 14 Oct 2015 23:48:03 +0000
Subject: [PATCH 236/380] NTB: Abstract doorbell register access

The doorbell registers (and associated mask) are 16-bit on Xeon but
64-bit on SoC.  Abstract IO access to doorbell registers with
'db_ioread' and 'db_iowrite' (names and idea borrowed from the dual
BSD/GPL Linux driver).

Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/ntb_hw/ntb_hw.c | 120 +++++++++++++++++++++---------------
 1 file changed, 71 insertions(+), 49 deletions(-)

diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c
index 3cddaeacfe7d..18526cf08996 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.c
@@ -211,6 +211,8 @@ static void handle_xeon_irq(void *arg);
 static void handle_xeon_event_irq(void *arg);
 static void ntb_handle_legacy_interrupt(void *arg);
 static void ntb_irq_work(void *arg);
+static uint64_t db_ioread(struct ntb_softc *, uint32_t regoff);
+static void db_iowrite(struct ntb_softc *, uint32_t regoff, uint64_t val);
 static void mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx);
 static void unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx);
 static int ntb_create_callbacks(struct ntb_softc *ntb, uint32_t num_vectors);
@@ -601,18 +603,19 @@ static int
 ntb_setup_interrupts(struct ntb_softc *ntb)
 {
 	uint32_t desired_vectors, num_vectors;
+	uint64_t mask;
 	int rc;
 
 	ntb->allocated_interrupts = 0;
+
 	/*
 	 * On SOC, disable all interrupts.  On XEON, disable all but Link
 	 * Interrupt.  The rest will be unmasked as callbacks are registered.
 	 */
-	if (ntb->type == NTB_SOC)
-		ntb_reg_write(8, ntb->reg_ofs.ldb_mask, ~0);
-	else
-		ntb_reg_write(2, ntb->reg_ofs.ldb_mask,
-		    (uint16_t) ~(1 << XEON_LINK_DB));
+	mask = 0;
+	if (ntb->type == NTB_XEON)
+		mask = (1 << XEON_LINK_DB);
+	db_iowrite(ntb, ntb->reg_ofs.ldb_mask, ~mask);
 
 	num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device),
 	    ntb->limits.max_db_bits);
@@ -700,24 +703,53 @@ ntb_teardown_interrupts(struct ntb_softc *ntb)
 	pci_release_msi(ntb->device);
 }
 
+/*
+ * Doorbell register and mask are 64-bit on SoC, 16-bit on Xeon.  Abstract it
+ * out to make code clearer.
+ */
+static uint64_t
+db_ioread(struct ntb_softc *ntb, uint32_t regoff)
+{
+
+	if (ntb->type == NTB_SOC)
+		return (ntb_reg_read(8, regoff));
+
+	KASSERT(ntb->type == NTB_XEON, ("bad ntb type"));
+
+	return (ntb_reg_read(2, regoff));
+}
+
+static void
+db_iowrite(struct ntb_softc *ntb, uint32_t regoff, uint64_t val)
+{
+
+	if (ntb->type == NTB_SOC) {
+		ntb_reg_write(8, regoff, val);
+		return;
+	}
+
+	KASSERT(ntb->type == NTB_XEON, ("bad ntb type"));
+	ntb_reg_write(2, regoff, (uint16_t)val);
+}
+
 static void
 mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx)
 {
-	unsigned long mask;
+	uint64_t mask;
 
-	mask = ntb_reg_read(2, ntb->reg_ofs.ldb_mask);
+	mask = db_ioread(ntb, ntb->reg_ofs.ldb_mask);
 	mask |= 1 << (idx * ntb->bits_per_vector);
-	ntb_reg_write(2, ntb->reg_ofs.ldb_mask, mask);
+	db_iowrite(ntb, ntb->reg_ofs.ldb_mask, mask);
 }
 
 static void
 unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx)
 {
-	unsigned long mask;
+	uint64_t mask;
 
-	mask = ntb_reg_read(2, ntb->reg_ofs.ldb_mask);
+	mask = db_ioread(ntb, ntb->reg_ofs.ldb_mask);
 	mask &= ~(1 << (idx * ntb->bits_per_vector));
-	ntb_reg_write(2, ntb->reg_ofs.ldb_mask, mask);
+	db_iowrite(ntb, ntb->reg_ofs.ldb_mask, mask);
 }
 
 static void
@@ -726,7 +758,7 @@ handle_soc_irq(void *arg)
 	struct ntb_db_cb *db_cb = arg;
 	struct ntb_softc *ntb = db_cb->ntb;
 
-	ntb_reg_write(8, ntb->reg_ofs.ldb, (uint64_t) 1 << db_cb->db_num);
+	db_iowrite(ntb, ntb->reg_ofs.ldb, (uint64_t) 1 << db_cb->db_num);
 
 	if (db_cb->callback != NULL) {
 		mask_ldb_interrupt(ntb, db_cb->db_num);
@@ -746,7 +778,7 @@ handle_xeon_irq(void *arg)
 	 * vectors, with the 4th having a single bit for link
 	 * interrupts.
 	 */
-	ntb_reg_write(2, ntb->reg_ofs.ldb,
+	db_iowrite(ntb, ntb->reg_ofs.ldb,
 	    ((1 << ntb->bits_per_vector) - 1) <<
 	    (db_cb->db_num * ntb->bits_per_vector));
 
@@ -768,40 +800,31 @@ handle_xeon_event_irq(void *arg)
 		device_printf(ntb->device, "Error determining link status\n");
 
 	/* bit 15 is always the link bit */
-	ntb_reg_write(2, ntb->reg_ofs.ldb, 1 << XEON_LINK_DB);
+	db_iowrite(ntb, ntb->reg_ofs.ldb, 1 << XEON_LINK_DB);
 }
 
 static void
 ntb_handle_legacy_interrupt(void *arg)
 {
 	struct ntb_softc *ntb = arg;
-	unsigned int i = 0;
-	uint64_t ldb64;
-	uint16_t ldb16;
+	unsigned int i;
+	uint64_t ldb;
 
-	if (ntb->type == NTB_SOC) {
-		ldb64 = ntb_reg_read(8, ntb->reg_ofs.ldb);
+	ldb = db_ioread(ntb, ntb->reg_ofs.ldb);
 
-		while (ldb64) {
-			i = ffs(ldb64);
-			ldb64 &= ldb64 - 1;
-			handle_soc_irq(&ntb->db_cb[i]);
-		}
-	} else {
-		ldb16 = ntb_reg_read(2, ntb->reg_ofs.ldb);
-
-		if ((ldb16 & XEON_DB_HW_LINK) != 0) {
-			handle_xeon_event_irq(ntb);
-			ldb16 &= ~XEON_DB_HW_LINK;
-		}
-
-		while (ldb16 != 0) {
-			i = ffs(ldb16);
-			ldb16 &= ldb16 - 1;
-			handle_xeon_irq(&ntb->db_cb[i]);
-		}
+	if (ntb->type == NTB_XEON && (ldb & XEON_DB_HW_LINK) != 0) {
+		handle_xeon_event_irq(ntb);
+		ldb &= ~XEON_DB_HW_LINK;
 	}
 
+	while (ldb != 0) {
+		i = ffs(ldb);
+		ldb &= ldb - 1;
+		if (ntb->type == NTB_SOC)
+			handle_soc_irq(&ntb->db_cb[i]);
+		else
+			handle_xeon_irq(&ntb->db_cb[i]);
+	}
 }
 
 static int
@@ -1669,25 +1692,24 @@ ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr)
  *
  * This function allows triggering of a doorbell on the secondary/external
  * side that will initiate an interrupt on the remote host
- *
- * RETURNS: An appropriate ERRNO error value on error, or zero for success.
  */
 void
 ntb_ring_doorbell(struct ntb_softc *ntb, unsigned int db)
 {
+	uint64_t bit;
 
 	if (ntb->type == NTB_SOC)
-		ntb_reg_write(8, ntb->reg_ofs.rdb, (uint64_t) 1 << db);
-	else {
-		if (HAS_FEATURE(NTB_REGS_THRU_MW))
-			ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET,
-			    ((1 << ntb->bits_per_vector) - 1) <<
-			    (db * ntb->bits_per_vector));
-		else
-			ntb_reg_write(2, ntb->reg_ofs.rdb,
-			    ((1 << ntb->bits_per_vector) - 1) <<
-			    (db * ntb->bits_per_vector));
+		bit = 1 << db;
+	else
+		bit = ((1 << ntb->bits_per_vector) - 1) <<
+		    (db * ntb->bits_per_vector);
+
+	if (HAS_FEATURE(NTB_REGS_THRU_MW)) {
+		ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, bit);
+		return;
 	}
+
+	db_iowrite(ntb, ntb->reg_ofs.rdb, bit);
 }
 
 /**

From 189b502e31fce6eb470b38d8215aa24ce587d898 Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Wed, 14 Oct 2015 23:48:16 +0000
Subject: [PATCH 237/380] NTB: MFV 1db97f25: Pull out platform detection logic

Pull out read of PPD and platform detection logic to new functions,
ntb_detect_xeon(), ntb_detect_soc().  No functional change -- mostly
this is just shuffling the code to more closely match the Linux driver.
Linux commit log:

To simplify some of the platform detection code. Move the platform
detection to a function to be called earlier.

Authored by:	Dave Jiang
Obtained from:	Linux (Dual BSD/GPL driver)
Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/ntb_hw/ntb_hw.c | 97 +++++++++++++++++++++++++------------
 1 file changed, 66 insertions(+), 31 deletions(-)

diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c
index 18526cf08996..58ff611ff7a0 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.c
@@ -146,6 +146,7 @@ struct ntb_softc {
 		uint32_t lnk_stat;
 		uint32_t spci_cmd;
 	} reg_ofs;
+	uint32_t ppd;
 	uint8_t conn_type;
 	uint8_t dev_type;
 	uint8_t bits_per_vector;
@@ -218,6 +219,8 @@ static void unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx);
 static int ntb_create_callbacks(struct ntb_softc *ntb, uint32_t num_vectors);
 static void ntb_free_callbacks(struct ntb_softc *ntb);
 static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id);
+static int ntb_detect_xeon(struct ntb_softc *ntb);
+static int ntb_detect_soc(struct ntb_softc *ntb);
 static int ntb_setup_xeon(struct ntb_softc *ntb);
 static int ntb_setup_soc(struct ntb_softc *ntb);
 static void ntb_teardown_xeon(struct ntb_softc *ntb);
@@ -311,6 +314,13 @@ ntb_attach(device_t device)
 	callout_init(&ntb->heartbeat_timer, 1);
 	callout_init(&ntb->lr_timer, 1);
 
+	if (ntb->type == NTB_SOC)
+		error = ntb_detect_soc(ntb);
+	else
+		error = ntb_detect_xeon(ntb);
+	if (error)
+		goto out;
+
 	error = ntb_map_pci_bars(ntb);
 	if (error)
 		goto out;
@@ -874,29 +884,70 @@ ntb_teardown_xeon(struct ntb_softc *ntb)
 }
 
 static int
-ntb_setup_xeon(struct ntb_softc *ntb)
+ntb_detect_xeon(struct ntb_softc *ntb)
 {
-	uint8_t val, connection_type;
+	uint8_t ppd, conn_type;
 
-	val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1);
+	ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1);
+	ntb->ppd = ppd;
 
-	connection_type = val & XEON_PPD_CONN_TYPE;
-
-	if ((val & XEON_PPD_DEV_TYPE) != 0)
+	if ((ppd & XEON_PPD_DEV_TYPE) != 0)
 		ntb->dev_type = NTB_DEV_USD;
 	else
 		ntb->dev_type = NTB_DEV_DSD;
 
+	conn_type = ppd & XEON_PPD_CONN_TYPE;
+	switch (conn_type) {
+	case NTB_CONN_B2B:
+		ntb->conn_type = conn_type;
+		break;
+	case NTB_CONN_RP:
+	case NTB_CONN_TRANSPARENT:
+	default:
+		device_printf(ntb->device, "Unsupported connection type: %u\n",
+		    (unsigned)conn_type);
+		return (ENXIO);
+	}
+	return (0);
+}
+
+static int
+ntb_detect_soc(struct ntb_softc *ntb)
+{
+	uint32_t ppd, conn_type;
+
+	ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4);
+	ntb->ppd = ppd;
+
+	if ((ppd & SOC_PPD_DEV_TYPE) != 0)
+		ntb->dev_type = NTB_DEV_DSD;
+	else
+		ntb->dev_type = NTB_DEV_USD;
+
+	conn_type = (ppd & SOC_PPD_CONN_TYPE) >> 8;
+	switch (conn_type) {
+	case NTB_CONN_B2B:
+		ntb->conn_type = conn_type;
+		break;
+	default:
+		device_printf(ntb->device, "Unsupported NTB configuration\n");
+		return (ENXIO);
+	}
+	return (0);
+}
+
+static int
+ntb_setup_xeon(struct ntb_softc *ntb)
+{
+
 	ntb->reg_ofs.ldb	= XEON_PDOORBELL_OFFSET;
 	ntb->reg_ofs.ldb_mask	= XEON_PDBMSK_OFFSET;
 	ntb->reg_ofs.spad_local	= XEON_SPAD_OFFSET;
 	ntb->reg_ofs.bar2_xlat	= XEON_SBAR2XLAT_OFFSET;
 	ntb->reg_ofs.bar4_xlat	= XEON_SBAR4XLAT_OFFSET;
 
-	switch (connection_type) {
+	switch (ntb->conn_type) {
 	case NTB_CONN_B2B:
-		ntb->conn_type = NTB_CONN_B2B;
-
 		/*
 		 * reg_ofs.rdb and reg_ofs.spad_remote are effectively ignored
 		 * with the NTB_REGS_THRU_MW errata mode enabled.  (See
@@ -922,7 +973,7 @@ ntb_setup_xeon(struct ntb_softc *ntb)
 	case NTB_CONN_TRANSPARENT:
 	default:
 		device_printf(ntb->device, "Connection type %d not supported\n",
-		    connection_type);
+		    ntb->conn_type);
 		return (ENXIO);
 	}
 
@@ -971,7 +1022,7 @@ ntb_setup_xeon(struct ntb_softc *ntb)
 	 */
 	if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14) &&
 	    !HAS_FEATURE(NTB_REGS_THRU_MW) &&
-	    connection_type == NTB_CONN_B2B)
+	    ntb->conn_type == NTB_CONN_B2B)
 		ntb->limits.max_db_bits = XEON_MAX_DB_BITS - 1;
 
 	configure_xeon_secondary_side_bars(ntb);
@@ -990,29 +1041,13 @@ ntb_setup_xeon(struct ntb_softc *ntb)
 static int
 ntb_setup_soc(struct ntb_softc *ntb)
 {
-	uint32_t val, connection_type;
 
-	val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4);
-
-	connection_type = (val & SOC_PPD_CONN_TYPE) >> 8;
-	switch (connection_type) {
-	case NTB_CONN_B2B:
-		ntb->conn_type = NTB_CONN_B2B;
-		break;
-	default:
-		device_printf(ntb->device,
-		    "Unsupported NTB configuration (%d)\n", connection_type);
-		return (ENXIO);
-	}
-
-	if ((val & SOC_PPD_DEV_TYPE) != 0)
-		ntb->dev_type = NTB_DEV_DSD;
-	else
-		ntb->dev_type = NTB_DEV_USD;
+	KASSERT(ntb->conn_type == NTB_CONN_B2B,
+	    ("Unsupported NTB configuration (%d)\n", ntb->conn_type));
 
 	/* Initiate PCI-E link training */
-	pci_write_config(ntb->device, NTB_PPD_OFFSET, val | SOC_PPD_INIT_LINK,
-	    4);
+	pci_write_config(ntb->device, NTB_PPD_OFFSET,
+	    ntb->ppd | SOC_PPD_INIT_LINK, 4);
 
 	ntb->reg_ofs.ldb	 = SOC_PDOORBELL_OFFSET;
 	ntb->reg_ofs.ldb_mask	 = SOC_PDBMSK_OFFSET;

From c4e448c582ca7ddf1b1034e3b6c7bd55251088a6 Mon Sep 17 00:00:00 2001
From: emaste <emaste@FreeBSD.org>
Date: Thu, 15 Oct 2015 01:09:14 +0000
Subject: [PATCH 238/380] mkimg: support fat16b partitions (MBR type 06h)

Reviewed by:	marcel
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D3894
---
 usr.bin/mkimg/ebr.c    | 4 ++++
 usr.bin/mkimg/mbr.c    | 4 ++++
 usr.bin/mkimg/scheme.c | 1 +
 usr.bin/mkimg/scheme.h | 1 +
 4 files changed, 10 insertions(+)

diff --git a/usr.bin/mkimg/ebr.c b/usr.bin/mkimg/ebr.c
index 28931ea4de8c..526c494084cf 100644
--- a/usr.bin/mkimg/ebr.c
+++ b/usr.bin/mkimg/ebr.c
@@ -39,11 +39,15 @@ __FBSDID("$FreeBSD$");
 #include "mkimg.h"
 #include "scheme.h"
 
+#ifndef DOSPTYP_FAT16B
+#define	DOSPTYP_FAT16B	0x06
+#endif
 #ifndef DOSPTYP_FAT32
 #define	DOSPTYP_FAT32	0x0b
 #endif
 
 static struct mkimg_alias ebr_aliases[] = {
+    {	ALIAS_FAT16B, ALIAS_INT2TYPE(DOSPTYP_FAT16B) },
     {	ALIAS_FAT32, ALIAS_INT2TYPE(DOSPTYP_FAT32) },
     {	ALIAS_FREEBSD, ALIAS_INT2TYPE(DOSPTYP_386BSD) },
     {	ALIAS_NONE, 0 }
diff --git a/usr.bin/mkimg/mbr.c b/usr.bin/mkimg/mbr.c
index 961ca45b45eb..20d4d91e2238 100644
--- a/usr.bin/mkimg/mbr.c
+++ b/usr.bin/mkimg/mbr.c
@@ -39,6 +39,9 @@ __FBSDID("$FreeBSD$");
 #include "mkimg.h"
 #include "scheme.h"
 
+#ifndef DOSPTYP_FAT16B
+#define	DOSPTYP_FAT16B	0x06
+#endif
 #ifndef DOSPTYP_FAT32
 #define	DOSPTYP_FAT32	0x0b
 #endif
@@ -49,6 +52,7 @@ __FBSDID("$FreeBSD$");
 static struct mkimg_alias mbr_aliases[] = {
     {	ALIAS_EBR, ALIAS_INT2TYPE(DOSPTYP_EXT) },
     {	ALIAS_EFI, ALIAS_INT2TYPE(DOSPTYP_EFI) },
+    {	ALIAS_FAT16B, ALIAS_INT2TYPE(DOSPTYP_FAT16B) },
     {	ALIAS_FAT32, ALIAS_INT2TYPE(DOSPTYP_FAT32) },
     {	ALIAS_FREEBSD, ALIAS_INT2TYPE(DOSPTYP_386BSD) },
     {	ALIAS_NTFS, ALIAS_INT2TYPE(DOSPTYP_NTFS) },
diff --git a/usr.bin/mkimg/scheme.c b/usr.bin/mkimg/scheme.c
index 9bdf8a5a5463..1e648557e75d 100644
--- a/usr.bin/mkimg/scheme.c
+++ b/usr.bin/mkimg/scheme.c
@@ -50,6 +50,7 @@ static struct {
 } scheme_alias[] = {
 	{ "ebr", ALIAS_EBR },
 	{ "efi", ALIAS_EFI },
+	{ "fat16b", ALIAS_FAT16B },
 	{ "fat32", ALIAS_FAT32 },
 	{ "freebsd", ALIAS_FREEBSD },
 	{ "freebsd-boot", ALIAS_FREEBSD_BOOT },
diff --git a/usr.bin/mkimg/scheme.h b/usr.bin/mkimg/scheme.h
index 73b06eb7a4fb..3ba4243bba90 100644
--- a/usr.bin/mkimg/scheme.h
+++ b/usr.bin/mkimg/scheme.h
@@ -36,6 +36,7 @@ enum alias {
 	/* start */
 	ALIAS_EBR,
 	ALIAS_EFI,
+	ALIAS_FAT16B,
 	ALIAS_FAT32,
 	ALIAS_FREEBSD,
 	ALIAS_FREEBSD_BOOT,

From 75679462b6d97f0a92683e973e15c3c02d5589b4 Mon Sep 17 00:00:00 2001
From: bz <bz@FreeBSD.org>
Date: Thu, 15 Oct 2015 01:44:32 +0000
Subject: [PATCH 239/380] Hopefully also unbreak VIMAGE kernels replacing the
 &V_... with &VNET_NAME(...). Everything else is just a whitespace wrapping
 change.

---
 sys/netinet/tcp_pcap.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sys/netinet/tcp_pcap.c b/sys/netinet/tcp_pcap.c
index c431f2444bd8..f0c651dab555 100644
--- a/sys/netinet/tcp_pcap.c
+++ b/sys/netinet/tcp_pcap.c
@@ -70,9 +70,9 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_pcap_alloc_new_mbuf,
 
 VNET_DEFINE(int, tcp_pcap_packets) = 0;
 #define V_tcp_pcap_packets	VNET(tcp_pcap_packets)
-SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_pcap_packets, CTLFLAG_RW,
-	&V_tcp_pcap_packets, 0, "Default number of packets saved per direction "
-	"per TCPCB");
+SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_pcap_packets,
+	CTLFLAG_RW, &VNET_NAME(tcp_pcap_packets), 0,
+	"Default number of packets saved per direction per TCPCB");
 
 /* Initialize the values. */
 static void

From a888cf588cf8e4f8b9923cefe83e67190f0726d0 Mon Sep 17 00:00:00 2001
From: emaste <emaste@FreeBSD.org>
Date: Thu, 15 Oct 2015 01:46:55 +0000
Subject: [PATCH 240/380] Create /usr/tests *.debug file directory hierarchy

Reviewed by:	bdrewery, ngie
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D3896
---
 Makefile.inc1            | 8 ++++++++
 etc/Makefile             | 1 +
 etc/mtree/BSD.debug.dist | 2 ++
 3 files changed, 11 insertions(+)

diff --git a/Makefile.inc1 b/Makefile.inc1
index 9332b300e060..5d1917ca2b5b 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -557,6 +557,10 @@ _worldtmp:
 	mkdir -p ${WORLDTMP}${TESTSBASE}
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.tests.dist \
 	    -p ${WORLDTMP}${TESTSBASE} >/dev/null
+.if ${MK_DEBUG_FILES} != "no"
+	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.tests.dist \
+	    -p ${WORLDTMP}/usr/lib/debug/${TESTSBASE} >/dev/null
+.endif
 .endif
 .for _mtree in ${LOCAL_MTREE}
 	mtree -deU -f ${.CURDIR}/${_mtree} -p ${WORLDTMP} > /dev/null
@@ -892,6 +896,10 @@ distributeworld installworld: _installcheck_world
 	-mkdir -p ${DESTDIR}/${DISTDIR}/${dist}${TESTSBASE}
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.tests.dist \
 	    -p ${DESTDIR}/${DISTDIR}/${dist}${TESTSBASE} >/dev/null
+.if ${MK_DEBUG_FILES} != "no"
+	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.tests.dist \
+	    -p ${DESTDIR}/${DISTDIR}/${dist}/usr/lib/debug/${TESTSBASE} >/dev/null
+.endif
 .endif
 .if defined(NO_ROOT)
 	${IMAKEENV} mtree -C -f ${.CURDIR}/etc/mtree/BSD.root.dist | \
diff --git a/etc/Makefile b/etc/Makefile
index 0112c617bc64..81b9a1973f2b 100644
--- a/etc/Makefile
+++ b/etc/Makefile
@@ -343,6 +343,7 @@ MTREES+=	mtree/BSD.groff.dist		/usr
 .endif
 .if ${MK_TESTS} != "no"
 MTREES+=	mtree/BSD.tests.dist		${TESTSBASE}
+MTREES+=	mtree/BSD.tests.dist		/usr/lib/debug/${TESTSBASE}
 .endif
 .if ${MK_SENDMAIL} != "no"
 MTREES+=	mtree/BSD.sendmail.dist		/
diff --git a/etc/mtree/BSD.debug.dist b/etc/mtree/BSD.debug.dist
index 9584e591eb47..8df0a725e5fe 100644
--- a/etc/mtree/BSD.debug.dist
+++ b/etc/mtree/BSD.debug.dist
@@ -61,6 +61,8 @@
             ..
             sbin
             ..
+            tests
+            ..
         ..
     ..
 ..

From c8d3be347752a6f84988ff725b31cc84888cffa4 Mon Sep 17 00:00:00 2001
From: emaste <emaste@FreeBSD.org>
Date: Thu, 15 Oct 2015 01:48:52 +0000
Subject: [PATCH 241/380] Add debug file extension to freebsd-update(8) after
 r288176

Reviewed by:	delphij
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D3784
---
 usr.sbin/freebsd-update/freebsd-update.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/usr.sbin/freebsd-update/freebsd-update.sh b/usr.sbin/freebsd-update/freebsd-update.sh
index 9fcc01214755..cce2af574f61 100644
--- a/usr.sbin/freebsd-update/freebsd-update.sh
+++ b/usr.sbin/freebsd-update/freebsd-update.sh
@@ -2736,7 +2736,7 @@ backup_kernel () {
 	if [ $BACKUPKERNELSYMBOLFILES = yes ]; then
 		FINDFILTER=""
 	else
-		FINDFILTER=-"a ! -name *.symbols"
+		FINDFILTER="-a ! -name *.debug -a ! -name *.symbols"
 	fi
 
 	# Backup all the kernel files using hardlinks.

From 9b00aa0101ef116656b80004aae21df78e1a01b9 Mon Sep 17 00:00:00 2001
From: bz <bz@FreeBSD.org>
Date: Thu, 15 Oct 2015 01:51:10 +0000
Subject: [PATCH 242/380] 2nd try, after r289319:

HWPMC depends on pmu.c even if device pmu is not specified.
Would be great if we could just automatically enabled "device pmu"
if we try to compile in HWPMC.

Also several old kernel cnfigurations seem to have HWPMC enabled but are
pre-FDT and thus fail.  So make pmu.c depend on fdt in case of hwpmc as
well.

MFC after:		2 weeks
Sponsored by:		DARPA/AFRL
Differential Revision:	https://reviews.freebsd.org/D3877
---
 sys/conf/files.arm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/conf/files.arm b/sys/conf/files.arm
index 85830bb6717c..a2abcc243c54 100644
--- a/sys/conf/files.arm
+++ b/sys/conf/files.arm
@@ -56,7 +56,7 @@ arm/arm/platform_if.m		optional	platform
 arm/arm/pmap.c			optional	!armv6
 arm/arm/pmap-v6.c		optional	armv6 !arm_new_pmap
 arm/arm/pmap-v6-new.c		optional	armv6 arm_new_pmap
-arm/arm/pmu.c			optional	pmu
+arm/arm/pmu.c			optional	pmu | fdt hwpmc
 arm/arm/sc_machdep.c		optional	sc
 arm/arm/setcpsr.S		standard
 arm/arm/setstack.s		standard

From e2f6f0979cb1fbee05e8c79f686773565d2e0e85 Mon Sep 17 00:00:00 2001
From: emaste <emaste@FreeBSD.org>
Date: Thu, 15 Oct 2015 02:52:30 +0000
Subject: [PATCH 243/380] /usr/tests is part of the base system (for *.debug
 files)

Sponsored by:	The FreeBSD Foundation
---
 share/mk/bsd.prog.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/share/mk/bsd.prog.mk b/share/mk/bsd.prog.mk
index b487541bc018..dea7661eab8c 100644
--- a/share/mk/bsd.prog.mk
+++ b/share/mk/bsd.prog.mk
@@ -58,7 +58,7 @@ PROG_FULL=${PROG}.full
     ${BINDIR} == "/bin" ||\
     ${BINDIR:C%/libexec(/.*)?%/libexec%} == "/libexec" ||\
     ${BINDIR} == "/sbin" ||\
-    ${BINDIR:C%/usr/(bin|bsdinstall|libexec|lpr|sendmail|sm.bin|sbin)(/.*)?%/usr/bin%} == "/usr/bin"\
+    ${BINDIR:C%/usr/(bin|bsdinstall|libexec|lpr|sendmail|sm.bin|sbin|tests)(/.*)?%/usr/bin%} == "/usr/bin"\
      )
 DEBUGFILEDIR=	${DEBUGDIR}${BINDIR}
 .else

From 4e7945e48ed65cb55ef9f9faea35fcf7c94e26b2 Mon Sep 17 00:00:00 2001
From: adrian <adrian@FreeBSD.org>
Date: Thu, 15 Oct 2015 04:22:56 +0000
Subject: [PATCH 244/380] Add support for the BCM57765 card reader.

This patch adds support for the BCM57765[2] card reader function included in
Broadcom's BCM57766 ethernet/sd3.0 controller. This controller is commonly
found in laptops and Apple hardware (MBP, iMac, etc).

The BCM57765 chipset is almost fully compatible with the SD3.0 spec, but
does not support deriving a frequency below 781KHz from its default base
clock via the standard SD3.0-configured 10-bit clock divisor.

If such a divisor is set, card identification (which requires a 400KHz
clock frequency) will time out[1].

As a work-around, I've made use of an undocumented device-specific clock
control register to switch the controller to a 63MHz clock source when
targeting clock speeds below 781KHz; the clock source is likewise switched
back to the 200MHz clock when targeting speeds greater than 781KHz.

Additionally, this patch fixes a small sdhci_pci bug; the
sdhci_pci_softc->quirks flag was not copied to the sdhci_slot, resulting in
`quirk` behavior not being applied by sdhci.c.

[1] A number of Linux/FreeBSD users have noted that bringing up the chipsets'
associated ethernet interface will allow SD cards to enumerate (slowly).
This is a controller implementation side-effect triggered by the ethernet
driver's reading of the hardware statistics registers.

[2] This may also fix card detection when using the BCM57785 chipset, but I
don't have access to the BCM57785 chipset and can't verify.

I actually snagged some BCM57785 hardware recently (2012 Retina MacBook Pro)
and can confirm that this also fixes card enumeration with the BCM57785
chipset; with the patch, I can boot off of the internal sdcard reader.

PR:		kern/203385
Submitted by:	Landon Fuller <landon@landonf.org>
---
 sys/dev/sdhci/sdhci.c     | 41 ++++++++++++++++++++++++++++++++++-----
 sys/dev/sdhci/sdhci.h     |  2 ++
 sys/dev/sdhci/sdhci_pci.c |  4 ++++
 3 files changed, 42 insertions(+), 5 deletions(-)

diff --git a/sys/dev/sdhci/sdhci.c b/sys/dev/sdhci/sdhci.c
index b1fae2916ccf..e625d4f11f32 100644
--- a/sys/dev/sdhci/sdhci.c
+++ b/sys/dev/sdhci/sdhci.c
@@ -89,6 +89,19 @@ static void sdhci_card_task(void *, int);
 #define	SDHCI_200_MAX_DIVIDER	256
 #define	SDHCI_300_MAX_DIVIDER	2046
 
+/*
+ * Broadcom BCM577xx Controller Constants
+ */
+#define BCM577XX_DEFAULT_MAX_DIVIDER	256		/* Maximum divider supported by the default clock source. */
+#define BCM577XX_ALT_CLOCK_BASE		63000000	/* Alternative clock's base frequency. */
+
+#define BCM577XX_HOST_CONTROL		0x198
+#define BCM577XX_CTRL_CLKSEL_MASK	0xFFFFCFFF
+#define BCM577XX_CTRL_CLKSEL_SHIFT	12
+#define BCM577XX_CTRL_CLKSEL_DEFAULT	0x0
+#define BCM577XX_CTRL_CLKSEL_64MHZ	0x3
+
+
 static void
 sdhci_getaddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 {
@@ -228,6 +241,8 @@ sdhci_init(struct sdhci_slot *slot)
 static void
 sdhci_set_clock(struct sdhci_slot *slot, uint32_t clock)
 {
+	uint32_t clk_base;
+	uint32_t clk_sel;
 	uint32_t res;
 	uint16_t clk;
 	uint16_t div;
@@ -243,6 +258,22 @@ sdhci_set_clock(struct sdhci_slot *slot, uint32_t clock)
 	/* If no clock requested - left it so. */
 	if (clock == 0)
 		return;
+	
+	/* Determine the clock base frequency */
+	clk_base = slot->max_clk;
+	if (slot->quirks & SDHCI_QUIRK_BCM577XX_400KHZ_CLKSRC) {
+		clk_sel = RD2(slot, BCM577XX_HOST_CONTROL) & BCM577XX_CTRL_CLKSEL_MASK;
+
+		/* Select clock source appropriate for the requested frequency. */
+		if ((clk_base / BCM577XX_DEFAULT_MAX_DIVIDER) > clock) {
+			clk_base = BCM577XX_ALT_CLOCK_BASE;
+			clk_sel |= (BCM577XX_CTRL_CLKSEL_64MHZ << BCM577XX_CTRL_CLKSEL_SHIFT);
+		} else {
+			clk_sel |= (BCM577XX_CTRL_CLKSEL_DEFAULT << BCM577XX_CTRL_CLKSEL_SHIFT);
+		}
+		
+		WR2(slot, BCM577XX_HOST_CONTROL, clk_sel);
+	}
 
 	/* Recalculate timeout clock frequency based on the new sd clock. */
 	if (slot->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK)
@@ -250,7 +281,7 @@ sdhci_set_clock(struct sdhci_slot *slot, uint32_t clock)
 
 	if (slot->version < SDHCI_SPEC_300) {
 		/* Looking for highest freq <= clock. */
-		res = slot->max_clk;
+		res = clk_base;
 		for (div = 1; div < SDHCI_200_MAX_DIVIDER; div <<= 1) {
 			if (res <= clock)
 				break;
@@ -261,11 +292,11 @@ sdhci_set_clock(struct sdhci_slot *slot, uint32_t clock)
 	}
 	else {
 		/* Version 3.0 divisors are multiples of two up to 1023*2 */
-		if (clock >= slot->max_clk)
+		if (clock >= clk_base)
 			div = 0;
 		else {
 			for (div = 2; div < SDHCI_300_MAX_DIVIDER; div += 2) { 
-				if ((slot->max_clk / div) <= clock) 
+				if ((clk_base / div) <= clock) 
 					break;
 			}
 		}
@@ -273,8 +304,8 @@ sdhci_set_clock(struct sdhci_slot *slot, uint32_t clock)
 	}
 
 	if (bootverbose || sdhci_debug)
-		slot_printf(slot, "Divider %d for freq %d (max %d)\n", 
-			div, clock, slot->max_clk);
+		slot_printf(slot, "Divider %d for freq %d (base %d)\n", 
+			div, clock, clk_base);
 
 	/* Now we have got divider, set it. */
 	clk = (div & SDHCI_DIVIDER_MASK) << SDHCI_DIVIDER_SHIFT;
diff --git a/sys/dev/sdhci/sdhci.h b/sys/dev/sdhci/sdhci.h
index 3c27d0c3cd22..7683c1dbf300 100644
--- a/sys/dev/sdhci/sdhci.h
+++ b/sys/dev/sdhci/sdhci.h
@@ -63,6 +63,8 @@
 #define	SDHCI_QUIRK_WAITFOR_RESET_ASSERTED		(1<<14)
 /* Leave controller in standard mode when putting card in HS mode. */
 #define	SDHCI_QUIRK_DONT_SET_HISPD_BIT			(1<<15)
+/* Alternate clock source is required when supplying a 400 KHz clock. */
+#define	SDHCI_QUIRK_BCM577XX_400KHZ_CLKSRC		(1<<16)
 
 /*
  * Controller registers
diff --git a/sys/dev/sdhci/sdhci_pci.c b/sys/dev/sdhci/sdhci_pci.c
index b818a50bde66..b149bb6fa490 100644
--- a/sys/dev/sdhci/sdhci_pci.c
+++ b/sys/dev/sdhci/sdhci_pci.c
@@ -105,6 +105,8 @@ static const struct sdhci_device {
 	{ 0x2381197B, 	0xffff,	"JMicron JMB38X SD",
 	    SDHCI_QUIRK_32BIT_DMA_SIZE |
 	    SDHCI_QUIRK_RESET_AFTER_REQUEST },
+	{ 0x16bc14e4,	0xffff,	"Broadcom BCM577xx SDXC/MMC Card Reader",
+	    SDHCI_QUIRK_BCM577XX_400KHZ_CLKSRC },
 	{ 0,		0xffff,	NULL,
 	    0 }
 };
@@ -334,6 +336,8 @@ sdhci_pci_attach(device_t dev)
 			device_printf(dev, "Can't allocate memory for slot %d\n", i);
 			continue;
 		}
+		
+		slot->quirks = sc->quirks;
 
 		if (sdhci_init_slot(dev, slot, i) != 0)
 			continue;

From 7da5ecb17db130dc5d4003e5d666e66123398475 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Thu, 15 Oct 2015 05:26:11 +0000
Subject: [PATCH 245/380] Add temporary workaround for .MAKE being applied to
 _worldtmp, since r251750.

This was causing files to be removed from the objdir when -n was used.
_worldtmp makes no sub-make calls.

A more comprehensive solution is coming involving fine-grained '+' where
appropriate.

Sponsored by:	EMC / Isilon Storage Division
MFC after:	1 week
---
 Makefile.inc1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile.inc1 b/Makefile.inc1
index 5d1917ca2b5b..4e6a873887ce 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -1062,7 +1062,7 @@ INSTALLKERNEL= ${_kernel}
 .endif
 .endfor
 
-buildkernel ${WMAKE_TGTS} ${.ALLTARGETS:M_*}: .MAKE
+buildkernel ${WMAKE_TGTS:N_worldtmp} ${.ALLTARGETS:M_*:N_worldtmp}: .MAKE
 
 #
 # buildkernel

From 504ba9ab05b649ed00b12d36a98f54a4b384e6b6 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Thu, 15 Oct 2015 05:32:56 +0000
Subject: [PATCH 246/380] Consider top-level targets to be .PHONY as bmake
 won't build them otherwise if a file with the same name is found in the
 directory.

MFC after:	1 week
Sponsored by:	EMC / Isilon Storage Division
---
 Makefile      | 2 +-
 Makefile.inc1 | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index a86df3b67236..aa9c05736ebb 100644
--- a/Makefile
+++ b/Makefile
@@ -250,7 +250,7 @@ ${TGTS}: .MAKE
 tinderbox toolchains kernel-toolchains: .MAKE
 .endif
 
-${TGTS}:
+${TGTS}: .PHONY
 	${_+_}@cd ${.CURDIR}; ${_MAKE} ${.TARGET}
 
 # The historic default "all" target creates files which may cause stale
diff --git a/Makefile.inc1 b/Makefile.inc1
index 4e6a873887ce..1cdefc948cd9 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -1062,7 +1062,7 @@ INSTALLKERNEL= ${_kernel}
 .endif
 .endfor
 
-buildkernel ${WMAKE_TGTS:N_worldtmp} ${.ALLTARGETS:M_*:N_worldtmp}: .MAKE
+buildkernel ${WMAKE_TGTS:N_worldtmp} ${.ALLTARGETS:M_*:N_worldtmp}: .MAKE .PHONY
 
 #
 # buildkernel

From c1c5997359a0319d0b6984203c5c3bb10c28274c Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Thu, 15 Oct 2015 08:47:32 +0000
Subject: [PATCH 247/380] MFV r289312: 2605 want to resume interrupted zfs send

Reviewed by: George Wilson <george.wilson@delphix.com>
Reviewed by: Paul Dagnelie <pcd@delphix.com>
Reviewed by: Richard Elling <Richard.Elling@RichardElling.com>
Reviewed by: Xin Li <delphij@freebsd.org>
Reviewed by: Arne Jansen <sensille@gmx.net>
Approved by: Dan McDonald <danmcd@omniti.com>
Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8

For more info, see:
 - slides http://www.slideshare.net/MatthewAhrens/openzfs-send-and-receive
 - video https://www.youtube.com/watch?v=iY44jPMvxog
 - manpage changes (for zfs resume -s and zfs send -t)
 - upcoming talk at the OpenZFS Developer Summit

The TL;DR is:
Use "zfs receive -s" to save the partially received state on failure.
On failure, get the receive token with "zfs get receive_resume_token <fs>"
Resume the send with "zfs send -t <token_value>"

Relnotes:	yes
---
 cddl/contrib/opensolaris/cmd/zfs/zfs.8        |  69 ++-
 cddl/contrib/opensolaris/cmd/zfs/zfs_main.c   | 120 +++-
 .../opensolaris/cmd/zstreamdump/zstreamdump.c |   5 +-
 .../opensolaris/lib/libzfs/common/libzfs.h    |  10 +
 .../lib/libzfs/common/libzfs_compat.c         |   3 +
 .../lib/libzfs/common/libzfs_dataset.c        |  27 +-
 .../lib/libzfs/common/libzfs_mount.c          |  11 +
 .../lib/libzfs/common/libzfs_sendrecv.c       | 445 ++++++++++++---
 .../lib/libzfs_core/common/libzfs_core.c      |  69 ++-
 .../lib/libzfs_core/common/libzfs_core.h      |   6 +-
 cddl/lib/libzfs/Makefile                      |   4 +-
 .../opensolaris/common/zfs/zfs_ioctl_compat.c | 125 +++-
 .../opensolaris/common/zfs/zfs_ioctl_compat.h |  47 +-
 .../contrib/opensolaris/common/zfs/zfs_prop.c |   4 +
 .../uts/common/fs/zfs/dmu_objset.c            |  20 +
 .../opensolaris/uts/common/fs/zfs/dmu_send.c  | 538 +++++++++++++++---
 .../uts/common/fs/zfs/dmu_traverse.c          |  82 +--
 .../uts/common/fs/zfs/dsl_dataset.c           | 139 +++++
 .../uts/common/fs/zfs/dsl_destroy.c           |  12 +-
 .../uts/common/fs/zfs/sys/dmu_impl.h          |   4 +-
 .../uts/common/fs/zfs/sys/dmu_send.h          |  18 +-
 .../uts/common/fs/zfs/sys/dmu_traverse.h      |   2 +
 .../uts/common/fs/zfs/sys/dsl_dataset.h       |  23 +
 .../uts/common/fs/zfs/sys/zfs_ioctl.h         |  22 +-
 .../opensolaris/uts/common/fs/zfs/zfs_ioctl.c |  26 +-
 .../opensolaris/uts/common/sys/fs/zfs.h       |   1 +
 26 files changed, 1538 insertions(+), 294 deletions(-)

diff --git a/cddl/contrib/opensolaris/cmd/zfs/zfs.8 b/cddl/contrib/opensolaris/cmd/zfs/zfs.8
index b9dd26bda106..9cfa4dec37f2 100644
--- a/cddl/contrib/opensolaris/cmd/zfs/zfs.8
+++ b/cddl/contrib/opensolaris/cmd/zfs/zfs.8
@@ -189,17 +189,25 @@
 .Op Fl i Ar snapshot Ns | Ns bookmark
 .Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot
 .Nm
+.Cm send
+.Op Fl Penv
+.Fl t Ar receive_resume_token
+.Nm
 .Cm receive Ns | Ns Cm recv
-.Op Fl vnFu
+.Op Fl vnsFu
 .Op Fl o Sy origin Ns = Ns Ar snapshot
 .Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot
 .Nm
 .Cm receive Ns | Ns Cm recv
-.Op Fl vnFu
+.Op Fl vnsFu
 .Op Fl d | e
 .Op Fl o Sy origin Ns = Ns Ar snapshot
 .Ar filesystem
 .Nm
+.Cm receive Ns | Ns Cm recv
+.Fl A
+.Ar filesystem Ns | Ns Ar volume
+.Nm
 .Cm allow
 .Ar filesystem Ns | Ns Ar volume
 .Nm
@@ -597,6 +605,13 @@ For cloned file systems or volumes, the snapshot from which the clone was
 created. See also the
 .Sy clones
 property.
+.It Sy receive_resume_token
+For filesystems or volumes which have saved partially-completed state from
+.Sy zfs receive -s ,
+this opaque token can be provided to
+.Sy zfs send -t
+to resume and complete the
+.Sy zfs receive .
 .It Sy referenced
 The amount of data that is accessible by this dataset, which may or may not be
 shared with other datasets in the pool. When a snapshot or clone is created, it
@@ -2714,15 +2729,28 @@ feature.
 .El
 .It Xo
 .Nm
+.Cm send
+.Op Fl Penv
+.Fl t
+.Ar receive_resume_token
+.Xc
+Creates a send stream which resumes an interrupted receive.  The
+.Ar receive_resume_token
+is the value of this property on the filesystem
+or volume that was being received into.  See the documentation for
+.Sy zfs receive -s
+for more details.
+.It Xo
+.Nm
 .Cm receive Ns | Ns Cm recv
-.Op Fl vnFu
+.Op Fl vnsFu
 .Op Fl o Sy origin Ns = Ns Ar snapshot
 .Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot
 .Xc
 .It Xo
 .Nm
 .Cm receive Ns | Ns Cm recv
-.Op Fl vnFu
+.Op Fl vnsFu
 .Op Fl d | e
 .Op Fl o Sy origin Ns = Ns Ar snapshot
 .Ar filesystem
@@ -2819,9 +2847,42 @@ performing the receive operation. If receiving an incremental replication
 stream (for example, one generated by
 .Qq Nm Cm send Fl R Bro Fl i | Fl I Brc ) ,
 destroy snapshots and file systems that do not exist on the sending side.
+.It Fl s
+If the receive is interrupted, save the partially received state, rather
+than deleting it.  Interruption may be due to premature termination of
+the stream
+.Po e.g. due to network failure or failure of the remote system
+if the stream is being read over a network connection
+.Pc ,
+a checksum error in the stream, termination of the
+.Nm zfs Cm receive
+process, or unclean shutdown of the system.
+.Pp
+The receive can be resumed with a stream generated by
+.Nm zfs Cm send Fl t Ar token ,
+where the
+.Ar token
+is the value of the
+.Sy receive_resume_token
+property of the filesystem or volume which is received into.
+.Pp
+To use this flag, the storage pool must have the
+.Sy extensible_dataset
+feature enabled.  See
+.Xr zpool-features 5
+for details on ZFS feature flags.
 .El
 .It Xo
 .Nm
+.Cm receive Ns | Ns Cm recv
+.Fl A
+.Ar filesystem Ns | Ns Ar volume
+.Xc
+Abort an interrupted
+.Nm zfs Cm receive Fl s ,
+deleting its saved partially received state.
+.It Xo
+.Nm
 .Cm allow
 .Ar filesystem Ns | Ns Ar volume
 .Xc
diff --git a/cddl/contrib/opensolaris/cmd/zfs/zfs_main.c b/cddl/contrib/opensolaris/cmd/zfs/zfs_main.c
index 9ea178e9c3de..9b123ced6671 100644
--- a/cddl/contrib/opensolaris/cmd/zfs/zfs_main.c
+++ b/cddl/contrib/opensolaris/cmd/zfs/zfs_main.c
@@ -263,10 +263,11 @@ get_usage(zfs_help_t idx)
 	case HELP_PROMOTE:
 		return (gettext("\tpromote <clone-filesystem>\n"));
 	case HELP_RECEIVE:
-		return (gettext("\treceive|recv [-vnFu] <filesystem|volume|"
+		return (gettext("\treceive|recv [-vnsFu] <filesystem|volume|"
 		    "snapshot>\n"
-		    "\treceive|recv [-vnFu] [-o origin=<snapshot>] [-d | -e] "
-		    "<filesystem>\n"));
+		    "\treceive|recv [-vnsFu] [-o origin=<snapshot>] [-d | -e] "
+		    "<filesystem>\n"
+		    "\treceive|recv -A <filesystem|volume>\n"));
 	case HELP_RENAME:
 		return (gettext("\trename [-f] <filesystem|volume|snapshot> "
 		    "<filesystem|volume|snapshot>\n"
@@ -279,7 +280,8 @@ get_usage(zfs_help_t idx)
 		return (gettext("\tsend [-DnPpRvLe] [-[iI] snapshot] "
 		    "<snapshot>\n"
 		    "\tsend [-Le] [-i snapshot|bookmark] "
-		    "<filesystem|volume|snapshot>\n"));
+		    "<filesystem|volume|snapshot>\n"
+		    "\tsend [-nvPe] -t <receive_resume_token>\n"));
 	case HELP_SET:
 		return (gettext("\tset <property=value> "
 		    "<filesystem|volume|snapshot> ...\n"));
@@ -3728,6 +3730,7 @@ zfs_do_send(int argc, char **argv)
 {
 	char *fromname = NULL;
 	char *toname = NULL;
+	char *resume_token = NULL;
 	char *cp;
 	zfs_handle_t *zhp;
 	sendflags_t flags = { 0 };
@@ -3736,7 +3739,7 @@ zfs_do_send(int argc, char **argv)
 	boolean_t extraverbose = B_FALSE;
 
 	/* check options */
-	while ((c = getopt(argc, argv, ":i:I:RDpvnPLe")) != -1) {
+	while ((c = getopt(argc, argv, ":i:I:RDpvnPLet:")) != -1) {
 		switch (c) {
 		case 'i':
 			if (fromname)
@@ -3777,6 +3780,9 @@ zfs_do_send(int argc, char **argv)
 		case 'e':
 			flags.embed_data = B_TRUE;
 			break;
+		case 't':
+			resume_token = optarg;
+			break;
 		case ':':
 			(void) fprintf(stderr, gettext("missing argument for "
 			    "'%c' option\n"), optopt);
@@ -3792,14 +3798,28 @@ zfs_do_send(int argc, char **argv)
 	argc -= optind;
 	argv += optind;
 
-	/* check number of arguments */
-	if (argc < 1) {
-		(void) fprintf(stderr, gettext("missing snapshot argument\n"));
-		usage(B_FALSE);
-	}
-	if (argc > 1) {
-		(void) fprintf(stderr, gettext("too many arguments\n"));
-		usage(B_FALSE);
+	if (resume_token != NULL) {
+		if (fromname != NULL || flags.replicate || flags.props ||
+		    flags.dedup) {
+			(void) fprintf(stderr,
+			    gettext("invalid flags combined with -t\n"));
+			usage(B_FALSE);
+		}
+		if (argc != 0) {
+			(void) fprintf(stderr, gettext("no additional "
+			    "arguments are permitted with -t\n"));
+			usage(B_FALSE);
+		}
+	} else {
+		if (argc < 1) {
+			(void) fprintf(stderr,
+			    gettext("missing snapshot argument\n"));
+			usage(B_FALSE);
+		}
+		if (argc > 1) {
+			(void) fprintf(stderr, gettext("too many arguments\n"));
+			usage(B_FALSE);
+		}
 	}
 
 	if (!flags.dryrun && isatty(STDOUT_FILENO)) {
@@ -3809,6 +3829,11 @@ zfs_do_send(int argc, char **argv)
 		return (1);
 	}
 
+	if (resume_token != NULL) {
+		return (zfs_send_resume(g_zfs, &flags, STDOUT_FILENO,
+		    resume_token));
+	}
+
 	/*
 	 * Special case sending a filesystem, or from a bookmark.
 	 */
@@ -3914,8 +3939,6 @@ zfs_do_send(int argc, char **argv)
 }
 
 /*
- * zfs receive [-vnFu] [-d | -e] <fs@snap>
- *
  * Restore a backup stream from stdin.
  */
 static int
@@ -3923,6 +3946,8 @@ zfs_do_receive(int argc, char **argv)
 {
 	int c, err;
 	recvflags_t flags = { 0 };
+	boolean_t abort_resumable = B_FALSE;
+
 	nvlist_t *props;
 	nvpair_t *nvp = NULL;
 
@@ -3930,7 +3955,7 @@ zfs_do_receive(int argc, char **argv)
 		nomem();
 
 	/* check options */
-	while ((c = getopt(argc, argv, ":o:denuvF")) != -1) {
+	while ((c = getopt(argc, argv, ":o:denuvFsA")) != -1) {
 		switch (c) {
 		case 'o':
 			if (parseprop(props, optarg) != 0)
@@ -3952,9 +3977,15 @@ zfs_do_receive(int argc, char **argv)
 		case 'v':
 			flags.verbose = B_TRUE;
 			break;
+		case 's':
+			flags.resumable = B_TRUE;
+			break;
 		case 'F':
 			flags.force = B_TRUE;
 			break;
+		case 'A':
+			abort_resumable = B_TRUE;
+			break;
 		case ':':
 			(void) fprintf(stderr, gettext("missing argument for "
 			    "'%c' option\n"), optopt);
@@ -3987,6 +4018,44 @@ zfs_do_receive(int argc, char **argv)
 		}
 	}
 
+	if (abort_resumable) {
+		if (flags.isprefix || flags.istail || flags.dryrun ||
+		    flags.resumable || flags.nomount) {
+			(void) fprintf(stderr, gettext("invalid option"));
+			usage(B_FALSE);
+		}
+
+		char namebuf[ZFS_MAXNAMELEN];
+		(void) snprintf(namebuf, sizeof (namebuf),
+		    "%s/%%recv", argv[0]);
+
+		if (zfs_dataset_exists(g_zfs, namebuf,
+		    ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME)) {
+			zfs_handle_t *zhp = zfs_open(g_zfs,
+			    namebuf, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME);
+			if (zhp == NULL)
+				return (1);
+			err = zfs_destroy(zhp, B_FALSE);
+		} else {
+			zfs_handle_t *zhp = zfs_open(g_zfs,
+			    argv[0], ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME);
+			if (zhp == NULL)
+				usage(B_FALSE);
+			if (!zfs_prop_get_int(zhp, ZFS_PROP_INCONSISTENT) ||
+			    zfs_prop_get(zhp, ZFS_PROP_RECEIVE_RESUME_TOKEN,
+			    NULL, 0, NULL, NULL, 0, B_TRUE) == -1) {
+				(void) fprintf(stderr,
+				    gettext("'%s' does not have any "
+				    "resumable receive state to abort\n"),
+				    argv[0]);
+				return (1);
+			}
+			err = zfs_destroy(zhp, B_FALSE);
+		}
+
+		return (err != 0);
+	}
+
 	if (isatty(STDIN_FILENO)) {
 		(void) fprintf(stderr,
 		    gettext("Error: Backup stream can not be read "
@@ -3994,7 +4063,6 @@ zfs_do_receive(int argc, char **argv)
 		    "You must redirect standard input.\n"));
 		return (1);
 	}
-
 	err = zfs_receive(g_zfs, argv[0], props, &flags, STDIN_FILENO, NULL);
 
 	return (err != 0);
@@ -5815,6 +5883,24 @@ share_mount_one(zfs_handle_t *zhp, int op, int flags, char *protocol,
 		return (0);
 	}
 
+	/*
+	 * If this filesystem is inconsistent and has a receive resume
+	 * token, we can not mount it.
+	 */
+	if (zfs_prop_get_int(zhp, ZFS_PROP_INCONSISTENT) &&
+	    zfs_prop_get(zhp, ZFS_PROP_RECEIVE_RESUME_TOKEN,
+	    NULL, 0, NULL, NULL, 0, B_TRUE) == 0) {
+		if (!explicit)
+			return (0);
+
+		(void) fprintf(stderr, gettext("cannot %s '%s': "
+		    "Contains partially-completed state from "
+		    "\"zfs receive -r\", which can be resumed with "
+		    "\"zfs send -t\"\n"),
+		    cmdname, zfs_get_name(zhp));
+		return (1);
+	}
+
 	/*
 	 * At this point, we have verified that the mountpoint and/or
 	 * shareopts are appropriate for auto management. If the
diff --git a/cddl/contrib/opensolaris/cmd/zstreamdump/zstreamdump.c b/cddl/contrib/opensolaris/cmd/zstreamdump/zstreamdump.c
index f6dedc2fa384..83a5b54b001d 100644
--- a/cddl/contrib/opensolaris/cmd/zstreamdump/zstreamdump.c
+++ b/cddl/contrib/opensolaris/cmd/zstreamdump/zstreamdump.c
@@ -125,7 +125,7 @@ read_hdr(dmu_replay_record_t *drr, zio_cksum_t *cksum)
 		    saved_cksum.zc_word[1],
 		    saved_cksum.zc_word[2],
 		    saved_cksum.zc_word[3]);
-		exit(1);
+		return (0);
 	}
 	return (sizeof (*drr));
 }
@@ -346,8 +346,7 @@ main(int argc, char *argv[])
 			if (verbose)
 				(void) printf("\n");
 
-			if ((DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) ==
-			    DMU_COMPOUNDSTREAM) && drr->drr_payloadlen != 0) {
+			if (drr->drr_payloadlen != 0) {
 				nvlist_t *nv;
 				int sz = drr->drr_payloadlen;
 
diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs.h b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs.h
index 968732ede989..48410408d0f8 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs.h
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs.h
@@ -621,6 +621,10 @@ typedef boolean_t (snapfilter_cb_t)(zfs_handle_t *, void *);
 extern int zfs_send(zfs_handle_t *, const char *, const char *,
     sendflags_t *, int, snapfilter_cb_t, void *, nvlist_t **);
 extern int zfs_send_one(zfs_handle_t *, const char *, int, enum lzc_send_flags);
+extern int zfs_send_resume(libzfs_handle_t *, sendflags_t *, int outfd,
+    const char *);
+extern nvlist_t *zfs_send_resume_token_to_nvlist(libzfs_handle_t *hdl,
+    const char *token);
 
 extern int zfs_promote(zfs_handle_t *);
 extern int zfs_hold(zfs_handle_t *, const char *, const char *,
@@ -661,6 +665,12 @@ typedef struct recvflags {
 	/* set "canmount=off" on all modified filesystems */
 	boolean_t canmountoff;
 
+	/*
+	 * Mark the file systems as "resumable" and do not destroy them if the
+	 * receive is interrupted
+	 */
+	boolean_t resumable;
+
 	/* byteswap flag is used internally; callers need not specify */
 	boolean_t byteswap;
 
diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_compat.c b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_compat.c
index a3f6129876c9..833e1b675741 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_compat.c
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_compat.c
@@ -74,6 +74,9 @@ zcmd_ioctl(int fd, int request, zfs_cmd_t *zc)
 
 	if (zfs_ioctl_version >= ZFS_IOCVER_DEADMAN) {
 		switch (zfs_ioctl_version) {
+		case ZFS_IOCVER_EDBP:
+			cflag = ZFS_CMD_COMPAT_EDBP;
+			break;
 		case ZFS_IOCVER_ZCMD:
 			cflag = ZFS_CMD_COMPAT_ZCMD;
 			break;
diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_dataset.c b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_dataset.c
index 6d9b68ffde49..f387d65db112 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_dataset.c
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_dataset.c
@@ -1778,22 +1778,21 @@ getprop_uint64(zfs_handle_t *zhp, zfs_prop_t prop, char **source)
 	return (value);
 }
 
-static char *
+static const char *
 getprop_string(zfs_handle_t *zhp, zfs_prop_t prop, char **source)
 {
 	nvlist_t *nv;
-	char *value;
+	const char *value;
 
 	*source = NULL;
 	if (nvlist_lookup_nvlist(zhp->zfs_props,
 	    zfs_prop_to_name(prop), &nv) == 0) {
-		verify(nvlist_lookup_string(nv, ZPROP_VALUE, &value) == 0);
+		value = fnvlist_lookup_string(nv, ZPROP_VALUE);
 		(void) nvlist_lookup_string(nv, ZPROP_SOURCE, source);
 	} else {
 		verify(!zhp->zfs_props_table ||
 		    zhp->zfs_props_table[prop] == B_TRUE);
-		if ((value = (char *)zfs_prop_default_string(prop)) == NULL)
-			value = "";
+		value = zfs_prop_default_string(prop);
 		*source = "";
 	}
 
@@ -2195,7 +2194,7 @@ zfs_prop_get(zfs_handle_t *zhp, zfs_prop_t prop, char *propbuf, size_t proplen,
 {
 	char *source = NULL;
 	uint64_t val;
-	char *str;
+	const char *str;
 	const char *strval;
 	boolean_t received = zfs_is_recvd_props_mode(zhp);
 
@@ -2300,14 +2299,10 @@ zfs_prop_get(zfs_handle_t *zhp, zfs_prop_t prop, char *propbuf, size_t proplen,
 		break;
 
 	case ZFS_PROP_ORIGIN:
-		(void) strlcpy(propbuf, getprop_string(zhp, prop, &source),
-		    proplen);
-		/*
-		 * If there is no parent at all, return failure to indicate that
-		 * it doesn't apply to this dataset.
-		 */
-		if (propbuf[0] == '\0')
+		str = getprop_string(zhp, prop, &source);
+		if (str == NULL)
 			return (-1);
+		(void) strlcpy(propbuf, str, proplen);
 		break;
 
 	case ZFS_PROP_CLONES:
@@ -2488,8 +2483,10 @@ zfs_prop_get(zfs_handle_t *zhp, zfs_prop_t prop, char *propbuf, size_t proplen,
 			break;
 
 		case PROP_TYPE_STRING:
-			(void) strlcpy(propbuf,
-			    getprop_string(zhp, prop, &source), proplen);
+			str = getprop_string(zhp, prop, &source);
+			if (str == NULL)
+				return (-1);
+			(void) strlcpy(propbuf, str, proplen);
 			break;
 
 		case PROP_TYPE_INDEX:
diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_mount.c b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_mount.c
index 6c115153bf07..f03da991ed66 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_mount.c
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_mount.c
@@ -1072,6 +1072,17 @@ mount_cb(zfs_handle_t *zhp, void *data)
 		return (0);
 	}
 
+	/*
+	 * If this filesystem is inconsistent and has a receive resume
+	 * token, we can not mount it.
+	 */
+	if (zfs_prop_get_int(zhp, ZFS_PROP_INCONSISTENT) &&
+	    zfs_prop_get(zhp, ZFS_PROP_RECEIVE_RESUME_TOKEN,
+	    NULL, 0, NULL, NULL, 0, B_TRUE) == 0) {
+		zfs_close(zhp);
+		return (0);
+	}
+
 	libzfs_add_handle(cbp, zhp);
 	if (zfs_iter_filesystems(zhp, mount_cb, cbp) != 0) {
 		zfs_close(zhp);
diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_sendrecv.c b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_sendrecv.c
index 51388814bd07..d10bc0a18c4f 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_sendrecv.c
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_sendrecv.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
  * Copyright (c) 2012 Pawel Jakub Dawidek <pawel@dawidek.net>.
  * All rights reserved.
@@ -51,6 +51,7 @@
 #include "zfs_prop.h"
 #include "zfs_fletcher.h"
 #include "libzfs_impl.h"
+#include <zlib.h>
 #include <sha2.h>
 #include <sys/zio_checksum.h>
 #include <sys/ddt.h>
@@ -67,6 +68,8 @@ extern void zfs_setprop_error(libzfs_handle_t *, zfs_prop_t, int, char *);
 static int zfs_receive_impl(libzfs_handle_t *, const char *, const char *,
     recvflags_t *, int, const char *, nvlist_t *, avl_tree_t *, char **, int,
     uint64_t *);
+static int guid_to_name(libzfs_handle_t *, const char *,
+    uint64_t, boolean_t, char *);
 
 static const zio_cksum_t zero_cksum = { 0 };
 
@@ -284,8 +287,7 @@ cksummer(void *arg)
 			    DMU_BACKUP_FEATURE_DEDUPPROPS);
 			DMU_SET_FEATUREFLAGS(drrb->drr_versioninfo, fflags);
 
-			if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) ==
-			    DMU_COMPOUNDSTREAM && drr->drr_payloadlen != 0) {
+			if (drr->drr_payloadlen != 0) {
 				sz = drr->drr_payloadlen;
 
 				if (sz > SPA_MAXBLOCKSIZE) {
@@ -995,17 +997,14 @@ static void *
 send_progress_thread(void *arg)
 {
 	progress_arg_t *pa = arg;
-
 	zfs_cmd_t zc = { 0 };
 	zfs_handle_t *zhp = pa->pa_zhp;
 	libzfs_handle_t *hdl = zhp->zfs_hdl;
 	unsigned long long bytes;
 	char buf[16];
-
 	time_t t;
 	struct tm *tm;
 
-	assert(zhp->zfs_type == ZFS_TYPE_SNAPSHOT);
 	(void) strlcpy(zc.zc_name, zhp->zfs_name, sizeof (zc.zc_name));
 
 	if (!pa->pa_parsable)
@@ -1038,6 +1037,51 @@ send_progress_thread(void *arg)
 	}
 }
 
+static void
+send_print_verbose(FILE *fout, const char *tosnap, const char *fromsnap,
+    uint64_t size, boolean_t parsable)
+{
+	if (parsable) {
+		if (fromsnap != NULL) {
+			(void) fprintf(fout, "incremental\t%s\t%s",
+			    fromsnap, tosnap);
+		} else {
+			(void) fprintf(fout, "full\t%s",
+			    tosnap);
+		}
+	} else {
+		if (fromsnap != NULL) {
+			if (strchr(fromsnap, '@') == NULL &&
+			    strchr(fromsnap, '#') == NULL) {
+				(void) fprintf(fout, dgettext(TEXT_DOMAIN,
+				    "send from @%s to %s"),
+				    fromsnap, tosnap);
+			} else {
+				(void) fprintf(fout, dgettext(TEXT_DOMAIN,
+				    "send from %s to %s"),
+				    fromsnap, tosnap);
+			}
+		} else {
+			(void) fprintf(fout, dgettext(TEXT_DOMAIN,
+			    "full send of %s"),
+			    tosnap);
+		}
+	}
+
+	if (size != 0) {
+		if (parsable) {
+			(void) fprintf(fout, "\t%llu",
+			    (longlong_t)size);
+		} else {
+			char buf[16];
+			zfs_nicenum(size, buf, sizeof (buf));
+			(void) fprintf(fout, dgettext(TEXT_DOMAIN,
+			    " estimated size is %s"), buf);
+		}
+	}
+	(void) fprintf(fout, "\n");
+}
+
 static int
 dump_snapshot(zfs_handle_t *zhp, void *arg)
 {
@@ -1117,37 +1161,14 @@ dump_snapshot(zfs_handle_t *zhp, void *arg)
 	    (sdd->fromorigin || sdd->replicate);
 
 	if (sdd->verbose) {
-		uint64_t size;
-		err = estimate_ioctl(zhp, sdd->prevsnap_obj,
+		uint64_t size = 0;
+		(void) estimate_ioctl(zhp, sdd->prevsnap_obj,
 		    fromorigin, &size);
 
-		if (sdd->parsable) {
-			if (sdd->prevsnap[0] != '\0') {
-				(void) fprintf(fout, "incremental\t%s\t%s",
-				    sdd->prevsnap, zhp->zfs_name);
-			} else {
-				(void) fprintf(fout, "full\t%s",
-				    zhp->zfs_name);
-			}
-		} else {
-			(void) fprintf(fout, dgettext(TEXT_DOMAIN,
-			    "send from @%s to %s"),
-			    sdd->prevsnap, zhp->zfs_name);
-		}
-		if (err == 0) {
-			if (sdd->parsable) {
-				(void) fprintf(fout, "\t%llu\n",
-				    (longlong_t)size);
-			} else {
-				char buf[16];
-				zfs_nicenum(size, buf, sizeof (buf));
-				(void) fprintf(fout, dgettext(TEXT_DOMAIN,
-				    " estimated size is %s\n"), buf);
-			}
-			sdd->size += size;
-		} else {
-			(void) fprintf(fout, "\n");
-		}
+		send_print_verbose(fout, zhp->zfs_name,
+		    sdd->prevsnap[0] ? sdd->prevsnap : NULL,
+		    size, sdd->parsable);
+		sdd->size += size;
 	}
 
 	if (!sdd->dryrun) {
@@ -1358,6 +1379,233 @@ again:
 	return (0);
 }
 
+nvlist_t *
+zfs_send_resume_token_to_nvlist(libzfs_handle_t *hdl, const char *token)
+{
+	unsigned int version;
+	int nread;
+	unsigned long long checksum, packed_len;
+
+	/*
+	 * Decode token header, which is:
+	 *   <token version>-<checksum of payload>-<uncompressed payload length>
+	 * Note that the only supported token version is 1.
+	 */
+	nread = sscanf(token, "%u-%llx-%llx-",
+	    &version, &checksum, &packed_len);
+	if (nread != 3) {
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "resume token is corrupt (invalid format)"));
+		return (NULL);
+	}
+
+	if (version != ZFS_SEND_RESUME_TOKEN_VERSION) {
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "resume token is corrupt (invalid version %u)"),
+		    version);
+		return (NULL);
+	}
+
+	/* convert hexadecimal representation to binary */
+	token = strrchr(token, '-') + 1;
+	int len = strlen(token) / 2;
+	unsigned char *compressed = zfs_alloc(hdl, len);
+	for (int i = 0; i < len; i++) {
+		nread = sscanf(token + i * 2, "%2hhx", compressed + i);
+		if (nread != 1) {
+			free(compressed);
+			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+			    "resume token is corrupt "
+			    "(payload is not hex-encoded)"));
+			return (NULL);
+		}
+	}
+
+	/* verify checksum */
+	zio_cksum_t cksum;
+	fletcher_4_native(compressed, len, &cksum);
+	if (cksum.zc_word[0] != checksum) {
+		free(compressed);
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "resume token is corrupt (incorrect checksum)"));
+		return (NULL);
+	}
+
+	/* uncompress */
+	void *packed = zfs_alloc(hdl, packed_len);
+	uLongf packed_len_long = packed_len;
+	if (uncompress(packed, &packed_len_long, compressed, len) != Z_OK ||
+	    packed_len_long != packed_len) {
+		free(packed);
+		free(compressed);
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "resume token is corrupt (decompression failed)"));
+		return (NULL);
+	}
+
+	/* unpack nvlist */
+	nvlist_t *nv;
+	int error = nvlist_unpack(packed, packed_len, &nv, KM_SLEEP);
+	free(packed);
+	free(compressed);
+	if (error != 0) {
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "resume token is corrupt (nvlist_unpack failed)"));
+		return (NULL);
+	}
+	return (nv);
+}
+
+int
+zfs_send_resume(libzfs_handle_t *hdl, sendflags_t *flags, int outfd,
+    const char *resume_token)
+{
+	char errbuf[1024];
+	char *toname;
+	char *fromname = NULL;
+	uint64_t resumeobj, resumeoff, toguid, fromguid, bytes;
+	zfs_handle_t *zhp;
+	int error = 0;
+	char name[ZFS_MAXNAMELEN];
+	enum lzc_send_flags lzc_flags = 0;
+
+	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
+	    "cannot resume send"));
+
+	nvlist_t *resume_nvl =
+	    zfs_send_resume_token_to_nvlist(hdl, resume_token);
+	if (resume_nvl == NULL) {
+		/*
+		 * zfs_error_aux has already been set by
+		 * zfs_send_resume_token_to_nvlist
+		 */
+		return (zfs_error(hdl, EZFS_FAULT, errbuf));
+	}
+	if (flags->verbose) {
+		(void) fprintf(stderr, dgettext(TEXT_DOMAIN,
+		    "resume token contents:\n"));
+		nvlist_print(stderr, resume_nvl);
+	}
+
+	if (nvlist_lookup_string(resume_nvl, "toname", &toname) != 0 ||
+	    nvlist_lookup_uint64(resume_nvl, "object", &resumeobj) != 0 ||
+	    nvlist_lookup_uint64(resume_nvl, "offset", &resumeoff) != 0 ||
+	    nvlist_lookup_uint64(resume_nvl, "bytes", &bytes) != 0 ||
+	    nvlist_lookup_uint64(resume_nvl, "toguid", &toguid) != 0) {
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "resume token is corrupt"));
+		return (zfs_error(hdl, EZFS_FAULT, errbuf));
+	}
+	fromguid = 0;
+	(void) nvlist_lookup_uint64(resume_nvl, "fromguid", &fromguid);
+
+	if (flags->embed_data || nvlist_exists(resume_nvl, "embedok"))
+		lzc_flags |= LZC_SEND_FLAG_EMBED_DATA;
+
+	if (guid_to_name(hdl, toname, toguid, B_FALSE, name) != 0) {
+		if (zfs_dataset_exists(hdl, toname, ZFS_TYPE_DATASET)) {
+			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+			    "'%s' is no longer the same snapshot used in "
+			    "the initial send"), toname);
+		} else {
+			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+			    "'%s' used in the initial send no longer exists"),
+			    toname);
+		}
+		return (zfs_error(hdl, EZFS_BADPATH, errbuf));
+	}
+	zhp = zfs_open(hdl, name, ZFS_TYPE_DATASET);
+	if (zhp == NULL) {
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "unable to access '%s'"), name);
+		return (zfs_error(hdl, EZFS_BADPATH, errbuf));
+	}
+
+	if (fromguid != 0) {
+		if (guid_to_name(hdl, toname, fromguid, B_TRUE, name) != 0) {
+			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+			    "incremental source %#llx no longer exists"),
+			    (longlong_t)fromguid);
+			return (zfs_error(hdl, EZFS_BADPATH, errbuf));
+		}
+		fromname = name;
+	}
+
+	if (flags->verbose) {
+		uint64_t size = 0;
+		error = lzc_send_space(zhp->zfs_name, fromname, &size);
+		if (error == 0)
+			size = MAX(0, (int64_t)(size - bytes));
+		send_print_verbose(stderr, zhp->zfs_name, fromname,
+		    size, flags->parsable);
+	}
+
+	if (!flags->dryrun) {
+		progress_arg_t pa = { 0 };
+		pthread_t tid;
+		/*
+		 * If progress reporting is requested, spawn a new thread to
+		 * poll ZFS_IOC_SEND_PROGRESS at a regular interval.
+		 */
+		if (flags->progress) {
+			pa.pa_zhp = zhp;
+			pa.pa_fd = outfd;
+			pa.pa_parsable = flags->parsable;
+
+			error = pthread_create(&tid, NULL,
+			    send_progress_thread, &pa);
+			if (error != 0) {
+				zfs_close(zhp);
+				return (error);
+			}
+		}
+
+		error = lzc_send_resume(zhp->zfs_name, fromname, outfd,
+		    lzc_flags, resumeobj, resumeoff);
+
+		if (flags->progress) {
+			(void) pthread_cancel(tid);
+			(void) pthread_join(tid, NULL);
+		}
+
+		char errbuf[1024];
+		(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
+		    "warning: cannot send '%s'"), zhp->zfs_name);
+
+		zfs_close(zhp);
+
+		switch (error) {
+		case 0:
+			return (0);
+		case EXDEV:
+		case ENOENT:
+		case EDQUOT:
+		case EFBIG:
+		case EIO:
+		case ENOLINK:
+		case ENOSPC:
+#ifdef illumos
+		case ENOSTR:
+#endif
+		case ENXIO:
+		case EPIPE:
+		case ERANGE:
+		case EFAULT:
+		case EROFS:
+			zfs_error_aux(hdl, strerror(errno));
+			return (zfs_error(hdl, EZFS_BADBACKUP, errbuf));
+
+		default:
+			return (zfs_standard_error(hdl, errno, errbuf));
+		}
+	}
+
+
+	zfs_close(zhp);
+
+	return (error);
+}
+
 /*
  * Generate a send stream for the dataset identified by the argument zhp.
  *
@@ -1897,6 +2145,7 @@ recv_destroy(libzfs_handle_t *hdl, const char *name, int baselen,
 
 typedef struct guid_to_name_data {
 	uint64_t guid;
+	boolean_t bookmark_ok;
 	char *name;
 	char *skip;
 } guid_to_name_data_t;
@@ -1905,20 +2154,25 @@ static int
 guid_to_name_cb(zfs_handle_t *zhp, void *arg)
 {
 	guid_to_name_data_t *gtnd = arg;
+	const char *slash;
 	int err;
 
 	if (gtnd->skip != NULL &&
-	    strcmp(zhp->zfs_name, gtnd->skip) == 0) {
+	    (slash = strrchr(zhp->zfs_name, '/')) != NULL &&
+	    strcmp(slash + 1, gtnd->skip) == 0) {
+		zfs_close(zhp);
 		return (0);
 	}
 
-	if (zhp->zfs_dmustats.dds_guid == gtnd->guid) {
+	if (zfs_prop_get_int(zhp, ZFS_PROP_GUID) == gtnd->guid) {
 		(void) strcpy(gtnd->name, zhp->zfs_name);
 		zfs_close(zhp);
 		return (EEXIST);
 	}
 
 	err = zfs_iter_children(zhp, guid_to_name_cb, gtnd);
+	if (err != EEXIST && gtnd->bookmark_ok)
+		err = zfs_iter_bookmarks(zhp, guid_to_name_cb, gtnd);
 	zfs_close(zhp);
 	return (err);
 }
@@ -1932,45 +2186,48 @@ guid_to_name_cb(zfs_handle_t *zhp, void *arg)
  */
 static int
 guid_to_name(libzfs_handle_t *hdl, const char *parent, uint64_t guid,
-    char *name)
+    boolean_t bookmark_ok, char *name)
 {
-	/* exhaustive search all local snapshots */
 	char pname[ZFS_MAXNAMELEN];
 	guid_to_name_data_t gtnd;
-	int err = 0;
-	zfs_handle_t *zhp;
-	char *cp;
 
 	gtnd.guid = guid;
+	gtnd.bookmark_ok = bookmark_ok;
 	gtnd.name = name;
 	gtnd.skip = NULL;
 
-	(void) strlcpy(pname, parent, sizeof (pname));
-
 	/*
-	 * Search progressively larger portions of the hierarchy.  This will
+	 * Search progressively larger portions of the hierarchy, starting
+	 * with the filesystem specified by 'parent'.  This will
 	 * select the "most local" version of the origin snapshot in the case
 	 * that there are multiple matching snapshots in the system.
 	 */
-	while ((cp = strrchr(pname, '/')) != NULL) {
-
+	(void) strlcpy(pname, parent, sizeof (pname));
+	char *cp = strrchr(pname, '@');
+	if (cp == NULL)
+		cp = strchr(pname, '\0');
+	for (; cp != NULL; cp = strrchr(pname, '/')) {
 		/* Chop off the last component and open the parent */
 		*cp = '\0';
-		zhp = make_dataset_handle(hdl, pname);
+		zfs_handle_t *zhp = make_dataset_handle(hdl, pname);
 
 		if (zhp == NULL)
 			continue;
-
-		err = zfs_iter_children(zhp, guid_to_name_cb, &gtnd);
+		int err = guid_to_name_cb(zfs_handle_dup(zhp), &gtnd);
+		if (err != EEXIST)
+			err = zfs_iter_children(zhp, guid_to_name_cb, &gtnd);
+		if (err != EEXIST && bookmark_ok)
+			err = zfs_iter_bookmarks(zhp, guid_to_name_cb, &gtnd);
 		zfs_close(zhp);
 		if (err == EEXIST)
 			return (0);
 
 		/*
-		 * Remember the dataset that we already searched, so we
-		 * skip it next time through.
+		 * Remember the last portion of the dataset so we skip it next
+		 * time through (as we've already searched that portion of the
+		 * hierarchy).
 		 */
-		gtnd.skip = pname;
+		gtnd.skip = strrchr(pname, '/') + 1;
 	}
 
 	return (ENOENT);
@@ -2568,11 +2825,9 @@ recv_skip(libzfs_handle_t *hdl, int fd, boolean_t byteswap)
 
 		switch (drr->drr_type) {
 		case DRR_BEGIN:
-			/* NB: not to be used on v2 stream packages */
 			if (drr->drr_payloadlen != 0) {
-				zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
-				    "invalid substream header"));
-				return (zfs_error(hdl, EZFS_BADSTREAM, errbuf));
+				(void) recv_read(hdl, fd, buf,
+				    drr->drr_payloadlen, B_FALSE, NULL);
 			}
 			break;
 
@@ -2633,6 +2888,40 @@ recv_skip(libzfs_handle_t *hdl, int fd, boolean_t byteswap)
 	return (-1);
 }
 
+static void
+recv_ecksum_set_aux(libzfs_handle_t *hdl, const char *target_snap,
+    boolean_t resumable)
+{
+	char target_fs[ZFS_MAXNAMELEN];
+
+	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+	    "checksum mismatch or incomplete stream"));
+
+	if (!resumable)
+		return;
+	(void) strlcpy(target_fs, target_snap, sizeof (target_fs));
+	*strchr(target_fs, '@') = '\0';
+	zfs_handle_t *zhp = zfs_open(hdl, target_fs,
+	    ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME);
+	if (zhp == NULL)
+		return;
+
+	char token_buf[ZFS_MAXPROPLEN];
+	int error = zfs_prop_get(zhp, ZFS_PROP_RECEIVE_RESUME_TOKEN,
+	    token_buf, sizeof (token_buf),
+	    NULL, NULL, 0, B_TRUE);
+	if (error == 0) {
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "checksum mismatch or incomplete stream.\n"
+		    "Partially received snapshot is saved.\n"
+		    "A resuming stream can be generated on the sending "
+		    "system by running:\n"
+		    "    zfs send -t %s"),
+		    token_buf);
+	}
+	zfs_close(zhp);
+}
+
 /*
  * Restores a backup of tosnap from the file descriptor specified by infd.
  */
@@ -2796,7 +3085,7 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap,
 	 */
 	if (drrb->drr_flags & DRR_FLAG_CLONE) {
 		if (guid_to_name(hdl, zc.zc_value,
-		    drrb->drr_fromguid, zc.zc_string) != 0) {
+		    drrb->drr_fromguid, B_FALSE, zc.zc_string) != 0) {
 			zcmd_free_nvlists(&zc);
 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
 			    "local origin for clone %s does not exist"),
@@ -2812,8 +3101,10 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap,
 			    zc.zc_string);
 	}
 
+	boolean_t resuming = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) &
+	    DMU_BACKUP_FEATURE_RESUMING;
 	stream_wantsnewfs = (drrb->drr_fromguid == 0 ||
-	    (drrb->drr_flags & DRR_FLAG_CLONE) || originsnap);
+	    (drrb->drr_flags & DRR_FLAG_CLONE) || originsnap) && !resuming;
 
 	if (stream_wantsnewfs) {
 		/*
@@ -2832,7 +3123,7 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap,
 			char suffix[ZFS_MAXNAMELEN];
 			(void) strcpy(suffix, strrchr(zc.zc_value, '/'));
 			if (guid_to_name(hdl, zc.zc_name, parent_snapguid,
-			    zc.zc_value) == 0) {
+			    B_FALSE, zc.zc_value) == 0) {
 				*strchr(zc.zc_value, '@') = '\0';
 				(void) strcat(zc.zc_value, suffix);
 			}
@@ -2859,7 +3150,7 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap,
 			char snap[ZFS_MAXNAMELEN];
 			(void) strcpy(snap, strchr(zc.zc_value, '@'));
 			if (guid_to_name(hdl, zc.zc_name, drrb->drr_fromguid,
-			    zc.zc_value) == 0) {
+			    B_FALSE, zc.zc_value) == 0) {
 				*strchr(zc.zc_value, '@') = '\0';
 				(void) strcat(zc.zc_value, snap);
 			}
@@ -2873,11 +3164,12 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap,
 		zfs_handle_t *zhp;
 
 		/*
-		 * Destination fs exists.  Therefore this should either
-		 * be an incremental, or the stream specifies a new fs
-		 * (full stream or clone) and they want us to blow it
-		 * away (and have therefore specified -F and removed any
-		 * snapshots).
+		 * Destination fs exists.  It must be one of these cases:
+		 *  - an incremental send stream
+		 *  - the stream specifies a new fs (full stream or clone)
+		 *    and they want us to blow away the existing fs (and
+		 *    have therefore specified -F and removed any snapshots)
+		 *  - we are resuming a failed receive.
 		 */
 		if (stream_wantsnewfs) {
 			if (!flags->force) {
@@ -2932,6 +3224,18 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap,
 				return (-1);
 			}
 		}
+
+		/*
+		 * If we are resuming a newfs, set newfs here so that we will
+		 * mount it if the recv succeeds this time.  We can tell
+		 * that it was a newfs on the first recv because the fs
+		 * itself will be inconsistent (if the fs existed when we
+		 * did the first recv, we would have received it into
+		 * .../%recv).
+		 */
+		if (resuming && zfs_prop_get_int(zhp, ZFS_PROP_INCONSISTENT))
+			newfs = B_TRUE;
+
 		zfs_close(zhp);
 	} else {
 		/*
@@ -2964,9 +3268,10 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap,
 		newfs = B_TRUE;
 	}
 
-	zc.zc_begin_record = drr_noswap->drr_u.drr_begin;
+	zc.zc_begin_record = *drr_noswap;
 	zc.zc_cookie = infd;
 	zc.zc_guid = flags->force;
+	zc.zc_resumable = flags->resumable;
 	if (flags->verbose) {
 		(void) printf("%s %s stream of %s into %s\n",
 		    flags->dryrun ? "would receive" : "receiving",
@@ -3103,8 +3408,7 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap,
 			(void) zfs_error(hdl, EZFS_BADSTREAM, errbuf);
 			break;
 		case ECKSUM:
-			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
-			    "invalid stream (checksum mismatch)"));
+			recv_ecksum_set_aux(hdl, zc.zc_value, flags->resumable);
 			(void) zfs_error(hdl, EZFS_BADSTREAM, errbuf);
 			break;
 		case ENOTSUP:
@@ -3306,7 +3610,8 @@ zfs_receive_impl(libzfs_handle_t *hdl, const char *tosnap,
  * Restores a backup of tosnap from the file descriptor specified by infd.
  * Return 0 on total success, -2 if some things couldn't be
  * destroyed/renamed/promoted, -1 if some things couldn't be received.
- * (-1 will override -2).
+ * (-1 will override -2, if -1 and the resumable flag was specified the
+ * transfer can be resumed if the sending side supports it).
  */
 int
 zfs_receive(libzfs_handle_t *hdl, const char *tosnap, nvlist_t *props,
diff --git a/cddl/contrib/opensolaris/lib/libzfs_core/common/libzfs_core.c b/cddl/contrib/opensolaris/lib/libzfs_core/common/libzfs_core.c
index 5ba660de6ba5..69d332a5f8eb 100644
--- a/cddl/contrib/opensolaris/lib/libzfs_core/common/libzfs_core.c
+++ b/cddl/contrib/opensolaris/lib/libzfs_core/common/libzfs_core.c
@@ -514,6 +514,13 @@ lzc_get_holds(const char *snapname, nvlist_t **holdsp)
 int
 lzc_send(const char *snapname, const char *from, int fd,
     enum lzc_send_flags flags)
+{
+	return (lzc_send_resume(snapname, from, fd, flags, 0, 0));
+}
+
+int
+lzc_send_resume(const char *snapname, const char *from, int fd,
+    enum lzc_send_flags flags, uint64_t resumeobj, uint64_t resumeoff)
 {
 	nvlist_t *args;
 	int err;
@@ -526,6 +533,10 @@ lzc_send(const char *snapname, const char *from, int fd,
 		fnvlist_add_boolean(args, "largeblockok");
 	if (flags & LZC_SEND_FLAG_EMBED_DATA)
 		fnvlist_add_boolean(args, "embedok");
+	if (resumeobj != 0 || resumeoff != 0) {
+		fnvlist_add_uint64(args, "resume_object", resumeobj);
+		fnvlist_add_uint64(args, "resume_offset", resumeoff);
+	}
 	err = lzc_ioctl(ZFS_IOC_SEND_NEW, snapname, args, NULL);
 	nvlist_free(args);
 	return (err);
@@ -583,22 +594,9 @@ recv_read(int fd, void *buf, int ilen)
 	return (0);
 }
 
-/*
- * The simplest receive case: receive from the specified fd, creating the
- * specified snapshot.  Apply the specified properties a "received" properties
- * (which can be overridden by locally-set properties).  If the stream is a
- * clone, its origin snapshot must be specified by 'origin'.  The 'force'
- * flag will cause the target filesystem to be rolled back or destroyed if
- * necessary to receive.
- *
- * Return 0 on success or an errno on failure.
- *
- * Note: this interface does not work on dedup'd streams
- * (those with DMU_BACKUP_FEATURE_DEDUP).
- */
-int
-lzc_receive(const char *snapname, nvlist_t *props, const char *origin,
-    boolean_t force, int fd)
+static int
+lzc_receive_impl(const char *snapname, nvlist_t *props, const char *origin,
+    boolean_t force, boolean_t resumable, int fd)
 {
 	/*
 	 * The receive ioctl is still legacy, so we need to construct our own
@@ -608,7 +606,6 @@ lzc_receive(const char *snapname, nvlist_t *props, const char *origin,
 	char *atp;
 	char *packed = NULL;
 	size_t size;
-	dmu_replay_record_t drr;
 	int error;
 
 	ASSERT3S(g_refcount, >, 0);
@@ -644,10 +641,9 @@ lzc_receive(const char *snapname, nvlist_t *props, const char *origin,
 		(void) strlcpy(zc.zc_string, origin, sizeof (zc.zc_string));
 
 	/* zc_begin_record is non-byteswapped BEGIN record */
-	error = recv_read(fd, &drr, sizeof (drr));
+	error = recv_read(fd, &zc.zc_begin_record, sizeof (zc.zc_begin_record));
 	if (error != 0)
 		goto out;
-	zc.zc_begin_record = drr.drr_u.drr_begin;
 
 	/* zc_cookie is fd to read from */
 	zc.zc_cookie = fd;
@@ -655,6 +651,8 @@ lzc_receive(const char *snapname, nvlist_t *props, const char *origin,
 	/* zc guid is force flag */
 	zc.zc_guid = force;
 
+	zc.zc_resumable = resumable;
+
 	/* zc_cleanup_fd is unused */
 	zc.zc_cleanup_fd = -1;
 
@@ -669,6 +667,39 @@ out:
 	return (error);
 }
 
+/*
+ * The simplest receive case: receive from the specified fd, creating the
+ * specified snapshot.  Apply the specified properties as "received" properties
+ * (which can be overridden by locally-set properties).  If the stream is a
+ * clone, its origin snapshot must be specified by 'origin'.  The 'force'
+ * flag will cause the target filesystem to be rolled back or destroyed if
+ * necessary to receive.
+ *
+ * Return 0 on success or an errno on failure.
+ *
+ * Note: this interface does not work on dedup'd streams
+ * (those with DMU_BACKUP_FEATURE_DEDUP).
+ */
+int
+lzc_receive(const char *snapname, nvlist_t *props, const char *origin,
+    boolean_t force, int fd)
+{
+	return (lzc_receive_impl(snapname, props, origin, force, B_FALSE, fd));
+}
+
+/*
+ * Like lzc_receive, but if the receive fails due to premature stream
+ * termination, the intermediate state will be preserved on disk.  In this
+ * case, ECKSUM will be returned.  The receive may subsequently be resumed
+ * with a resuming send stream generated by lzc_send_resume().
+ */
+int
+lzc_receive_resumable(const char *snapname, nvlist_t *props, const char *origin,
+    boolean_t force, int fd)
+{
+	return (lzc_receive_impl(snapname, props, origin, force, B_TRUE, fd));
+}
+
 /*
  * Roll back this filesystem or volume to its most recent snapshot.
  * If snapnamebuf is not NULL, it will be filled in with the name
diff --git a/cddl/contrib/opensolaris/lib/libzfs_core/common/libzfs_core.h b/cddl/contrib/opensolaris/lib/libzfs_core/common/libzfs_core.h
index b6a4c12f2500..b9235d1eb10b 100644
--- a/cddl/contrib/opensolaris/lib/libzfs_core/common/libzfs_core.h
+++ b/cddl/contrib/opensolaris/lib/libzfs_core/common/libzfs_core.h
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Martin Matuska <mm@FreeBSD.org>. All rights reserved.
  */
 
@@ -59,7 +59,11 @@ enum lzc_send_flags {
 };
 
 int lzc_send(const char *, const char *, int, enum lzc_send_flags);
+int lzc_send_resume(const char *, const char *, int,
+    enum lzc_send_flags, uint64_t, uint64_t);
 int lzc_receive(const char *, nvlist_t *, const char *, boolean_t, int);
+int lzc_receive_resumable(const char *, nvlist_t *, const char *,
+    boolean_t, int);
 int lzc_send_space(const char *, const char *, uint64_t *);
 
 boolean_t lzc_exists(const char *);
diff --git a/cddl/lib/libzfs/Makefile b/cddl/lib/libzfs/Makefile
index 19a3f2e1b1b5..b960c9e97d9f 100644
--- a/cddl/lib/libzfs/Makefile
+++ b/cddl/lib/libzfs/Makefile
@@ -8,10 +8,10 @@
 LIB=	zfs
 DPADD=	${LIBMD} ${LIBPTHREAD} ${LIBUMEM} ${LIBUTIL} ${LIBM} ${LIBNVPAIR} \
 	${LIBAVL} ${LIBZFS_CORE} ${LIBUUTIL} ${LIBBSDXML} ${LIBGEOM} \
-	${LIBNVPAIR}
+	${LIBNVPAIR} ${LIBZ}
 
 LDADD=	-lmd -lpthread -lumem -lutil -luutil -lm -lnvpair -lavl \
-	-lbsdxml -lgeom -lnvpair -lzfs_core
+	-lbsdxml -lgeom -lnvpair -lz -lzfs_core
 
 SRCS=	deviceid.c \
 	fsshare.c \
diff --git a/sys/cddl/contrib/opensolaris/common/zfs/zfs_ioctl_compat.c b/sys/cddl/contrib/opensolaris/common/zfs/zfs_ioctl_compat.c
index 62fed7b6ef27..4cb1804aad66 100644
--- a/sys/cddl/contrib/opensolaris/common/zfs/zfs_ioctl_compat.c
+++ b/sys/cddl/contrib/opensolaris/common/zfs/zfs_ioctl_compat.c
@@ -53,8 +53,51 @@ zfs_cmd_compat_get(zfs_cmd_t *zc, caddr_t addr, const int cflag)
 	zfs_cmd_v28_t *zc28_c;
 	zfs_cmd_deadman_t *zcdm_c;
 	zfs_cmd_zcmd_t *zcmd_c;
+	zfs_cmd_edbp_t *edbp_c;
 
 	switch (cflag) {
+	case ZFS_CMD_COMPAT_EDBP:
+		edbp_c = (void *)addr;
+		/* zc */
+		strlcpy(zc->zc_name, edbp_c->zc_name, MAXPATHLEN);
+		strlcpy(zc->zc_value, edbp_c->zc_value, MAXPATHLEN * 2);
+		strlcpy(zc->zc_string, edbp_c->zc_string, MAXPATHLEN);
+
+#define ZCMD_COPY(field) zc->field = edbp_c->field
+		ZCMD_COPY(zc_nvlist_src);
+		ZCMD_COPY(zc_nvlist_src_size);
+		ZCMD_COPY(zc_nvlist_dst);
+		ZCMD_COPY(zc_nvlist_dst_size);
+		ZCMD_COPY(zc_nvlist_dst_filled);
+		ZCMD_COPY(zc_pad2);
+		ZCMD_COPY(zc_history);
+		ZCMD_COPY(zc_guid);
+		ZCMD_COPY(zc_nvlist_conf);
+		ZCMD_COPY(zc_nvlist_conf_size);
+		ZCMD_COPY(zc_cookie);
+		ZCMD_COPY(zc_objset_type);
+		ZCMD_COPY(zc_perm_action);
+		ZCMD_COPY(zc_history_len);
+		ZCMD_COPY(zc_history_offset);
+		ZCMD_COPY(zc_obj);
+		ZCMD_COPY(zc_iflags);
+		ZCMD_COPY(zc_share);
+		ZCMD_COPY(zc_jailid);
+		ZCMD_COPY(zc_objset_stats);
+		ZCMD_COPY(zc_inject_record);
+		ZCMD_COPY(zc_defer_destroy);
+		ZCMD_COPY(zc_flags);
+		ZCMD_COPY(zc_action_handle);
+		ZCMD_COPY(zc_cleanup_fd);
+		ZCMD_COPY(zc_simple);
+		zc->zc_resumable = B_FALSE;
+		ZCMD_COPY(zc_sendobj);
+		ZCMD_COPY(zc_fromobj);
+		ZCMD_COPY(zc_createtxg);
+		ZCMD_COPY(zc_stat);
+#undef ZCMD_COPY
+		break;
+
 	case ZFS_CMD_COMPAT_ZCMD:
 		zcmd_c = (void *)addr;
 		/* zc */
@@ -83,14 +126,6 @@ zfs_cmd_compat_get(zfs_cmd_t *zc, caddr_t addr, const int cflag)
 		ZCMD_COPY(zc_share);
 		ZCMD_COPY(zc_jailid);
 		ZCMD_COPY(zc_objset_stats);
-
-		/*
-		 * zc_begin_record, zc_inject_record didn't change in embedeed-data
-		 * block pointers
-		 *
-		 * TODO: CTASSERT?
-		 */
-		ZCMD_COPY(zc_begin_record);
 		ZCMD_COPY(zc_inject_record);
 
 		/* boolean_t -> uint32_t */
@@ -100,7 +135,7 @@ zfs_cmd_compat_get(zfs_cmd_t *zc, caddr_t addr, const int cflag)
 		ZCMD_COPY(zc_action_handle);
 		ZCMD_COPY(zc_cleanup_fd);
 		ZCMD_COPY(zc_simple);
-		bcopy(zcmd_c->zc_pad, zc->zc_pad, sizeof(zc->zc_pad));
+		zc->zc_resumable = B_FALSE;
 		ZCMD_COPY(zc_sendobj);
 		ZCMD_COPY(zc_fromobj);
 		ZCMD_COPY(zc_createtxg);
@@ -133,13 +168,12 @@ zfs_cmd_compat_get(zfs_cmd_t *zc, caddr_t addr, const int cflag)
 		zc->zc_share = zcdm_c->zc_share;
 		zc->zc_jailid = zcdm_c->zc_jailid;
 		zc->zc_objset_stats = zcdm_c->zc_objset_stats;
-		zc->zc_begin_record = zcdm_c->zc_begin_record;
 		zc->zc_defer_destroy = zcdm_c->zc_defer_destroy;
 		(void)zcdm_c->zc_temphold;
 		zc->zc_action_handle = zcdm_c->zc_action_handle;
 		zc->zc_cleanup_fd = zcdm_c->zc_cleanup_fd;
 		zc->zc_simple = zcdm_c->zc_simple;
-		bcopy(zcdm_c->zc_pad, zc->zc_pad, sizeof(zc->zc_pad));
+		zc->zc_resumable = B_FALSE;
 		zc->zc_sendobj = zcdm_c->zc_sendobj;
 		zc->zc_fromobj = zcdm_c->zc_fromobj;
 		zc->zc_createtxg = zcdm_c->zc_createtxg;
@@ -177,13 +211,12 @@ zfs_cmd_compat_get(zfs_cmd_t *zc, caddr_t addr, const int cflag)
 		zc->zc_share = zc28_c->zc_share;
 		zc->zc_jailid = zc28_c->zc_jailid;
 		zc->zc_objset_stats = zc28_c->zc_objset_stats;
-		zc->zc_begin_record = zc28_c->zc_begin_record;
 		zc->zc_defer_destroy = zc28_c->zc_defer_destroy;
 		(void)zc28_c->zc_temphold;
 		zc->zc_action_handle = zc28_c->zc_action_handle;
 		zc->zc_cleanup_fd = zc28_c->zc_cleanup_fd;
 		zc->zc_simple = zc28_c->zc_simple;
-		bcopy(zc28_c->zc_pad, zc->zc_pad, sizeof(zc->zc_pad));
+		zc->zc_resumable = B_FALSE;
 		zc->zc_sendobj = zc28_c->zc_sendobj;
 		zc->zc_fromobj = zc28_c->zc_fromobj;
 		zc->zc_createtxg = zc28_c->zc_createtxg;
@@ -246,7 +279,6 @@ zfs_cmd_compat_get(zfs_cmd_t *zc, caddr_t addr, const int cflag)
 		zc->zc_share = zc_c->zc_share;
 		zc->zc_jailid = zc_c->zc_jailid;
 		zc->zc_objset_stats = zc_c->zc_objset_stats;
-		zc->zc_begin_record = zc_c->zc_begin_record;
 
 		/* zc->zc_inject_record */
 		zc->zc_inject_record.zi_objset =
@@ -281,8 +313,49 @@ zfs_cmd_compat_put(zfs_cmd_t *zc, caddr_t addr, const int request,
 	zfs_cmd_v28_t *zc28_c;
 	zfs_cmd_deadman_t *zcdm_c;
 	zfs_cmd_zcmd_t *zcmd_c;
+	zfs_cmd_edbp_t *edbp_c;
 
 	switch (cflag) {
+	case ZFS_CMD_COMPAT_EDBP:
+		edbp_c = (void *)addr;
+		strlcpy(edbp_c->zc_name, zc->zc_name, MAXPATHLEN);
+		strlcpy(edbp_c->zc_value, zc->zc_value, MAXPATHLEN * 2);
+		strlcpy(edbp_c->zc_string, zc->zc_string, MAXPATHLEN);
+
+#define ZCMD_COPY(field) edbp_c->field = zc->field
+		ZCMD_COPY(zc_nvlist_src);
+		ZCMD_COPY(zc_nvlist_src_size);
+		ZCMD_COPY(zc_nvlist_dst);
+		ZCMD_COPY(zc_nvlist_dst_size);
+		ZCMD_COPY(zc_nvlist_dst_filled);
+		ZCMD_COPY(zc_pad2);
+		ZCMD_COPY(zc_history);
+		ZCMD_COPY(zc_guid);
+		ZCMD_COPY(zc_nvlist_conf);
+		ZCMD_COPY(zc_nvlist_conf_size);
+		ZCMD_COPY(zc_cookie);
+		ZCMD_COPY(zc_objset_type);
+		ZCMD_COPY(zc_perm_action);
+		ZCMD_COPY(zc_history_len);
+		ZCMD_COPY(zc_history_offset);
+		ZCMD_COPY(zc_obj);
+		ZCMD_COPY(zc_iflags);
+		ZCMD_COPY(zc_share);
+		ZCMD_COPY(zc_jailid);
+		ZCMD_COPY(zc_objset_stats);
+		ZCMD_COPY(zc_inject_record);
+		ZCMD_COPY(zc_defer_destroy);
+		ZCMD_COPY(zc_flags);
+		ZCMD_COPY(zc_action_handle);
+		ZCMD_COPY(zc_cleanup_fd);
+		ZCMD_COPY(zc_simple);
+		ZCMD_COPY(zc_sendobj);
+		ZCMD_COPY(zc_fromobj);
+		ZCMD_COPY(zc_createtxg);
+		ZCMD_COPY(zc_stat);
+#undef ZCMD_COPY
+		break;
+
 	case ZFS_CMD_COMPAT_ZCMD:
 		zcmd_c = (void *)addr;
 		/* zc */
@@ -311,14 +384,6 @@ zfs_cmd_compat_put(zfs_cmd_t *zc, caddr_t addr, const int request,
 		ZCMD_COPY(zc_share);
 		ZCMD_COPY(zc_jailid);
 		ZCMD_COPY(zc_objset_stats);
-
-		/*
-		 * zc_begin_record, zc_inject_record didn't change in embedeed-data
-		 * block pointers
-		 *
-		 * TODO: CTASSERT?
-		 */
-		ZCMD_COPY(zc_begin_record);
 		ZCMD_COPY(zc_inject_record);
 
 		/* boolean_t -> uint32_t */
@@ -328,7 +393,6 @@ zfs_cmd_compat_put(zfs_cmd_t *zc, caddr_t addr, const int request,
 		ZCMD_COPY(zc_action_handle);
 		ZCMD_COPY(zc_cleanup_fd);
 		ZCMD_COPY(zc_simple);
-		bcopy(zc->zc_pad, zcmd_c->zc_pad, sizeof(zcmd_c->zc_pad));
 		ZCMD_COPY(zc_sendobj);
 		ZCMD_COPY(zc_fromobj);
 		ZCMD_COPY(zc_createtxg);
@@ -361,13 +425,11 @@ zfs_cmd_compat_put(zfs_cmd_t *zc, caddr_t addr, const int request,
 		zcdm_c->zc_share = zc->zc_share;
 		zcdm_c->zc_jailid = zc->zc_jailid;
 		zcdm_c->zc_objset_stats = zc->zc_objset_stats;
-		zcdm_c->zc_begin_record = zc->zc_begin_record;
 		zcdm_c->zc_defer_destroy = zc->zc_defer_destroy;
 		zcdm_c->zc_temphold = 0;
 		zcdm_c->zc_action_handle = zc->zc_action_handle;
 		zcdm_c->zc_cleanup_fd = zc->zc_cleanup_fd;
 		zcdm_c->zc_simple = zc->zc_simple;
-		bcopy(zc->zc_pad, zcdm_c->zc_pad, sizeof(zcdm_c->zc_pad));
 		zcdm_c->zc_sendobj = zc->zc_sendobj;
 		zcdm_c->zc_fromobj = zc->zc_fromobj;
 		zcdm_c->zc_createtxg = zc->zc_createtxg;
@@ -407,13 +469,11 @@ zfs_cmd_compat_put(zfs_cmd_t *zc, caddr_t addr, const int request,
 		zc28_c->zc_share = zc->zc_share;
 		zc28_c->zc_jailid = zc->zc_jailid;
 		zc28_c->zc_objset_stats = zc->zc_objset_stats;
-		zc28_c->zc_begin_record = zc->zc_begin_record;
 		zc28_c->zc_defer_destroy = zc->zc_defer_destroy;
 		zc28_c->zc_temphold = 0;
 		zc28_c->zc_action_handle = zc->zc_action_handle;
 		zc28_c->zc_cleanup_fd = zc->zc_cleanup_fd;
 		zc28_c->zc_simple = zc->zc_simple;
-		bcopy(zc->zc_pad, zc28_c->zc_pad, sizeof(zc28_c->zc_pad));
 		zc28_c->zc_sendobj = zc->zc_sendobj;
 		zc28_c->zc_fromobj = zc->zc_fromobj;
 		zc28_c->zc_createtxg = zc->zc_createtxg;
@@ -479,7 +539,6 @@ zfs_cmd_compat_put(zfs_cmd_t *zc, caddr_t addr, const int request,
 		zc_c->zc_share = zc->zc_share;
 		zc_c->zc_jailid = zc->zc_jailid;
 		zc_c->zc_objset_stats = zc->zc_objset_stats;
-		zc_c->zc_begin_record = zc->zc_begin_record;
 
 		/* zc_inject_record */
 		zc_c->zc_inject_record.zi_objset =
@@ -697,6 +756,12 @@ zcmd_ioctl_compat(int fd, int request, zfs_cmd_t *zc, const int cflag)
 		zp.zfs_cmd_size = sizeof(zfs_cmd_t);
 		zp.zfs_ioctl_version = ZFS_IOCVER_CURRENT;
 		return (ioctl(fd, ncmd, &zp));
+	case ZFS_CMD_COMPAT_EDBP:
+		ncmd = _IOWR('Z', request, struct zfs_iocparm);
+		zp.zfs_cmd = (uint64_t)zc;
+		zp.zfs_cmd_size = sizeof(zfs_cmd_edbp_t);
+		zp.zfs_ioctl_version = ZFS_IOCVER_EDBP;
+		return (ioctl(fd, ncmd, &zp));
 	case ZFS_CMD_COMPAT_ZCMD:
 		ncmd = _IOWR('Z', request, struct zfs_iocparm);
 		zp.zfs_cmd = (uint64_t)zc;
@@ -801,7 +866,7 @@ zfs_ioctl_compat_innvl(zfs_cmd_t *zc, nvlist_t * innvl, const int vec,
 	int err;
 
 	if (cflag == ZFS_CMD_COMPAT_NONE || cflag == ZFS_CMD_COMPAT_LZC ||
-	    cflag == ZFS_CMD_COMPAT_ZCMD)
+	    cflag == ZFS_CMD_COMPAT_ZCMD || cflag == ZFS_CMD_COMPAT_EDBP)
 		goto out;
 
 	switch (vec) {
@@ -953,7 +1018,7 @@ zfs_ioctl_compat_outnvl(zfs_cmd_t *zc, nvlist_t * outnvl, const int vec,
 	nvlist_t *tmpnvl;
 
 	if (cflag == ZFS_CMD_COMPAT_NONE || cflag == ZFS_CMD_COMPAT_LZC ||
-	    cflag == ZFS_CMD_COMPAT_ZCMD)
+	    cflag == ZFS_CMD_COMPAT_ZCMD || cflag == ZFS_CMD_COMPAT_EDBP)
 		return (outnvl);
 
 	switch (vec) {
diff --git a/sys/cddl/contrib/opensolaris/common/zfs/zfs_ioctl_compat.h b/sys/cddl/contrib/opensolaris/common/zfs/zfs_ioctl_compat.h
index bdcac6f9c3be..8361aa32b459 100644
--- a/sys/cddl/contrib/opensolaris/common/zfs/zfs_ioctl_compat.h
+++ b/sys/cddl/contrib/opensolaris/common/zfs/zfs_ioctl_compat.h
@@ -52,7 +52,8 @@ extern "C" {
 #define	ZFS_IOCVER_LZC		2
 #define	ZFS_IOCVER_ZCMD		3
 #define	ZFS_IOCVER_EDBP		4
-#define	ZFS_IOCVER_CURRENT	ZFS_IOCVER_EDBP
+#define	ZFS_IOCVER_RESUME	5
+#define	ZFS_IOCVER_CURRENT	ZFS_IOCVER_RESUME
 
 /* compatibility conversion flag */
 #define	ZFS_CMD_COMPAT_NONE	0
@@ -61,6 +62,7 @@ extern "C" {
 #define	ZFS_CMD_COMPAT_DEADMAN	3
 #define	ZFS_CMD_COMPAT_LZC	4
 #define	ZFS_CMD_COMPAT_ZCMD	5
+#define	ZFS_CMD_COMPAT_EDBP	6
 
 #define	ZFS_IOC_COMPAT_PASS	254
 #define	ZFS_IOC_COMPAT_FAIL	255
@@ -246,6 +248,49 @@ typedef struct zfs_cmd_zcmd {
 	zfs_stat_t	zc_stat;
 } zfs_cmd_zcmd_t;
 
+typedef struct zfs_cmd_edbp {
+	char		zc_name[MAXPATHLEN];	/* name of pool or dataset */
+	uint64_t	zc_nvlist_src;		/* really (char *) */
+	uint64_t	zc_nvlist_src_size;
+	uint64_t	zc_nvlist_dst;		/* really (char *) */
+	uint64_t	zc_nvlist_dst_size;
+	boolean_t	zc_nvlist_dst_filled;	/* put an nvlist in dst? */
+	int		zc_pad2;
+
+	/*
+	 * The following members are for legacy ioctls which haven't been
+	 * converted to the new method.
+	 */
+	uint64_t	zc_history;		/* really (char *) */
+	char		zc_value[MAXPATHLEN * 2];
+	char		zc_string[MAXNAMELEN];
+	uint64_t	zc_guid;
+	uint64_t	zc_nvlist_conf;		/* really (char *) */
+	uint64_t	zc_nvlist_conf_size;
+	uint64_t	zc_cookie;
+	uint64_t	zc_objset_type;
+	uint64_t	zc_perm_action;
+	uint64_t	zc_history_len;
+	uint64_t	zc_history_offset;
+	uint64_t	zc_obj;
+	uint64_t	zc_iflags;		/* internal to zfs(7fs) */
+	zfs_share_t	zc_share;
+	uint64_t	zc_jailid;
+	dmu_objset_stats_t zc_objset_stats;
+	struct drr_begin zc_begin_record;
+	zinject_record_t zc_inject_record;
+	uint32_t	zc_defer_destroy;
+	uint32_t	zc_flags;
+	uint64_t	zc_action_handle;
+	int		zc_cleanup_fd;
+	uint8_t		zc_simple;
+	uint8_t		zc_pad[3];		/* alignment */
+	uint64_t	zc_sendobj;
+	uint64_t	zc_fromobj;
+	uint64_t	zc_createtxg;
+	zfs_stat_t	zc_stat;
+} zfs_cmd_edbp_t;
+
 #ifdef _KERNEL
 unsigned static long zfs_ioctl_v15_to_v28[] = {
 	0,	/*  0 ZFS_IOC_POOL_CREATE */
diff --git a/sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c b/sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c
index dda72de8737e..9e4fd25f10b9 100644
--- a/sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c
+++ b/sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c
@@ -343,6 +343,10 @@ zfs_prop_init(void)
 	zprop_register_string(ZFS_PROP_MLSLABEL, "mlslabel",
 	    ZFS_MLSLABEL_DEFAULT, PROP_INHERIT, ZFS_TYPE_DATASET,
 	    "<sensitivity label>", "MLSLABEL");
+	zprop_register_string(ZFS_PROP_RECEIVE_RESUME_TOKEN,
+	    "receive_resume_token",
+	    NULL, PROP_READONLY, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME,
+	    "<string token>", "RESUMETOK");
 
 	/* readonly number properties */
 	zprop_register_number(ZFS_PROP_USED, "used", 0, PROP_READONLY,
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_objset.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_objset.c
index 6846572f8d5c..79de1d127d42 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_objset.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_objset.c
@@ -362,6 +362,17 @@ dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
 	 * checksum/compression/copies.
 	 */
 	if (ds != NULL) {
+		boolean_t needlock = B_FALSE;
+
+		/*
+		 * Note: it's valid to open the objset if the dataset is
+		 * long-held, in which case the pool_config lock will not
+		 * be held.
+		 */
+		if (!dsl_pool_config_held(dmu_objset_pool(os))) {
+			needlock = B_TRUE;
+			dsl_pool_config_enter(dmu_objset_pool(os), FTAG);
+		}
 		err = dsl_prop_register(ds,
 		    zfs_prop_to_name(ZFS_PROP_PRIMARYCACHE),
 		    primary_cache_changed_cb, os);
@@ -413,6 +424,8 @@ dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
 				    recordsize_changed_cb, os);
 			}
 		}
+		if (needlock)
+			dsl_pool_config_exit(dmu_objset_pool(os), FTAG);
 		if (err != 0) {
 			VERIFY(arc_buf_remove_ref(os->os_phys_buf,
 			    &os->os_phys_buf));
@@ -469,6 +482,13 @@ dmu_objset_from_ds(dsl_dataset_t *ds, objset_t **osp)
 {
 	int err = 0;
 
+	/*
+	 * We shouldn't be doing anything with dsl_dataset_t's unless the
+	 * pool_config lock is held, or the dataset is long-held.
+	 */
+	ASSERT(dsl_pool_config_held(ds->ds_dir->dd_pool) ||
+	    dsl_dataset_long_held(ds));
+
 	mutex_enter(&ds->ds_opening_lock);
 	if (ds->ds_objset == NULL) {
 		objset_t *os;
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_send.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_send.c
index ef139616d0e9..024db77d08a6 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_send.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_send.c
@@ -66,12 +66,14 @@ int zfs_send_queue_length = 16 * 1024 * 1024;
 int zfs_recv_queue_length = 16 * 1024 * 1024;
 
 static char *dmu_recv_tag = "dmu_recv_tag";
-static const char *recv_clone_name = "%recv";
+const char *recv_clone_name = "%recv";
 
 #define	BP_SPAN(datablkszsec, indblkshift, level) \
 	(((uint64_t)datablkszsec) << (SPA_MINBLOCKSHIFT + \
 	(level) * (indblkshift - SPA_BLKPTRSHIFT)))
 
+static void byteswap_record(dmu_replay_record_t *drr);
+
 struct send_thread_arg {
 	bqueue_t	q;
 	dsl_dataset_t	*ds;		/* Dataset to traverse */
@@ -79,6 +81,7 @@ struct send_thread_arg {
 	int		flags;		/* flags to pass to traverse_dataset */
 	int		error_code;
 	boolean_t	cancel;
+	zbookmark_phys_t resume;
 };
 
 struct send_block_record {
@@ -93,7 +96,7 @@ struct send_block_record {
 static int
 dump_bytes(dmu_sendarg_t *dsp, void *buf, int len)
 {
-	dsl_dataset_t *ds = dsp->dsa_os->os_dsl_dataset;
+	dsl_dataset_t *ds = dmu_objset_ds(dsp->dsa_os);
 	struct uio auio;
 	struct iovec aiov;
 	ASSERT0(len % 8);
@@ -166,7 +169,7 @@ dump_free(dmu_sendarg_t *dsp, uint64_t object, uint64_t offset,
 	 * that the receiving system doesn't have any dbufs in the range
 	 * being freed.  This is always true because there is a one-record
 	 * constraint: we only send one WRITE record for any given
-	 * object+offset.  We know that the one-record constraint is
+	 * object,offset.  We know that the one-record constraint is
 	 * true because we always send data in increasing order by
 	 * object,offset.
 	 *
@@ -414,6 +417,19 @@ dump_dnode(dmu_sendarg_t *dsp, uint64_t object, dnode_phys_t *dnp)
 {
 	struct drr_object *drro = &(dsp->dsa_drr->drr_u.drr_object);
 
+	if (object < dsp->dsa_resume_object) {
+		/*
+		 * Note: when resuming, we will visit all the dnodes in
+		 * the block of dnodes that we are resuming from.  In
+		 * this case it's unnecessary to send the dnodes prior to
+		 * the one we are resuming from.  We should be at most one
+		 * block's worth of dnodes behind the resume point.
+		 */
+		ASSERT3U(dsp->dsa_resume_object - object, <,
+		    1 << (DNODE_BLOCK_SHIFT - DNODE_SHIFT));
+		return (0);
+	}
+
 	if (dnp == NULL || dnp->dn_type == DMU_OT_NONE)
 		return (dump_freeobjects(dsp, object, 1));
 
@@ -494,6 +510,9 @@ send_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 	uint64_t record_size;
 	int err = 0;
 
+	ASSERT(zb->zb_object == DMU_META_DNODE_OBJECT ||
+	    zb->zb_object >= sta->resume.zb_object);
+
 	if (sta->cancel)
 		return (SET_ERROR(EINTR));
 
@@ -530,8 +549,10 @@ send_traverse_thread(void *arg)
 	struct send_block_record *data;
 
 	if (st_arg->ds != NULL) {
-		err = traverse_dataset(st_arg->ds, st_arg->fromtxg,
-		    st_arg->flags, send_cb, arg);
+		err = traverse_dataset_resume(st_arg->ds,
+		    st_arg->fromtxg, &st_arg->resume,
+		    st_arg->flags, send_cb, st_arg);
+
 		if (err != EINTR)
 			st_arg->error_code = err;
 	}
@@ -560,6 +581,9 @@ do_dump(dmu_sendarg_t *dsa, struct send_block_record *data)
 
 	ASSERT3U(zb->zb_level, >=, 0);
 
+	ASSERT(zb->zb_object == DMU_META_DNODE_OBJECT ||
+	    zb->zb_object >= dsa->dsa_resume_object);
+
 	if (zb->zb_object != DMU_META_DNODE_OBJECT &&
 	    DMU_OBJECT_IS_SPECIAL(zb->zb_object)) {
 		return (0);
@@ -620,6 +644,10 @@ do_dump(dmu_sendarg_t *dsa, struct send_block_record *data)
 		uint64_t offset;
 
 		ASSERT0(zb->zb_level);
+		ASSERT(zb->zb_object > dsa->dsa_resume_object ||
+		    (zb->zb_object == dsa->dsa_resume_object &&
+		    zb->zb_blkid * blksz >= dsa->dsa_resume_offset));
+
 		if (arc_read(NULL, spa, bp, arc_getbuf_func, &abuf,
 		    ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL,
 		    &aflags, zb) != 0) {
@@ -680,11 +708,13 @@ get_next_record(bqueue_t *bq, struct send_block_record *data)
  */
 static int
 dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *to_ds,
-    zfs_bookmark_phys_t *ancestor_zb, boolean_t is_clone, boolean_t embedok,
+    zfs_bookmark_phys_t *ancestor_zb,
+    boolean_t is_clone, boolean_t embedok, boolean_t large_block_ok, int outfd,
+    uint64_t resumeobj, uint64_t resumeoff,
 #ifdef illumos
-    boolean_t large_block_ok, int outfd, vnode_t *vp, offset_t *off)
+    vnode_t *vp, offset_t *off)
 #else
-    boolean_t large_block_ok, int outfd, struct file *fp, offset_t *off)
+    struct file *fp, offset_t *off)
 #endif
 {
 	objset_t *os;
@@ -693,7 +723,7 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *to_ds,
 	int err;
 	uint64_t fromtxg = 0;
 	uint64_t featureflags = 0;
-	struct send_thread_arg to_arg;
+	struct send_thread_arg to_arg = { 0 };
 
 	err = dmu_objset_from_ds(to_ds, &os);
 	if (err != 0) {
@@ -730,6 +760,10 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *to_ds,
 			featureflags |= DMU_BACKUP_FEATURE_EMBED_DATA_LZ4;
 	}
 
+	if (resumeobj != 0 || resumeoff != 0) {
+		featureflags |= DMU_BACKUP_FEATURE_RESUMING;
+	}
+
 	DMU_SET_FEATUREFLAGS(drr->drr_u.drr_begin.drr_versioninfo,
 	    featureflags);
 
@@ -766,6 +800,8 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *to_ds,
 	dsp->dsa_pending_op = PENDING_NONE;
 	dsp->dsa_incremental = (ancestor_zb != NULL);
 	dsp->dsa_featureflags = featureflags;
+	dsp->dsa_resume_object = resumeobj;
+	dsp->dsa_resume_offset = resumeoff;
 
 	mutex_enter(&to_ds->ds_sendstream_lock);
 	list_insert_head(&to_ds->ds_sendstreams, dsp);
@@ -774,7 +810,27 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *to_ds,
 	dsl_dataset_long_hold(to_ds, FTAG);
 	dsl_pool_rele(dp, tag);
 
-	if (dump_record(dsp, NULL, 0) != 0) {
+	void *payload = NULL;
+	size_t payload_len = 0;
+	if (resumeobj != 0 || resumeoff != 0) {
+		dmu_object_info_t to_doi;
+		err = dmu_object_info(os, resumeobj, &to_doi);
+		if (err != 0)
+			goto out;
+		SET_BOOKMARK(&to_arg.resume, to_ds->ds_object, resumeobj, 0,
+		    resumeoff / to_doi.doi_data_block_size);
+
+		nvlist_t *nvl = fnvlist_alloc();
+		fnvlist_add_uint64(nvl, "resume_object", resumeobj);
+		fnvlist_add_uint64(nvl, "resume_offset", resumeoff);
+		payload = fnvlist_pack(nvl, &payload_len);
+		drr->drr_payloadlen = payload_len;
+		fnvlist_free(nvl);
+	}
+
+	err = dump_record(dsp, payload, payload_len);
+	fnvlist_pack_free(payload, payload_len);
+	if (err != 0) {
 		err = dsp->dsa_err;
 		goto out;
 	}
@@ -889,22 +945,22 @@ dmu_send_obj(const char *pool, uint64_t tosnap, uint64_t fromsnap,
 		is_clone = (fromds->ds_dir != ds->ds_dir);
 		dsl_dataset_rele(fromds, FTAG);
 		err = dmu_send_impl(FTAG, dp, ds, &zb, is_clone,
-		    embedok, large_block_ok, outfd, fp, off);
+		    embedok, large_block_ok, outfd, 0, 0, fp, off);
 	} else {
 		err = dmu_send_impl(FTAG, dp, ds, NULL, B_FALSE,
-		    embedok, large_block_ok, outfd, fp, off);
+		    embedok, large_block_ok, outfd, 0, 0, fp, off);
 	}
 	dsl_dataset_rele(ds, FTAG);
 	return (err);
 }
 
 int
-dmu_send(const char *tosnap, const char *fromsnap,
-    boolean_t embedok, boolean_t large_block_ok,
+dmu_send(const char *tosnap, const char *fromsnap, boolean_t embedok,
+    boolean_t large_block_ok, int outfd, uint64_t resumeobj, uint64_t resumeoff,
 #ifdef illumos
-    int outfd, vnode_t *vp, offset_t *off)
+    vnode_t *vp, offset_t *off)
 #else
-    int outfd, struct file *fp, offset_t *off)
+    struct file *fp, offset_t *off)
 #endif
 {
 	dsl_pool_t *dp;
@@ -972,10 +1028,12 @@ dmu_send(const char *tosnap, const char *fromsnap,
 			return (err);
 		}
 		err = dmu_send_impl(FTAG, dp, ds, &zb, is_clone,
-		    embedok, large_block_ok, outfd, fp, off);
+		    embedok, large_block_ok,
+		    outfd, resumeobj, resumeoff, fp, off);
 	} else {
 		err = dmu_send_impl(FTAG, dp, ds, NULL, B_FALSE,
-		    embedok, large_block_ok, outfd, fp, off);
+		    embedok, large_block_ok,
+		    outfd, resumeobj, resumeoff, fp, off);
 	}
 	if (owned)
 		dsl_dataset_disown(ds, FTAG);
@@ -1218,6 +1276,7 @@ dmu_recv_begin_check(void *arg, dmu_tx_t *tx)
 
 	/* already checked */
 	ASSERT3U(drrb->drr_magic, ==, DMU_BACKUP_MAGIC);
+	ASSERT(!(featureflags & DMU_BACKUP_FEATURE_RESUMING));
 
 	if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) ==
 	    DMU_COMPOUNDSTREAM ||
@@ -1230,6 +1289,10 @@ dmu_recv_begin_check(void *arg, dmu_tx_t *tx)
 	    spa_version(dp->dp_spa) < SPA_VERSION_SA)
 		return (SET_ERROR(ENOTSUP));
 
+	if (drba->drba_cookie->drc_resumable &&
+	    !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_EXTENSIBLE_DATASET))
+		return (SET_ERROR(ENOTSUP));
+
 	/*
 	 * The receiving code doesn't know how to translate a WRITE_EMBEDDED
 	 * record to a plan WRITE record, so the pool must have the
@@ -1333,15 +1396,16 @@ dmu_recv_begin_sync(void *arg, dmu_tx_t *tx)
 {
 	dmu_recv_begin_arg_t *drba = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
+	objset_t *mos = dp->dp_meta_objset;
 	struct drr_begin *drrb = drba->drba_cookie->drc_drrb;
 	const char *tofs = drba->drba_cookie->drc_tofs;
 	dsl_dataset_t *ds, *newds;
 	uint64_t dsobj;
 	int error;
-	uint64_t crflags;
+	uint64_t crflags = 0;
 
-	crflags = (drrb->drr_flags & DRR_FLAG_CI_DATA) ?
-	    DS_FLAG_CI_DATASET : 0;
+	if (drrb->drr_flags & DRR_FLAG_CI_DATA)
+		crflags |= DS_FLAG_CI_DATASET;
 
 	error = dsl_dataset_hold(dp, tofs, FTAG, &ds);
 	if (error == 0) {
@@ -1379,6 +1443,31 @@ dmu_recv_begin_sync(void *arg, dmu_tx_t *tx)
 	}
 	VERIFY0(dsl_dataset_own_obj(dp, dsobj, dmu_recv_tag, &newds));
 
+	if (drba->drba_cookie->drc_resumable) {
+		dsl_dataset_zapify(newds, tx);
+		if (drrb->drr_fromguid != 0) {
+			VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_FROMGUID,
+			    8, 1, &drrb->drr_fromguid, tx));
+		}
+		VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_TOGUID,
+		    8, 1, &drrb->drr_toguid, tx));
+		VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_TONAME,
+		    1, strlen(drrb->drr_toname) + 1, drrb->drr_toname, tx));
+		uint64_t one = 1;
+		uint64_t zero = 0;
+		VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_OBJECT,
+		    8, 1, &one, tx));
+		VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_OFFSET,
+		    8, 1, &zero, tx));
+		VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_BYTES,
+		    8, 1, &zero, tx));
+		if (DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) &
+		    DMU_BACKUP_FEATURE_EMBED_DATA) {
+			VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_EMBEDOK,
+			    8, 1, &one, tx));
+		}
+	}
+
 	dmu_buf_will_dirty(newds->ds_dbuf, tx);
 	dsl_dataset_phys(newds)->ds_flags |= DS_FLAG_INCONSISTENT;
 
@@ -1396,56 +1485,192 @@ dmu_recv_begin_sync(void *arg, dmu_tx_t *tx)
 	spa_history_log_internal_ds(newds, "receive", tx, "");
 }
 
+static int
+dmu_recv_resume_begin_check(void *arg, dmu_tx_t *tx)
+{
+	dmu_recv_begin_arg_t *drba = arg;
+	dsl_pool_t *dp = dmu_tx_pool(tx);
+	struct drr_begin *drrb = drba->drba_cookie->drc_drrb;
+	int error;
+	uint64_t featureflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo);
+	dsl_dataset_t *ds;
+	const char *tofs = drba->drba_cookie->drc_tofs;
+
+	/* already checked */
+	ASSERT3U(drrb->drr_magic, ==, DMU_BACKUP_MAGIC);
+	ASSERT(featureflags & DMU_BACKUP_FEATURE_RESUMING);
+
+	if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) ==
+	    DMU_COMPOUNDSTREAM ||
+	    drrb->drr_type >= DMU_OST_NUMTYPES)
+		return (SET_ERROR(EINVAL));
+
+	/* Verify pool version supports SA if SA_SPILL feature set */
+	if ((featureflags & DMU_BACKUP_FEATURE_SA_SPILL) &&
+	    spa_version(dp->dp_spa) < SPA_VERSION_SA)
+		return (SET_ERROR(ENOTSUP));
+
+	/*
+	 * The receiving code doesn't know how to translate a WRITE_EMBEDDED
+	 * record to a plain WRITE record, so the pool must have the
+	 * EMBEDDED_DATA feature enabled if the stream has WRITE_EMBEDDED
+	 * records.  Same with WRITE_EMBEDDED records that use LZ4 compression.
+	 */
+	if ((featureflags & DMU_BACKUP_FEATURE_EMBED_DATA) &&
+	    !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_EMBEDDED_DATA))
+		return (SET_ERROR(ENOTSUP));
+	if ((featureflags & DMU_BACKUP_FEATURE_EMBED_DATA_LZ4) &&
+	    !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_LZ4_COMPRESS))
+		return (SET_ERROR(ENOTSUP));
+
+	char recvname[ZFS_MAXNAMELEN];
+
+	(void) snprintf(recvname, sizeof (recvname), "%s/%s",
+	    tofs, recv_clone_name);
+
+	if (dsl_dataset_hold(dp, recvname, FTAG, &ds) != 0) {
+		/* %recv does not exist; continue in tofs */
+		error = dsl_dataset_hold(dp, tofs, FTAG, &ds);
+		if (error != 0)
+			return (error);
+	}
+
+	/* check that ds is marked inconsistent */
+	if (!DS_IS_INCONSISTENT(ds)) {
+		dsl_dataset_rele(ds, FTAG);
+		return (SET_ERROR(EINVAL));
+	}
+
+	/* check that there is resuming data, and that the toguid matches */
+	if (!dsl_dataset_is_zapified(ds)) {
+		dsl_dataset_rele(ds, FTAG);
+		return (SET_ERROR(EINVAL));
+	}
+	uint64_t val;
+	error = zap_lookup(dp->dp_meta_objset, ds->ds_object,
+	    DS_FIELD_RESUME_TOGUID, sizeof (val), 1, &val);
+	if (error != 0 || drrb->drr_toguid != val) {
+		dsl_dataset_rele(ds, FTAG);
+		return (SET_ERROR(EINVAL));
+	}
+
+	/*
+	 * Check if the receive is still running.  If so, it will be owned.
+	 * Note that nothing else can own the dataset (e.g. after the receive
+	 * fails) because it will be marked inconsistent.
+	 */
+	if (dsl_dataset_has_owner(ds)) {
+		dsl_dataset_rele(ds, FTAG);
+		return (SET_ERROR(EBUSY));
+	}
+
+	/* There should not be any snapshots of this fs yet. */
+	if (ds->ds_prev != NULL && ds->ds_prev->ds_dir == ds->ds_dir) {
+		dsl_dataset_rele(ds, FTAG);
+		return (SET_ERROR(EINVAL));
+	}
+
+	/*
+	 * Note: resume point will be checked when we process the first WRITE
+	 * record.
+	 */
+
+	/* check that the origin matches */
+	val = 0;
+	(void) zap_lookup(dp->dp_meta_objset, ds->ds_object,
+	    DS_FIELD_RESUME_FROMGUID, sizeof (val), 1, &val);
+	if (drrb->drr_fromguid != val) {
+		dsl_dataset_rele(ds, FTAG);
+		return (SET_ERROR(EINVAL));
+	}
+
+	dsl_dataset_rele(ds, FTAG);
+	return (0);
+}
+
+static void
+dmu_recv_resume_begin_sync(void *arg, dmu_tx_t *tx)
+{
+	dmu_recv_begin_arg_t *drba = arg;
+	dsl_pool_t *dp = dmu_tx_pool(tx);
+	const char *tofs = drba->drba_cookie->drc_tofs;
+	dsl_dataset_t *ds;
+	uint64_t dsobj;
+	char recvname[ZFS_MAXNAMELEN];
+
+	(void) snprintf(recvname, sizeof (recvname), "%s/%s",
+	    tofs, recv_clone_name);
+
+	if (dsl_dataset_hold(dp, recvname, FTAG, &ds) != 0) {
+		/* %recv does not exist; continue in tofs */
+		VERIFY0(dsl_dataset_hold(dp, tofs, FTAG, &ds));
+		drba->drba_cookie->drc_newfs = B_TRUE;
+	}
+
+	/* clear the inconsistent flag so that we can own it */
+	ASSERT(DS_IS_INCONSISTENT(ds));
+	dmu_buf_will_dirty(ds->ds_dbuf, tx);
+	dsl_dataset_phys(ds)->ds_flags &= ~DS_FLAG_INCONSISTENT;
+	dsobj = ds->ds_object;
+	dsl_dataset_rele(ds, FTAG);
+
+	VERIFY0(dsl_dataset_own_obj(dp, dsobj, dmu_recv_tag, &ds));
+
+	dmu_buf_will_dirty(ds->ds_dbuf, tx);
+	dsl_dataset_phys(ds)->ds_flags |= DS_FLAG_INCONSISTENT;
+
+	ASSERT(!BP_IS_HOLE(dsl_dataset_get_blkptr(ds)));
+
+	drba->drba_cookie->drc_ds = ds;
+
+	spa_history_log_internal_ds(ds, "resume receive", tx, "");
+}
+
 /*
  * NB: callers *MUST* call dmu_recv_stream() if dmu_recv_begin()
  * succeeds; otherwise we will leak the holds on the datasets.
  */
 int
-dmu_recv_begin(char *tofs, char *tosnap, struct drr_begin *drrb,
-    boolean_t force, char *origin, dmu_recv_cookie_t *drc)
+dmu_recv_begin(char *tofs, char *tosnap, dmu_replay_record_t *drr_begin,
+    boolean_t force, boolean_t resumable, char *origin, dmu_recv_cookie_t *drc)
 {
 	dmu_recv_begin_arg_t drba = { 0 };
-	dmu_replay_record_t *drr;
 
 	bzero(drc, sizeof (dmu_recv_cookie_t));
-	drc->drc_drrb = drrb;
+	drc->drc_drr_begin = drr_begin;
+	drc->drc_drrb = &drr_begin->drr_u.drr_begin;
 	drc->drc_tosnap = tosnap;
 	drc->drc_tofs = tofs;
 	drc->drc_force = force;
+	drc->drc_resumable = resumable;
 	drc->drc_cred = CRED();
 
-	if (drrb->drr_magic == BSWAP_64(DMU_BACKUP_MAGIC))
+	if (drc->drc_drrb->drr_magic == BSWAP_64(DMU_BACKUP_MAGIC)) {
 		drc->drc_byteswap = B_TRUE;
-	else if (drrb->drr_magic != DMU_BACKUP_MAGIC)
-		return (SET_ERROR(EINVAL));
-
-	drr = kmem_zalloc(sizeof (dmu_replay_record_t), KM_SLEEP);
-	drr->drr_type = DRR_BEGIN;
-	drr->drr_u.drr_begin = *drc->drc_drrb;
-	if (drc->drc_byteswap) {
-		fletcher_4_incremental_byteswap(drr,
+		fletcher_4_incremental_byteswap(drr_begin,
+		    sizeof (dmu_replay_record_t), &drc->drc_cksum);
+		byteswap_record(drr_begin);
+	} else if (drc->drc_drrb->drr_magic == DMU_BACKUP_MAGIC) {
+		fletcher_4_incremental_native(drr_begin,
 		    sizeof (dmu_replay_record_t), &drc->drc_cksum);
 	} else {
-		fletcher_4_incremental_native(drr,
-		    sizeof (dmu_replay_record_t), &drc->drc_cksum);
-	}
-	kmem_free(drr, sizeof (dmu_replay_record_t));
-
-	if (drc->drc_byteswap) {
-		drrb->drr_magic = BSWAP_64(drrb->drr_magic);
-		drrb->drr_versioninfo = BSWAP_64(drrb->drr_versioninfo);
-		drrb->drr_creation_time = BSWAP_64(drrb->drr_creation_time);
-		drrb->drr_type = BSWAP_32(drrb->drr_type);
-		drrb->drr_toguid = BSWAP_64(drrb->drr_toguid);
-		drrb->drr_fromguid = BSWAP_64(drrb->drr_fromguid);
+		return (SET_ERROR(EINVAL));
 	}
 
 	drba.drba_origin = origin;
 	drba.drba_cookie = drc;
 	drba.drba_cred = CRED();
 
-	return (dsl_sync_task(tofs, dmu_recv_begin_check, dmu_recv_begin_sync,
-	    &drba, 5, ZFS_SPACE_CHECK_NORMAL));
+	if (DMU_GET_FEATUREFLAGS(drc->drc_drrb->drr_versioninfo) &
+	    DMU_BACKUP_FEATURE_RESUMING) {
+		return (dsl_sync_task(tofs,
+		    dmu_recv_resume_begin_check, dmu_recv_resume_begin_sync,
+		    &drba, 5, ZFS_SPACE_CHECK_NORMAL));
+	} else  {
+		return (dsl_sync_task(tofs,
+		    dmu_recv_begin_check, dmu_recv_begin_sync,
+		    &drba, 5, ZFS_SPACE_CHECK_NORMAL));
+	}
 }
 
 struct receive_record_arg {
@@ -1457,6 +1682,7 @@ struct receive_record_arg {
 	 */
 	arc_buf_t *write_buf;
 	int payload_size;
+	uint64_t bytes_read; /* bytes read from stream when record created */
 	boolean_t eos_marker; /* Marks the end of the stream */
 	bqueue_node_t node;
 };
@@ -1465,6 +1691,7 @@ struct receive_writer_arg {
 	objset_t *os;
 	boolean_t byteswap;
 	bqueue_t q;
+
 	/*
 	 * These three args are used to signal to the main thread that we're
 	 * done.
@@ -1472,9 +1699,13 @@ struct receive_writer_arg {
 	kmutex_t mutex;
 	kcondvar_t cv;
 	boolean_t done;
+
 	int err;
 	/* A map from guid to dataset to help handle dedup'd streams. */
 	avl_tree_t *guid_to_ds_map;
+	boolean_t resumable;
+	uint64_t last_object, last_offset;
+	uint64_t bytes_read; /* bytes read when current record created */
 };
 
 struct receive_arg  {
@@ -1482,6 +1713,7 @@ struct receive_arg  {
 	kthread_t *td;
 	struct file *fp;
 	uint64_t voff; /* The current offset in the stream */
+	uint64_t bytes_read;
 	/*
 	 * A record that has had its payload read in, but hasn't yet been handed
 	 * off to the worker thread.
@@ -1577,14 +1809,21 @@ receive_read(struct receive_arg *ra, int len, void *buf)
 		ra->err = restore_bytes(ra, buf + done,
 		    len - done, ra->voff, &resid);
 
-		if (resid == len - done)
-			ra->err = SET_ERROR(EINVAL);
+		if (resid == len - done) {
+			/*
+			 * Note: ECKSUM indicates that the receive
+			 * was interrupted and can potentially be resumed.
+			 */
+			ra->err = SET_ERROR(ECKSUM);
+		}
 		ra->voff += len - done - resid;
 		done = len - resid;
 		if (ra->err != 0)
 			return (ra->err);
 	}
 
+	ra->bytes_read += len;
+
 	ASSERT3U(done, ==, len);
 	return (0);
 }
@@ -1685,6 +1924,43 @@ deduce_nblkptr(dmu_object_type_t bonus_type, uint64_t bonus_size)
 	}
 }
 
+static void
+save_resume_state(struct receive_writer_arg *rwa,
+    uint64_t object, uint64_t offset, dmu_tx_t *tx)
+{
+	int txgoff = dmu_tx_get_txg(tx) & TXG_MASK;
+
+	if (!rwa->resumable)
+		return;
+
+	/*
+	 * We use ds_resume_bytes[] != 0 to indicate that we need to
+	 * update this on disk, so it must not be 0.
+	 */
+	ASSERT(rwa->bytes_read != 0);
+
+	/*
+	 * We only resume from write records, which have a valid
+	 * (non-meta-dnode) object number.
+	 */
+	ASSERT(object != 0);
+
+	/*
+	 * For resuming to work correctly, we must receive records in order,
+	 * sorted by object,offset.  This is checked by the callers, but
+	 * assert it here for good measure.
+	 */
+	ASSERT3U(object, >=, rwa->os->os_dsl_dataset->ds_resume_object[txgoff]);
+	ASSERT(object != rwa->os->os_dsl_dataset->ds_resume_object[txgoff] ||
+	    offset >= rwa->os->os_dsl_dataset->ds_resume_offset[txgoff]);
+	ASSERT3U(rwa->bytes_read, >=,
+	    rwa->os->os_dsl_dataset->ds_resume_bytes[txgoff]);
+
+	rwa->os->os_dsl_dataset->ds_resume_object[txgoff] = object;
+	rwa->os->os_dsl_dataset->ds_resume_offset[txgoff] = offset;
+	rwa->os->os_dsl_dataset->ds_resume_bytes[txgoff] = rwa->bytes_read;
+}
+
 static int
 receive_object(struct receive_writer_arg *rwa, struct drr_object *drro,
     void *data)
@@ -1781,6 +2057,7 @@ receive_object(struct receive_writer_arg *rwa, struct drr_object *drro,
 		dmu_buf_rele(db, FTAG);
 	}
 	dmu_tx_commit(tx);
+
 	return (0);
 }
 
@@ -1806,6 +2083,7 @@ receive_freeobjects(struct receive_writer_arg *rwa,
 		if (err != 0)
 			return (err);
 	}
+
 	return (0);
 }
 
@@ -1820,6 +2098,18 @@ receive_write(struct receive_writer_arg *rwa, struct drr_write *drrw,
 	    !DMU_OT_IS_VALID(drrw->drr_type))
 		return (SET_ERROR(EINVAL));
 
+	/*
+	 * For resuming to work, records must be in increasing order
+	 * by (object, offset).
+	 */
+	if (drrw->drr_object < rwa->last_object ||
+	    (drrw->drr_object == rwa->last_object &&
+	    drrw->drr_offset < rwa->last_offset)) {
+		return (SET_ERROR(EINVAL));
+	}
+	rwa->last_object = drrw->drr_object;
+	rwa->last_offset = drrw->drr_offset;
+
 	if (dmu_object_info(rwa->os, drrw->drr_object, NULL) != 0)
 		return (SET_ERROR(EINVAL));
 
@@ -1843,8 +2133,17 @@ receive_write(struct receive_writer_arg *rwa, struct drr_write *drrw,
 	if (dmu_bonus_hold(rwa->os, drrw->drr_object, FTAG, &bonus) != 0)
 		return (SET_ERROR(EINVAL));
 	dmu_assign_arcbuf(bonus, drrw->drr_offset, abuf, tx);
+
+	/*
+	 * Note: If the receive fails, we want the resume stream to start
+	 * with the same record that we last successfully received (as opposed
+	 * to the next record), so that we can verify that we are
+	 * resuming from the correct location.
+	 */
+	save_resume_state(rwa, drrw->drr_object, drrw->drr_offset, tx);
 	dmu_tx_commit(tx);
 	dmu_buf_rele(bonus, FTAG);
+
 	return (0);
 }
 
@@ -1903,43 +2202,48 @@ receive_write_byref(struct receive_writer_arg *rwa,
 	dmu_write(rwa->os, drrwbr->drr_object,
 	    drrwbr->drr_offset, drrwbr->drr_length, dbp->db_data, tx);
 	dmu_buf_rele(dbp, FTAG);
+
+	/* See comment in restore_write. */
+	save_resume_state(rwa, drrwbr->drr_object, drrwbr->drr_offset, tx);
 	dmu_tx_commit(tx);
 	return (0);
 }
 
 static int
 receive_write_embedded(struct receive_writer_arg *rwa,
-    struct drr_write_embedded *drrwnp, void *data)
+    struct drr_write_embedded *drrwe, void *data)
 {
 	dmu_tx_t *tx;
 	int err;
 
-	if (drrwnp->drr_offset + drrwnp->drr_length < drrwnp->drr_offset)
+	if (drrwe->drr_offset + drrwe->drr_length < drrwe->drr_offset)
 		return (EINVAL);
 
-	if (drrwnp->drr_psize > BPE_PAYLOAD_SIZE)
+	if (drrwe->drr_psize > BPE_PAYLOAD_SIZE)
 		return (EINVAL);
 
-	if (drrwnp->drr_etype >= NUM_BP_EMBEDDED_TYPES)
+	if (drrwe->drr_etype >= NUM_BP_EMBEDDED_TYPES)
 		return (EINVAL);
-	if (drrwnp->drr_compression >= ZIO_COMPRESS_FUNCTIONS)
+	if (drrwe->drr_compression >= ZIO_COMPRESS_FUNCTIONS)
 		return (EINVAL);
 
 	tx = dmu_tx_create(rwa->os);
 
-	dmu_tx_hold_write(tx, drrwnp->drr_object,
-	    drrwnp->drr_offset, drrwnp->drr_length);
+	dmu_tx_hold_write(tx, drrwe->drr_object,
+	    drrwe->drr_offset, drrwe->drr_length);
 	err = dmu_tx_assign(tx, TXG_WAIT);
 	if (err != 0) {
 		dmu_tx_abort(tx);
 		return (err);
 	}
 
-	dmu_write_embedded(rwa->os, drrwnp->drr_object,
-	    drrwnp->drr_offset, data, drrwnp->drr_etype,
-	    drrwnp->drr_compression, drrwnp->drr_lsize, drrwnp->drr_psize,
+	dmu_write_embedded(rwa->os, drrwe->drr_object,
+	    drrwe->drr_offset, data, drrwe->drr_etype,
+	    drrwe->drr_compression, drrwe->drr_lsize, drrwe->drr_psize,
 	    rwa->byteswap ^ ZFS_HOST_BYTEORDER, tx);
 
+	/* See comment in restore_write. */
+	save_resume_state(rwa, drrwe->drr_object, drrwe->drr_offset, tx);
 	dmu_tx_commit(tx);
 	return (0);
 }
@@ -2013,10 +2317,16 @@ receive_free(struct receive_writer_arg *rwa, struct drr_free *drrf)
 static void
 dmu_recv_cleanup_ds(dmu_recv_cookie_t *drc)
 {
-	char name[MAXNAMELEN];
-	dsl_dataset_name(drc->drc_ds, name);
-	dsl_dataset_disown(drc->drc_ds, dmu_recv_tag);
-	(void) dsl_destroy_head(name);
+	if (drc->drc_resumable) {
+		/* wait for our resume state to be written to disk */
+		txg_wait_synced(drc->drc_ds->ds_dir->dd_pool, 0);
+		dsl_dataset_disown(drc->drc_ds, dmu_recv_tag);
+	} else {
+		char name[MAXNAMELEN];
+		dsl_dataset_name(drc->drc_ds, name);
+		dsl_dataset_disown(drc->drc_ds, dmu_recv_tag);
+		(void) dsl_destroy_head(name);
+	}
 }
 
 static void
@@ -2043,12 +2353,17 @@ receive_read_payload_and_next_header(struct receive_arg *ra, int len, void *buf)
 
 	if (len != 0) {
 		ASSERT3U(len, <=, SPA_MAXBLOCKSIZE);
-		ra->rrd->payload = buf;
-		ra->rrd->payload_size = len;
-		err = receive_read(ra, len, ra->rrd->payload);
+		err = receive_read(ra, len, buf);
 		if (err != 0)
 			return (err);
-		receive_cksum(ra, len, ra->rrd->payload);
+		receive_cksum(ra, len, buf);
+
+		/* note: rrd is NULL when reading the begin record's payload */
+		if (ra->rrd != NULL) {
+			ra->rrd->payload = buf;
+			ra->rrd->payload_size = len;
+			ra->rrd->bytes_read = ra->bytes_read;
+		}
 	}
 
 	ra->prev_cksum = ra->cksum;
@@ -2056,6 +2371,7 @@ receive_read_payload_and_next_header(struct receive_arg *ra, int len, void *buf)
 	ra->next_rrd = kmem_zalloc(sizeof (*ra->next_rrd), KM_SLEEP);
 	err = receive_read(ra, sizeof (ra->next_rrd->header),
 	    &ra->next_rrd->header);
+	ra->next_rrd->bytes_read = ra->bytes_read;
 	if (err != 0) {
 		kmem_free(ra->next_rrd, sizeof (*ra->next_rrd));
 		ra->next_rrd = NULL;
@@ -2235,7 +2551,7 @@ receive_read_record(struct receive_arg *ra)
 	{
 		struct drr_end *drre = &ra->rrd->header.drr_u.drr_end;
 		if (!ZIO_CHECKSUM_EQUAL(ra->prev_cksum, drre->drr_checksum))
-			return (SET_ERROR(EINVAL));
+			return (SET_ERROR(ECKSUM));
 		return (0);
 	}
 	case DRR_SPILL:
@@ -2262,6 +2578,10 @@ receive_process_record(struct receive_writer_arg *rwa,
 {
 	int err;
 
+	/* Processing in order, therefore bytes_read should be increasing. */
+	ASSERT3U(rrd->bytes_read, >=, rwa->bytes_read);
+	rwa->bytes_read = rrd->bytes_read;
+
 	switch (rrd->header.drr_type) {
 	case DRR_OBJECT:
 	{
@@ -2357,6 +2677,33 @@ receive_writer_thread(void *arg)
 	thread_exit();
 }
 
+static int
+resume_check(struct receive_arg *ra, nvlist_t *begin_nvl)
+{
+	uint64_t val;
+	objset_t *mos = dmu_objset_pool(ra->os)->dp_meta_objset;
+	uint64_t dsobj = dmu_objset_id(ra->os);
+	uint64_t resume_obj, resume_off;
+
+	if (nvlist_lookup_uint64(begin_nvl,
+	    "resume_object", &resume_obj) != 0 ||
+	    nvlist_lookup_uint64(begin_nvl,
+	    "resume_offset", &resume_off) != 0) {
+		return (SET_ERROR(EINVAL));
+	}
+	VERIFY0(zap_lookup(mos, dsobj,
+	    DS_FIELD_RESUME_OBJECT, sizeof (val), 1, &val));
+	if (resume_obj != val)
+		return (SET_ERROR(EINVAL));
+	VERIFY0(zap_lookup(mos, dsobj,
+	    DS_FIELD_RESUME_OFFSET, sizeof (val), 1, &val));
+	if (resume_off != val)
+		return (SET_ERROR(EINVAL));
+
+	return (0);
+}
+
+
 /*
  * Read in the stream's records, one by one, and apply them to the pool.  There
  * are two threads involved; the thread that calls this function will spin up a
@@ -2377,12 +2724,20 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, struct file *fp, offset_t *voffp,
 	struct receive_arg ra = { 0 };
 	struct receive_writer_arg rwa = { 0 };
 	int featureflags;
+	nvlist_t *begin_nvl = NULL;
 
 	ra.byteswap = drc->drc_byteswap;
 	ra.cksum = drc->drc_cksum;
 	ra.td = curthread;
 	ra.fp = fp;
 	ra.voff = *voffp;
+
+	if (dsl_dataset_is_zapified(drc->drc_ds)) {
+		(void) zap_lookup(drc->drc_ds->ds_dir->dd_pool->dp_meta_objset,
+		    drc->drc_ds->ds_object, DS_FIELD_RESUME_BYTES,
+		    sizeof (ra.bytes_read), 1, &ra.bytes_read);
+	}
+
 	list_create(&ra.ignore_obj_list, sizeof (struct receive_ign_obj_node),
 	    offsetof(struct receive_ign_obj_node, node));
 
@@ -2435,9 +2790,29 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, struct file *fp, offset_t *voffp,
 		drc->drc_guid_to_ds_map = rwa.guid_to_ds_map;
 	}
 
-	err = receive_read_payload_and_next_header(&ra, 0, NULL);
-	if (err)
+	uint32_t payloadlen = drc->drc_drr_begin->drr_payloadlen;
+	void *payload = NULL;
+	if (payloadlen != 0)
+		payload = kmem_alloc(payloadlen, KM_SLEEP);
+
+	err = receive_read_payload_and_next_header(&ra, payloadlen, payload);
+	if (err != 0) {
+		if (payloadlen != 0)
+			kmem_free(payload, payloadlen);
 		goto out;
+	}
+	if (payloadlen != 0) {
+		err = nvlist_unpack(payload, payloadlen, &begin_nvl, KM_SLEEP);
+		kmem_free(payload, payloadlen);
+		if (err != 0)
+			goto out;
+	}
+
+	if (featureflags & DMU_BACKUP_FEATURE_RESUMING) {
+		err = resume_check(&ra, begin_nvl);
+		if (err != 0)
+			goto out;
+	}
 
 	(void) bqueue_init(&rwa.q, zfs_recv_queue_length,
 	    offsetof(struct receive_record_arg, node));
@@ -2445,6 +2820,7 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, struct file *fp, offset_t *voffp,
 	mutex_init(&rwa.mutex, NULL, MUTEX_DEFAULT, NULL);
 	rwa.os = ra.os;
 	rwa.byteswap = drc->drc_byteswap;
+	rwa.resumable = drc->drc_resumable;
 
 	(void) thread_create(NULL, 0, receive_writer_thread, &rwa, 0, &p0,
 	    TS_RUN, minclsyspri);
@@ -2503,13 +2879,15 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, struct file *fp, offset_t *voffp,
 		err = rwa.err;
 
 out:
+	nvlist_free(begin_nvl);
 	if ((featureflags & DMU_BACKUP_FEATURE_DEDUP) && (cleanup_fd != -1))
 		zfs_onexit_fd_rele(cleanup_fd);
 
 	if (err != 0) {
 		/*
-		 * destroy what we created, so we don't leave it in the
-		 * inconsistent restoring state.
+		 * Clean up references. If receive is not resumable,
+		 * destroy what we created, so we don't leave it in
+		 * the inconsistent state.
 		 */
 		dmu_recv_cleanup_ds(drc);
 	}
@@ -2669,6 +3047,20 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx)
 
 		dmu_buf_will_dirty(ds->ds_dbuf, tx);
 		dsl_dataset_phys(ds)->ds_flags &= ~DS_FLAG_INCONSISTENT;
+		if (dsl_dataset_has_resume_receive_state(ds)) {
+			(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
+			    DS_FIELD_RESUME_FROMGUID, tx);
+			(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
+			    DS_FIELD_RESUME_OBJECT, tx);
+			(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
+			    DS_FIELD_RESUME_OFFSET, tx);
+			(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
+			    DS_FIELD_RESUME_BYTES, tx);
+			(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
+			    DS_FIELD_RESUME_TOGUID, tx);
+			(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
+			    DS_FIELD_RESUME_TONAME, tx);
+		}
 	}
 	drc->drc_newsnapobj = dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj;
 	/*
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_traverse.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_traverse.c
index 151d04c0fb92..2c718df04897 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_traverse.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_traverse.c
@@ -48,6 +48,7 @@ typedef struct prefetch_data {
 	int pd_flags;
 	boolean_t pd_cancel;
 	boolean_t pd_exited;
+	zbookmark_phys_t pd_resume;
 } prefetch_data_t;
 
 typedef struct traverse_data {
@@ -307,59 +308,52 @@ traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
 		arc_flags_t flags = ARC_FLAG_WAIT;
 		int i;
 		int epb = BP_GET_LSIZE(bp) >> DNODE_SHIFT;
-		dnode_phys_t *cdnp;
 
 		err = arc_read(NULL, td->td_spa, bp, arc_getbuf_func, &buf,
 		    ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL, &flags, zb);
 		if (err != 0)
 			goto post;
-		cdnp = buf->b_data;
+		dnode_phys_t *child_dnp = buf->b_data;
 
 		for (i = 0; i < epb; i++) {
-			prefetch_dnode_metadata(td, &cdnp[i], zb->zb_objset,
-			    zb->zb_blkid * epb + i);
+			prefetch_dnode_metadata(td, &child_dnp[i],
+			    zb->zb_objset, zb->zb_blkid * epb + i);
 		}
 
 		/* recursively visitbp() blocks below this */
 		for (i = 0; i < epb; i++) {
-			err = traverse_dnode(td, &cdnp[i], zb->zb_objset,
-			    zb->zb_blkid * epb + i);
+			err = traverse_dnode(td, &child_dnp[i],
+			    zb->zb_objset, zb->zb_blkid * epb + i);
 			if (err != 0)
 				break;
 		}
 	} else if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) {
 		arc_flags_t flags = ARC_FLAG_WAIT;
-		objset_phys_t *osp;
-		dnode_phys_t *mdnp, *gdnp, *udnp;
 
 		err = arc_read(NULL, td->td_spa, bp, arc_getbuf_func, &buf,
 		    ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL, &flags, zb);
 		if (err != 0)
 			goto post;
 
-		osp = buf->b_data;
-		mdnp = &osp->os_meta_dnode;
-		gdnp = &osp->os_groupused_dnode;
-		udnp = &osp->os_userused_dnode;
-
-		prefetch_dnode_metadata(td, mdnp, zb->zb_objset,
+		objset_phys_t *osp = buf->b_data;
+		prefetch_dnode_metadata(td, &osp->os_meta_dnode, zb->zb_objset,
 		    DMU_META_DNODE_OBJECT);
 		if (arc_buf_size(buf) >= sizeof (objset_phys_t)) {
-			prefetch_dnode_metadata(td, gdnp, zb->zb_objset,
-			    DMU_GROUPUSED_OBJECT);
-			prefetch_dnode_metadata(td, udnp, zb->zb_objset,
-			    DMU_USERUSED_OBJECT);
+			prefetch_dnode_metadata(td, &osp->os_groupused_dnode,
+			    zb->zb_objset, DMU_GROUPUSED_OBJECT);
+			prefetch_dnode_metadata(td, &osp->os_userused_dnode,
+			    zb->zb_objset, DMU_USERUSED_OBJECT);
 		}
 
-		err = traverse_dnode(td, mdnp, zb->zb_objset,
+		err = traverse_dnode(td, &osp->os_meta_dnode, zb->zb_objset,
 		    DMU_META_DNODE_OBJECT);
 		if (err == 0 && arc_buf_size(buf) >= sizeof (objset_phys_t)) {
-			err = traverse_dnode(td, gdnp, zb->zb_objset,
-			    DMU_GROUPUSED_OBJECT);
+			err = traverse_dnode(td, &osp->os_groupused_dnode,
+			    zb->zb_objset, DMU_GROUPUSED_OBJECT);
 		}
 		if (err == 0 && arc_buf_size(buf) >= sizeof (objset_phys_t)) {
-			err = traverse_dnode(td, udnp, zb->zb_objset,
-			    DMU_USERUSED_OBJECT);
+			err = traverse_dnode(td, &osp->os_userused_dnode,
+			    zb->zb_objset, DMU_USERUSED_OBJECT);
 		}
 	}
 
@@ -391,9 +385,15 @@ post:
 		 * Set the bookmark to the first level-0 block that we need
 		 * to visit.  This way, the resuming code does not need to
 		 * deal with resuming from indirect blocks.
+		 *
+		 * Note, if zb_level <= 0, dnp may be NULL, so we don't want
+		 * to dereference it.
 		 */
-		td->td_resume->zb_blkid = zb->zb_blkid <<
-		    (zb->zb_level * (dnp->dn_indblkshift - SPA_BLKPTRSHIFT));
+		td->td_resume->zb_blkid = zb->zb_blkid;
+		if (zb->zb_level > 0) {
+			td->td_resume->zb_blkid <<= zb->zb_level *
+			    (dnp->dn_indblkshift - SPA_BLKPTRSHIFT);
+		}
 		td->td_paused = B_TRUE;
 	}
 
@@ -425,6 +425,10 @@ traverse_dnode(traverse_data_t *td, const dnode_phys_t *dnp,
 	int j, err = 0;
 	zbookmark_phys_t czb;
 
+	if (object != DMU_META_DNODE_OBJECT && td->td_resume != NULL &&
+	    object < td->td_resume->zb_object)
+		return (0);
+
 	if (td->td_flags & TRAVERSE_PRE) {
 		SET_BOOKMARK(&czb, objset, object, ZB_DNODE_LEVEL,
 		    ZB_DNODE_BLKID);
@@ -501,6 +505,7 @@ traverse_prefetch_thread(void *arg)
 	td.td_func = traverse_prefetcher;
 	td.td_arg = td_main->td_pfd;
 	td.td_pfd = NULL;
+	td.td_resume = &td_main->td_pfd->pd_resume;
 
 	SET_BOOKMARK(&czb, td.td_objset,
 	    ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID);
@@ -529,12 +534,6 @@ traverse_impl(spa_t *spa, dsl_dataset_t *ds, uint64_t objset, blkptr_t *rootbp,
 	ASSERT(ds == NULL || objset == ds->ds_object);
 	ASSERT(!(flags & TRAVERSE_PRE) || !(flags & TRAVERSE_POST));
 
-	/*
-	 * The data prefetching mechanism (the prefetch thread) is incompatible
-	 * with resuming from a bookmark.
-	 */
-	ASSERT(resume == NULL || !(flags & TRAVERSE_PREFETCH_DATA));
-
 	td.td_spa = spa;
 	td.td_objset = objset;
 	td.td_rootbp = rootbp;
@@ -554,6 +553,8 @@ traverse_impl(spa_t *spa, dsl_dataset_t *ds, uint64_t objset, blkptr_t *rootbp,
 	}
 
 	pd.pd_flags = flags;
+	if (resume != NULL)
+		pd.pd_resume = *resume;
 	mutex_init(&pd.pd_mtx, NULL, MUTEX_DEFAULT, NULL);
 	cv_init(&pd.pd_cv, NULL, CV_DEFAULT, NULL);
 
@@ -601,11 +602,19 @@ traverse_impl(spa_t *spa, dsl_dataset_t *ds, uint64_t objset, blkptr_t *rootbp,
  * in syncing context).
  */
 int
-traverse_dataset(dsl_dataset_t *ds, uint64_t txg_start, int flags,
-    blkptr_cb_t func, void *arg)
+traverse_dataset_resume(dsl_dataset_t *ds, uint64_t txg_start,
+    zbookmark_phys_t *resume,
+    int flags, blkptr_cb_t func, void *arg)
 {
 	return (traverse_impl(ds->ds_dir->dd_pool->dp_spa, ds, ds->ds_object,
-	    &dsl_dataset_phys(ds)->ds_bp, txg_start, NULL, flags, func, arg));
+	    &dsl_dataset_phys(ds)->ds_bp, txg_start, resume, flags, func, arg));
+}
+
+int
+traverse_dataset(dsl_dataset_t *ds, uint64_t txg_start,
+    int flags, blkptr_cb_t func, void *arg)
+{
+	return (traverse_dataset_resume(ds, txg_start, NULL, flags, func, arg));
 }
 
 int
@@ -625,7 +634,6 @@ traverse_pool(spa_t *spa, uint64_t txg_start, int flags,
     blkptr_cb_t func, void *arg)
 {
 	int err;
-	uint64_t obj;
 	dsl_pool_t *dp = spa_get_dsl(spa);
 	objset_t *mos = dp->dp_meta_objset;
 	boolean_t hard = (flags & TRAVERSE_HARD);
@@ -637,8 +645,8 @@ traverse_pool(spa_t *spa, uint64_t txg_start, int flags,
 		return (err);
 
 	/* visit each dataset */
-	for (obj = 1; err == 0;
-	    err = dmu_object_next(mos, &obj, FALSE, txg_start)) {
+	for (uint64_t obj = 1; err == 0;
+	    err = dmu_object_next(mos, &obj, B_FALSE, txg_start)) {
 		dmu_object_info_t doi;
 
 		err = dmu_object_info(mos, obj, &doi);
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c
index 4fbbe7c4e805..47ec6d52cf49 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c
@@ -34,6 +34,7 @@
 #include <sys/dsl_synctask.h>
 #include <sys/dmu_traverse.h>
 #include <sys/dmu_impl.h>
+#include <sys/dmu_send.h>
 #include <sys/dmu_tx.h>
 #include <sys/arc.h>
 #include <sys/zio.h>
@@ -51,6 +52,8 @@
 #include <sys/dsl_destroy.h>
 #include <sys/dsl_userhold.h>
 #include <sys/dsl_bookmark.h>
+#include <sys/zio_compress.h>
+#include <zfs_fletcher.h>
 
 SYSCTL_DECL(_vfs_zfs);
 
@@ -701,6 +704,7 @@ dsl_dataset_tryown(dsl_dataset_t *ds, void *tag)
 {
 	boolean_t gotit = FALSE;
 
+	ASSERT(dsl_pool_config_held(ds->ds_dir->dd_pool));
 	mutex_enter(&ds->ds_lock);
 	if (ds->ds_owner == NULL && !DS_IS_INCONSISTENT(ds)) {
 		ds->ds_owner = tag;
@@ -711,6 +715,16 @@ dsl_dataset_tryown(dsl_dataset_t *ds, void *tag)
 	return (gotit);
 }
 
+boolean_t
+dsl_dataset_has_owner(dsl_dataset_t *ds)
+{
+	boolean_t rv;
+	mutex_enter(&ds->ds_lock);
+	rv = (ds->ds_owner != NULL);
+	mutex_exit(&ds->ds_lock);
+	return (rv);
+}
+
 static void
 dsl_dataset_activate_feature(uint64_t dsobj, spa_feature_t f, dmu_tx_t *tx)
 {
@@ -1657,6 +1671,21 @@ dsl_dataset_sync(dsl_dataset_t *ds, zio_t *zio, dmu_tx_t *tx)
 	dmu_buf_will_dirty(ds->ds_dbuf, tx);
 	dsl_dataset_phys(ds)->ds_fsid_guid = ds->ds_fsid_guid;
 
+	if (ds->ds_resume_bytes[tx->tx_txg & TXG_MASK] != 0) {
+		VERIFY0(zap_update(tx->tx_pool->dp_meta_objset,
+		    ds->ds_object, DS_FIELD_RESUME_OBJECT, 8, 1,
+		    &ds->ds_resume_object[tx->tx_txg & TXG_MASK], tx));
+		VERIFY0(zap_update(tx->tx_pool->dp_meta_objset,
+		    ds->ds_object, DS_FIELD_RESUME_OFFSET, 8, 1,
+		    &ds->ds_resume_offset[tx->tx_txg & TXG_MASK], tx));
+		VERIFY0(zap_update(tx->tx_pool->dp_meta_objset,
+		    ds->ds_object, DS_FIELD_RESUME_BYTES, 8, 1,
+		    &ds->ds_resume_bytes[tx->tx_txg & TXG_MASK], tx));
+		ds->ds_resume_object[tx->tx_txg & TXG_MASK] = 0;
+		ds->ds_resume_offset[tx->tx_txg & TXG_MASK] = 0;
+		ds->ds_resume_bytes[tx->tx_txg & TXG_MASK] = 0;
+	}
+
 	dmu_objset_sync(ds->ds_objset, zio, tx);
 
 	for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
@@ -1712,6 +1741,76 @@ fail:
 	nvlist_free(propval);
 }
 
+static void
+get_receive_resume_stats(dsl_dataset_t *ds, nvlist_t *nv)
+{
+	dsl_pool_t *dp = ds->ds_dir->dd_pool;
+
+	if (dsl_dataset_has_resume_receive_state(ds)) {
+		char *str;
+		void *packed;
+		uint8_t *compressed;
+		uint64_t val;
+		nvlist_t *token_nv = fnvlist_alloc();
+		size_t packed_size, compressed_size;
+
+		if (zap_lookup(dp->dp_meta_objset, ds->ds_object,
+		    DS_FIELD_RESUME_FROMGUID, sizeof (val), 1, &val) == 0) {
+			fnvlist_add_uint64(token_nv, "fromguid", val);
+		}
+		if (zap_lookup(dp->dp_meta_objset, ds->ds_object,
+		    DS_FIELD_RESUME_OBJECT, sizeof (val), 1, &val) == 0) {
+			fnvlist_add_uint64(token_nv, "object", val);
+		}
+		if (zap_lookup(dp->dp_meta_objset, ds->ds_object,
+		    DS_FIELD_RESUME_OFFSET, sizeof (val), 1, &val) == 0) {
+			fnvlist_add_uint64(token_nv, "offset", val);
+		}
+		if (zap_lookup(dp->dp_meta_objset, ds->ds_object,
+		    DS_FIELD_RESUME_BYTES, sizeof (val), 1, &val) == 0) {
+			fnvlist_add_uint64(token_nv, "bytes", val);
+		}
+		if (zap_lookup(dp->dp_meta_objset, ds->ds_object,
+		    DS_FIELD_RESUME_TOGUID, sizeof (val), 1, &val) == 0) {
+			fnvlist_add_uint64(token_nv, "toguid", val);
+		}
+		char buf[256];
+		if (zap_lookup(dp->dp_meta_objset, ds->ds_object,
+		    DS_FIELD_RESUME_TONAME, 1, sizeof (buf), buf) == 0) {
+			fnvlist_add_string(token_nv, "toname", buf);
+		}
+		if (zap_contains(dp->dp_meta_objset, ds->ds_object,
+		    DS_FIELD_RESUME_EMBEDOK) == 0) {
+			fnvlist_add_boolean(token_nv, "embedok");
+		}
+		packed = fnvlist_pack(token_nv, &packed_size);
+		fnvlist_free(token_nv);
+		compressed = kmem_alloc(packed_size, KM_SLEEP);
+
+		compressed_size = gzip_compress(packed, compressed,
+		    packed_size, packed_size, 6);
+
+		zio_cksum_t cksum;
+		fletcher_4_native(compressed, compressed_size, &cksum);
+
+		str = kmem_alloc(compressed_size * 2 + 1, KM_SLEEP);
+		for (int i = 0; i < compressed_size; i++) {
+			(void) sprintf(str + i * 2, "%02x", compressed[i]);
+		}
+		str[compressed_size * 2] = '\0';
+		char *propval = kmem_asprintf("%u-%llx-%llx-%s",
+		    ZFS_SEND_RESUME_TOKEN_VERSION,
+		    (longlong_t)cksum.zc_word[0],
+		    (longlong_t)packed_size, str);
+		dsl_prop_nvlist_add_string(nv,
+		    ZFS_PROP_RECEIVE_RESUME_TOKEN, propval);
+		kmem_free(packed, packed_size);
+		kmem_free(str, compressed_size * 2 + 1);
+		kmem_free(compressed, packed_size);
+		strfree(propval);
+	}
+}
+
 void
 dsl_dataset_stats(dsl_dataset_t *ds, nvlist_t *nv)
 {
@@ -1783,6 +1882,29 @@ dsl_dataset_stats(dsl_dataset_t *ds, nvlist_t *nv)
 			}
 		}
 	}
+
+	if (!dsl_dataset_is_snapshot(ds)) {
+		/*
+		 * A failed "newfs" (e.g. full) resumable receive leaves
+		 * the stats set on this dataset.  Check here for the prop.
+		 */
+		get_receive_resume_stats(ds, nv);
+
+		/*
+		 * A failed incremental resumable receive leaves the
+		 * stats set on our child named "%recv".  Check the child
+		 * for the prop.
+		 */
+		char recvname[ZFS_MAXNAMELEN];
+		dsl_dataset_t *recv_ds;
+		dsl_dataset_name(ds, recvname);
+		(void) strcat(recvname, "/");
+		(void) strcat(recvname, recv_clone_name);
+		if (dsl_dataset_hold(dp, recvname, FTAG, &recv_ds) == 0) {
+			get_receive_resume_stats(recv_ds, nv);
+			dsl_dataset_rele(recv_ds, FTAG);
+		}
+	}
 }
 
 void
@@ -3467,3 +3589,20 @@ dsl_dataset_zapify(dsl_dataset_t *ds, dmu_tx_t *tx)
 	objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset;
 	dmu_object_zapify(mos, ds->ds_object, DMU_OT_DSL_DATASET, tx);
 }
+
+boolean_t
+dsl_dataset_is_zapified(dsl_dataset_t *ds)
+{
+	dmu_object_info_t doi;
+
+	dmu_object_info_from_db(ds->ds_dbuf, &doi);
+	return (doi.doi_type == DMU_OTN_ZAP_METADATA);
+}
+
+boolean_t
+dsl_dataset_has_resume_receive_state(dsl_dataset_t *ds)
+{
+	return (dsl_dataset_is_zapified(ds) &&
+	    zap_contains(ds->ds_dir->dd_pool->dp_meta_objset,
+	    ds->ds_object, DS_FIELD_RESUME_TOGUID) == 0);
+}
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_destroy.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_destroy.c
index c7a623c8291b..7de98450797f 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_destroy.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_destroy.c
@@ -968,9 +968,17 @@ dsl_destroy_inconsistent(const char *dsname, void *arg)
 	objset_t *os;
 
 	if (dmu_objset_hold(dsname, FTAG, &os) == 0) {
-		boolean_t inconsistent = DS_IS_INCONSISTENT(dmu_objset_ds(os));
+		boolean_t need_destroy = DS_IS_INCONSISTENT(dmu_objset_ds(os));
+
+		/*
+		 * If the dataset is inconsistent because a resumable receive
+		 * has failed, then do not destroy it.
+		 */
+		if (dsl_dataset_has_resume_receive_state(dmu_objset_ds(os)))
+			need_destroy = B_FALSE;
+
 		dmu_objset_rele(os, FTAG);
-		if (inconsistent)
+		if (need_destroy)
 			(void) dsl_destroy_head(dsname);
 	}
 	return (0);
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dmu_impl.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dmu_impl.h
index 6f67b5a0b9bb..e8d6294b9815 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dmu_impl.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dmu_impl.h
@@ -25,7 +25,7 @@
 /*
  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
  * Copyright (c) 2012, Martin Matuska <mm@FreeBSD.org>. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
  */
 
 #ifndef _SYS_DMU_IMPL_H
@@ -300,6 +300,8 @@ typedef struct dmu_sendarg {
 	uint64_t dsa_featureflags;
 	uint64_t dsa_last_data_object;
 	uint64_t dsa_last_data_offset;
+	uint64_t dsa_resume_object;
+	uint64_t dsa_resume_offset;
 } dmu_sendarg_t;
 
 void dmu_object_zapify(objset_t *, uint64_t, dmu_object_type_t, dmu_tx_t *);
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dmu_send.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dmu_send.h
index 143d43f037fe..2865e82913a2 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dmu_send.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dmu_send.h
@@ -35,13 +35,16 @@ struct vnode;
 struct dsl_dataset;
 struct drr_begin;
 struct avl_tree;
+struct dmu_replay_record;
 
-int dmu_send(const char *tosnap, const char *fromsnap,
-    boolean_t embedok, boolean_t large_block_ok,
+extern const char *recv_clone_name;
+
+int dmu_send(const char *tosnap, const char *fromsnap, boolean_t embedok,
+    boolean_t large_block_ok, int outfd, uint64_t resumeobj, uint64_t resumeoff,
 #ifdef illumos
-    int outfd, struct vnode *vp, offset_t *off);
+    struct vnode *vp, offset_t *off);
 #else
-    int outfd, struct file *fp, offset_t *off);
+    struct file *fp, offset_t *off);
 #endif
 int dmu_send_estimate(struct dsl_dataset *ds, struct dsl_dataset *fromds,
     uint64_t *sizep);
@@ -57,12 +60,14 @@ int dmu_send_obj(const char *pool, uint64_t tosnap, uint64_t fromsnap,
 
 typedef struct dmu_recv_cookie {
 	struct dsl_dataset *drc_ds;
+	struct dmu_replay_record *drc_drr_begin;
 	struct drr_begin *drc_drrb;
 	const char *drc_tofs;
 	const char *drc_tosnap;
 	boolean_t drc_newfs;
 	boolean_t drc_byteswap;
 	boolean_t drc_force;
+	boolean_t drc_resumable;
 	struct avl_tree *drc_guid_to_ds_map;
 	zio_cksum_t drc_cksum;
 	uint64_t drc_newsnapobj;
@@ -70,8 +75,9 @@ typedef struct dmu_recv_cookie {
 	cred_t *drc_cred;
 } dmu_recv_cookie_t;
 
-int dmu_recv_begin(char *tofs, char *tosnap, struct drr_begin *drrb,
-    boolean_t force, char *origin, dmu_recv_cookie_t *drc);
+int dmu_recv_begin(char *tofs, char *tosnap,
+    struct dmu_replay_record *drr_begin,
+    boolean_t force, boolean_t resumable, char *origin, dmu_recv_cookie_t *drc);
 #ifdef illumos
 int dmu_recv_stream(dmu_recv_cookie_t *drc, struct vnode *vp, offset_t *voffp,
 #else
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dmu_traverse.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dmu_traverse.h
index 544b721e4612..c010edd440d9 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dmu_traverse.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dmu_traverse.h
@@ -54,6 +54,8 @@ typedef int (blkptr_cb_t)(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 
 int traverse_dataset(struct dsl_dataset *ds,
     uint64_t txg_start, int flags, blkptr_cb_t func, void *arg);
+int traverse_dataset_resume(struct dsl_dataset *ds, uint64_t txg_start,
+    zbookmark_phys_t *resume, int flags, blkptr_cb_t func, void *arg);
 int traverse_dataset_destroyed(spa_t *spa, blkptr_t *blkptr,
     uint64_t txg_start, zbookmark_phys_t *resume, int flags,
     blkptr_cb_t func, void *arg);
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_dataset.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_dataset.h
index c9cd5890aa74..d7df05b309be 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_dataset.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_dataset.h
@@ -91,6 +91,18 @@ struct dsl_pool;
  */
 #define	DS_FIELD_LARGE_BLOCKS "org.open-zfs:large_blocks"
 
+/*
+ * These fields are set on datasets that are in the middle of a resumable
+ * receive, and allow the sender to resume the send if it is interrupted.
+ */
+#define	DS_FIELD_RESUME_FROMGUID "com.delphix:resume_fromguid"
+#define	DS_FIELD_RESUME_TONAME "com.delphix:resume_toname"
+#define	DS_FIELD_RESUME_TOGUID "com.delphix:resume_toguid"
+#define	DS_FIELD_RESUME_OBJECT "com.delphix:resume_object"
+#define	DS_FIELD_RESUME_OFFSET "com.delphix:resume_offset"
+#define	DS_FIELD_RESUME_BYTES "com.delphix:resume_bytes"
+#define	DS_FIELD_RESUME_EMBEDOK "com.delphix:resume_embedok"
+
 /*
  * DS_FLAG_CI_DATASET is set if the dataset contains a file system whose
  * name lookups should be performed case-insensitively.
@@ -184,6 +196,14 @@ typedef struct dsl_dataset {
 	kmutex_t ds_sendstream_lock;
 	list_t ds_sendstreams;
 
+	/*
+	 * When in the middle of a resumable receive, tracks how much
+	 * progress we have made.
+	 */
+	uint64_t ds_resume_object[TXG_SIZE];
+	uint64_t ds_resume_offset[TXG_SIZE];
+	uint64_t ds_resume_bytes[TXG_SIZE];
+
 	/* Protected by our dsl_dir's dd_lock */
 	list_t ds_prop_cbs;
 
@@ -235,6 +255,7 @@ int dsl_dataset_own_obj(struct dsl_pool *dp, uint64_t dsobj,
 void dsl_dataset_disown(dsl_dataset_t *ds, void *tag);
 void dsl_dataset_name(dsl_dataset_t *ds, char *name);
 boolean_t dsl_dataset_tryown(dsl_dataset_t *ds, void *tag);
+boolean_t dsl_dataset_has_owner(dsl_dataset_t *ds);
 uint64_t dsl_dataset_create_sync(dsl_dir_t *pds, const char *lastname,
     dsl_dataset_t *origin, uint64_t flags, cred_t *, dmu_tx_t *);
 uint64_t dsl_dataset_create_sync_dd(dsl_dir_t *dd, dsl_dataset_t *origin,
@@ -315,6 +336,8 @@ int dsl_dataset_snap_remove(dsl_dataset_t *ds, const char *name, dmu_tx_t *tx,
 void dsl_dataset_set_refreservation_sync_impl(dsl_dataset_t *ds,
     zprop_source_t source, uint64_t value, dmu_tx_t *tx);
 void dsl_dataset_zapify(dsl_dataset_t *ds, dmu_tx_t *tx);
+boolean_t dsl_dataset_is_zapified(dsl_dataset_t *ds);
+boolean_t dsl_dataset_has_resume_receive_state(dsl_dataset_t *ds);
 int dsl_dataset_rollback(const char *fsname, void *owner, nvlist_t *result);
 
 void dsl_dataset_deactivate_feature(uint64_t dsobj,
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_ioctl.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_ioctl.h
index f9eca27075b1..20bf54553329 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_ioctl.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_ioctl.h
@@ -79,14 +79,15 @@ typedef enum drr_headertype {
  * Feature flags for zfs send streams (flags in drr_versioninfo)
  */
 
-#define	DMU_BACKUP_FEATURE_DEDUP		(1<<0)
-#define	DMU_BACKUP_FEATURE_DEDUPPROPS		(1<<1)
-#define	DMU_BACKUP_FEATURE_SA_SPILL		(1<<2)
+#define	DMU_BACKUP_FEATURE_DEDUP		(1 << 0)
+#define	DMU_BACKUP_FEATURE_DEDUPPROPS		(1 << 1)
+#define	DMU_BACKUP_FEATURE_SA_SPILL		(1 << 2)
 /* flags #3 - #15 are reserved for incompatible closed-source implementations */
-#define	DMU_BACKUP_FEATURE_EMBED_DATA		(1<<16)
-#define	DMU_BACKUP_FEATURE_EMBED_DATA_LZ4	(1<<17)
+#define	DMU_BACKUP_FEATURE_EMBED_DATA		(1 << 16)
+#define	DMU_BACKUP_FEATURE_EMBED_DATA_LZ4	(1 << 17)
 /* flag #18 is reserved for a Delphix feature */
-#define	DMU_BACKUP_FEATURE_LARGE_BLOCKS		(1<<19)
+#define	DMU_BACKUP_FEATURE_LARGE_BLOCKS		(1 << 19)
+#define	DMU_BACKUP_FEATURE_RESUMING		(1 << 20)
 
 /*
  * Mask of all supported backup features
@@ -94,11 +95,16 @@ typedef enum drr_headertype {
 #define	DMU_BACKUP_FEATURE_MASK	(DMU_BACKUP_FEATURE_DEDUP | \
     DMU_BACKUP_FEATURE_DEDUPPROPS | DMU_BACKUP_FEATURE_SA_SPILL | \
     DMU_BACKUP_FEATURE_EMBED_DATA | DMU_BACKUP_FEATURE_EMBED_DATA_LZ4 | \
+    DMU_BACKUP_FEATURE_RESUMING | \
     DMU_BACKUP_FEATURE_LARGE_BLOCKS)
 
 /* Are all features in the given flag word currently supported? */
 #define	DMU_STREAM_SUPPORTED(x)	(!((x) & ~DMU_BACKUP_FEATURE_MASK))
 
+typedef enum dmu_send_resume_token_version {
+	ZFS_SEND_RESUME_TOKEN_VERSION = 1
+} dmu_send_resume_token_version_t;
+
 /*
  * The drr_versioninfo field of the dmu_replay_record has the
  * following layout:
@@ -358,14 +364,14 @@ typedef struct zfs_cmd {
 	zfs_share_t	zc_share;
 	uint64_t	zc_jailid;
 	dmu_objset_stats_t zc_objset_stats;
-	struct drr_begin zc_begin_record;
+	dmu_replay_record_t zc_begin_record;
 	zinject_record_t zc_inject_record;
 	uint32_t	zc_defer_destroy;
 	uint32_t	zc_flags;
 	uint64_t	zc_action_handle;
 	int		zc_cleanup_fd;
 	uint8_t		zc_simple;
-	uint8_t		zc_pad[3];		/* alignment */
+	boolean_t	zc_resumable;
 	uint64_t	zc_sendobj;
 	uint64_t	zc_fromobj;
 	uint64_t	zc_createtxg;
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c
index 9ead0b71fc02..5ae90f6634ec 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c
@@ -4156,6 +4156,7 @@ static boolean_t zfs_ioc_recv_inject_err;
  * zc_guid		force flag
  * zc_cleanup_fd	cleanup-on-exit file descriptor
  * zc_action_handle	handle for this guid/ds mapping (or zero on first call)
+ * zc_resumable		if data is incomplete assume sender will resume
  *
  * outputs:
  * zc_cookie		number of bytes read
@@ -4207,13 +4208,13 @@ zfs_ioc_recv(zfs_cmd_t *zc)
 		return (SET_ERROR(EBADF));
 	}
 
-	VERIFY(nvlist_alloc(&errors, NV_UNIQUE_NAME, KM_SLEEP) == 0);
+	errors = fnvlist_alloc();
 
 	if (zc->zc_string[0])
 		origin = zc->zc_string;
 
 	error = dmu_recv_begin(tofs, tosnap,
-	    &zc->zc_begin_record, force, origin, &drc);
+	    &zc->zc_begin_record, force, zc->zc_resumable, origin, &drc);
 	if (error != 0)
 		goto out;
 
@@ -5432,6 +5433,8 @@ zfs_ioc_unjail(zfs_cmd_t *zc)
  *         indicates that blocks > 128KB are permitted
  *     (optional) "embedok" -> (value ignored)
  *         presence indicates DRR_WRITE_EMBEDDED records are permitted
+ *     (optional) "resume_object" and "resume_offset" -> (uint64)
+ *         if present, resume send stream from specified object and offset.
  * }
  *
  * outnvl is unused
@@ -5448,6 +5451,8 @@ zfs_ioc_send_new(const char *snapname, nvlist_t *innvl, nvlist_t *outnvl)
 	int fd;
 	boolean_t largeblockok;
 	boolean_t embedok;
+	uint64_t resumeobj = 0;
+	uint64_t resumeoff = 0;
 
 	error = nvlist_lookup_int32(innvl, "fd", &fd);
 	if (error != 0)
@@ -5458,6 +5463,9 @@ zfs_ioc_send_new(const char *snapname, nvlist_t *innvl, nvlist_t *outnvl)
 	largeblockok = nvlist_exists(innvl, "largeblockok");
 	embedok = nvlist_exists(innvl, "embedok");
 
+	(void) nvlist_lookup_uint64(innvl, "resume_object", &resumeobj);
+	(void) nvlist_lookup_uint64(innvl, "resume_offset", &resumeoff);
+
 #ifdef illumos
 	file_t *fp = getf(fd);
 #else
@@ -5467,11 +5475,11 @@ zfs_ioc_send_new(const char *snapname, nvlist_t *innvl, nvlist_t *outnvl)
 		return (SET_ERROR(EBADF));
 
 	off = fp->f_offset;
-	error = dmu_send(snapname, fromname, embedok, largeblockok,
+	error = dmu_send(snapname, fromname, embedok, largeblockok, fd,
 #ifdef illumos
-	    fd, fp->f_vnode, &off);
+	    resumeobj, resumeoff, fp->f_vnode, &off);
 #else
-	    fd, fp, &off);
+	    resumeobj, resumeoff, fp, &off);
 #endif
 
 #ifdef illumos
@@ -6100,6 +6108,14 @@ zfsdev_ioctl(struct cdev *dev, u_long zcmd, caddr_t arg, int flag,
 				goto out;
 			}
 			break;
+		case ZFS_IOCVER_EDBP:
+			if (zc_iocparm->zfs_cmd_size != sizeof(zfs_cmd_edbp_t)) {
+				error = SET_ERROR(EFAULT);
+				goto out;
+			}
+			compat = B_TRUE;
+			cflag = ZFS_CMD_COMPAT_EDBP;
+			break;
 		case ZFS_IOCVER_ZCMD:
 			if (zc_iocparm->zfs_cmd_size > sizeof(zfs_cmd_t) ||
 			    zc_iocparm->zfs_cmd_size < sizeof(zfs_cmd_zcmd_t)) {
diff --git a/sys/cddl/contrib/opensolaris/uts/common/sys/fs/zfs.h b/sys/cddl/contrib/opensolaris/uts/common/sys/fs/zfs.h
index 16d528e025d5..809ce9bc5ffe 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/sys/fs/zfs.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/sys/fs/zfs.h
@@ -152,6 +152,7 @@ typedef enum {
 	ZFS_PROP_SNAPSHOT_COUNT,
 	ZFS_PROP_REDUNDANT_METADATA,
 	ZFS_PROP_PREV_SNAP,
+	ZFS_PROP_RECEIVE_RESUME_TOKEN,
 	ZFS_NUM_PROPS
 } zfs_prop_t;
 

From e0a76fc624ff8f59328873dcf100f69d94d852f8 Mon Sep 17 00:00:00 2001
From: emaste <emaste@FreeBSD.org>
Date: Thu, 15 Oct 2015 16:19:00 +0000
Subject: [PATCH 248/380] newfs_msdos: rework option parsing to match NetBSD

NetBSD split newfs_msdos in two so that they can reuse the file system
creation part in makefs. This change is a step on the path of bringing
that support to FreeBSD.

Reviewed by:	kib, pfg
Sponsored by:	The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D3905
---
 sbin/newfs_msdos/newfs_msdos.c | 397 ++++++++++++++++++---------------
 1 file changed, 217 insertions(+), 180 deletions(-)

diff --git a/sbin/newfs_msdos/newfs_msdos.c b/sbin/newfs_msdos/newfs_msdos.c
index 468578287671..a3994257ea06 100644
--- a/sbin/newfs_msdos/newfs_msdos.c
+++ b/sbin/newfs_msdos/newfs_msdos.c
@@ -214,6 +214,35 @@ static const u_int8_t bootcode[] = {
     0
 };
 
+struct msdos_options {
+    const char *bootstrap;
+    const char *volume_label;
+    const char *OEM_string;
+    const char *floppy;
+    u_int fat_type;
+    u_int volume_id;
+    u_int bytes_per_sector;
+    u_int sectors_per_fat;
+    u_int block_size;
+    u_int sectors_per_cluster;
+    u_int directory_entries;
+    u_int drive_heads;
+    u_int info_sector;
+    u_int backup_sector;
+    u_int media_descriptor;
+    u_int num_FAT;
+    u_int hidden_sectors;
+    u_int reserved_sectors;
+    u_int size;
+    u_int sectors_per_track;
+    int no_create;
+    off_t create_size;
+    off_t offset;
+    int volume_id_set;
+    int media_descriptor_set;
+    int hidden_sectors_set;
+};
+
 static volatile sig_atomic_t got_siginfo;
 static void infohandler(int);
 
@@ -226,6 +255,7 @@ static u_int ckgeom(const char *, u_int, const char *);
 static u_int argtou(const char *, u_int, u_int, const char *);
 static off_t argtooff(const char *, const char *);
 static int oklabel(const char *);
+static int mkfs_msdos(const char *, const char *, const struct msdos_options *);
 static void mklabel(u_int8_t *, const char *);
 static void setstr(u_int8_t *, const char *, size_t);
 static void usage(void);
@@ -237,12 +267,117 @@ int
 main(int argc, char *argv[])
 {
     static const char opts[] = "@:NB:C:F:I:L:O:S:a:b:c:e:f:h:i:k:m:n:o:r:s:u:";
-    const char *opt_B = NULL, *opt_L = NULL, *opt_O = NULL, *opt_f = NULL;
-    u_int opt_F = 0, opt_I = 0, opt_S = 0, opt_a = 0, opt_b = 0, opt_c = 0;
-    u_int opt_e = 0, opt_h = 0, opt_i = 0, opt_k = 0, opt_m = 0, opt_n = 0;
-    u_int opt_o = 0, opt_r = 0, opt_s = 0, opt_u = 0;
-    int opt_N = 0;
-    int Iflag = 0, mflag = 0, oflag = 0;
+    struct msdos_options o;
+    const char *fname, *dtype;
+    char buf[MAXPATHLEN];
+    int ch;
+
+    memset(&o, 0, sizeof(o));
+
+    while ((ch = getopt(argc, argv, opts)) != -1)
+	switch (ch) {
+	case '@':
+	    o.offset = argtooff(optarg, "offset");
+	    break;
+	case 'N':
+	    o.no_create = 1;
+	    break;
+	case 'B':
+	    o.bootstrap = optarg;
+	    break;
+	case 'C':
+	    o.create_size = argtooff(optarg, "create size");
+	    break;
+	case 'F':
+	    if (strcmp(optarg, "12") &&
+		strcmp(optarg, "16") &&
+		strcmp(optarg, "32"))
+		errx(1, "%s: bad FAT type", optarg);
+	    o.fat_type = atoi(optarg);
+	    break;
+	case 'I':
+	    o.volume_id = argto4(optarg, 0, "volume ID");
+	    o.volume_id_set = 1;
+	    break;
+	case 'L':
+	    if (!oklabel(optarg))
+		errx(1, "%s: bad volume label", optarg);
+	    o.volume_label = optarg;
+	    break;
+	case 'O':
+	    if (strlen(optarg) > 8)
+		errx(1, "%s: bad OEM string", optarg);
+	    o.OEM_string = optarg;
+	    break;
+	case 'S':
+	    o.bytes_per_sector = argto2(optarg, 1, "bytes/sector");
+	    break;
+	case 'a':
+	    o.sectors_per_fat = argto4(optarg, 1, "sectors/FAT");
+	    break;
+	case 'b':
+	    o.block_size = argtox(optarg, 1, "block size");
+	    o.sectors_per_cluster = 0;
+	    break;
+	case 'c':
+	    o.sectors_per_cluster = argto1(optarg, 1, "sectors/cluster");
+	    o.block_size = 0;
+	    break;
+	case 'e':
+	    o.directory_entries = argto2(optarg, 1, "directory entries");
+	    break;
+	case 'f':
+	    o.floppy = optarg;
+	    break;
+	case 'h':
+	    o.drive_heads = argto2(optarg, 1, "drive heads");
+	    break;
+	case 'i':
+	    o.info_sector = argto2(optarg, 1, "info sector");
+	    break;
+	case 'k':
+	    o.backup_sector = argto2(optarg, 1, "backup sector");
+	    break;
+	case 'm':
+	    o.media_descriptor = argto1(optarg, 0, "media descriptor");
+	    o.media_descriptor_set = 1;
+	    break;
+	case 'n':
+	    o.num_FAT = argto1(optarg, 1, "number of FATs");
+	    break;
+	case 'o':
+	    o.hidden_sectors = argto4(optarg, 0, "hidden sectors");
+	    o.hidden_sectors_set = 1;
+	    break;
+	case 'r':
+	    o.reserved_sectors = argto2(optarg, 1, "reserved sectors");
+	    break;
+	case 's':
+	    o.size = argto4(optarg, 1, "file system size");
+	    break;
+	case 'u':
+	    o.sectors_per_track = argto2(optarg, 1, "sectors/track");
+	    break;
+	default:
+	    usage();
+	}
+    argc -= optind;
+    argv += optind;
+    if (argc < 1 || argc > 2)
+	usage();
+    fname = *argv++;
+    if (!o.create_size && !strchr(fname, '/')) {
+	snprintf(buf, sizeof(buf), "%s%s", _PATH_DEV, fname);
+	if (!(fname = strdup(buf)))
+	    err(1, NULL);
+    }
+    dtype = *argv;
+    return mkfs_msdos(fname, dtype, &o);
+}
+
+int mkfs_msdos(const char *fname, const char *dtype,
+    const struct msdos_options *op)
+{
     char buf[MAXPATHLEN];
     struct sigaction si_sa;
     struct stat sb;
@@ -255,156 +390,58 @@ main(int argc, char *argv[])
     struct bsx *bsx;
     struct de *de;
     u_int8_t *img;
-    const char *fname, *dtype, *bname;
+    const char *bname;
     ssize_t n;
     time_t now;
     u_int fat, bss, rds, cls, dir, lsn, x, x1, x2;
-    int ch, fd, fd1;
-    off_t opt_create = 0, opt_ofs = 0;
+    int fd, fd1;
+    struct msdos_options o = *op;
 
-    while ((ch = getopt(argc, argv, opts)) != -1)
-	switch (ch) {
-	case '@':
-	    opt_ofs = argtooff(optarg, "offset");
-	    break;
-	case 'N':
-	    opt_N = 1;
-	    break;
-	case 'B':
-	    opt_B = optarg;
-	    break;
-	case 'C':
-	    opt_create = argtooff(optarg, "create size");
-	    break;
-	case 'F':
-	    if (strcmp(optarg, "12") &&
-		strcmp(optarg, "16") &&
-		strcmp(optarg, "32"))
-		errx(1, "%s: bad FAT type", optarg);
-	    opt_F = atoi(optarg);
-	    break;
-	case 'I':
-	    opt_I = argto4(optarg, 0, "volume ID");
-	    Iflag = 1;
-	    break;
-	case 'L':
-	    if (!oklabel(optarg))
-		errx(1, "%s: bad volume label", optarg);
-	    opt_L = optarg;
-	    break;
-	case 'O':
-	    if (strlen(optarg) > 8)
-		errx(1, "%s: bad OEM string", optarg);
-	    opt_O = optarg;
-	    break;
-	case 'S':
-	    opt_S = argto2(optarg, 1, "bytes/sector");
-	    break;
-	case 'a':
-	    opt_a = argto4(optarg, 1, "sectors/FAT");
-	    break;
-	case 'b':
-	    opt_b = argtox(optarg, 1, "block size");
-	    opt_c = 0;
-	    break;
-	case 'c':
-	    opt_c = argto1(optarg, 1, "sectors/cluster");
-	    opt_b = 0;
-	    break;
-	case 'e':
-	    opt_e = argto2(optarg, 1, "directory entries");
-	    break;
-	case 'f':
-	    opt_f = optarg;
-	    break;
-	case 'h':
-	    opt_h = argto2(optarg, 1, "drive heads");
-	    break;
-	case 'i':
-	    opt_i = argto2(optarg, 1, "info sector");
-	    break;
-	case 'k':
-	    opt_k = argto2(optarg, 1, "backup sector");
-	    break;
-	case 'm':
-	    opt_m = argto1(optarg, 0, "media descriptor");
-	    mflag = 1;
-	    break;
-	case 'n':
-	    opt_n = argto1(optarg, 1, "number of FATs");
-	    break;
-	case 'o':
-	    opt_o = argto4(optarg, 0, "hidden sectors");
-	    oflag = 1;
-	    break;
-	case 'r':
-	    opt_r = argto2(optarg, 1, "reserved sectors");
-	    break;
-	case 's':
-	    opt_s = argto4(optarg, 1, "file system size");
-	    break;
-	case 'u':
-	    opt_u = argto2(optarg, 1, "sectors/track");
-	    break;
-	default:
-	    usage();
-	}
-    argc -= optind;
-    argv += optind;
-    if (argc < 1 || argc > 2)
-	usage();
-    fname = *argv++;
-    if (!opt_create && !strchr(fname, '/')) {
-	snprintf(buf, sizeof(buf), "%s%s", _PATH_DEV, fname);
-	if (!(fname = strdup(buf)))
-	    err(1, NULL);
-    }
-    dtype = *argv;
-    if (opt_create) {
-	if (opt_N)
+    if (o.create_size) {
+	if (o.no_create)
 	    errx(1, "create (-C) is incompatible with -N");
 	fd = open(fname, O_RDWR | O_CREAT | O_TRUNC, 0644);
 	if (fd == -1)
 	    errx(1, "failed to create %s", fname);
-	if (ftruncate(fd, opt_create))
-	    errx(1, "failed to initialize %jd bytes", (intmax_t)opt_create);
-    } else if ((fd = open(fname, opt_N ? O_RDONLY : O_RDWR)) == -1)
+	if (ftruncate(fd, o.create_size))
+	    errx(1, "failed to initialize %jd bytes", (intmax_t)o.create_size);
+    } else if ((fd = open(fname, o.no_create ? O_RDONLY : O_RDWR)) == -1)
 	err(1, "%s", fname);
     if (fstat(fd, &sb))
 	err(1, "%s", fname);
-    if (opt_create) {
+    if (o.create_size) {
 	if (!S_ISREG(sb.st_mode))
 	    warnx("warning, %s is not a regular file", fname);
     } else {
 	if (!S_ISCHR(sb.st_mode))
 	    warnx("warning, %s is not a character device", fname);
     }
-    if (!opt_N)
+    if (!o.no_create)
 	check_mounted(fname, sb.st_mode);
-    if (opt_ofs && opt_ofs != lseek(fd, opt_ofs, SEEK_SET))
-	errx(1, "cannot seek to %jd", (intmax_t)opt_ofs);
+    if (o.offset && o.offset != lseek(fd, o.offset, SEEK_SET))
+	errx(1, "cannot seek to %jd", (intmax_t)o.offset);
     memset(&bpb, 0, sizeof(bpb));
-    if (opt_f) {
-	getstdfmt(opt_f, &bpb);
+    if (o.floppy) {
+	getstdfmt(o.floppy, &bpb);
 	bpb.bpbHugeSectors = bpb.bpbSectors;
 	bpb.bpbSectors = 0;
 	bpb.bpbBigFATsecs = bpb.bpbFATsecs;
 	bpb.bpbFATsecs = 0;
     }
-    if (opt_h)
-	bpb.bpbHeads = opt_h;
-    if (opt_u)
-	bpb.bpbSecPerTrack = opt_u;
-    if (opt_S)
-	bpb.bpbBytesPerSec = opt_S;
-    if (opt_s)
-	bpb.bpbHugeSectors = opt_s;
-    if (oflag)
-	bpb.bpbHiddenSecs = opt_o;
-    if (!(opt_f || (opt_h && opt_u && opt_S && opt_s && oflag))) {
+    if (o.drive_heads)
+	bpb.bpbHeads = o.drive_heads;
+    if (o.sectors_per_track)
+	bpb.bpbSecPerTrack = o.sectors_per_track;
+    if (o.bytes_per_sector)
+	bpb.bpbBytesPerSec = o.bytes_per_sector;
+    if (o.size)
+	bpb.bpbHugeSectors = o.size;
+    if (o.hidden_sectors_set)
+	bpb.bpbHiddenSecs = o.hidden_sectors;
+    if (!(o.floppy || (o.drive_heads && o.sectors_per_track && o.bytes_per_sector && o.size && o.hidden_sectors_set))) {
 	off_t delta;
-	getdiskinfo(fd, fname, dtype, oflag, &bpb);
-	bpb.bpbHugeSectors -= (opt_ofs / bpb.bpbBytesPerSec);
+	getdiskinfo(fd, fname, dtype, o.hidden_sectors_set, &bpb);
+	bpb.bpbHugeSectors -= (o.offset / bpb.bpbBytesPerSec);
 	delta = bpb.bpbHugeSectors % bpb.bpbSecPerTrack;
 	if (delta != 0) {
 	    warnx("trim %d sectors to adjust to a multiple of %d",
@@ -429,60 +466,60 @@ main(int argc, char *argv[])
     if (bpb.bpbBytesPerSec < MINBPS)
 	errx(1, "bytes/sector (%u) is too small; minimum is %u",
 	     bpb.bpbBytesPerSec, MINBPS);
-    if (!(fat = opt_F)) {
-	if (opt_f)
+    if (!(fat = o.fat_type)) {
+	if (o.floppy)
 	    fat = 12;
-	else if (!opt_e && (opt_i || opt_k))
+	else if (!o.directory_entries && (o.info_sector || o.backup_sector))
 	    fat = 32;
     }
-    if ((fat == 32 && opt_e) || (fat != 32 && (opt_i || opt_k)))
+    if ((fat == 32 && o.directory_entries) || (fat != 32 && (o.info_sector || o.backup_sector)))
 	errx(1, "-%c is not a legal FAT%s option",
-	     fat == 32 ? 'e' : opt_i ? 'i' : 'k',
+	     fat == 32 ? 'e' : o.info_sector ? 'i' : 'k',
 	     fat == 32 ? "32" : "12/16");
-    if (opt_f && fat == 32)
+    if (o.floppy && fat == 32)
 	bpb.bpbRootDirEnts = 0;
-    if (opt_b) {
-	if (!powerof2(opt_b))
-	    errx(1, "block size (%u) is not a power of 2", opt_b);
-	if (opt_b < bpb.bpbBytesPerSec)
+    if (o.block_size) {
+	if (!powerof2(o.block_size))
+	    errx(1, "block size (%u) is not a power of 2", o.block_size);
+	if (o.block_size < bpb.bpbBytesPerSec)
 	    errx(1, "block size (%u) is too small; minimum is %u",
-		 opt_b, bpb.bpbBytesPerSec);
-	if (opt_b > bpb.bpbBytesPerSec * MAXSPC)
+		 o.block_size, bpb.bpbBytesPerSec);
+	if (o.block_size > bpb.bpbBytesPerSec * MAXSPC)
 	    errx(1, "block size (%u) is too large; maximum is %u",
-		 opt_b, bpb.bpbBytesPerSec * MAXSPC);
-	bpb.bpbSecPerClust = opt_b / bpb.bpbBytesPerSec;
+		 o.block_size, bpb.bpbBytesPerSec * MAXSPC);
+	bpb.bpbSecPerClust = o.block_size / bpb.bpbBytesPerSec;
     }
-    if (opt_c) {
-	if (!powerof2(opt_c))
-	    errx(1, "sectors/cluster (%u) is not a power of 2", opt_c);
-	bpb.bpbSecPerClust = opt_c;
+    if (o.sectors_per_cluster) {
+	if (!powerof2(o.sectors_per_cluster))
+	    errx(1, "sectors/cluster (%u) is not a power of 2", o.sectors_per_cluster);
+	bpb.bpbSecPerClust = o.sectors_per_cluster;
     }
-    if (opt_r)
-	bpb.bpbResSectors = opt_r;
-    if (opt_n) {
-	if (opt_n > MAXNFT)
+    if (o.reserved_sectors)
+	bpb.bpbResSectors = o.reserved_sectors;
+    if (o.num_FAT) {
+	if (o.num_FAT > MAXNFT)
 	    errx(1, "number of FATs (%u) is too large; maximum is %u",
-		 opt_n, MAXNFT);
-	bpb.bpbFATs = opt_n;
+		 o.num_FAT, MAXNFT);
+	bpb.bpbFATs = o.num_FAT;
     }
-    if (opt_e)
-	bpb.bpbRootDirEnts = opt_e;
-    if (mflag) {
-	if (opt_m < 0xf0)
-	    errx(1, "illegal media descriptor (%#x)", opt_m);
-	bpb.bpbMedia = opt_m;
+    if (o.directory_entries)
+	bpb.bpbRootDirEnts = o.directory_entries;
+    if (o.media_descriptor_set) {
+	if (o.media_descriptor < 0xf0)
+	    errx(1, "illegal media descriptor (%#x)", o.media_descriptor);
+	bpb.bpbMedia = o.media_descriptor;
     }
-    if (opt_a)
-	bpb.bpbBigFATsecs = opt_a;
-    if (opt_i)
-	bpb.bpbFSInfo = opt_i;
-    if (opt_k)
-	bpb.bpbBackup = opt_k;
+    if (o.sectors_per_fat)
+	bpb.bpbBigFATsecs = o.sectors_per_fat;
+    if (o.info_sector)
+	bpb.bpbFSInfo = o.info_sector;
+    if (o.backup_sector)
+	bpb.bpbBackup = o.backup_sector;
     bss = 1;
     bname = NULL;
     fd1 = -1;
-    if (opt_B) {
-	bname = opt_B;
+    if (o.bootstrap) {
+	bname = o.bootstrap;
 	if (!strchr(bname, '/')) {
 	    snprintf(buf, sizeof(buf), "/boot/%s", bname);
 	    if (!(bname = strdup(buf)))
@@ -612,7 +649,7 @@ main(int argc, char *argv[])
 	bpb.bpbBigFATsecs = 0;
     }
     print_bpb(&bpb);
-    if (!opt_N) {
+    if (!o.no_create) {
 	gettimeofday(&tv, NULL);
 	now = tv.tv_sec;
 	tm = localtime(&now);
@@ -634,14 +671,14 @@ main(int argc, char *argv[])
 		    got_siginfo = 0;
 	    }
 	    x = lsn;
-	    if (opt_B &&
+	    if (o.bootstrap &&
 		fat == 32 && bpb.bpbBackup != MAXU16 &&
 		bss <= bpb.bpbBackup && x >= bpb.bpbBackup) {
 		x -= bpb.bpbBackup;
-		if (!x && lseek(fd1, opt_ofs, SEEK_SET))
+		if (!x && lseek(fd1, o.offset, SEEK_SET))
 		    err(1, "%s", bname);
 	    }
-	    if (opt_B && x < bss) {
+	    if (o.bootstrap && x < bss) {
 		if ((n = read(fd1, img, bpb.bpbBytesPerSec)) == -1)
 		    err(1, "%s", bname);
 		if ((unsigned)n != bpb.bpbBytesPerSec)
@@ -678,8 +715,8 @@ main(int argc, char *argv[])
 		}
 		bsx = (struct bsx *)(img + x1);
 		mk1(bsx->exBootSignature, 0x29);
-		if (Iflag)
-		    x = opt_I;
+		if (o.volume_id_set)
+		    x = o.volume_id;
 		else
 		    x = (((u_int)(1 + tm->tm_mon) << 8 |
 			  (u_int)tm->tm_mday) +
@@ -689,16 +726,16 @@ main(int argc, char *argv[])
 			 ((u_int)tm->tm_hour << 8 |
 			  (u_int)tm->tm_min));
 		mk4(bsx->exVolumeID, x);
-		mklabel(bsx->exVolumeLabel, opt_L ? opt_L : "NO NAME");
+		mklabel(bsx->exVolumeLabel, o.volume_label ? o.volume_label : "NO NAME");
 		sprintf(buf, "FAT%u", fat);
 		setstr(bsx->exFileSysType, buf, sizeof(bsx->exFileSysType));
-		if (!opt_B) {
+		if (!o.bootstrap) {
 		    x1 += sizeof(struct bsx);
 		    bs = (struct bs *)img;
 		    mk1(bs->bsJump[0], 0xeb);
 		    mk1(bs->bsJump[1], x1 - 2);
 		    mk1(bs->bsJump[2], 0x90);
-		    setstr(bs->bsOemName, opt_O ? opt_O : "BSD4.4  ",
+		    setstr(bs->bsOemName, o.OEM_string ? o.OEM_string : "BSD4.4  ",
 			   sizeof(bs->bsOemName));
 		    memcpy(img + x1, bootcode, sizeof(bootcode));
 		    mk2(img + MINBPS - 2, DOSMAGIC);
@@ -719,9 +756,9 @@ main(int argc, char *argv[])
 		mk1(img[0], bpb.bpbMedia);
 		for (x = 1; x < fat * (fat == 32 ? 3 : 2) / 8; x++)
 		    mk1(img[x], fat == 32 && x % 4 == 3 ? 0x0f : 0xff);
-	    } else if (lsn == dir && opt_L) {
+	    } else if (lsn == dir && o.volume_label) {
 		de = (struct de *)img;
-		mklabel(de->deName, opt_L);
+		mklabel(de->deName, o.volume_label);
 		mk1(de->deAttributes, 050);
 		x = (u_int)tm->tm_hour << 11 |
 		    (u_int)tm->tm_min << 5 |

From 7540fa5d76e4f200821971a7f5c873af75724bf4 Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Thu, 15 Oct 2015 17:40:39 +0000
Subject: [PATCH 249/380] ARM userspace accessors, e.g. {s,f}uword(9),
 copy{in,out}(9), casuword(9) and others, use LDRT and STRT instructions to
 access memory with the privileges of userspace.  If the *RT instruction
 faults on the kernel address, then additional checks must be done to not
 confuse the VM system with invalid kernel-mode faults.

Put ARM on line with other FreeBSD architectures and disallow usermode
buffers which intersect with the kernel address space in advance,
before any accesses are performed.  In other words, vm_fault(9) is no
longer called when e.g. suword(9) stores to invalid (i.e. not
userspace) address.

Also, switch ARM to use fueword(9) and casueword(9).

Note: there is a pending patch in D3617, which adds the special
processing for faults from LDRT and STRT.  The addition of the
processing is useful for potential other uses of the instructions and
for completeness, but standard userspace accessors are better served
by not allowing such faults beforehand.

Reviewed by:	andrew
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D3816
MFC after:	2 weeks
---
 sys/arm/arm/bcopyinout.S        |  18 +++++
 sys/arm/arm/bcopyinout_xscale.S |  18 +++++
 sys/arm/arm/copystr.S           |  16 ++++-
 sys/arm/arm/fusu.S              | 124 +++++++++++++++++++++-----------
 sys/arm/arm/genassym.c          |   1 +
 sys/arm/include/param.h         |   4 --
 6 files changed, 133 insertions(+), 48 deletions(-)

diff --git a/sys/arm/arm/bcopyinout.S b/sys/arm/arm/bcopyinout.S
index df0484e685c6..1c7e0fd6341a 100644
--- a/sys/arm/arm/bcopyinout.S
+++ b/sys/arm/arm/bcopyinout.S
@@ -94,6 +94,15 @@ ENTRY(copyin)
 	moveq	r0, #0
 	RETeq
 
+	adds	r3, r0, r2
+	movcs	r0, #EFAULT
+	RETc(cs)
+
+	ldr	r12, =(VM_MAXUSER_ADDRESS + 1)
+	cmp	r3, r12
+	movcs	r0, #EFAULT
+	RETc(cs)
+
 	ldr	r3, .L_arm_memcpy
 	ldr	r3, [r3]
 	cmp	r3, #0
@@ -332,6 +341,15 @@ ENTRY(copyout)
 	moveq	r0, #0
 	RETeq
 
+	adds	r3, r1, r2
+	movcs	r0, #EFAULT
+	RETc(cs)
+
+	ldr	r12, =(VM_MAXUSER_ADDRESS + 1)
+	cmp	r3, r12
+	movcs	r0, #EFAULT
+	RETc(cs)
+
 	ldr	r3, .L_arm_memcpy
 	ldr	r3, [r3]
 	cmp	r3, #0
diff --git a/sys/arm/arm/bcopyinout_xscale.S b/sys/arm/arm/bcopyinout_xscale.S
index 7a5abb588d63..79a5027356c0 100644
--- a/sys/arm/arm/bcopyinout_xscale.S
+++ b/sys/arm/arm/bcopyinout_xscale.S
@@ -67,6 +67,15 @@ ENTRY(copyin)
 	movle	r0, #0x00
 	movle	pc, lr			/* Bail early if length is <= 0 */
 
+	adds	r3, r0, r2
+	movcs	r0, #EFAULT
+	RETc(cs)
+
+	ldr	r12, =(VM_MAXUSER_ADDRESS + 1)
+	cmp	r3, r12
+	movcs	r0, #EFAULT
+	RETc(cs)
+
 	ldr	r3, .L_arm_memcpy
 	ldr	r3, [r3]
 	cmp	r3, #0
@@ -509,6 +518,15 @@ ENTRY(copyout)
 	movle	r0, #0x00
 	movle	pc, lr			/* Bail early if length is <= 0 */
 
+	adds	r3, r1, r2
+	movcs	r0, #EFAULT
+	RETc(cs)
+
+	ldr	r12, =(VM_MAXUSER_ADDRESS + 1)
+	cmp	r3, r12
+	movcs	r0, #EFAULT
+	RETc(cs)
+
 	ldr	r3, .L_arm_memcpy
 	ldr	r3, [r3]
 	cmp	r3, #0
diff --git a/sys/arm/arm/copystr.S b/sys/arm/arm/copystr.S
index 48c296b09292..4cb8469903d8 100644
--- a/sys/arm/arm/copystr.S
+++ b/sys/arm/arm/copystr.S
@@ -113,6 +113,8 @@ ENTRY(copyinstr)
 	moveq	r0, #ENAMETOOLONG
 	beq	2f
 
+	ldr	r12, =VM_MAXUSER_ADDRESS
+
 	GET_PCB(r4)
 	ldr	r4, [r4]
 
@@ -124,7 +126,10 @@ ENTRY(copyinstr)
 	adr	r5, .Lcopystrfault
 	str	r5, [r4, #PCB_ONFAULT]
 
-1:	ldrbt	r5, [r0], #0x0001
+1:
+	cmp	r0, r12
+	bcs	.Lcopystrfault
+	ldrbt	r5, [r0], #0x0001
 	add	r6, r6, #0x00000001
 	teq	r5, #0x00000000
 	strb	r5, [r1], #0x0001
@@ -161,6 +166,8 @@ ENTRY(copyoutstr)
 	moveq	r0, #ENAMETOOLONG
 	beq	2f
 
+	ldr	r12, =VM_MAXUSER_ADDRESS
+
 	GET_PCB(r4)
 	ldr	r4, [r4]
 
@@ -172,7 +179,10 @@ ENTRY(copyoutstr)
 	adr	r5, .Lcopystrfault
 	str	r5, [r4, #PCB_ONFAULT]
 
-1:	ldrb	r5, [r0], #0x0001
+1:
+	cmp	r0, r12
+	bcs	.Lcopystrfault
+	ldrb	r5, [r0], #0x0001
 	add	r6, r6, #0x00000001
 	teq	r5, #0x00000000
 	strbt	r5, [r1], #0x0001
@@ -195,9 +205,9 @@ END(copyoutstr)
 
 /* A fault occurred during the copy */
 .Lcopystrfault:
-	mov	r0, #EFAULT
 	mov	r1, #0x00000000
 	str	r1, [r4, #PCB_ONFAULT]
+	mov	r0, #EFAULT
 	RESTORE_REGS
 	RET
 
diff --git a/sys/arm/arm/fusu.S b/sys/arm/arm/fusu.S
index 06853c456948..dee2455a0d34 100644
--- a/sys/arm/arm/fusu.S
+++ b/sys/arm/arm/fusu.S
@@ -53,49 +53,51 @@ __FBSDID("$FreeBSD$");
 #endif
 
 /*
- * fuword(caddr_t uaddr);
- * Fetch an int from the user's address space.
+ * casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
+ *    uint32_t newval);
  */
 
-ENTRY(casuword)
-EENTRY_NP(casuword32)
-	GET_PCB(r3)
-	ldr	r3, [r3]
+ENTRY(casueword)
+EENTRY_NP(casueword32)
+	stmfd	sp!, {r4, r5, r6}
+
+	ldr	r4, =(VM_MAXUSER_ADDRESS-3)
+	cmp	r0, r4
+	mvncs	r0, #0
+	bcs	2f
+
+	GET_PCB(r6)
+	ldr	r6, [r6]
 
 #ifdef DIAGNOSTIC
-	teq	r3, #0x00000000
+	teq	r6, #0x00000000
+	ldmfdeq	sp!, {r4, r5, r6}
 	beq	.Lfusupcbfault
 #endif
-	stmfd	sp!, {r4, r5}
+
 	adr	r4, .Lcasuwordfault
-	str	r4, [r3, #PCB_ONFAULT]
+	str	r4, [r6, #PCB_ONFAULT]
+
 #if __ARM_ARCH >= 6
 1:
-	cmp     r0, #KERNBASE
-	mvnhs   r0, #0
-	bhs     2f
-
-	ldrex   r5, [r0]
-	cmp     r5, r1
-	movne   r0, r5
-	bne     2f
-	strex   r5, r2, [r0]
-	cmp     r5, #0
-	bne     1b
+	ldrex	r4, [r0]
+	cmp	r4, r1
+	strexeq	r5, r3, [r0]
+	cmpeq	r5, #1
+	beq	1b
 #else
-	ldrt	r5, [r0]
-	cmp	r5, r1
-	movne	r0, r5
-	strteq	r2, [r0]
+	ldrt	r4, [r0]
+	cmp	r4, r1
+	strteq	r3, [r0]
 #endif
-	moveq	r0, r1
+	str	r4, [r2]
+	mov	r0, #0
+	str	r0, [r6, #PCB_ONFAULT]
 2:
-	ldmfd	sp!, {r4, r5}
-	mov	r1, #0x00000000
-	str	r1, [r3, #PCB_ONFAULT]
+	ldmfd	sp!, {r4, r5, r6}
 	RET
-EEND(casuword32)
-END(casuword)
+EEND(casueword32)
+END(casueword)
 
 /*
  * Handle faults from casuword.  Clean up and return -1.
@@ -103,18 +105,23 @@ END(casuword)
 
 .Lcasuwordfault:
 	mov	r0, #0x00000000
-	str	r0, [r3, #PCB_ONFAULT]
-	mvn	r0, #0x00000000
-	ldmfd	sp!, {r4, r5}
+	str	r0, [r6, #PCB_ONFAULT]
+	mvn	r0, #0
+	ldmfd	sp!, {r4, r5, r6}
 	RET
 
 /*
- * fuword(caddr_t uaddr);
+ * fueword(caddr_t uaddr, long *val);
  * Fetch an int from the user's address space.
  */
 
-ENTRY(fuword)
-EENTRY_NP(fuword32)
+ENTRY(fueword)
+EENTRY_NP(fueword32)
+	ldr	r3, =(VM_MAXUSER_ADDRESS-3)
+	cmp	r0, r3
+	mvncs	r0, #0
+	RETc(cs)
+
 	GET_PCB(r2)
 	ldr	r2, [r2]
 
@@ -123,14 +130,14 @@ EENTRY_NP(fuword32)
 	beq	.Lfusupcbfault
 #endif
 
-	adr	r1, .Lfusufault
-	str	r1, [r2, #PCB_ONFAULT]
+	adr	r3, .Lfusufault
+	str	r3, [r2, #PCB_ONFAULT]
 
 	ldrt	r3, [r0]
+	str	r3, [r1]
 
-	mov	r1, #0x00000000
-	str	r1, [r2, #PCB_ONFAULT]
-	mov	r0, r3
+	mov	r0, #0x00000000
+	str	r0, [r2, #PCB_ONFAULT]
 	RET
 EEND(fuword32)
 END(fuword)
@@ -141,6 +148,11 @@ END(fuword)
  */
 
 ENTRY(fusword)
+	ldr	r3, =(VM_MAXUSER_ADDRESS-1)
+	cmp	r0, r3
+	mvncs	r0, #0
+	RETc(cs)
+
 	GET_PCB(r2)
 	ldr	r2, [r2]
 
@@ -171,6 +183,11 @@ END(fusword)
  */
 
 ENTRY(fuswintr)
+	ldr	r3, =(VM_MAXUSER_ADDRESS-1)
+	cmp	r0, r3
+	mvncs	r0, #0
+	RETc(cs)
+
 	ldr	r2, Lblock_userspace_access
 	ldr	r2, [r2]
 	teq	r2, #0
@@ -217,6 +234,11 @@ _C_LABEL(block_userspace_access):
  */
 
 ENTRY(fubyte)
+	ldr	r3, =VM_MAXUSER_ADDRESS
+	cmp	r0, r3
+	mvncs	r0, #0
+	RETc(cs)
+
 	GET_PCB(r2)
 	ldr	r2, [r2]
 
@@ -282,6 +304,11 @@ fusupcbfaulttext:
 
 ENTRY(suword)
 EENTRY_NP(suword32)
+	ldr	r3, =(VM_MAXUSER_ADDRESS-3)
+	cmp	r0, r3
+	mvncs	r0, #0
+	RETc(cs)
+
 	GET_PCB(r2)
 	ldr	r2, [r2]
 
@@ -308,6 +335,11 @@ END(suword)
  */
 
 ENTRY(suswintr)
+	ldr	r3, =(VM_MAXUSER_ADDRESS-1)
+	cmp	r0, r3
+	mvncs	r0, #0
+	RETc(cs)
+
 	ldr	r2, Lblock_userspace_access
 	ldr	r2, [r2]
 	teq	r2, #0
@@ -345,6 +377,11 @@ END(suswintr)
  */
 
 ENTRY(susword)
+	ldr	r3, =(VM_MAXUSER_ADDRESS-1)
+	cmp	r0, r3
+	mvncs	r0, #0
+	RETc(cs)
+
 	GET_PCB(r2)
 	ldr	r2, [r2]
 
@@ -376,6 +413,11 @@ END(susword)
  */
 
 ENTRY(subyte)
+	ldr	r3, =VM_MAXUSER_ADDRESS
+	cmp	r0, r3
+	mvncs	r0, #0
+	RETc(cs)
+
 	GET_PCB(r2)
 	ldr	r2, [r2]
 
diff --git a/sys/arm/arm/genassym.c b/sys/arm/arm/genassym.c
index 900fc4183942..4e2b7e91f235 100644
--- a/sys/arm/arm/genassym.c
+++ b/sys/arm/arm/genassym.c
@@ -161,6 +161,7 @@ ASSYM(P_VMSPACE, offsetof(struct proc, p_vmspace));
 ASSYM(VM_PMAP, offsetof(struct vmspace, vm_pmap));
 ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active));
 ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid));
+ASSYM(VM_MAXUSER_ADDRESS, VM_MAXUSER_ADDRESS);
 
 ASSYM(DCACHE_LINE_SIZE, offsetof(struct cpuinfo, dcache_line_size));
 ASSYM(DCACHE_LINE_MASK, offsetof(struct cpuinfo, dcache_line_mask));
diff --git a/sys/arm/include/param.h b/sys/arm/include/param.h
index d3aa01b7ce1c..bbe9bcb08325 100644
--- a/sys/arm/include/param.h
+++ b/sys/arm/include/param.h
@@ -149,8 +149,4 @@
 
 #define	pgtok(x)		((x) * (PAGE_SIZE / 1024))
 
-#ifdef _KERNEL
-#define	NO_FUEWORD	1
-#endif
-
 #endif /* !_ARM_INCLUDE_PARAM_H_ */

From 9797ba242b63e6da85022e8b72ea7d475c2c5fcd Mon Sep 17 00:00:00 2001
From: ed <ed@FreeBSD.org>
Date: Thu, 15 Oct 2015 17:50:28 +0000
Subject: [PATCH 250/380] Properly set the return value for casueword to 0 upon
 success.

While trying to get multithreading working for CloudABI on aarch64, I
noticed that compare-and-exchange operations in kernelspace would always
fail. It turns out that we don't properly set the return value to 0 when
the compare and exchange succeeds.

Approved by:	andrew
Differential Revision:	https://reviews.freebsd.org/D3899
---
 sys/arm64/arm64/support.S | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/sys/arm64/arm64/support.S b/sys/arm64/arm64/support.S
index 73ef6a4ac581..914a6e3145e9 100644
--- a/sys/arm64/arm64/support.S
+++ b/sys/arm64/arm64/support.S
@@ -59,6 +59,7 @@ ENTRY(casueword32)
 	ldrb	w0, [x0]		/* Try loading the data */
 2:	SET_FAULT_HANDLER(xzr, x5)	/* Reset the fault handler */
 	str	w4, [x2]		/* Store the read data */
+	mov	x0, #0			/* Success */
 	ret				/* Return */
 END(casueword32)
 
@@ -76,6 +77,7 @@ ENTRY(casueword)
 	ldrb	w0, [x0]		/* Try loading the data */
 2:	SET_FAULT_HANDLER(xzr, x5)	/* Reset the fault handler */
 	str	x4, [x2]		/* Store the read data */
+	mov	x0, #0			/* Success */
 	ret				/* Return */
 END(casueword)
 

From 952d11b6c6103e7ef093ccc7e47783c8a6310e56 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Thu, 15 Oct 2015 18:53:26 +0000
Subject: [PATCH 251/380] Remove excess .else

MFC after:	1 week
Sponsored by:	EMC / Isilon Storage Division
---
 share/mk/bsd.doc.mk | 1 -
 1 file changed, 1 deletion(-)

diff --git a/share/mk/bsd.doc.mk b/share/mk/bsd.doc.mk
index 9c6b00579122..ef319aca066c 100644
--- a/share/mk/bsd.doc.mk
+++ b/share/mk/bsd.doc.mk
@@ -184,7 +184,6 @@ ${DFILE.html}: ${SRCS}
 .else # unroff(1) requires a macro package as an argument
 	cd ${SRCDIR}; ${UNROFF} -ms ${UNROFFFLAGS} \
 	    document=${DOC} ${SRCS}
-.else
 .endif
 .endif
 .endfor

From fc88096acf7ec91e10bb0e587f85f2d6a63cc133 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Thu, 15 Oct 2015 18:55:43 +0000
Subject: [PATCH 252/380] Fix wrong use of .for; the iteration variable is not
 used in the loop.

MFC after:	1 week
Sponsored by:	EMC / Isilon Storage Division
---
 share/mk/bsd.doc.mk | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/share/mk/bsd.doc.mk b/share/mk/bsd.doc.mk
index ef319aca066c..c89c09d1c4aa 100644
--- a/share/mk/bsd.doc.mk
+++ b/share/mk/bsd.doc.mk
@@ -133,11 +133,11 @@ CLEANFILES+=	${DOC}.ascii ${DOC}.ascii${DCOMPRESS_EXT} \
 		${DOC}.html ${DOC}-*.html
 
 realinstall:
-.for _dev in ${PRINTERDEVICE:Mhtml}
+.if ${PRINTERDEVICE:Mhtml}
 	cd ${SRCDIR}; \
 	    ${INSTALL} -o ${BINOWN} -g ${BINGRP} -m ${BINMODE} \
 	    ${DOC}*.html ${DESTDIR}${BINDIR}/${VOLUME}
-.endfor
+.endif
 .for _dev in ${PRINTERDEVICE:Nhtml}
 	${INSTALL} -o ${BINOWN} -g ${BINGRP} -m ${BINMODE} \
 	    ${DFILE.${_dev}} ${DESTDIR}${BINDIR}/${VOLUME}

From 86f2836d37468318da95f77eb25bda9c577e29e1 Mon Sep 17 00:00:00 2001
From: emaste <emaste@FreeBSD.org>
Date: Thu, 15 Oct 2015 19:00:33 +0000
Subject: [PATCH 253/380] newfs_msdos: move mkfs_msdos to separate file for
 later use in makefs

Sponsored by:	The FreeBSD Foundation
---
 sbin/newfs_msdos/Makefile      |   1 +
 sbin/newfs_msdos/mkfs_msdos.c  | 854 +++++++++++++++++++++++++++++++++
 sbin/newfs_msdos/mkfs_msdos.h  |  61 +++
 sbin/newfs_msdos/newfs_msdos.c | 837 +-------------------------------
 4 files changed, 918 insertions(+), 835 deletions(-)
 create mode 100644 sbin/newfs_msdos/mkfs_msdos.c
 create mode 100644 sbin/newfs_msdos/mkfs_msdos.h

diff --git a/sbin/newfs_msdos/Makefile b/sbin/newfs_msdos/Makefile
index 0a9ffb4340a1..ec5e84878aec 100644
--- a/sbin/newfs_msdos/Makefile
+++ b/sbin/newfs_msdos/Makefile
@@ -2,6 +2,7 @@
 
 PROG=	newfs_msdos
 MAN=	newfs_msdos.8
+SRCS=	newfs_msdos.c mkfs_msdos.c
 
 # XXX - this is verboten
 .if ${MACHINE_CPUARCH} == "arm"
diff --git a/sbin/newfs_msdos/mkfs_msdos.c b/sbin/newfs_msdos/mkfs_msdos.c
new file mode 100644
index 000000000000..f4222700365b
--- /dev/null
+++ b/sbin/newfs_msdos/mkfs_msdos.c
@@ -0,0 +1,854 @@
+/*
+ * Copyright (c) 1998 Robert Nordier
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+ * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef lint
+static const char rcsid[] =
+  "$FreeBSD$";
+#endif /* not lint */
+
+#include <sys/param.h>
+#include <sys/fdcio.h>
+#include <sys/disk.h>
+#include <sys/disklabel.h>
+#include <sys/mount.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+
+#include <ctype.h>
+#include <err.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <inttypes.h>
+#include <paths.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+
+#include "mkfs_msdos.h"
+
+#define	MAXU16	  0xffff	/* maximum unsigned 16-bit quantity */
+#define	BPN	  4		/* bits per nibble */
+#define	NPB	  2		/* nibbles per byte */
+
+#define	DOSMAGIC  0xaa55	/* DOS magic number */
+#define	MINBPS	  512		/* minimum bytes per sector */
+#define	MAXSPC	  128		/* maximum sectors per cluster */
+#define	MAXNFT	  16		/* maximum number of FATs */
+#define	DEFBLK	  4096		/* default block size */
+#define	DEFBLK16  2048		/* default block size FAT16 */
+#define	DEFRDE	  512		/* default root directory entries */
+#define	RESFTE	  2		/* reserved FAT entries */
+#define	MINCLS12  1U		/* minimum FAT12 clusters */
+#define	MINCLS16  0xff5U	/* minimum FAT16 clusters */
+#define	MINCLS32  0xfff5U	/* minimum FAT32 clusters */
+#define	MAXCLS12  0xff4U	/* maximum FAT12 clusters */
+#define	MAXCLS16  0xfff4U	/* maximum FAT16 clusters */
+#define	MAXCLS32  0xffffff4U	/* maximum FAT32 clusters */
+
+#define	mincls(fat)  ((fat) == 12 ? MINCLS12 :	\
+		      (fat) == 16 ? MINCLS16 :	\
+				    MINCLS32)
+
+#define	maxcls(fat)  ((fat) == 12 ? MAXCLS12 :	\
+		      (fat) == 16 ? MAXCLS16 :	\
+				    MAXCLS32)
+
+#define	mk1(p, x)				\
+    (p) = (u_int8_t)(x)
+
+#define	mk2(p, x)				\
+    (p)[0] = (u_int8_t)(x),			\
+    (p)[1] = (u_int8_t)((x) >> 010)
+
+#define	mk4(p, x)				\
+    (p)[0] = (u_int8_t)(x),			\
+    (p)[1] = (u_int8_t)((x) >> 010),		\
+    (p)[2] = (u_int8_t)((x) >> 020),		\
+    (p)[3] = (u_int8_t)((x) >> 030)
+
+struct bs {
+    u_int8_t bsJump[3];			/* bootstrap entry point */
+    u_int8_t bsOemName[8];		/* OEM name and version */
+} __packed;
+
+struct bsbpb {
+    u_int8_t bpbBytesPerSec[2];		/* bytes per sector */
+    u_int8_t bpbSecPerClust;		/* sectors per cluster */
+    u_int8_t bpbResSectors[2];		/* reserved sectors */
+    u_int8_t bpbFATs;			/* number of FATs */
+    u_int8_t bpbRootDirEnts[2];		/* root directory entries */
+    u_int8_t bpbSectors[2];		/* total sectors */
+    u_int8_t bpbMedia;			/* media descriptor */
+    u_int8_t bpbFATsecs[2];		/* sectors per FAT */
+    u_int8_t bpbSecPerTrack[2];		/* sectors per track */
+    u_int8_t bpbHeads[2];		/* drive heads */
+    u_int8_t bpbHiddenSecs[4];		/* hidden sectors */
+    u_int8_t bpbHugeSectors[4];		/* big total sectors */
+} __packed;
+
+struct bsxbpb {
+    u_int8_t bpbBigFATsecs[4];		/* big sectors per FAT */
+    u_int8_t bpbExtFlags[2];		/* FAT control flags */
+    u_int8_t bpbFSVers[2];		/* file system version */
+    u_int8_t bpbRootClust[4];		/* root directory start cluster */
+    u_int8_t bpbFSInfo[2];		/* file system info sector */
+    u_int8_t bpbBackup[2];		/* backup boot sector */
+    u_int8_t bpbReserved[12];		/* reserved */
+} __packed;
+
+struct bsx {
+    u_int8_t exDriveNumber;		/* drive number */
+    u_int8_t exReserved1;		/* reserved */
+    u_int8_t exBootSignature;		/* extended boot signature */
+    u_int8_t exVolumeID[4];		/* volume ID number */
+    u_int8_t exVolumeLabel[11];		/* volume label */
+    u_int8_t exFileSysType[8];		/* file system type */
+} __packed;
+
+struct de {
+    u_int8_t deName[11];		/* name and extension */
+    u_int8_t deAttributes;		/* attributes */
+    u_int8_t rsvd[10];			/* reserved */
+    u_int8_t deMTime[2];		/* last-modified time */
+    u_int8_t deMDate[2];		/* last-modified date */
+    u_int8_t deStartCluster[2];		/* starting cluster */
+    u_int8_t deFileSize[4];		/* size */
+} __packed;
+
+struct bpb {
+    u_int bpbBytesPerSec;		/* bytes per sector */
+    u_int bpbSecPerClust;		/* sectors per cluster */
+    u_int bpbResSectors;		/* reserved sectors */
+    u_int bpbFATs;			/* number of FATs */
+    u_int bpbRootDirEnts;		/* root directory entries */
+    u_int bpbSectors;			/* total sectors */
+    u_int bpbMedia;			/* media descriptor */
+    u_int bpbFATsecs;			/* sectors per FAT */
+    u_int bpbSecPerTrack;		/* sectors per track */
+    u_int bpbHeads;			/* drive heads */
+    u_int bpbHiddenSecs;		/* hidden sectors */
+    u_int bpbHugeSectors; 		/* big total sectors */
+    u_int bpbBigFATsecs; 		/* big sectors per FAT */
+    u_int bpbRootClust; 		/* root directory start cluster */
+    u_int bpbFSInfo; 			/* file system info sector */
+    u_int bpbBackup; 			/* backup boot sector */
+};
+
+#define	BPBGAP 0, 0, 0, 0, 0, 0
+
+static struct {
+    const char *name;
+    struct bpb bpb;
+} const stdfmt[] = {
+    {"160",  {512, 1, 1, 2,  64,  320, 0xfe, 1,  8, 1, BPBGAP}},
+    {"180",  {512, 1, 1, 2,  64,  360, 0xfc, 2,  9, 1, BPBGAP}},
+    {"320",  {512, 2, 1, 2, 112,  640, 0xff, 1,  8, 2, BPBGAP}},
+    {"360",  {512, 2, 1, 2, 112,  720, 0xfd, 2,  9, 2, BPBGAP}},
+    {"640",  {512, 2, 1, 2, 112, 1280, 0xfb, 2,  8, 2, BPBGAP}},
+    {"720",  {512, 2, 1, 2, 112, 1440, 0xf9, 3,  9, 2, BPBGAP}},
+    {"1200", {512, 1, 1, 2, 224, 2400, 0xf9, 7, 15, 2, BPBGAP}},
+    {"1232", {1024,1, 1, 2, 192, 1232, 0xfe, 2,  8, 2, BPBGAP}},
+    {"1440", {512, 1, 1, 2, 224, 2880, 0xf0, 9, 18, 2, BPBGAP}},
+    {"2880", {512, 2, 1, 2, 240, 5760, 0xf0, 9, 36, 2, BPBGAP}}
+};
+
+static const u_int8_t bootcode[] = {
+    0xfa,			/* cli		    */
+    0x31, 0xc0, 		/* xor	   ax,ax    */
+    0x8e, 0xd0, 		/* mov	   ss,ax    */
+    0xbc, 0x00, 0x7c,		/* mov	   sp,7c00h */
+    0xfb,			/* sti		    */
+    0x8e, 0xd8, 		/* mov	   ds,ax    */
+    0xe8, 0x00, 0x00,		/* call    $ + 3    */
+    0x5e,			/* pop	   si	    */
+    0x83, 0xc6, 0x19,		/* add	   si,+19h  */
+    0xbb, 0x07, 0x00,		/* mov	   bx,0007h */
+    0xfc,			/* cld		    */
+    0xac,			/* lodsb	    */
+    0x84, 0xc0, 		/* test    al,al    */
+    0x74, 0x06, 		/* jz	   $ + 8    */
+    0xb4, 0x0e, 		/* mov	   ah,0eh   */
+    0xcd, 0x10, 		/* int	   10h	    */
+    0xeb, 0xf5, 		/* jmp	   $ - 9    */
+    0x30, 0xe4, 		/* xor	   ah,ah    */
+    0xcd, 0x16, 		/* int	   16h	    */
+    0xcd, 0x19, 		/* int	   19h	    */
+    0x0d, 0x0a,
+    'N', 'o', 'n', '-', 's', 'y', 's', 't',
+    'e', 'm', ' ', 'd', 'i', 's', 'k',
+    0x0d, 0x0a,
+    'P', 'r', 'e', 's', 's', ' ', 'a', 'n',
+    'y', ' ', 'k', 'e', 'y', ' ', 't', 'o',
+    ' ', 'r', 'e', 'b', 'o', 'o', 't',
+    0x0d, 0x0a,
+    0
+};
+
+static volatile sig_atomic_t got_siginfo;
+static void infohandler(int);
+
+static void check_mounted(const char *, mode_t);
+static void getstdfmt(const char *, struct bpb *);
+static void getdiskinfo(int, const char *, const char *, int,
+			struct bpb *);
+static void print_bpb(struct bpb *);
+static u_int ckgeom(const char *, u_int, const char *);
+static void mklabel(u_int8_t *, const char *);
+static int oklabel(const char *);
+static void setstr(u_int8_t *, const char *, size_t);
+
+int mkfs_msdos(const char *fname, const char *dtype,
+    const struct msdos_options *op)
+{
+    char buf[MAXPATHLEN];
+    struct sigaction si_sa;
+    struct stat sb;
+    struct timeval tv;
+    struct bpb bpb;
+    struct tm *tm;
+    struct bs *bs;
+    struct bsbpb *bsbpb;
+    struct bsxbpb *bsxbpb;
+    struct bsx *bsx;
+    struct de *de;
+    u_int8_t *img;
+    const char *bname;
+    ssize_t n;
+    time_t now;
+    u_int fat, bss, rds, cls, dir, lsn, x, x1, x2;
+    int fd, fd1;
+    struct msdos_options o = *op;
+
+    if (o.OEM_string && strlen(o.OEM_string) > 8) {
+	errx(1, "%s: bad OEM string", o.OEM_string);
+    }
+    if (o.create_size) {
+	if (o.no_create)
+	    errx(1, "create (-C) is incompatible with -N");
+	fd = open(fname, O_RDWR | O_CREAT | O_TRUNC, 0644);
+	if (fd == -1)
+	    errx(1, "failed to create %s", fname);
+	if (ftruncate(fd, o.create_size))
+	    errx(1, "failed to initialize %jd bytes", (intmax_t)o.create_size);
+    } else if ((fd = open(fname, o.no_create ? O_RDONLY : O_RDWR)) == -1)
+	err(1, "%s", fname);
+    if (fstat(fd, &sb))
+	err(1, "%s", fname);
+    if (o.create_size) {
+	if (!S_ISREG(sb.st_mode))
+	    warnx("warning, %s is not a regular file", fname);
+    } else {
+	if (!S_ISCHR(sb.st_mode))
+	    warnx("warning, %s is not a character device", fname);
+    }
+    if (!o.no_create)
+	check_mounted(fname, sb.st_mode);
+    if (o.offset && o.offset != lseek(fd, o.offset, SEEK_SET))
+	errx(1, "cannot seek to %jd", (intmax_t)o.offset);
+    memset(&bpb, 0, sizeof(bpb));
+    if (o.floppy) {
+	getstdfmt(o.floppy, &bpb);
+	bpb.bpbHugeSectors = bpb.bpbSectors;
+	bpb.bpbSectors = 0;
+	bpb.bpbBigFATsecs = bpb.bpbFATsecs;
+	bpb.bpbFATsecs = 0;
+    }
+    if (o.drive_heads)
+	bpb.bpbHeads = o.drive_heads;
+    if (o.sectors_per_track)
+	bpb.bpbSecPerTrack = o.sectors_per_track;
+    if (o.bytes_per_sector)
+	bpb.bpbBytesPerSec = o.bytes_per_sector;
+    if (o.size)
+	bpb.bpbHugeSectors = o.size;
+    if (o.hidden_sectors_set)
+	bpb.bpbHiddenSecs = o.hidden_sectors;
+    if (!(o.floppy || (o.drive_heads && o.sectors_per_track && o.bytes_per_sector && o.size && o.hidden_sectors_set))) {
+	off_t delta;
+	getdiskinfo(fd, fname, dtype, o.hidden_sectors_set, &bpb);
+	bpb.bpbHugeSectors -= (o.offset / bpb.bpbBytesPerSec);
+	delta = bpb.bpbHugeSectors % bpb.bpbSecPerTrack;
+	if (delta != 0) {
+	    warnx("trim %d sectors to adjust to a multiple of %d",
+		(int)delta, bpb.bpbSecPerTrack);
+	    bpb.bpbHugeSectors -= delta;
+	}
+	if (bpb.bpbSecPerClust == 0) {	/* set defaults */
+	    if (bpb.bpbHugeSectors <= 6000)	/* about 3MB -> 512 bytes */
+		bpb.bpbSecPerClust = 1;
+	    else if (bpb.bpbHugeSectors <= (1<<17)) /* 64M -> 4k */
+		bpb.bpbSecPerClust = 8;
+	    else if (bpb.bpbHugeSectors <= (1<<19)) /* 256M -> 8k */
+		bpb.bpbSecPerClust = 16;
+	    else if (bpb.bpbHugeSectors <= (1<<21)) /* 1G -> 16k */
+		bpb.bpbSecPerClust = 32;
+	    else
+		bpb.bpbSecPerClust = 64;		/* otherwise 32k */
+	}
+    }
+    if (!powerof2(bpb.bpbBytesPerSec))
+	errx(1, "bytes/sector (%u) is not a power of 2", bpb.bpbBytesPerSec);
+    if (bpb.bpbBytesPerSec < MINBPS)
+	errx(1, "bytes/sector (%u) is too small; minimum is %u",
+	     bpb.bpbBytesPerSec, MINBPS);
+    if (o.volume_label && !oklabel(o.volume_label))
+	errx(1, "%s: bad volume label", o.volume_label);
+    if (!(fat = o.fat_type)) {
+	if (o.floppy)
+	    fat = 12;
+	else if (!o.directory_entries && (o.info_sector || o.backup_sector))
+	    fat = 32;
+    }
+    if ((fat == 32 && o.directory_entries) || (fat != 32 && (o.info_sector || o.backup_sector)))
+	errx(1, "-%c is not a legal FAT%s option",
+	     fat == 32 ? 'e' : o.info_sector ? 'i' : 'k',
+	     fat == 32 ? "32" : "12/16");
+    if (o.floppy && fat == 32)
+	bpb.bpbRootDirEnts = 0;
+    if (fat != 0 && fat != 12 && fat != 16 && fat != 32) {
+	    errx(1, "%d: bad FAT type", fat);
+    }
+
+    if (o.block_size) {
+	if (!powerof2(o.block_size))
+	    errx(1, "block size (%u) is not a power of 2", o.block_size);
+	if (o.block_size < bpb.bpbBytesPerSec)
+	    errx(1, "block size (%u) is too small; minimum is %u",
+		 o.block_size, bpb.bpbBytesPerSec);
+	if (o.block_size > bpb.bpbBytesPerSec * MAXSPC)
+	    errx(1, "block size (%u) is too large; maximum is %u",
+		 o.block_size, bpb.bpbBytesPerSec * MAXSPC);
+	bpb.bpbSecPerClust = o.block_size / bpb.bpbBytesPerSec;
+    }
+    if (o.sectors_per_cluster) {
+	if (!powerof2(o.sectors_per_cluster))
+	    errx(1, "sectors/cluster (%u) is not a power of 2", o.sectors_per_cluster);
+	bpb.bpbSecPerClust = o.sectors_per_cluster;
+    }
+    if (o.reserved_sectors)
+	bpb.bpbResSectors = o.reserved_sectors;
+    if (o.num_FAT) {
+	if (o.num_FAT > MAXNFT)
+	    errx(1, "number of FATs (%u) is too large; maximum is %u",
+		 o.num_FAT, MAXNFT);
+	bpb.bpbFATs = o.num_FAT;
+    }
+    if (o.directory_entries)
+	bpb.bpbRootDirEnts = o.directory_entries;
+    if (o.media_descriptor_set) {
+	if (o.media_descriptor < 0xf0)
+	    errx(1, "illegal media descriptor (%#x)", o.media_descriptor);
+	bpb.bpbMedia = o.media_descriptor;
+    }
+    if (o.sectors_per_fat)
+	bpb.bpbBigFATsecs = o.sectors_per_fat;
+    if (o.info_sector)
+	bpb.bpbFSInfo = o.info_sector;
+    if (o.backup_sector)
+	bpb.bpbBackup = o.backup_sector;
+    bss = 1;
+    bname = NULL;
+    fd1 = -1;
+    if (o.bootstrap) {
+	bname = o.bootstrap;
+	if (!strchr(bname, '/')) {
+	    snprintf(buf, sizeof(buf), "/boot/%s", bname);
+	    if (!(bname = strdup(buf)))
+		err(1, NULL);
+	}
+	if ((fd1 = open(bname, O_RDONLY)) == -1 || fstat(fd1, &sb))
+	    err(1, "%s", bname);
+	if (!S_ISREG(sb.st_mode) || sb.st_size % bpb.bpbBytesPerSec ||
+	    sb.st_size < bpb.bpbBytesPerSec ||
+	    sb.st_size > bpb.bpbBytesPerSec * MAXU16)
+	    errx(1, "%s: inappropriate file type or format", bname);
+	bss = sb.st_size / bpb.bpbBytesPerSec;
+    }
+    if (!bpb.bpbFATs)
+	bpb.bpbFATs = 2;
+    if (!fat) {
+	if (bpb.bpbHugeSectors < (bpb.bpbResSectors ? bpb.bpbResSectors : bss) +
+	    howmany((RESFTE + (bpb.bpbSecPerClust ? MINCLS16 : MAXCLS12 + 1)) *
+		(bpb.bpbSecPerClust ? 16 : 12) / BPN,
+		bpb.bpbBytesPerSec * NPB) *
+	    bpb.bpbFATs +
+	    howmany(bpb.bpbRootDirEnts ? bpb.bpbRootDirEnts : DEFRDE,
+		    bpb.bpbBytesPerSec / sizeof(struct de)) +
+	    (bpb.bpbSecPerClust ? MINCLS16 : MAXCLS12 + 1) *
+	    (bpb.bpbSecPerClust ? bpb.bpbSecPerClust :
+	     howmany(DEFBLK, bpb.bpbBytesPerSec)))
+	    fat = 12;
+	else if (bpb.bpbRootDirEnts || bpb.bpbHugeSectors <
+		 (bpb.bpbResSectors ? bpb.bpbResSectors : bss) +
+		 howmany((RESFTE + MAXCLS16) * 2, bpb.bpbBytesPerSec) *
+		 bpb.bpbFATs +
+		 howmany(DEFRDE, bpb.bpbBytesPerSec / sizeof(struct de)) +
+		 (MAXCLS16 + 1) *
+		 (bpb.bpbSecPerClust ? bpb.bpbSecPerClust :
+		  howmany(8192, bpb.bpbBytesPerSec)))
+	    fat = 16;
+	else
+	    fat = 32;
+    }
+    x = bss;
+    if (fat == 32) {
+	if (!bpb.bpbFSInfo) {
+	    if (x == MAXU16 || x == bpb.bpbBackup)
+		errx(1, "no room for info sector");
+	    bpb.bpbFSInfo = x;
+	}
+	if (bpb.bpbFSInfo != MAXU16 && x <= bpb.bpbFSInfo)
+	    x = bpb.bpbFSInfo + 1;
+	if (!bpb.bpbBackup) {
+	    if (x == MAXU16)
+		errx(1, "no room for backup sector");
+	    bpb.bpbBackup = x;
+	} else if (bpb.bpbBackup != MAXU16 && bpb.bpbBackup == bpb.bpbFSInfo)
+	    errx(1, "backup sector would overwrite info sector");
+	if (bpb.bpbBackup != MAXU16 && x <= bpb.bpbBackup)
+	    x = bpb.bpbBackup + 1;
+    }
+    if (!bpb.bpbResSectors)
+	bpb.bpbResSectors = fat == 32 ?
+	    MAX(x, MAX(16384 / bpb.bpbBytesPerSec, 4)) : x;
+    else if (bpb.bpbResSectors < x)
+	errx(1, "too few reserved sectors (need %d have %d)", x,
+	     bpb.bpbResSectors);
+    if (fat != 32 && !bpb.bpbRootDirEnts)
+	bpb.bpbRootDirEnts = DEFRDE;
+    rds = howmany(bpb.bpbRootDirEnts, bpb.bpbBytesPerSec / sizeof(struct de));
+    if (!bpb.bpbSecPerClust)
+	for (bpb.bpbSecPerClust = howmany(fat == 16 ? DEFBLK16 :
+					  DEFBLK, bpb.bpbBytesPerSec);
+	     bpb.bpbSecPerClust < MAXSPC &&
+	     bpb.bpbResSectors +
+	     howmany((RESFTE + maxcls(fat)) * (fat / BPN),
+		     bpb.bpbBytesPerSec * NPB) *
+	     bpb.bpbFATs +
+	     rds +
+	     (u_int64_t) (maxcls(fat) + 1) *
+	     bpb.bpbSecPerClust <= bpb.bpbHugeSectors;
+	     bpb.bpbSecPerClust <<= 1)
+	    continue;
+    if (fat != 32 && bpb.bpbBigFATsecs > MAXU16)
+	errx(1, "too many sectors/FAT for FAT12/16");
+    x1 = bpb.bpbResSectors + rds;
+    x = bpb.bpbBigFATsecs ? bpb.bpbBigFATsecs : 1;
+    if (x1 + (u_int64_t)x * bpb.bpbFATs > bpb.bpbHugeSectors)
+	errx(1, "meta data exceeds file system size");
+    x1 += x * bpb.bpbFATs;
+    x = (u_int64_t)(bpb.bpbHugeSectors - x1) * bpb.bpbBytesPerSec * NPB /
+	(bpb.bpbSecPerClust * bpb.bpbBytesPerSec * NPB + fat /
+	 BPN * bpb.bpbFATs);
+    x2 = howmany((RESFTE + MIN(x, maxcls(fat))) * (fat / BPN),
+		 bpb.bpbBytesPerSec * NPB);
+    if (!bpb.bpbBigFATsecs) {
+	bpb.bpbBigFATsecs = x2;
+	x1 += (bpb.bpbBigFATsecs - 1) * bpb.bpbFATs;
+    }
+    cls = (bpb.bpbHugeSectors - x1) / bpb.bpbSecPerClust;
+    x = (u_int64_t)bpb.bpbBigFATsecs * bpb.bpbBytesPerSec * NPB / (fat / BPN) -
+	RESFTE;
+    if (cls > x)
+	cls = x;
+    if (bpb.bpbBigFATsecs < x2)
+	warnx("warning: sectors/FAT limits file system to %u clusters",
+	      cls);
+    if (cls < mincls(fat))
+	errx(1, "%u clusters too few clusters for FAT%u, need %u", cls, fat,
+	    mincls(fat));
+    if (cls > maxcls(fat)) {
+	cls = maxcls(fat);
+	bpb.bpbHugeSectors = x1 + (cls + 1) * bpb.bpbSecPerClust - 1;
+	warnx("warning: FAT type limits file system to %u sectors",
+	      bpb.bpbHugeSectors);
+    }
+    printf("%s: %u sector%s in %u FAT%u cluster%s "
+	   "(%u bytes/cluster)\n", fname, cls * bpb.bpbSecPerClust,
+	   cls * bpb.bpbSecPerClust == 1 ? "" : "s", cls, fat,
+	   cls == 1 ? "" : "s", bpb.bpbBytesPerSec * bpb.bpbSecPerClust);
+    if (!bpb.bpbMedia)
+	bpb.bpbMedia = !bpb.bpbHiddenSecs ? 0xf0 : 0xf8;
+    if (fat == 32)
+	bpb.bpbRootClust = RESFTE;
+    if (bpb.bpbHiddenSecs + bpb.bpbHugeSectors <= MAXU16) {
+	bpb.bpbSectors = bpb.bpbHugeSectors;
+	bpb.bpbHugeSectors = 0;
+    }
+    if (fat != 32) {
+	bpb.bpbFATsecs = bpb.bpbBigFATsecs;
+	bpb.bpbBigFATsecs = 0;
+    }
+    print_bpb(&bpb);
+    if (!o.no_create) {
+	gettimeofday(&tv, NULL);
+	now = tv.tv_sec;
+	tm = localtime(&now);
+	if (!(img = malloc(bpb.bpbBytesPerSec)))
+	    err(1, NULL);
+	dir = bpb.bpbResSectors + (bpb.bpbFATsecs ? bpb.bpbFATsecs :
+				   bpb.bpbBigFATsecs) * bpb.bpbFATs;
+	memset(&si_sa, 0, sizeof(si_sa));
+	si_sa.sa_handler = infohandler;
+	if (sigaction(SIGINFO, &si_sa, NULL) == -1)
+		err(1, "sigaction SIGINFO");
+	for (lsn = 0; lsn < dir + (fat == 32 ? bpb.bpbSecPerClust : rds); lsn++) {
+	    if (got_siginfo) {
+		    fprintf(stderr,"%s: writing sector %u of %u (%u%%)\n",
+			fname, lsn,
+			(dir + (fat == 32 ? bpb.bpbSecPerClust: rds)),
+			(lsn * 100) / (dir +
+			    (fat == 32 ? bpb.bpbSecPerClust: rds)));
+		    got_siginfo = 0;
+	    }
+	    x = lsn;
+	    if (o.bootstrap &&
+		fat == 32 && bpb.bpbBackup != MAXU16 &&
+		bss <= bpb.bpbBackup && x >= bpb.bpbBackup) {
+		x -= bpb.bpbBackup;
+		if (!x && lseek(fd1, o.offset, SEEK_SET))
+		    err(1, "%s", bname);
+	    }
+	    if (o.bootstrap && x < bss) {
+		if ((n = read(fd1, img, bpb.bpbBytesPerSec)) == -1)
+		    err(1, "%s", bname);
+		if ((unsigned)n != bpb.bpbBytesPerSec)
+		    errx(1, "%s: can't read sector %u", bname, x);
+	    } else
+		memset(img, 0, bpb.bpbBytesPerSec);
+	    if (!lsn ||
+		(fat == 32 && bpb.bpbBackup != MAXU16 &&
+		 lsn == bpb.bpbBackup)) {
+		x1 = sizeof(struct bs);
+		bsbpb = (struct bsbpb *)(img + x1);
+		mk2(bsbpb->bpbBytesPerSec, bpb.bpbBytesPerSec);
+		mk1(bsbpb->bpbSecPerClust, bpb.bpbSecPerClust);
+		mk2(bsbpb->bpbResSectors, bpb.bpbResSectors);
+		mk1(bsbpb->bpbFATs, bpb.bpbFATs);
+		mk2(bsbpb->bpbRootDirEnts, bpb.bpbRootDirEnts);
+		mk2(bsbpb->bpbSectors, bpb.bpbSectors);
+		mk1(bsbpb->bpbMedia, bpb.bpbMedia);
+		mk2(bsbpb->bpbFATsecs, bpb.bpbFATsecs);
+		mk2(bsbpb->bpbSecPerTrack, bpb.bpbSecPerTrack);
+		mk2(bsbpb->bpbHeads, bpb.bpbHeads);
+		mk4(bsbpb->bpbHiddenSecs, bpb.bpbHiddenSecs);
+		mk4(bsbpb->bpbHugeSectors, bpb.bpbHugeSectors);
+		x1 += sizeof(struct bsbpb);
+		if (fat == 32) {
+		    bsxbpb = (struct bsxbpb *)(img + x1);
+		    mk4(bsxbpb->bpbBigFATsecs, bpb.bpbBigFATsecs);
+		    mk2(bsxbpb->bpbExtFlags, 0);
+		    mk2(bsxbpb->bpbFSVers, 0);
+		    mk4(bsxbpb->bpbRootClust, bpb.bpbRootClust);
+		    mk2(bsxbpb->bpbFSInfo, bpb.bpbFSInfo);
+		    mk2(bsxbpb->bpbBackup, bpb.bpbBackup);
+		    x1 += sizeof(struct bsxbpb);
+		}
+		bsx = (struct bsx *)(img + x1);
+		mk1(bsx->exBootSignature, 0x29);
+		if (o.volume_id_set)
+		    x = o.volume_id;
+		else
+		    x = (((u_int)(1 + tm->tm_mon) << 8 |
+			  (u_int)tm->tm_mday) +
+			 ((u_int)tm->tm_sec << 8 |
+			  (u_int)(tv.tv_usec / 10))) << 16 |
+			((u_int)(1900 + tm->tm_year) +
+			 ((u_int)tm->tm_hour << 8 |
+			  (u_int)tm->tm_min));
+		mk4(bsx->exVolumeID, x);
+		mklabel(bsx->exVolumeLabel, o.volume_label ? o.volume_label : "NO NAME");
+		sprintf(buf, "FAT%u", fat);
+		setstr(bsx->exFileSysType, buf, sizeof(bsx->exFileSysType));
+		if (!o.bootstrap) {
+		    x1 += sizeof(struct bsx);
+		    bs = (struct bs *)img;
+		    mk1(bs->bsJump[0], 0xeb);
+		    mk1(bs->bsJump[1], x1 - 2);
+		    mk1(bs->bsJump[2], 0x90);
+		    setstr(bs->bsOemName, o.OEM_string ? o.OEM_string : "BSD4.4  ",
+			   sizeof(bs->bsOemName));
+		    memcpy(img + x1, bootcode, sizeof(bootcode));
+		    mk2(img + MINBPS - 2, DOSMAGIC);
+		}
+	    } else if (fat == 32 && bpb.bpbFSInfo != MAXU16 &&
+		       (lsn == bpb.bpbFSInfo ||
+			(bpb.bpbBackup != MAXU16 &&
+			 lsn == bpb.bpbBackup + bpb.bpbFSInfo))) {
+		mk4(img, 0x41615252);
+		mk4(img + MINBPS - 28, 0x61417272);
+		mk4(img + MINBPS - 24, 0xffffffff);
+		mk4(img + MINBPS - 20, bpb.bpbRootClust);
+		mk2(img + MINBPS - 2, DOSMAGIC);
+	    } else if (lsn >= bpb.bpbResSectors && lsn < dir &&
+		       !((lsn - bpb.bpbResSectors) %
+			 (bpb.bpbFATsecs ? bpb.bpbFATsecs :
+			  bpb.bpbBigFATsecs))) {
+		mk1(img[0], bpb.bpbMedia);
+		for (x = 1; x < fat * (fat == 32 ? 3 : 2) / 8; x++)
+		    mk1(img[x], fat == 32 && x % 4 == 3 ? 0x0f : 0xff);
+	    } else if (lsn == dir && o.volume_label) {
+		de = (struct de *)img;
+		mklabel(de->deName, o.volume_label);
+		mk1(de->deAttributes, 050);
+		x = (u_int)tm->tm_hour << 11 |
+		    (u_int)tm->tm_min << 5 |
+		    (u_int)tm->tm_sec >> 1;
+		mk2(de->deMTime, x);
+		x = (u_int)(tm->tm_year - 80) << 9 |
+		    (u_int)(tm->tm_mon + 1) << 5 |
+		    (u_int)tm->tm_mday;
+		mk2(de->deMDate, x);
+	    }
+	    if ((n = write(fd, img, bpb.bpbBytesPerSec)) == -1)
+		err(1, "%s", fname);
+	    if ((unsigned)n != bpb.bpbBytesPerSec)
+		errx(1, "%s: can't write sector %u", fname, lsn);
+	}
+    }
+    return 0;
+}
+
+/*
+ * Exit with error if file system is mounted.
+ */
+static void
+check_mounted(const char *fname, mode_t mode)
+{
+    struct statfs *mp;
+    const char *s1, *s2;
+    size_t len;
+    int n, r;
+
+    if (!(n = getmntinfo(&mp, MNT_NOWAIT)))
+	err(1, "getmntinfo");
+    len = strlen(_PATH_DEV);
+    s1 = fname;
+    if (!strncmp(s1, _PATH_DEV, len))
+	s1 += len;
+    r = S_ISCHR(mode) && s1 != fname && *s1 == 'r';
+    for (; n--; mp++) {
+	s2 = mp->f_mntfromname;
+	if (!strncmp(s2, _PATH_DEV, len))
+	    s2 += len;
+	if ((r && s2 != mp->f_mntfromname && !strcmp(s1 + 1, s2)) ||
+	    !strcmp(s1, s2))
+	    errx(1, "%s is mounted on %s", fname, mp->f_mntonname);
+    }
+}
+
+/*
+ * Get a standard format.
+ */
+static void
+getstdfmt(const char *fmt, struct bpb *bpb)
+{
+    u_int x, i;
+
+    x = sizeof(stdfmt) / sizeof(stdfmt[0]);
+    for (i = 0; i < x && strcmp(fmt, stdfmt[i].name); i++);
+    if (i == x)
+	errx(1, "%s: unknown standard format", fmt);
+    *bpb = stdfmt[i].bpb;
+}
+
+/*
+ * Get disk slice, partition, and geometry information.
+ */
+static void
+getdiskinfo(int fd, const char *fname, const char *dtype, __unused int oflag,
+	    struct bpb *bpb)
+{
+    struct disklabel *lp, dlp;
+    struct fd_type type;
+    off_t ms, hs = 0;
+
+    lp = NULL;
+
+    /* If the user specified a disk type, try to use that */
+    if (dtype != NULL) {
+	lp = getdiskbyname(dtype);
+    }
+
+    /* Maybe it's a floppy drive */
+    if (lp == NULL) {
+	if (ioctl(fd, DIOCGMEDIASIZE, &ms) == -1) {
+	    struct stat st;
+
+	    if (fstat(fd, &st))
+		err(1, "cannot get disk size");
+	    /* create a fake geometry for a file image */
+	    ms = st.st_size;
+	    dlp.d_secsize = 512;
+	    dlp.d_nsectors = 63;
+	    dlp.d_ntracks = 255;
+	    dlp.d_secperunit = ms / dlp.d_secsize;
+	    lp = &dlp;
+	} else if (ioctl(fd, FD_GTYPE, &type) != -1) {
+	    dlp.d_secsize = 128 << type.secsize;
+	    dlp.d_nsectors = type.sectrac;
+	    dlp.d_ntracks = type.heads;
+	    dlp.d_secperunit = ms / dlp.d_secsize;
+	    lp = &dlp;
+	}
+    }
+
+    /* Maybe it's a fixed drive */
+    if (lp == NULL) {
+	if (bpb->bpbBytesPerSec)
+	    dlp.d_secsize = bpb->bpbBytesPerSec;
+	if (bpb->bpbBytesPerSec == 0 && ioctl(fd, DIOCGSECTORSIZE,
+					      &dlp.d_secsize) == -1)
+	    err(1, "cannot get sector size");
+
+	dlp.d_secperunit = ms / dlp.d_secsize;
+
+	if (bpb->bpbSecPerTrack == 0 && ioctl(fd, DIOCGFWSECTORS,
+					      &dlp.d_nsectors) == -1) {
+	    warn("cannot get number of sectors per track");
+	    dlp.d_nsectors = 63;
+	}
+	if (bpb->bpbHeads == 0 &&
+	    ioctl(fd, DIOCGFWHEADS, &dlp.d_ntracks) == -1) {
+	    warn("cannot get number of heads");
+	    if (dlp.d_secperunit <= 63*1*1024)
+		dlp.d_ntracks = 1;
+	    else if (dlp.d_secperunit <= 63*16*1024)
+		dlp.d_ntracks = 16;
+	    else
+		dlp.d_ntracks = 255;
+	}
+
+	hs = (ms / dlp.d_secsize) - dlp.d_secperunit;
+	lp = &dlp;
+    }
+
+    if (bpb->bpbBytesPerSec == 0)
+	bpb->bpbBytesPerSec = ckgeom(fname, lp->d_secsize, "bytes/sector");
+    if (bpb->bpbSecPerTrack == 0)
+	bpb->bpbSecPerTrack = ckgeom(fname, lp->d_nsectors, "sectors/track");
+    if (bpb->bpbHeads == 0)
+	bpb->bpbHeads = ckgeom(fname, lp->d_ntracks, "drive heads");
+    if (bpb->bpbHugeSectors == 0)
+	bpb->bpbHugeSectors = lp->d_secperunit;
+    if (bpb->bpbHiddenSecs == 0)
+	bpb->bpbHiddenSecs = hs;
+}
+
+/*
+ * Print out BPB values.
+ */
+static void
+print_bpb(struct bpb *bpb)
+{
+    printf("BytesPerSec=%u SecPerClust=%u ResSectors=%u FATs=%u",
+	   bpb->bpbBytesPerSec, bpb->bpbSecPerClust, bpb->bpbResSectors,
+	   bpb->bpbFATs);
+    if (bpb->bpbRootDirEnts)
+	printf(" RootDirEnts=%u", bpb->bpbRootDirEnts);
+    if (bpb->bpbSectors)
+	printf(" Sectors=%u", bpb->bpbSectors);
+    printf(" Media=%#x", bpb->bpbMedia);
+    if (bpb->bpbFATsecs)
+	printf(" FATsecs=%u", bpb->bpbFATsecs);
+    printf(" SecPerTrack=%u Heads=%u HiddenSecs=%u", bpb->bpbSecPerTrack,
+	   bpb->bpbHeads, bpb->bpbHiddenSecs);
+    if (bpb->bpbHugeSectors)
+	printf(" HugeSectors=%u", bpb->bpbHugeSectors);
+    if (!bpb->bpbFATsecs) {
+	printf(" FATsecs=%u RootCluster=%u", bpb->bpbBigFATsecs,
+	       bpb->bpbRootClust);
+	printf(" FSInfo=");
+	printf(bpb->bpbFSInfo == MAXU16 ? "%#x" : "%u", bpb->bpbFSInfo);
+	printf(" Backup=");
+	printf(bpb->bpbBackup == MAXU16 ? "%#x" : "%u", bpb->bpbBackup);
+    }
+    printf("\n");
+}
+
+/*
+ * Check a disk geometry value.
+ */
+static u_int
+ckgeom(const char *fname, u_int val, const char *msg)
+{
+    if (!val)
+	errx(1, "%s: no default %s", fname, msg);
+    if (val > MAXU16)
+	errx(1, "%s: illegal %s %d", fname, msg, val);
+    return val;
+}
+
+/*
+ * Check a volume label.
+ */
+static int
+oklabel(const char *src)
+{
+    int c, i;
+
+    for (i = 0; i <= 11; i++) {
+	c = (u_char)*src++;
+	if (c < ' ' + !i || strchr("\"*+,./:;<=>?[\\]|", c))
+	    break;
+    }
+    return i && !c;
+}
+
+/*
+ * Make a volume label.
+ */
+static void
+mklabel(u_int8_t *dest, const char *src)
+{
+    int c, i;
+
+    for (i = 0; i < 11; i++) {
+	c = *src ? toupper(*src++) : ' ';
+	*dest++ = !i && c == '\xe5' ? 5 : c;
+    }
+}
+
+/*
+ * Copy string, padding with spaces.
+ */
+static void
+setstr(u_int8_t *dest, const char *src, size_t len)
+{
+    while (len--)
+	*dest++ = *src ? *src++ : ' ';
+}
+
+static void
+infohandler(int sig __unused)
+{
+
+	got_siginfo = 1;
+}
diff --git a/sbin/newfs_msdos/mkfs_msdos.h b/sbin/newfs_msdos/mkfs_msdos.h
new file mode 100644
index 000000000000..e9529cee8e1a
--- /dev/null
+++ b/sbin/newfs_msdos/mkfs_msdos.h
@@ -0,0 +1,61 @@
+/*-
+ * Copyright (c) 2015 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by Ed Maste under sponsorship from
+ * the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+struct msdos_options {
+    const char *bootstrap;
+    const char *volume_label;
+    const char *OEM_string;
+    const char *floppy;
+    u_int fat_type;
+    u_int volume_id;
+    u_int bytes_per_sector;
+    u_int sectors_per_fat;
+    u_int block_size;
+    u_int sectors_per_cluster;
+    u_int directory_entries;
+    u_int drive_heads;
+    u_int info_sector;
+    u_int backup_sector;
+    u_int media_descriptor;
+    u_int num_FAT;
+    u_int hidden_sectors;
+    u_int reserved_sectors;
+    u_int size;
+    u_int sectors_per_track;
+    int no_create;
+    off_t create_size;
+    off_t offset;
+    int volume_id_set;
+    int media_descriptor_set;
+    int hidden_sectors_set;
+};
+
+int mkfs_msdos(const char *, const char *, const struct msdos_options *);
diff --git a/sbin/newfs_msdos/newfs_msdos.c b/sbin/newfs_msdos/newfs_msdos.c
index a3994257ea06..79529f284205 100644
--- a/sbin/newfs_msdos/newfs_msdos.c
+++ b/sbin/newfs_msdos/newfs_msdos.c
@@ -31,233 +31,24 @@ static const char rcsid[] =
 #endif /* not lint */
 
 #include <sys/param.h>
-#include <sys/fdcio.h>
-#include <sys/disk.h>
-#include <sys/disklabel.h>
-#include <sys/mount.h>
-#include <sys/stat.h>
-#include <sys/time.h>
 
-#include <ctype.h>
 #include <err.h>
 #include <errno.h>
-#include <fcntl.h>
-#include <inttypes.h>
 #include <paths.h>
-#include <signal.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <time.h>
 #include <unistd.h>
 
-#define	MAXU16	  0xffff	/* maximum unsigned 16-bit quantity */
-#define	BPN	  4		/* bits per nibble */
-#define	NPB	  2		/* nibbles per byte */
-
-#define	DOSMAGIC  0xaa55	/* DOS magic number */
-#define	MINBPS	  512		/* minimum bytes per sector */
-#define	MAXSPC	  128		/* maximum sectors per cluster */
-#define	MAXNFT	  16		/* maximum number of FATs */
-#define	DEFBLK	  4096		/* default block size */
-#define	DEFBLK16  2048		/* default block size FAT16 */
-#define	DEFRDE	  512		/* default root directory entries */
-#define	RESFTE	  2		/* reserved FAT entries */
-#define	MINCLS12  1U		/* minimum FAT12 clusters */
-#define	MINCLS16  0xff5U	/* minimum FAT16 clusters */
-#define	MINCLS32  0xfff5U	/* minimum FAT32 clusters */
-#define	MAXCLS12  0xff4U	/* maximum FAT12 clusters */
-#define	MAXCLS16  0xfff4U	/* maximum FAT16 clusters */
-#define	MAXCLS32  0xffffff4U	/* maximum FAT32 clusters */
-
-#define	mincls(fat)  ((fat) == 12 ? MINCLS12 :	\
-		      (fat) == 16 ? MINCLS16 :	\
-				    MINCLS32)
-
-#define	maxcls(fat)  ((fat) == 12 ? MAXCLS12 :	\
-		      (fat) == 16 ? MAXCLS16 :	\
-				    MAXCLS32)
-
-#define	mk1(p, x)				\
-    (p) = (u_int8_t)(x)
-
-#define	mk2(p, x)				\
-    (p)[0] = (u_int8_t)(x),			\
-    (p)[1] = (u_int8_t)((x) >> 010)
-
-#define	mk4(p, x)				\
-    (p)[0] = (u_int8_t)(x),			\
-    (p)[1] = (u_int8_t)((x) >> 010),		\
-    (p)[2] = (u_int8_t)((x) >> 020),		\
-    (p)[3] = (u_int8_t)((x) >> 030)
+#include "mkfs_msdos.h"
 
 #define	argto1(arg, lo, msg)  argtou(arg, lo, 0xff, msg)
 #define	argto2(arg, lo, msg)  argtou(arg, lo, 0xffff, msg)
 #define	argto4(arg, lo, msg)  argtou(arg, lo, 0xffffffff, msg)
 #define	argtox(arg, lo, msg)  argtou(arg, lo, UINT_MAX, msg)
 
-struct bs {
-    u_int8_t bsJump[3];			/* bootstrap entry point */
-    u_int8_t bsOemName[8];		/* OEM name and version */
-} __packed;
-
-struct bsbpb {
-    u_int8_t bpbBytesPerSec[2];		/* bytes per sector */
-    u_int8_t bpbSecPerClust;		/* sectors per cluster */
-    u_int8_t bpbResSectors[2];		/* reserved sectors */
-    u_int8_t bpbFATs;			/* number of FATs */
-    u_int8_t bpbRootDirEnts[2];		/* root directory entries */
-    u_int8_t bpbSectors[2];		/* total sectors */
-    u_int8_t bpbMedia;			/* media descriptor */
-    u_int8_t bpbFATsecs[2];		/* sectors per FAT */
-    u_int8_t bpbSecPerTrack[2];		/* sectors per track */
-    u_int8_t bpbHeads[2];		/* drive heads */
-    u_int8_t bpbHiddenSecs[4];		/* hidden sectors */
-    u_int8_t bpbHugeSectors[4];		/* big total sectors */
-} __packed;
-
-struct bsxbpb {
-    u_int8_t bpbBigFATsecs[4];		/* big sectors per FAT */
-    u_int8_t bpbExtFlags[2];		/* FAT control flags */
-    u_int8_t bpbFSVers[2];		/* file system version */
-    u_int8_t bpbRootClust[4];		/* root directory start cluster */
-    u_int8_t bpbFSInfo[2];		/* file system info sector */
-    u_int8_t bpbBackup[2];		/* backup boot sector */
-    u_int8_t bpbReserved[12];		/* reserved */
-} __packed;
-
-struct bsx {
-    u_int8_t exDriveNumber;		/* drive number */
-    u_int8_t exReserved1;		/* reserved */
-    u_int8_t exBootSignature;		/* extended boot signature */
-    u_int8_t exVolumeID[4];		/* volume ID number */
-    u_int8_t exVolumeLabel[11];		/* volume label */
-    u_int8_t exFileSysType[8];		/* file system type */
-} __packed;
-
-struct de {
-    u_int8_t deName[11];		/* name and extension */
-    u_int8_t deAttributes;		/* attributes */
-    u_int8_t rsvd[10];			/* reserved */
-    u_int8_t deMTime[2];		/* last-modified time */
-    u_int8_t deMDate[2];		/* last-modified date */
-    u_int8_t deStartCluster[2];		/* starting cluster */
-    u_int8_t deFileSize[4];		/* size */
-} __packed;
-
-struct bpb {
-    u_int bpbBytesPerSec;		/* bytes per sector */
-    u_int bpbSecPerClust;		/* sectors per cluster */
-    u_int bpbResSectors;		/* reserved sectors */
-    u_int bpbFATs;			/* number of FATs */
-    u_int bpbRootDirEnts;		/* root directory entries */
-    u_int bpbSectors;			/* total sectors */
-    u_int bpbMedia;			/* media descriptor */
-    u_int bpbFATsecs;			/* sectors per FAT */
-    u_int bpbSecPerTrack;		/* sectors per track */
-    u_int bpbHeads;			/* drive heads */
-    u_int bpbHiddenSecs;		/* hidden sectors */
-    u_int bpbHugeSectors; 		/* big total sectors */
-    u_int bpbBigFATsecs; 		/* big sectors per FAT */
-    u_int bpbRootClust; 		/* root directory start cluster */
-    u_int bpbFSInfo; 			/* file system info sector */
-    u_int bpbBackup; 			/* backup boot sector */
-};
-
-#define	BPBGAP 0, 0, 0, 0, 0, 0
-
-static struct {
-    const char *name;
-    struct bpb bpb;
-} const stdfmt[] = {
-    {"160",  {512, 1, 1, 2,  64,  320, 0xfe, 1,  8, 1, BPBGAP}},
-    {"180",  {512, 1, 1, 2,  64,  360, 0xfc, 2,  9, 1, BPBGAP}},
-    {"320",  {512, 2, 1, 2, 112,  640, 0xff, 1,  8, 2, BPBGAP}},
-    {"360",  {512, 2, 1, 2, 112,  720, 0xfd, 2,  9, 2, BPBGAP}},
-    {"640",  {512, 2, 1, 2, 112, 1280, 0xfb, 2,  8, 2, BPBGAP}},
-    {"720",  {512, 2, 1, 2, 112, 1440, 0xf9, 3,  9, 2, BPBGAP}},
-    {"1200", {512, 1, 1, 2, 224, 2400, 0xf9, 7, 15, 2, BPBGAP}},
-    {"1232", {1024,1, 1, 2, 192, 1232, 0xfe, 2,  8, 2, BPBGAP}},
-    {"1440", {512, 1, 1, 2, 224, 2880, 0xf0, 9, 18, 2, BPBGAP}},
-    {"2880", {512, 2, 1, 2, 240, 5760, 0xf0, 9, 36, 2, BPBGAP}}
-};
-
-static const u_int8_t bootcode[] = {
-    0xfa,			/* cli		    */
-    0x31, 0xc0, 		/* xor	   ax,ax    */
-    0x8e, 0xd0, 		/* mov	   ss,ax    */
-    0xbc, 0x00, 0x7c,		/* mov	   sp,7c00h */
-    0xfb,			/* sti		    */
-    0x8e, 0xd8, 		/* mov	   ds,ax    */
-    0xe8, 0x00, 0x00,		/* call    $ + 3    */
-    0x5e,			/* pop	   si	    */
-    0x83, 0xc6, 0x19,		/* add	   si,+19h  */
-    0xbb, 0x07, 0x00,		/* mov	   bx,0007h */
-    0xfc,			/* cld		    */
-    0xac,			/* lodsb	    */
-    0x84, 0xc0, 		/* test    al,al    */
-    0x74, 0x06, 		/* jz	   $ + 8    */
-    0xb4, 0x0e, 		/* mov	   ah,0eh   */
-    0xcd, 0x10, 		/* int	   10h	    */
-    0xeb, 0xf5, 		/* jmp	   $ - 9    */
-    0x30, 0xe4, 		/* xor	   ah,ah    */
-    0xcd, 0x16, 		/* int	   16h	    */
-    0xcd, 0x19, 		/* int	   19h	    */
-    0x0d, 0x0a,
-    'N', 'o', 'n', '-', 's', 'y', 's', 't',
-    'e', 'm', ' ', 'd', 'i', 's', 'k',
-    0x0d, 0x0a,
-    'P', 'r', 'e', 's', 's', ' ', 'a', 'n',
-    'y', ' ', 'k', 'e', 'y', ' ', 't', 'o',
-    ' ', 'r', 'e', 'b', 'o', 'o', 't',
-    0x0d, 0x0a,
-    0
-};
-
-struct msdos_options {
-    const char *bootstrap;
-    const char *volume_label;
-    const char *OEM_string;
-    const char *floppy;
-    u_int fat_type;
-    u_int volume_id;
-    u_int bytes_per_sector;
-    u_int sectors_per_fat;
-    u_int block_size;
-    u_int sectors_per_cluster;
-    u_int directory_entries;
-    u_int drive_heads;
-    u_int info_sector;
-    u_int backup_sector;
-    u_int media_descriptor;
-    u_int num_FAT;
-    u_int hidden_sectors;
-    u_int reserved_sectors;
-    u_int size;
-    u_int sectors_per_track;
-    int no_create;
-    off_t create_size;
-    off_t offset;
-    int volume_id_set;
-    int media_descriptor_set;
-    int hidden_sectors_set;
-};
-
-static volatile sig_atomic_t got_siginfo;
-static void infohandler(int);
-
-static void check_mounted(const char *, mode_t);
-static void getstdfmt(const char *, struct bpb *);
-static void getdiskinfo(int, const char *, const char *, int,
-			struct bpb *);
-static void print_bpb(struct bpb *);
-static u_int ckgeom(const char *, u_int, const char *);
 static u_int argtou(const char *, u_int, u_int, const char *);
 static off_t argtooff(const char *, const char *);
-static int oklabel(const char *);
-static int mkfs_msdos(const char *, const char *, const struct msdos_options *);
-static void mklabel(u_int8_t *, const char *);
-static void setstr(u_int8_t *, const char *, size_t);
 static void usage(void);
 
 /*
@@ -300,13 +91,9 @@ main(int argc, char *argv[])
 	    o.volume_id_set = 1;
 	    break;
 	case 'L':
-	    if (!oklabel(optarg))
-		errx(1, "%s: bad volume label", optarg);
 	    o.volume_label = optarg;
 	    break;
 	case 'O':
-	    if (strlen(optarg) > 8)
-		errx(1, "%s: bad OEM string", optarg);
 	    o.OEM_string = optarg;
 	    break;
 	case 'S':
@@ -375,579 +162,6 @@ main(int argc, char *argv[])
     return mkfs_msdos(fname, dtype, &o);
 }
 
-int mkfs_msdos(const char *fname, const char *dtype,
-    const struct msdos_options *op)
-{
-    char buf[MAXPATHLEN];
-    struct sigaction si_sa;
-    struct stat sb;
-    struct timeval tv;
-    struct bpb bpb;
-    struct tm *tm;
-    struct bs *bs;
-    struct bsbpb *bsbpb;
-    struct bsxbpb *bsxbpb;
-    struct bsx *bsx;
-    struct de *de;
-    u_int8_t *img;
-    const char *bname;
-    ssize_t n;
-    time_t now;
-    u_int fat, bss, rds, cls, dir, lsn, x, x1, x2;
-    int fd, fd1;
-    struct msdos_options o = *op;
-
-    if (o.create_size) {
-	if (o.no_create)
-	    errx(1, "create (-C) is incompatible with -N");
-	fd = open(fname, O_RDWR | O_CREAT | O_TRUNC, 0644);
-	if (fd == -1)
-	    errx(1, "failed to create %s", fname);
-	if (ftruncate(fd, o.create_size))
-	    errx(1, "failed to initialize %jd bytes", (intmax_t)o.create_size);
-    } else if ((fd = open(fname, o.no_create ? O_RDONLY : O_RDWR)) == -1)
-	err(1, "%s", fname);
-    if (fstat(fd, &sb))
-	err(1, "%s", fname);
-    if (o.create_size) {
-	if (!S_ISREG(sb.st_mode))
-	    warnx("warning, %s is not a regular file", fname);
-    } else {
-	if (!S_ISCHR(sb.st_mode))
-	    warnx("warning, %s is not a character device", fname);
-    }
-    if (!o.no_create)
-	check_mounted(fname, sb.st_mode);
-    if (o.offset && o.offset != lseek(fd, o.offset, SEEK_SET))
-	errx(1, "cannot seek to %jd", (intmax_t)o.offset);
-    memset(&bpb, 0, sizeof(bpb));
-    if (o.floppy) {
-	getstdfmt(o.floppy, &bpb);
-	bpb.bpbHugeSectors = bpb.bpbSectors;
-	bpb.bpbSectors = 0;
-	bpb.bpbBigFATsecs = bpb.bpbFATsecs;
-	bpb.bpbFATsecs = 0;
-    }
-    if (o.drive_heads)
-	bpb.bpbHeads = o.drive_heads;
-    if (o.sectors_per_track)
-	bpb.bpbSecPerTrack = o.sectors_per_track;
-    if (o.bytes_per_sector)
-	bpb.bpbBytesPerSec = o.bytes_per_sector;
-    if (o.size)
-	bpb.bpbHugeSectors = o.size;
-    if (o.hidden_sectors_set)
-	bpb.bpbHiddenSecs = o.hidden_sectors;
-    if (!(o.floppy || (o.drive_heads && o.sectors_per_track && o.bytes_per_sector && o.size && o.hidden_sectors_set))) {
-	off_t delta;
-	getdiskinfo(fd, fname, dtype, o.hidden_sectors_set, &bpb);
-	bpb.bpbHugeSectors -= (o.offset / bpb.bpbBytesPerSec);
-	delta = bpb.bpbHugeSectors % bpb.bpbSecPerTrack;
-	if (delta != 0) {
-	    warnx("trim %d sectors to adjust to a multiple of %d",
-		(int)delta, bpb.bpbSecPerTrack);
-	    bpb.bpbHugeSectors -= delta;
-	}
-	if (bpb.bpbSecPerClust == 0) {	/* set defaults */
-	    if (bpb.bpbHugeSectors <= 6000)	/* about 3MB -> 512 bytes */
-		bpb.bpbSecPerClust = 1;
-	    else if (bpb.bpbHugeSectors <= (1<<17)) /* 64M -> 4k */
-		bpb.bpbSecPerClust = 8;
-	    else if (bpb.bpbHugeSectors <= (1<<19)) /* 256M -> 8k */
-		bpb.bpbSecPerClust = 16;
-	    else if (bpb.bpbHugeSectors <= (1<<21)) /* 1G -> 16k */
-		bpb.bpbSecPerClust = 32;
-	    else
-		bpb.bpbSecPerClust = 64;		/* otherwise 32k */
-	}
-    }
-    if (!powerof2(bpb.bpbBytesPerSec))
-	errx(1, "bytes/sector (%u) is not a power of 2", bpb.bpbBytesPerSec);
-    if (bpb.bpbBytesPerSec < MINBPS)
-	errx(1, "bytes/sector (%u) is too small; minimum is %u",
-	     bpb.bpbBytesPerSec, MINBPS);
-    if (!(fat = o.fat_type)) {
-	if (o.floppy)
-	    fat = 12;
-	else if (!o.directory_entries && (o.info_sector || o.backup_sector))
-	    fat = 32;
-    }
-    if ((fat == 32 && o.directory_entries) || (fat != 32 && (o.info_sector || o.backup_sector)))
-	errx(1, "-%c is not a legal FAT%s option",
-	     fat == 32 ? 'e' : o.info_sector ? 'i' : 'k',
-	     fat == 32 ? "32" : "12/16");
-    if (o.floppy && fat == 32)
-	bpb.bpbRootDirEnts = 0;
-    if (o.block_size) {
-	if (!powerof2(o.block_size))
-	    errx(1, "block size (%u) is not a power of 2", o.block_size);
-	if (o.block_size < bpb.bpbBytesPerSec)
-	    errx(1, "block size (%u) is too small; minimum is %u",
-		 o.block_size, bpb.bpbBytesPerSec);
-	if (o.block_size > bpb.bpbBytesPerSec * MAXSPC)
-	    errx(1, "block size (%u) is too large; maximum is %u",
-		 o.block_size, bpb.bpbBytesPerSec * MAXSPC);
-	bpb.bpbSecPerClust = o.block_size / bpb.bpbBytesPerSec;
-    }
-    if (o.sectors_per_cluster) {
-	if (!powerof2(o.sectors_per_cluster))
-	    errx(1, "sectors/cluster (%u) is not a power of 2", o.sectors_per_cluster);
-	bpb.bpbSecPerClust = o.sectors_per_cluster;
-    }
-    if (o.reserved_sectors)
-	bpb.bpbResSectors = o.reserved_sectors;
-    if (o.num_FAT) {
-	if (o.num_FAT > MAXNFT)
-	    errx(1, "number of FATs (%u) is too large; maximum is %u",
-		 o.num_FAT, MAXNFT);
-	bpb.bpbFATs = o.num_FAT;
-    }
-    if (o.directory_entries)
-	bpb.bpbRootDirEnts = o.directory_entries;
-    if (o.media_descriptor_set) {
-	if (o.media_descriptor < 0xf0)
-	    errx(1, "illegal media descriptor (%#x)", o.media_descriptor);
-	bpb.bpbMedia = o.media_descriptor;
-    }
-    if (o.sectors_per_fat)
-	bpb.bpbBigFATsecs = o.sectors_per_fat;
-    if (o.info_sector)
-	bpb.bpbFSInfo = o.info_sector;
-    if (o.backup_sector)
-	bpb.bpbBackup = o.backup_sector;
-    bss = 1;
-    bname = NULL;
-    fd1 = -1;
-    if (o.bootstrap) {
-	bname = o.bootstrap;
-	if (!strchr(bname, '/')) {
-	    snprintf(buf, sizeof(buf), "/boot/%s", bname);
-	    if (!(bname = strdup(buf)))
-		err(1, NULL);
-	}
-	if ((fd1 = open(bname, O_RDONLY)) == -1 || fstat(fd1, &sb))
-	    err(1, "%s", bname);
-	if (!S_ISREG(sb.st_mode) || sb.st_size % bpb.bpbBytesPerSec ||
-	    sb.st_size < bpb.bpbBytesPerSec ||
-	    sb.st_size > bpb.bpbBytesPerSec * MAXU16)
-	    errx(1, "%s: inappropriate file type or format", bname);
-	bss = sb.st_size / bpb.bpbBytesPerSec;
-    }
-    if (!bpb.bpbFATs)
-	bpb.bpbFATs = 2;
-    if (!fat) {
-	if (bpb.bpbHugeSectors < (bpb.bpbResSectors ? bpb.bpbResSectors : bss) +
-	    howmany((RESFTE + (bpb.bpbSecPerClust ? MINCLS16 : MAXCLS12 + 1)) *
-		(bpb.bpbSecPerClust ? 16 : 12) / BPN,
-		bpb.bpbBytesPerSec * NPB) *
-	    bpb.bpbFATs +
-	    howmany(bpb.bpbRootDirEnts ? bpb.bpbRootDirEnts : DEFRDE,
-		    bpb.bpbBytesPerSec / sizeof(struct de)) +
-	    (bpb.bpbSecPerClust ? MINCLS16 : MAXCLS12 + 1) *
-	    (bpb.bpbSecPerClust ? bpb.bpbSecPerClust :
-	     howmany(DEFBLK, bpb.bpbBytesPerSec)))
-	    fat = 12;
-	else if (bpb.bpbRootDirEnts || bpb.bpbHugeSectors <
-		 (bpb.bpbResSectors ? bpb.bpbResSectors : bss) +
-		 howmany((RESFTE + MAXCLS16) * 2, bpb.bpbBytesPerSec) *
-		 bpb.bpbFATs +
-		 howmany(DEFRDE, bpb.bpbBytesPerSec / sizeof(struct de)) +
-		 (MAXCLS16 + 1) *
-		 (bpb.bpbSecPerClust ? bpb.bpbSecPerClust :
-		  howmany(8192, bpb.bpbBytesPerSec)))
-	    fat = 16;
-	else
-	    fat = 32;
-    }
-    x = bss;
-    if (fat == 32) {
-	if (!bpb.bpbFSInfo) {
-	    if (x == MAXU16 || x == bpb.bpbBackup)
-		errx(1, "no room for info sector");
-	    bpb.bpbFSInfo = x;
-	}
-	if (bpb.bpbFSInfo != MAXU16 && x <= bpb.bpbFSInfo)
-	    x = bpb.bpbFSInfo + 1;
-	if (!bpb.bpbBackup) {
-	    if (x == MAXU16)
-		errx(1, "no room for backup sector");
-	    bpb.bpbBackup = x;
-	} else if (bpb.bpbBackup != MAXU16 && bpb.bpbBackup == bpb.bpbFSInfo)
-	    errx(1, "backup sector would overwrite info sector");
-	if (bpb.bpbBackup != MAXU16 && x <= bpb.bpbBackup)
-	    x = bpb.bpbBackup + 1;
-    }
-    if (!bpb.bpbResSectors)
-	bpb.bpbResSectors = fat == 32 ? 
-	    MAX(x, MAX(16384 / bpb.bpbBytesPerSec, 4)) : x;
-    else if (bpb.bpbResSectors < x)
-	errx(1, "too few reserved sectors (need %d have %d)", x,
-	     bpb.bpbResSectors);
-    if (fat != 32 && !bpb.bpbRootDirEnts)
-	bpb.bpbRootDirEnts = DEFRDE;
-    rds = howmany(bpb.bpbRootDirEnts, bpb.bpbBytesPerSec / sizeof(struct de));
-    if (!bpb.bpbSecPerClust)
-	for (bpb.bpbSecPerClust = howmany(fat == 16 ? DEFBLK16 :
-					  DEFBLK, bpb.bpbBytesPerSec);
-	     bpb.bpbSecPerClust < MAXSPC &&
-	     bpb.bpbResSectors +
-	     howmany((RESFTE + maxcls(fat)) * (fat / BPN),
-		     bpb.bpbBytesPerSec * NPB) *
-	     bpb.bpbFATs +
-	     rds +
-	     (u_int64_t) (maxcls(fat) + 1) *
-	     bpb.bpbSecPerClust <= bpb.bpbHugeSectors;
-	     bpb.bpbSecPerClust <<= 1)
-	    continue;
-    if (fat != 32 && bpb.bpbBigFATsecs > MAXU16)
-	errx(1, "too many sectors/FAT for FAT12/16");
-    x1 = bpb.bpbResSectors + rds;
-    x = bpb.bpbBigFATsecs ? bpb.bpbBigFATsecs : 1;
-    if (x1 + (u_int64_t)x * bpb.bpbFATs > bpb.bpbHugeSectors)
-	errx(1, "meta data exceeds file system size");
-    x1 += x * bpb.bpbFATs;
-    x = (u_int64_t)(bpb.bpbHugeSectors - x1) * bpb.bpbBytesPerSec * NPB /
-	(bpb.bpbSecPerClust * bpb.bpbBytesPerSec * NPB + fat /
-	 BPN * bpb.bpbFATs);
-    x2 = howmany((RESFTE + MIN(x, maxcls(fat))) * (fat / BPN),
-		 bpb.bpbBytesPerSec * NPB);
-    if (!bpb.bpbBigFATsecs) {
-	bpb.bpbBigFATsecs = x2;
-	x1 += (bpb.bpbBigFATsecs - 1) * bpb.bpbFATs;
-    }
-    cls = (bpb.bpbHugeSectors - x1) / bpb.bpbSecPerClust;
-    x = (u_int64_t)bpb.bpbBigFATsecs * bpb.bpbBytesPerSec * NPB / (fat / BPN) -
-	RESFTE;
-    if (cls > x)
-	cls = x;
-    if (bpb.bpbBigFATsecs < x2)
-	warnx("warning: sectors/FAT limits file system to %u clusters",
-	      cls);
-    if (cls < mincls(fat))
-	errx(1, "%u clusters too few clusters for FAT%u, need %u", cls, fat,
-	    mincls(fat));
-    if (cls > maxcls(fat)) {
-	cls = maxcls(fat);
-	bpb.bpbHugeSectors = x1 + (cls + 1) * bpb.bpbSecPerClust - 1;
-	warnx("warning: FAT type limits file system to %u sectors",
-	      bpb.bpbHugeSectors);
-    }
-    printf("%s: %u sector%s in %u FAT%u cluster%s "
-	   "(%u bytes/cluster)\n", fname, cls * bpb.bpbSecPerClust,
-	   cls * bpb.bpbSecPerClust == 1 ? "" : "s", cls, fat,
-	   cls == 1 ? "" : "s", bpb.bpbBytesPerSec * bpb.bpbSecPerClust);
-    if (!bpb.bpbMedia)
-	bpb.bpbMedia = !bpb.bpbHiddenSecs ? 0xf0 : 0xf8;
-    if (fat == 32)
-	bpb.bpbRootClust = RESFTE;
-    if (bpb.bpbHiddenSecs + bpb.bpbHugeSectors <= MAXU16) {
-	bpb.bpbSectors = bpb.bpbHugeSectors;
-	bpb.bpbHugeSectors = 0;
-    }
-    if (fat != 32) {
-	bpb.bpbFATsecs = bpb.bpbBigFATsecs;
-	bpb.bpbBigFATsecs = 0;
-    }
-    print_bpb(&bpb);
-    if (!o.no_create) {
-	gettimeofday(&tv, NULL);
-	now = tv.tv_sec;
-	tm = localtime(&now);
-	if (!(img = malloc(bpb.bpbBytesPerSec)))
-	    err(1, NULL);
-	dir = bpb.bpbResSectors + (bpb.bpbFATsecs ? bpb.bpbFATsecs :
-				   bpb.bpbBigFATsecs) * bpb.bpbFATs;
-	memset(&si_sa, 0, sizeof(si_sa));
-	si_sa.sa_handler = infohandler;
-	if (sigaction(SIGINFO, &si_sa, NULL) == -1)
-		err(1, "sigaction SIGINFO");
-	for (lsn = 0; lsn < dir + (fat == 32 ? bpb.bpbSecPerClust : rds); lsn++) {
-	    if (got_siginfo) {
-		    fprintf(stderr,"%s: writing sector %u of %u (%u%%)\n",
-			fname, lsn,
-			(dir + (fat == 32 ? bpb.bpbSecPerClust: rds)),
-			(lsn * 100) / (dir +
-			    (fat == 32 ? bpb.bpbSecPerClust: rds)));
-		    got_siginfo = 0;
-	    }
-	    x = lsn;
-	    if (o.bootstrap &&
-		fat == 32 && bpb.bpbBackup != MAXU16 &&
-		bss <= bpb.bpbBackup && x >= bpb.bpbBackup) {
-		x -= bpb.bpbBackup;
-		if (!x && lseek(fd1, o.offset, SEEK_SET))
-		    err(1, "%s", bname);
-	    }
-	    if (o.bootstrap && x < bss) {
-		if ((n = read(fd1, img, bpb.bpbBytesPerSec)) == -1)
-		    err(1, "%s", bname);
-		if ((unsigned)n != bpb.bpbBytesPerSec)
-		    errx(1, "%s: can't read sector %u", bname, x);
-	    } else
-		memset(img, 0, bpb.bpbBytesPerSec);
-	    if (!lsn ||
-		(fat == 32 && bpb.bpbBackup != MAXU16 &&
-		 lsn == bpb.bpbBackup)) {
-		x1 = sizeof(struct bs);
-		bsbpb = (struct bsbpb *)(img + x1);
-		mk2(bsbpb->bpbBytesPerSec, bpb.bpbBytesPerSec);
-		mk1(bsbpb->bpbSecPerClust, bpb.bpbSecPerClust);
-		mk2(bsbpb->bpbResSectors, bpb.bpbResSectors);
-		mk1(bsbpb->bpbFATs, bpb.bpbFATs);
-		mk2(bsbpb->bpbRootDirEnts, bpb.bpbRootDirEnts);
-		mk2(bsbpb->bpbSectors, bpb.bpbSectors);
-		mk1(bsbpb->bpbMedia, bpb.bpbMedia);
-		mk2(bsbpb->bpbFATsecs, bpb.bpbFATsecs);
-		mk2(bsbpb->bpbSecPerTrack, bpb.bpbSecPerTrack);
-		mk2(bsbpb->bpbHeads, bpb.bpbHeads);
-		mk4(bsbpb->bpbHiddenSecs, bpb.bpbHiddenSecs);
-		mk4(bsbpb->bpbHugeSectors, bpb.bpbHugeSectors);
-		x1 += sizeof(struct bsbpb);
-		if (fat == 32) {
-		    bsxbpb = (struct bsxbpb *)(img + x1);
-		    mk4(bsxbpb->bpbBigFATsecs, bpb.bpbBigFATsecs);
-		    mk2(bsxbpb->bpbExtFlags, 0);
-		    mk2(bsxbpb->bpbFSVers, 0);
-		    mk4(bsxbpb->bpbRootClust, bpb.bpbRootClust);
-		    mk2(bsxbpb->bpbFSInfo, bpb.bpbFSInfo);
-		    mk2(bsxbpb->bpbBackup, bpb.bpbBackup);
-		    x1 += sizeof(struct bsxbpb);
-		}
-		bsx = (struct bsx *)(img + x1);
-		mk1(bsx->exBootSignature, 0x29);
-		if (o.volume_id_set)
-		    x = o.volume_id;
-		else
-		    x = (((u_int)(1 + tm->tm_mon) << 8 |
-			  (u_int)tm->tm_mday) +
-			 ((u_int)tm->tm_sec << 8 |
-			  (u_int)(tv.tv_usec / 10))) << 16 |
-			((u_int)(1900 + tm->tm_year) +
-			 ((u_int)tm->tm_hour << 8 |
-			  (u_int)tm->tm_min));
-		mk4(bsx->exVolumeID, x);
-		mklabel(bsx->exVolumeLabel, o.volume_label ? o.volume_label : "NO NAME");
-		sprintf(buf, "FAT%u", fat);
-		setstr(bsx->exFileSysType, buf, sizeof(bsx->exFileSysType));
-		if (!o.bootstrap) {
-		    x1 += sizeof(struct bsx);
-		    bs = (struct bs *)img;
-		    mk1(bs->bsJump[0], 0xeb);
-		    mk1(bs->bsJump[1], x1 - 2);
-		    mk1(bs->bsJump[2], 0x90);
-		    setstr(bs->bsOemName, o.OEM_string ? o.OEM_string : "BSD4.4  ",
-			   sizeof(bs->bsOemName));
-		    memcpy(img + x1, bootcode, sizeof(bootcode));
-		    mk2(img + MINBPS - 2, DOSMAGIC);
-		}
-	    } else if (fat == 32 && bpb.bpbFSInfo != MAXU16 &&
-		       (lsn == bpb.bpbFSInfo ||
-			(bpb.bpbBackup != MAXU16 &&
-			 lsn == bpb.bpbBackup + bpb.bpbFSInfo))) {
-		mk4(img, 0x41615252);
-		mk4(img + MINBPS - 28, 0x61417272);
-		mk4(img + MINBPS - 24, 0xffffffff);
-		mk4(img + MINBPS - 20, bpb.bpbRootClust);
-		mk2(img + MINBPS - 2, DOSMAGIC);
-	    } else if (lsn >= bpb.bpbResSectors && lsn < dir &&
-		       !((lsn - bpb.bpbResSectors) %
-			 (bpb.bpbFATsecs ? bpb.bpbFATsecs :
-			  bpb.bpbBigFATsecs))) {
-		mk1(img[0], bpb.bpbMedia);
-		for (x = 1; x < fat * (fat == 32 ? 3 : 2) / 8; x++)
-		    mk1(img[x], fat == 32 && x % 4 == 3 ? 0x0f : 0xff);
-	    } else if (lsn == dir && o.volume_label) {
-		de = (struct de *)img;
-		mklabel(de->deName, o.volume_label);
-		mk1(de->deAttributes, 050);
-		x = (u_int)tm->tm_hour << 11 |
-		    (u_int)tm->tm_min << 5 |
-		    (u_int)tm->tm_sec >> 1;
-		mk2(de->deMTime, x);
-		x = (u_int)(tm->tm_year - 80) << 9 |
-		    (u_int)(tm->tm_mon + 1) << 5 |
-		    (u_int)tm->tm_mday;
-		mk2(de->deMDate, x);
-	    }
-	    if ((n = write(fd, img, bpb.bpbBytesPerSec)) == -1)
-		err(1, "%s", fname);
-	    if ((unsigned)n != bpb.bpbBytesPerSec)
-		errx(1, "%s: can't write sector %u", fname, lsn);
-	}
-    }
-    return 0;
-}
-
-/*
- * Exit with error if file system is mounted.
- */
-static void
-check_mounted(const char *fname, mode_t mode)
-{
-    struct statfs *mp;
-    const char *s1, *s2;
-    size_t len;
-    int n, r;
-
-    if (!(n = getmntinfo(&mp, MNT_NOWAIT)))
-	err(1, "getmntinfo");
-    len = strlen(_PATH_DEV);
-    s1 = fname;
-    if (!strncmp(s1, _PATH_DEV, len))
-	s1 += len;
-    r = S_ISCHR(mode) && s1 != fname && *s1 == 'r';
-    for (; n--; mp++) {
-	s2 = mp->f_mntfromname;
-	if (!strncmp(s2, _PATH_DEV, len))
-	    s2 += len;
-	if ((r && s2 != mp->f_mntfromname && !strcmp(s1 + 1, s2)) ||
-	    !strcmp(s1, s2))
-	    errx(1, "%s is mounted on %s", fname, mp->f_mntonname);
-    }
-}
-
-/*
- * Get a standard format.
- */
-static void
-getstdfmt(const char *fmt, struct bpb *bpb)
-{
-    u_int x, i;
-
-    x = sizeof(stdfmt) / sizeof(stdfmt[0]);
-    for (i = 0; i < x && strcmp(fmt, stdfmt[i].name); i++);
-    if (i == x)
-	errx(1, "%s: unknown standard format", fmt);
-    *bpb = stdfmt[i].bpb;
-}
-
-/*
- * Get disk slice, partition, and geometry information.
- */
-static void
-getdiskinfo(int fd, const char *fname, const char *dtype, __unused int oflag,
-	    struct bpb *bpb)
-{
-    struct disklabel *lp, dlp;
-    struct fd_type type;
-    off_t ms, hs = 0;
-
-    lp = NULL;
-
-    /* If the user specified a disk type, try to use that */
-    if (dtype != NULL) {
-	lp = getdiskbyname(dtype);
-    }
-
-    /* Maybe it's a floppy drive */
-    if (lp == NULL) {
-	if (ioctl(fd, DIOCGMEDIASIZE, &ms) == -1) {
-	    struct stat st;
-
-	    if (fstat(fd, &st))
-		err(1, "cannot get disk size");
-	    /* create a fake geometry for a file image */
-	    ms = st.st_size;
-	    dlp.d_secsize = 512;
-	    dlp.d_nsectors = 63;
-	    dlp.d_ntracks = 255;
-	    dlp.d_secperunit = ms / dlp.d_secsize;
-	    lp = &dlp;
-	} else if (ioctl(fd, FD_GTYPE, &type) != -1) {
-	    dlp.d_secsize = 128 << type.secsize;
-	    dlp.d_nsectors = type.sectrac;
-	    dlp.d_ntracks = type.heads;
-	    dlp.d_secperunit = ms / dlp.d_secsize;
-	    lp = &dlp;
-	}
-    }
-
-    /* Maybe it's a fixed drive */
-    if (lp == NULL) {
-	if (bpb->bpbBytesPerSec)
-	    dlp.d_secsize = bpb->bpbBytesPerSec;
-	if (bpb->bpbBytesPerSec == 0 && ioctl(fd, DIOCGSECTORSIZE,
-					      &dlp.d_secsize) == -1)
-	    err(1, "cannot get sector size");
-
-	dlp.d_secperunit = ms / dlp.d_secsize;
-
-	if (bpb->bpbSecPerTrack == 0 && ioctl(fd, DIOCGFWSECTORS,
-					      &dlp.d_nsectors) == -1) {
-	    warn("cannot get number of sectors per track");
-	    dlp.d_nsectors = 63;
-	}
-	if (bpb->bpbHeads == 0 &&
-	    ioctl(fd, DIOCGFWHEADS, &dlp.d_ntracks) == -1) {
-	    warn("cannot get number of heads");
-	    if (dlp.d_secperunit <= 63*1*1024)
-		dlp.d_ntracks = 1;
-	    else if (dlp.d_secperunit <= 63*16*1024)
-		dlp.d_ntracks = 16;
-	    else
-		dlp.d_ntracks = 255;
-	}
-
-	hs = (ms / dlp.d_secsize) - dlp.d_secperunit;
-	lp = &dlp;
-    }
-
-    if (bpb->bpbBytesPerSec == 0)
-	bpb->bpbBytesPerSec = ckgeom(fname, lp->d_secsize, "bytes/sector");
-    if (bpb->bpbSecPerTrack == 0)
-	bpb->bpbSecPerTrack = ckgeom(fname, lp->d_nsectors, "sectors/track");
-    if (bpb->bpbHeads == 0)
-	bpb->bpbHeads = ckgeom(fname, lp->d_ntracks, "drive heads");
-    if (bpb->bpbHugeSectors == 0)
-	bpb->bpbHugeSectors = lp->d_secperunit;
-    if (bpb->bpbHiddenSecs == 0)
-	bpb->bpbHiddenSecs = hs;
-}
-
-/*
- * Print out BPB values.
- */
-static void
-print_bpb(struct bpb *bpb)
-{
-    printf("BytesPerSec=%u SecPerClust=%u ResSectors=%u FATs=%u",
-	   bpb->bpbBytesPerSec, bpb->bpbSecPerClust, bpb->bpbResSectors,
-	   bpb->bpbFATs);
-    if (bpb->bpbRootDirEnts)
-	printf(" RootDirEnts=%u", bpb->bpbRootDirEnts);
-    if (bpb->bpbSectors)
-	printf(" Sectors=%u", bpb->bpbSectors);
-    printf(" Media=%#x", bpb->bpbMedia);
-    if (bpb->bpbFATsecs)
-	printf(" FATsecs=%u", bpb->bpbFATsecs);
-    printf(" SecPerTrack=%u Heads=%u HiddenSecs=%u", bpb->bpbSecPerTrack,
-	   bpb->bpbHeads, bpb->bpbHiddenSecs);
-    if (bpb->bpbHugeSectors)
-	printf(" HugeSectors=%u", bpb->bpbHugeSectors);
-    if (!bpb->bpbFATsecs) {
-	printf(" FATsecs=%u RootCluster=%u", bpb->bpbBigFATsecs,
-	       bpb->bpbRootClust);
-	printf(" FSInfo=");
-	printf(bpb->bpbFSInfo == MAXU16 ? "%#x" : "%u", bpb->bpbFSInfo);
-	printf(" Backup=");
-	printf(bpb->bpbBackup == MAXU16 ? "%#x" : "%u", bpb->bpbBackup);
-    }
-    printf("\n");
-}
-
-/*
- * Check a disk geometry value.
- */
-static u_int
-ckgeom(const char *fname, u_int val, const char *msg)
-{
-    if (!val)
-	errx(1, "%s: no default %s", fname, msg);
-    if (val > MAXU16)
-	errx(1, "%s: illegal %s %d", fname, msg, val);
-    return val;
-}
-
 /*
  * Convert and check a numeric option argument.
  */
@@ -983,7 +197,7 @@ argtooff(const char *arg, const char *msg)
 	default:
 	    errx(1, "%s: bad %s", arg, msg);
 	    /* notreached */
-	
+
 	case 's':		/* sector */
 	case 'S':
 	    x <<= 9;		/* times 512 */
@@ -1015,46 +229,6 @@ argtooff(const char *arg, const char *msg)
     return x;
 }
 
-/*
- * Check a volume label.
- */
-static int
-oklabel(const char *src)
-{
-    int c, i;
-
-    for (i = 0; i <= 11; i++) {
-	c = (u_char)*src++;
-	if (c < ' ' + !i || strchr("\"*+,./:;<=>?[\\]|", c))
-	    break;
-    }
-    return i && !c;
-}
-
-/*
- * Make a volume label.
- */
-static void
-mklabel(u_int8_t *dest, const char *src)
-{
-    int c, i;
-
-    for (i = 0; i < 11; i++) {
-	c = *src ? toupper(*src++) : ' ';
-	*dest++ = !i && c == '\xe5' ? 5 : c;
-    }
-}
-
-/*
- * Copy string, padding with spaces.
- */
-static void
-setstr(u_int8_t *dest, const char *src, size_t len)
-{
-    while (len--)
-	*dest++ = *src ? *src++ : ' ';
-}
-
 /*
  * Print usage message.
  */
@@ -1089,10 +263,3 @@ usage(void)
 	    "\t-u sectors/track\n");
 	exit(1);
 }
-
-static void
-infohandler(int sig __unused)
-{
-
-	got_siginfo = 1;
-}

From 3b70fdfb1d5aad9b451d851ad230e613004570f9 Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Thu, 15 Oct 2015 19:07:38 +0000
Subject: [PATCH 254/380] Revert r289302, invalid pages can be queued, e.g. by
 vfs_vmio_unwire().

Found by:	alc
Tested by:	pho
Sponsored by:	The FreeBSD Foundation
---
 sys/vm/vm_pageout.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c
index 5611ae66694a..e0d00f5172f3 100644
--- a/sys/vm/vm_pageout.c
+++ b/sys/vm/vm_pageout.c
@@ -1174,12 +1174,11 @@ unlock_page:
 		queues_locked = FALSE;
 
 		/*
-		 * Invalid pages cannot appear on a queue.  If
-		 * vm_pageout_fallback_object_lock() allowed a window
-		 * where the page could be invalidated, it should
-		 * detect this.
+		 * Invalid pages can be easily freed. They cannot be
+		 * mapped, vm_page_free() asserts this.
 		 */
-		KASSERT(m->valid != 0, ("Invalid page %p on inact queue", m));
+		if (m->valid == 0)
+			goto free_page;
 
 		/*
 		 * If the page has been referenced and the object is not dead,
@@ -1232,6 +1231,7 @@ unlock_page:
 			/*
 			 * Clean pages can be freed.
 			 */
+free_page:
 			vm_page_free(m);
 			PCPU_INC(cnt.v_dfree);
 			--page_shortage;

From 215c57e962d3627a34d57aeba530aacf17613c9a Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Thu, 15 Oct 2015 19:13:53 +0000
Subject: [PATCH 255/380] Mark sub-make targets as .MAKE and .PHONY to handle
 -n and always-build properly.

MFC after:	1 week
Sponsored by:	EMC / Isilon Storage Division
---
 secure/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/secure/Makefile b/secure/Makefile
index 4e2cc4f72523..12778538cf2c 100644
--- a/secure/Makefile
+++ b/secure/Makefile
@@ -17,7 +17,7 @@ SPROGS+=usr.sbin/sendmail
 .endif
 
 # This target is used to rebuild these programs with crypto.
-secure:
+secure: .MAKE .PHONY
 .for entry in ${SPROGS}
 	cd ${.CURDIR}/../${entry}; \
 	${MAKE} cleandir; \
@@ -28,7 +28,7 @@ secure:
 .endfor
 
 # This target is used to rebuild these programs without crypto.
-insecure:
+insecure: .MAKE .PHONY
 .for entry in ${SPROGS}
 	cd ${.CURDIR}/../${entry}; \
 	${MAKE} MK_CRYPT=no cleandir; \

From 60d1a410e8fe931f05bef64610f797c94e75184f Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Thu, 15 Oct 2015 19:49:09 +0000
Subject: [PATCH 256/380] Remove directories disconnected since r169718.

MFC after:	1 week
Sponsored by:	EMC / Isilon Storage Division
---
 gnu/usr.bin/cc/collect2/Makefile | 11 -----------
 gnu/usr.bin/cc/protoize/Makefile | 21 ---------------------
 2 files changed, 32 deletions(-)
 delete mode 100644 gnu/usr.bin/cc/collect2/Makefile
 delete mode 100644 gnu/usr.bin/cc/protoize/Makefile

diff --git a/gnu/usr.bin/cc/collect2/Makefile b/gnu/usr.bin/cc/collect2/Makefile
deleted file mode 100644
index 0b07b857a444..000000000000
--- a/gnu/usr.bin/cc/collect2/Makefile
+++ /dev/null
@@ -1,11 +0,0 @@
-# $FreeBSD$
-
-.include "../Makefile.inc"
-
-.PATH: ${GCCDIR}
-
-PROG=	collect2
-SRCS=	collect2.c tlink.c version.c
-MAN=
-
-.include <bsd.prog.mk>
diff --git a/gnu/usr.bin/cc/protoize/Makefile b/gnu/usr.bin/cc/protoize/Makefile
deleted file mode 100644
index b851bed18b2f..000000000000
--- a/gnu/usr.bin/cc/protoize/Makefile
+++ /dev/null
@@ -1,21 +0,0 @@
-# $FreeBSD$
-
-.include "../Makefile.inc"
-
-.PATH: ${.CURDIR}/../cc_tools ${GCCDIR}
-
-PROG=	protoize
-MAN=
-
-# things are rather hard-coded, we work around that here
-CFLAGS+= -DDEFAULT_TARGET_VERSION=\"\"
-CFLAGS+= -DDEFAULT_TARGET_MACHINE=\"../libdata/gcc\"
-
-DPADD=	${LIBCC_INT}
-LDADD=	${LIBCC_INT}
-
-CLEANFILES=	config.h
-config.h: auto-host.h freebsd-native.h
-	cat ${.ALLSRC} >${.TARGET}
-
-.include <bsd.prog.mk>

From 56a5aca5e1f6388362f8173ea1d4a91030678696 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Thu, 15 Oct 2015 19:51:34 +0000
Subject: [PATCH 257/380] Remove empty directory c++filt leftover from r286332.

Sponsored by:	EMC / Isilon Storage Division

From 1746acfd5f8491d7ad60f0d5399d6ac383a9e48d Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Thu, 15 Oct 2015 20:27:15 +0000
Subject: [PATCH 258/380] Let the SUBDIR_DEPEND*c++ variables actually work
 rather than being parsed as a +=.  These were safe due to a .WAIT very early
 on.

Sponsored by:	EMC / Isilon Storage Division
---
 lib/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/Makefile b/lib/Makefile
index 031c1b46bde6..aae401e0c76a 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -129,7 +129,7 @@ SUBDIR_DEPEND_libarchive= libz libbz2 libexpat liblzma libmd
 SUBDIR_DEPEND_libatm= libmd
 SUBDIR_DEPEND_libauditdm= libbsm
 SUBDIR_DEPEND_libbsnmp= ${_libnetgraph}
-SUBDIR_DEPEND_libc++= libcxxrt
+SUBDIR_DEPEND_libc++:= libcxxrt
 SUBDIR_DEPEND_libc= libcompiler_rt
 SUBDIR_DEPEND_libcam= libsbuf
 SUBDIR_DEPEND_libcapsicum= libnv
@@ -148,7 +148,7 @@ SUBDIR_DEPEND_libpjdlog= libutil
 SUBDIR_DEPEND_libprocstat= libkvm libutil
 SUBDIR_DEPEND_libradius= libmd
 SUBDIR_DEPEND_libsmb= libkiconv
-SUBDIR_DEPEND_libstdc++= msun
+SUBDIR_DEPEND_libstdc++:= msun
 SUBDIR_DEPEND_libtacplus= libmd
 SUBDIR_DEPEND_libulog= libmd
 SUBDIR_DEPEND_libunbound= ${_libldns}

From 39eb138e10e573929c410cd472344e36c0fe54a4 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Thu, 15 Oct 2015 20:40:23 +0000
Subject: [PATCH 259/380] Fix another ++= parsed as '+=', missed in r289384.

Sponsored by:	EMC / Isilon Storage Division
---
 lib/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/Makefile b/lib/Makefile
index aae401e0c76a..e5d56552d8c6 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -137,7 +137,7 @@ SUBDIR_DEPEND_libcasper= libcapsicum libnv libpjdlog
 SUBDIR_DEPEND_libdevstat= libkvm
 SUBDIR_DEPEND_libdpv= libfigpar ncurses libutil
 SUBDIR_DEPEND_libedit= ncurses
-SUBDIR_DEPEND_libg++= msun
+SUBDIR_DEPEND_libg++:= msun
 SUBDIR_DEPEND_libgeom= libexpat libsbuf
 SUBDIR_DEPEND_liblibrpcsec_gss= libgssapi
 SUBDIR_DEPEND_libmagic= libz

From 82e3ddf057c47542bb2204e29fc535d7bb470009 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Thu, 15 Oct 2015 20:46:34 +0000
Subject: [PATCH 260/380] Remove unneeded libg++ reference that came in with
 r267511 based on a removed comment.

Sponsored by:	EMC / Isilon Storage Division
---
 lib/Makefile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/Makefile b/lib/Makefile
index e5d56552d8c6..c25c508576ae 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -137,7 +137,6 @@ SUBDIR_DEPEND_libcasper= libcapsicum libnv libpjdlog
 SUBDIR_DEPEND_libdevstat= libkvm
 SUBDIR_DEPEND_libdpv= libfigpar ncurses libutil
 SUBDIR_DEPEND_libedit= ncurses
-SUBDIR_DEPEND_libg++:= msun
 SUBDIR_DEPEND_libgeom= libexpat libsbuf
 SUBDIR_DEPEND_liblibrpcsec_gss= libgssapi
 SUBDIR_DEPEND_libmagic= libz

From 2d53ea5e740794c089c6e3aa9d0d3b70f7b1c1ce Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Thu, 15 Oct 2015 20:49:20 +0000
Subject: [PATCH 261/380] libstdc++ also snook in incorrectly in r267511,
 despite not being a real subdir.

Sponsored by:	EMC / Isilon Storage Division
---
 lib/Makefile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/Makefile b/lib/Makefile
index c25c508576ae..a3321af9aed0 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -147,7 +147,6 @@ SUBDIR_DEPEND_libpjdlog= libutil
 SUBDIR_DEPEND_libprocstat= libkvm libutil
 SUBDIR_DEPEND_libradius= libmd
 SUBDIR_DEPEND_libsmb= libkiconv
-SUBDIR_DEPEND_libstdc++:= msun
 SUBDIR_DEPEND_libtacplus= libmd
 SUBDIR_DEPEND_libulog= libmd
 SUBDIR_DEPEND_libunbound= ${_libldns}

From 2dbdb62ad1b182f1a173c7e46f0fc55e6cfe3a5d Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Thu, 15 Oct 2015 21:21:41 +0000
Subject: [PATCH 262/380] Fix buildworld with clean objdir after r289351.

Sponsored by:	EMC / Isilon Storage Division
---
 Makefile.inc1 | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile.inc1 b/Makefile.inc1
index 1cdefc948cd9..32be188ccb26 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -558,6 +558,7 @@ _worldtmp:
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.tests.dist \
 	    -p ${WORLDTMP}${TESTSBASE} >/dev/null
 .if ${MK_DEBUG_FILES} != "no"
+	mkdir -p ${WORLDTMP}/usr/lib/debug/${TESTSBASE}
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.tests.dist \
 	    -p ${WORLDTMP}/usr/lib/debug/${TESTSBASE} >/dev/null
 .endif

From 1b224bebd4cbe6d294a9ab56a4578d337ea4f611 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Thu, 15 Oct 2015 22:49:56 +0000
Subject: [PATCH 263/380] Make installing to a non-existent directory an error.

Before this, if a file was installed to DESTDIR/some/dir and that directory
was missing due to not having ran 'make distrib-dirs' yet, the file would
be installed as 'some/dir'.  For something like bsd.incs.mk with INCLUDEDIR
being a sub-directory of /usr/include, this could result in all of the headers
being installed to a file rather than getting a directory of them.

Now it will error that the file/directory does not exist rather than hide
the issue.

Another option being discussed is to implement GNU's install -D flag which
would auto create any missing directories.

This is a mitigation of the problem.  The proper order to the build is to
run 'make distrib-dirs' first, but that can be forgotten if building from
a sub-directory after updating the source code to the latest revision.

MFC after:	2 weeks
Sponsored by:	EMC / Isilon Storage Division
---
 share/mk/bsd.confs.mk |  2 +-
 share/mk/bsd.files.mk |  2 +-
 share/mk/bsd.incs.mk  |  2 +-
 share/mk/bsd.info.mk  |  4 ++--
 share/mk/bsd.lib.mk   | 14 +++++++-------
 share/mk/bsd.man.mk   |  4 ++--
 share/mk/bsd.prog.mk  |  2 +-
 7 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/share/mk/bsd.confs.mk b/share/mk/bsd.confs.mk
index 0750a4e4fa57..36298d7da02f 100644
--- a/share/mk/bsd.confs.mk
+++ b/share/mk/bsd.confs.mk
@@ -69,7 +69,7 @@ _${group}INS: ${_${group}CONFS}
 	    ${.ALLSRC} ${DESTDIR}${${group}DIR}/${${group}NAME}
 .else
 	${INSTALL} -C -o ${${group}OWN} -g ${${group}GRP} -m ${${group}MODE} \
-	    ${.ALLSRC} ${DESTDIR}${${group}DIR}
+	    ${.ALLSRC} ${DESTDIR}${${group}DIR}/
 .endif
 .endif
 
diff --git a/share/mk/bsd.files.mk b/share/mk/bsd.files.mk
index b9379d92b59a..6a6c66a682ff 100644
--- a/share/mk/bsd.files.mk
+++ b/share/mk/bsd.files.mk
@@ -75,7 +75,7 @@ _${group}INS: ${_${group}FILES}
 	    ${DESTDIR}${${group}DIR}/${${group}NAME}
 .else
 	${INSTALL} -o ${${group}OWN} -g ${${group}GRP} \
-	    -m ${${group}MODE} ${.ALLSRC} ${DESTDIR}${${group}DIR}
+	    -m ${${group}MODE} ${.ALLSRC} ${DESTDIR}${${group}DIR}/
 .endif
 .endif
 
diff --git a/share/mk/bsd.incs.mk b/share/mk/bsd.incs.mk
index e1398c95dcd1..8ea97ba14fc8 100644
--- a/share/mk/bsd.incs.mk
+++ b/share/mk/bsd.incs.mk
@@ -70,7 +70,7 @@ _${group}INS: ${_${group}INCS}
 	    ${.ALLSRC} ${DESTDIR}${${group}DIR}/${${group}NAME}
 .else
 	${INSTALL} -C -o ${${group}OWN} -g ${${group}GRP} -m ${${group}MODE} \
-	    ${.ALLSRC} ${DESTDIR}${${group}DIR}
+	    ${.ALLSRC} ${DESTDIR}${${group}DIR}/
 .endif
 .endif
 
diff --git a/share/mk/bsd.info.mk b/share/mk/bsd.info.mk
index 8f9131585419..7720d286d3b5 100644
--- a/share/mk/bsd.info.mk
+++ b/share/mk/bsd.info.mk
@@ -177,11 +177,11 @@ CLEANFILES+=	${INFO:S/$/.info.*.html/} ${INFO:S/$/.info/}
 install: ${INSTALLINFODIRS}
 .if !empty(IFILES:N*.html)
 	${INSTALL} -o ${INFOOWN} -g ${INFOGRP} -m ${INFOMODE} \
-		${IFILES:N*.html} ${DESTDIR}${INFODIR}
+		${IFILES:N*.html} ${DESTDIR}${INFODIR}/
 .endif
 .if !empty(FORMATS:Mhtml)
 	${INSTALL} -o ${INFOOWN} -g ${INFOGRP} -m ${INFOMODE} \
-		${INFO:S/$/.info.*.html/} ${DESTDIR}${INFODIR}
+		${INFO:S/$/.info.*.html/} ${DESTDIR}${INFODIR}/
 .endif
 .else
 # The indirection in the following is to avoid the null install rule
diff --git a/share/mk/bsd.lib.mk b/share/mk/bsd.lib.mk
index 0ed83454ebf0..8c8a0141e186 100644
--- a/share/mk/bsd.lib.mk
+++ b/share/mk/bsd.lib.mk
@@ -339,23 +339,23 @@ realinstall: _libinstall
 _libinstall:
 .if defined(LIB) && !empty(LIB) && ${MK_INSTALLLIB} != "no"
 	${INSTALL} -C -o ${LIBOWN} -g ${LIBGRP} -m ${LIBMODE} \
-	    ${_INSTALLFLAGS} lib${LIB_PRIVATE}${LIB}.a ${DESTDIR}${_LIBDIR}
+	    ${_INSTALLFLAGS} lib${LIB_PRIVATE}${LIB}.a ${DESTDIR}${_LIBDIR}/
 .endif
 .if ${MK_PROFILE} != "no" && defined(LIB) && !empty(LIB)
 	${INSTALL} -C -o ${LIBOWN} -g ${LIBGRP} -m ${LIBMODE} \
-	    ${_INSTALLFLAGS} lib${LIB_PRIVATE}${LIB}_p.a ${DESTDIR}${_LIBDIR}
+	    ${_INSTALLFLAGS} lib${LIB_PRIVATE}${LIB}_p.a ${DESTDIR}${_LIBDIR}/
 .endif
 .if defined(SHLIB_NAME)
 	${INSTALL} ${STRIP} -o ${LIBOWN} -g ${LIBGRP} -m ${LIBMODE} \
 	    ${_INSTALLFLAGS} ${_SHLINSTALLFLAGS} \
-	    ${SHLIB_NAME} ${DESTDIR}${_SHLIBDIR}
+	    ${SHLIB_NAME} ${DESTDIR}${_SHLIBDIR}/
 .if ${MK_DEBUG_FILES} != "no"
 .if defined(DEBUGMKDIR)
-	${INSTALL} -T debug -d ${DESTDIR}${DEBUGFILEDIR}
+	${INSTALL} -T debug -d ${DESTDIR}${DEBUGFILEDIR}/
 .endif
 	${INSTALL} -T debug -o ${LIBOWN} -g ${LIBGRP} -m ${DEBUGMODE} \
 	    ${_INSTALLFLAGS} \
-	    ${SHLIB_NAME}.debug ${DESTDIR}${DEBUGFILEDIR}
+	    ${SHLIB_NAME}.debug ${DESTDIR}${DEBUGFILEDIR}/
 .endif
 .if defined(SHLIB_LINK)
 .if commands(${SHLIB_LINK:R}.ld)
@@ -378,11 +378,11 @@ _libinstall:
 .endif # SHIB_NAME
 .if defined(INSTALL_PIC_ARCHIVE) && defined(LIB) && !empty(LIB) && ${MK_TOOLCHAIN} != "no"
 	${INSTALL} -o ${LIBOWN} -g ${LIBGRP} -m ${LIBMODE} \
-	    ${_INSTALLFLAGS} lib${LIB}_pic.a ${DESTDIR}${_LIBDIR}
+	    ${_INSTALLFLAGS} lib${LIB}_pic.a ${DESTDIR}${_LIBDIR}/
 .endif
 .if defined(WANT_LINT) && !defined(NO_LINT) && defined(LIB) && !empty(LIB)
 	${INSTALL} -o ${LIBOWN} -g ${LIBGRP} -m ${LIBMODE} \
-	    ${_INSTALLFLAGS} ${LINTLIB} ${DESTDIR}${LINTLIBDIR}
+	    ${_INSTALLFLAGS} ${LINTLIB} ${DESTDIR}${LINTLIBDIR}/
 .endif
 .endif # !defined(INTERNALLIB)
 
diff --git a/share/mk/bsd.man.mk b/share/mk/bsd.man.mk
index bbb47b9df933..72aedfb5423e 100644
--- a/share/mk/bsd.man.mk
+++ b/share/mk/bsd.man.mk
@@ -187,7 +187,7 @@ _maninstall: ${MAN}
 .endfor
 .else
 .for _page _sect in ${.ALLSRC:C/\.([^.]*)$/.\1 \1/}
-	@d=${DESTDIR}${MANDIR}${_sect}${MANSUBDIR}; \
+	@d=${DESTDIR}${MANDIR}${_sect}${MANSUBDIR}/; \
 	${ECHO} ${MINSTALL} ${_page} $${d}; \
 	${MINSTALL} $${page} $${d};
 .endfor
@@ -201,7 +201,7 @@ _maninstall: ${MAN}
 .else
 .for __page in ${MAN}
 	${MINSTALL} ${__page:T:S/$/${MCOMPRESS_EXT}/g} \
-		${DESTDIR}${MANDIR}${__page:E}${MANSUBDIR}
+		${DESTDIR}${MANDIR}${__page:E}${MANSUBDIR}/
 .if defined(MANBUILDCAT) && !empty(MANBUILDCAT)
 	${MINSTALL} ${__page:T:S/$/${CATEXT}${MCOMPRESS_EXT}/g} \
 		${DESTDIR}${CATDIR}${__page:E}${MANSUBDIR}/${__page:T:S/$/${MCOMPRESS_EXT}/}
diff --git a/share/mk/bsd.prog.mk b/share/mk/bsd.prog.mk
index dea7661eab8c..fbb6922f7f61 100644
--- a/share/mk/bsd.prog.mk
+++ b/share/mk/bsd.prog.mk
@@ -205,7 +205,7 @@ _proginstall:
 	    ${_INSTALLFLAGS} ${PROG} ${DESTDIR}${BINDIR}/${PROGNAME}
 .if ${MK_DEBUG_FILES} != "no"
 .if defined(DEBUGMKDIR)
-	${INSTALL} -T debug -d ${DESTDIR}${DEBUGFILEDIR}
+	${INSTALL} -T debug -d ${DESTDIR}${DEBUGFILEDIR}/
 .endif
 	${INSTALL} -T debug -o ${BINOWN} -g ${BINGRP} -m ${DEBUGMODE} \
 	    ${PROGNAME}.debug ${DESTDIR}${DEBUGFILEDIR}/${PROGNAME}.debug

From 153205e33f5afb4965e71c9a5bab7edd7ec215af Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Thu, 15 Oct 2015 22:55:08 +0000
Subject: [PATCH 264/380] Add more SUBDIR_PARALLEL.

MFC after:	3 weeks
Sponsored by:	EMC / Isilon Storage Division
---
 bin/sh/tests/Makefile              |  2 ++
 cddl/Makefile                      |  5 ++++-
 etc/periodic/Makefile              |  1 +
 gnu/Makefile                       |  5 ++++-
 gnu/lib/Makefile                   |  3 +++
 gnu/usr.bin/cc/Makefile            | 12 +++++++++++-
 gnu/usr.bin/groff/Makefile         |  2 ++
 gnu/usr.bin/rcs/Makefile           |  4 +++-
 include/Makefile                   |  1 +
 lib/libiconv_modules/Makefile      |  1 +
 secure/Makefile                    |  4 +++-
 secure/libexec/Makefile            |  2 ++
 secure/usr.bin/Makefile            |  2 ++
 secure/usr.sbin/Makefile           |  2 ++
 share/Makefile                     |  2 ++
 share/doc/Makefile                 |  2 ++
 share/doc/legal/Makefile           |  2 ++
 share/doc/papers/Makefile          |  2 ++
 share/doc/psd/Makefile             |  2 ++
 share/doc/smm/Makefile             |  2 ++
 share/doc/usd/Makefile             |  1 +
 share/examples/Makefile            |  2 ++
 share/i18n/Makefile                |  1 +
 share/i18n/csmapper/Makefile       |  1 +
 share/i18n/esdb/Makefile           |  1 +
 share/man/Makefile                 |  1 +
 share/syscons/Makefile             |  1 +
 tests/Makefile                     |  1 +
 tests/sys/Makefile                 |  2 ++
 tests/sys/pjdfstest/Makefile       |  1 +
 tests/sys/pjdfstest/tests/Makefile |  2 ++
 31 files changed, 67 insertions(+), 5 deletions(-)

diff --git a/bin/sh/tests/Makefile b/bin/sh/tests/Makefile
index d93e19aea7e1..a9abc80311d1 100644
--- a/bin/sh/tests/Makefile
+++ b/bin/sh/tests/Makefile
@@ -10,4 +10,6 @@ TESTS_SUBDIRS+=	parameters
 TESTS_SUBDIRS+=	parser
 TESTS_SUBDIRS+=	set-e
 
+SUBDIR_PARALLEL=
+
 .include <bsd.test.mk>
diff --git a/cddl/Makefile b/cddl/Makefile
index 44b778dbfc6b..26b502a12082 100644
--- a/cddl/Makefile
+++ b/cddl/Makefile
@@ -2,10 +2,13 @@
 
 .include <src.opts.mk>
 
-SUBDIR=	lib sbin usr.bin usr.sbin
+SUBDIR=	lib .WAIT \
+	sbin usr.bin usr.sbin
 
 .if ${MK_TESTS} != "no"
 SUBDIR+=tests
 .endif
 
+SUBDIR_PARALLEL=
+
 .include <bsd.subdir.mk>
diff --git a/etc/periodic/Makefile b/etc/periodic/Makefile
index 8fb56dff051a..a2d99020fd10 100644
--- a/etc/periodic/Makefile
+++ b/etc/periodic/Makefile
@@ -1,5 +1,6 @@
 # $FreeBSD$
 
 SUBDIR=	daily security weekly monthly
+SUBDIR_PARALLEL=
 
 .include <bsd.subdir.mk>
diff --git a/gnu/Makefile b/gnu/Makefile
index 25480c7b951a..bd84bc426981 100644
--- a/gnu/Makefile
+++ b/gnu/Makefile
@@ -3,10 +3,13 @@
 
 .include <src.opts.mk>
 
-SUBDIR= lib ${_tests} usr.bin
+SUBDIR= lib .WAIT \
+	${_tests} usr.bin
 
 .if ${MK_TESTS} != "no"
 _tests=	tests
 .endif
 
+SUBDIR_PARALLEL=
+
 .include <bsd.subdir.mk>
diff --git a/gnu/lib/Makefile b/gnu/lib/Makefile
index 4fff36a48725..b77f9e3e74dd 100644
--- a/gnu/lib/Makefile
+++ b/gnu/lib/Makefile
@@ -24,6 +24,9 @@ SUBDIR+=	libreadline
 # have taken care of that already.
 .if ${MK_GNUCXX} != "no"
 SUBDIR+= libstdc++ libsupc++
+SUBDIR_DEPENDS_libsupc++:= libstdc++
 .endif
 
+SUBDIR_PARALLEL=
+
 .include <bsd.subdir.mk>
diff --git a/gnu/usr.bin/cc/Makefile b/gnu/usr.bin/cc/Makefile
index 41349cfa1b9b..378ababfcde8 100644
--- a/gnu/usr.bin/cc/Makefile
+++ b/gnu/usr.bin/cc/Makefile
@@ -5,7 +5,8 @@
 # The order of some of these are rather important.  Some depend on previous
 # subdirs.
 
-SUBDIR= cc_tools libiberty libcpp libdecnumber cc_int cc cc1 include
+SUBDIR= cc_tools .WAIT \
+	libiberty libcpp libdecnumber cc_int cc cc1 include
 
 .if ${MK_CPP} != "no"
 SUBDIR+= cpp
@@ -19,4 +20,13 @@ SUBDIR+= cc1plus c++
 SUBDIR+= gcov
 .endif
 
+SUBDIR_DEPEND_c++:=	libcpp libiberty
+SUBDIR_DEPEND_cc=	libcpp libiberty
+SUBDIR_DEPEND_cpp=	libcpp libiberty
+SUBDIR_DEPEND_cc1plus=	cc_int libcpp libdecnumber libiberty
+SUBDIR_DEPEND_cc1=	cc_int libcpp libdecnumber libiberty
+SUBDIR_DEPEND_gcov=	libiberty
+
+SUBDIR_PARALLEL=
+
 .include <bsd.subdir.mk>
diff --git a/gnu/usr.bin/groff/Makefile b/gnu/usr.bin/groff/Makefile
index cb83deaf10de..ba760258ee6e 100644
--- a/gnu/usr.bin/groff/Makefile
+++ b/gnu/usr.bin/groff/Makefile
@@ -6,4 +6,6 @@ SUBDIR=		contrib font man src tmac
 SUBDIR_DEPEND_${subdir}=	src
 .endfor
 
+SUBDIR_PARALLEL=
+
 .include <bsd.subdir.mk>
diff --git a/gnu/usr.bin/rcs/Makefile b/gnu/usr.bin/rcs/Makefile
index d6a960b729f2..6b2bd1c3a8f1 100644
--- a/gnu/usr.bin/rcs/Makefile
+++ b/gnu/usr.bin/rcs/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
-SUBDIR= lib ci co merge rcs rcsclean rcsdiff rcsmerge rlog rcsfreeze
+SUBDIR= lib .WAIT \
+	ci co merge rcs rcsclean rcsdiff rcsmerge rlog rcsfreeze
+SUBDIR_PARALLEL=
 
 .include <bsd.subdir.mk>
diff --git a/include/Makefile b/include/Makefile
index 1cecff751b00..eed1c926d84c 100644
--- a/include/Makefile
+++ b/include/Makefile
@@ -7,6 +7,7 @@
 
 CLEANFILES= osreldate.h version vers.c
 SUBDIR= arpa protocols rpcsvc rpc xlocale
+SUBDIR_PARALLEL=
 INCS=	a.out.h ar.h assert.h bitstring.h complex.h cpio.h _ctype.h ctype.h \
 	db.h \
 	dirent.h dlfcn.h elf.h elf-hints.h err.h fmtmsg.h fnmatch.h fstab.h \
diff --git a/lib/libiconv_modules/Makefile b/lib/libiconv_modules/Makefile
index 1a7213612b8f..3af91c746c28 100644
--- a/lib/libiconv_modules/Makefile
+++ b/lib/libiconv_modules/Makefile
@@ -5,5 +5,6 @@
 SUBDIR=	BIG5 DECHanyu EUC EUCTW GBK2K HZ ISO2022 JOHAB MSKanji UES UTF1632 \
 	UTF7 UTF8 VIQR ZW iconv_none iconv_std mapper_646 mapper_none \
 	mapper_parallel mapper_serial mapper_std mapper_zone
+SUBDIR_PARALLEL=
 
 .include <bsd.subdir.mk>
diff --git a/secure/Makefile b/secure/Makefile
index 12778538cf2c..0b8ec0106eca 100644
--- a/secure/Makefile
+++ b/secure/Makefile
@@ -2,7 +2,9 @@
 
 .include <src.opts.mk>
 
-SUBDIR= lib libexec ${_tests} usr.bin usr.sbin
+SUBDIR= lib .WAIT \
+	libexec ${_tests} usr.bin usr.sbin
+SUBDIR_PARALLEL=
 
 .if ${MK_TESTS} != "no"
 _tests=	tests
diff --git a/secure/libexec/Makefile b/secure/libexec/Makefile
index 07aa3dd4bdc4..2e2eaf96da1d 100644
--- a/secure/libexec/Makefile
+++ b/secure/libexec/Makefile
@@ -11,4 +11,6 @@ SUBDIR+=sftp-server ssh-keysign ssh-pkcs11-helper
 SUBDIR+=tests
 .endif
 
+SUBDIR_PARALLEL=
+
 .include <bsd.subdir.mk>
diff --git a/secure/usr.bin/Makefile b/secure/usr.bin/Makefile
index e14ebe3000e9..f85fced7501a 100644
--- a/secure/usr.bin/Makefile
+++ b/secure/usr.bin/Makefile
@@ -14,4 +14,6 @@ SUBDIR+=scp sftp ssh ssh-add ssh-agent ssh-keygen ssh-keyscan
 SUBDIR+=tests
 .endif
 
+SUBDIR_PARALLEL=
+
 .include <bsd.subdir.mk>
diff --git a/secure/usr.sbin/Makefile b/secure/usr.sbin/Makefile
index e42f456b12d0..9b2f910f9c49 100644
--- a/secure/usr.sbin/Makefile
+++ b/secure/usr.sbin/Makefile
@@ -11,4 +11,6 @@ SUBDIR+=sshd
 SUBDIR+=tests
 .endif
 
+SUBDIR_PARALLEL=
+
 .include <bsd.subdir.mk>
diff --git a/share/Makefile b/share/Makefile
index 97d1dac540ca..216ad1e0335d 100644
--- a/share/Makefile
+++ b/share/Makefile
@@ -98,4 +98,6 @@ _vt=		vt
 _zoneinfo=	zoneinfo
 .endif
 
+SUBDIR_PARALLEL=
+
 .include <bsd.subdir.mk>
diff --git a/share/doc/Makefile b/share/doc/Makefile
index c752c51b33eb..7a02b2952598 100644
--- a/share/doc/Makefile
+++ b/share/doc/Makefile
@@ -28,6 +28,8 @@ _IPv6=	IPv6
 _roffdocs=	papers psd smm usd
 .endif
 
+SUBDIR_PARALLEL=
+
 # Default output format for troff documents is ascii.
 # To generate postscript versions of troff documents, use:
 #  make PRINTERDEVICE=ps
diff --git a/share/doc/legal/Makefile b/share/doc/legal/Makefile
index 9590800a67d7..345eafc5b4a6 100644
--- a/share/doc/legal/Makefile
+++ b/share/doc/legal/Makefile
@@ -6,4 +6,6 @@ SUBDIR=	intel_ipw \
 	intel_wpi \
 	realtek
 
+SUBDIR_PARALLEL=
+
 .include <bsd.subdir.mk>
diff --git a/share/doc/papers/Makefile b/share/doc/papers/Makefile
index eaf097f56d38..ddf1323725da 100644
--- a/share/doc/papers/Makefile
+++ b/share/doc/papers/Makefile
@@ -15,4 +15,6 @@ SUBDIR=	beyond4.3 \
 	sysperf \
 	timecounter
 
+SUBDIR_PARALLEL=
+
 .include <bsd.subdir.mk>
diff --git a/share/doc/psd/Makefile b/share/doc/psd/Makefile
index 243ba9992e21..6b6d9cd727b2 100644
--- a/share/doc/psd/Makefile
+++ b/share/doc/psd/Makefile
@@ -37,4 +37,6 @@ SUBDIR+=22.rpcgen \
 	26.rpcrfc \
 	27.nfsrpc
 
+SUBDIR_PARALLEL=
+
 .include <bsd.subdir.mk>
diff --git a/share/doc/smm/Makefile b/share/doc/smm/Makefile
index 1d99b052a351..c6c9c51dc5e2 100644
--- a/share/doc/smm/Makefile
+++ b/share/doc/smm/Makefile
@@ -32,4 +32,6 @@ _08.sendmailop=	08.sendmailop
 _07.lpd=	07.lpd
 .endif
 
+SUBDIR_PARALLEL=
+
 .include <bsd.subdir.mk>
diff --git a/share/doc/usd/Makefile b/share/doc/usd/Makefile
index 5fcb6b305017..863815803631 100644
--- a/share/doc/usd/Makefile
+++ b/share/doc/usd/Makefile
@@ -19,5 +19,6 @@ SUBDIR=	title \
 	20.meref \
 	21.troff \
 	22.trofftut
+SUBDIR_PARALLEL=
 
 .include <bsd.subdir.mk>
diff --git a/share/examples/Makefile b/share/examples/Makefile
index f20ea7f2b5bc..63ad5b381762 100644
--- a/share/examples/Makefile
+++ b/share/examples/Makefile
@@ -261,4 +261,6 @@ SUBDIR+=pf
 SUBDIR+=tests
 .endif
 
+SUBDIR_PARALLEL=
+
 .include <bsd.subdir.mk>
diff --git a/share/i18n/Makefile b/share/i18n/Makefile
index 1cadd547c0b8..b166d9464803 100644
--- a/share/i18n/Makefile
+++ b/share/i18n/Makefile
@@ -4,5 +4,6 @@
 .include <bsd.own.mk>
 
 SUBDIR=	csmapper esdb
+SUBDIR_PARALLEL=
 
 .include <bsd.subdir.mk>
diff --git a/share/i18n/csmapper/Makefile b/share/i18n/csmapper/Makefile
index 7bca3ab36494..18ee8fcef6e0 100644
--- a/share/i18n/csmapper/Makefile
+++ b/share/i18n/csmapper/Makefile
@@ -5,6 +5,7 @@ FILESDIR= ${CSMAPPERDIR}
 
 SUBDIR=	APPLE AST BIG5 CNS CP EBCDIC GB GEORGIAN ISO646 ISO-8859 JIS \
 	KAZAKH KOI KS MISC TCVN
+SUBDIR_PARALLEL=
 
 mapper.dir: ${SUBDIR}
 	newfile=$$(for i in ${SUBDIR}; do \
diff --git a/share/i18n/esdb/Makefile b/share/i18n/esdb/Makefile
index 2b16d90288ab..69d44454a9b4 100644
--- a/share/i18n/esdb/Makefile
+++ b/share/i18n/esdb/Makefile
@@ -5,6 +5,7 @@ FILESDIR= ${ESDBDIR}
 
 SUBDIR=	APPLE AST BIG5 CP DEC EUC EBCDIC GB GEORGIAN ISO-2022 ISO-8859 \
 	ISO646 KAZAKH KOI MISC TCVN UTF
+SUBDIR_PARALLEL=
 
 FILES+=	esdb.dir esdb.dir.db esdb.alias esdb.alias.db
 CLEANFILES= ${FILES}
diff --git a/share/man/Makefile b/share/man/Makefile
index c3bd5f22aca4..e1dc9cf06c80 100644
--- a/share/man/Makefile
+++ b/share/man/Makefile
@@ -5,6 +5,7 @@
 
 # XXX MISSING:	man3f
 SUBDIR=	man1 man3 man4 man5 man6 man7 man8 man9
+SUBDIR_PARALLEL=
 
 MAKEWHATIS?=	makewhatis
 
diff --git a/share/syscons/Makefile b/share/syscons/Makefile
index 9cbf100f13b9..c5993a8eaf18 100644
--- a/share/syscons/Makefile
+++ b/share/syscons/Makefile
@@ -1,5 +1,6 @@
 # $FreeBSD$
 
 SUBDIR= fonts keymaps scrnmaps
+SUBDIR_PARALLEL=
 
 .include <bsd.subdir.mk>
diff --git a/tests/Makefile b/tests/Makefile
index 7fdc5d782b40..8b3ccb1452ae 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -4,6 +4,7 @@
 
 SUBDIR+= etc
 SUBDIR+= sys
+SUBDIR_PARALLEL=
 
 TESTSDIR= ${TESTSBASE}
 KYUAFILE= yes
diff --git a/tests/sys/Makefile b/tests/sys/Makefile
index 0f20aa6aa9ae..41eb34d53bf7 100644
--- a/tests/sys/Makefile
+++ b/tests/sys/Makefile
@@ -18,4 +18,6 @@ TESTS_SUBDIRS+=		vm
 # Items not integrated into kyua runs by default
 SUBDIR+=		pjdfstest
 
+SUBDIR_PARALLEL=
+
 .include <bsd.test.mk>
diff --git a/tests/sys/pjdfstest/Makefile b/tests/sys/pjdfstest/Makefile
index 7047811528c2..a398d394ea2d 100644
--- a/tests/sys/pjdfstest/Makefile
+++ b/tests/sys/pjdfstest/Makefile
@@ -2,5 +2,6 @@
 
 SUBDIR+=	pjdfstest
 SUBDIR+=	tests
+SUBDIR_PARALLEL=
 
 .include <bsd.subdir.mk>
diff --git a/tests/sys/pjdfstest/tests/Makefile b/tests/sys/pjdfstest/tests/Makefile
index 95b4bbe29b41..61b15dd12c25 100644
--- a/tests/sys/pjdfstest/tests/Makefile
+++ b/tests/sys/pjdfstest/tests/Makefile
@@ -36,4 +36,6 @@ TESTS_SUBDIRS+=	symlink
 TESTS_SUBDIRS+=	truncate
 TESTS_SUBDIRS+=	unlink
 
+SUBDIR_PARALLEL=
+
 .include <bsd.test.mk>

From 8a00aac2a683f0f9af2a46f727b57c37108ce717 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Thu, 15 Oct 2015 23:41:47 +0000
Subject: [PATCH 265/380] Add missing directory for
 /usr/share/doc/legal/intel_iwn/ used by share/doc/legal/intel_iwn.  Exposed
 by r289391.

MFC after:	3 weeks
X-MFC-With:	289391
Sponsored by:	EMC / Isilon Storage Division
---
 etc/mtree/BSD.usr.dist | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/etc/mtree/BSD.usr.dist b/etc/mtree/BSD.usr.dist
index f786438956ba..6b63a7ef53f2 100644
--- a/etc/mtree/BSD.usr.dist
+++ b/etc/mtree/BSD.usr.dist
@@ -192,6 +192,8 @@
             legal
                 intel_ipw
                 ..
+                intel_iwn
+                ..
                 intel_iwi
                 ..
                 intel_wpi

From 8076e3efdfc682559434fcbbc1633099cb891f90 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Thu, 15 Oct 2015 23:45:00 +0000
Subject: [PATCH 266/380] Add missing /usr/share/doc/legal/realtek/ for
 share/doc/legal/realtek, exposed by r289391.

MFC after:	3 weeks
X-MFC-With:	289391
Sponsored by:	EMC / Isilon Storage Division
---
 etc/mtree/BSD.usr.dist | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/etc/mtree/BSD.usr.dist b/etc/mtree/BSD.usr.dist
index 6b63a7ef53f2..59cfa514f952 100644
--- a/etc/mtree/BSD.usr.dist
+++ b/etc/mtree/BSD.usr.dist
@@ -198,6 +198,8 @@
                 ..
                 intel_wpi
                 ..
+                realtek
+                ..
             ..
             llvm
                 clang

From a2d3d01d7af652bb22508e147d58a7be415b5a8b Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Thu, 15 Oct 2015 23:45:43 +0000
Subject: [PATCH 267/380] NTB: Add variable number MW, DB CB support code

This is a follow-up to r289208: "Xeon Errata Workaround."

Add logic to support a variable number of memory windows and doorbell
callbacks.  This was added to the Linux driver in the "Xeon Errata
Workaround" commit, but I skipped it because it didn't look neccessary
at the time.  It is needed for future Haswell split-BAR support, so
bring it in now.

A new tunable was added for if_ntb, 'hw.ntb.max_num_clients'.  By
default, it is set to zero -- infer the number of clients from the
number of memory windows available from the hardware.  Any other
positive value can specify a different number of clients, limited by the
number of doorbell callbacks available (4 under MSI-X, or 15 (Xeon) or
34 (SoC) under legacy INTx).

Obtained from:	Linux (Dual BSD/GPL driver)
Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/if_ntb/if_ntb.c | 55 +++++++++++++++++--------------
 sys/dev/ntb/ntb_hw/ntb_hw.c | 66 +++++++++++++++++++++++++++++--------
 sys/dev/ntb/ntb_hw/ntb_hw.h |  4 ++-
 3 files changed, 86 insertions(+), 39 deletions(-)

diff --git a/sys/dev/ntb/if_ntb/if_ntb.c b/sys/dev/ntb/if_ntb/if_ntb.c
index 169296564d74..8b1fa7a5bd63 100644
--- a/sys/dev/ntb/if_ntb/if_ntb.c
+++ b/sys/dev/ntb/if_ntb/if_ntb.c
@@ -80,11 +80,11 @@ __FBSDID("$FreeBSD$");
 
 static unsigned int transport_mtu = 0x4000 + ETHER_HDR_LEN + ETHER_CRC_LEN;
 
-/*
- * This is an oversimplification to work around Xeon Errata.  The second client
- * may be usable for unidirectional traffic.
- */
-static unsigned int max_num_clients = 1;
+static unsigned int max_num_clients;
+SYSCTL_UINT(_hw_ntb, OID_AUTO, max_num_clients, CTLFLAG_RDTUN,
+    &max_num_clients, 0, "Maximum number of NTB transport clients.  "
+    "0 (default) - use all available NTB memory windows; "
+    "positive integer N - Limit to N memory windows.");
 
 STAILQ_HEAD(ntb_queue_list, ntb_queue_entry);
 
@@ -174,7 +174,7 @@ struct ntb_transport_mw {
 struct ntb_netdev {
 	struct ntb_softc	*ntb;
 	struct ifnet		*ifp;
-	struct ntb_transport_mw	mw[NTB_NUM_MW];
+	struct ntb_transport_mw	mw[NTB_MAX_NUM_MW];
 	struct ntb_transport_qp	*qps;
 	uint64_t		max_qps;
 	uint64_t		qp_bitmap;
@@ -220,7 +220,7 @@ enum {
 	IF_NTB_MAX_SPAD,
 };
 
-#define QP_TO_MW(qp)		((qp) % NTB_NUM_MW)
+#define QP_TO_MW(ntb, qp)	((qp) % ntb_get_max_mw(ntb))
 #define NTB_QP_DEF_NUM_ENTRIES	100
 #define NTB_LINK_DOWN_TIMEOUT	10
 
@@ -492,7 +492,11 @@ ntb_transport_init(struct ntb_softc *ntb)
 	struct ntb_netdev *nt = &net_softc;
 	int rc, i;
 
-	nt->max_qps = max_num_clients;
+	if (max_num_clients == 0)
+		nt->max_qps = MIN(ntb_get_max_cbs(ntb), ntb_get_max_mw(ntb));
+	else
+		nt->max_qps = MIN(ntb_get_max_cbs(ntb), max_num_clients);
+
 	ntb_register_transport(ntb, nt);
 	mtx_init(&nt->tx_lock, "ntb transport tx", NULL, MTX_DEF);
 	mtx_init(&nt->rx_lock, "ntb transport rx", NULL, MTX_DEF);
@@ -544,7 +548,7 @@ ntb_transport_free(void *transport)
 
 	ntb_unregister_event_callback(ntb);
 
-	for (i = 0; i < NTB_NUM_MW; i++)
+	for (i = 0; i < NTB_MAX_NUM_MW; i++)
 		ntb_free_mw(nt, i);
 
 	free(nt->qps, M_NTB_IF);
@@ -556,7 +560,10 @@ ntb_transport_init_queue(struct ntb_netdev *nt, unsigned int qp_num)
 {
 	struct ntb_transport_qp *qp;
 	unsigned int num_qps_mw, tx_size;
-	uint8_t mw_num = QP_TO_MW(qp_num);
+	uint8_t mw_num, mw_max;
+
+	mw_max = ntb_get_max_mw(nt->ntb);
+	mw_num = QP_TO_MW(nt->ntb, qp_num);
 
 	qp = &nt->qps[qp_num];
 	qp->qp_num = qp_num;
@@ -566,15 +573,15 @@ ntb_transport_init_queue(struct ntb_netdev *nt, unsigned int qp_num)
 	qp->client_ready = NTB_LINK_DOWN;
 	qp->event_handler = NULL;
 
-	if (nt->max_qps % NTB_NUM_MW && mw_num + 1 < nt->max_qps / NTB_NUM_MW)
-		num_qps_mw = nt->max_qps / NTB_NUM_MW + 1;
+	if (nt->max_qps % mw_max && mw_num + 1 < nt->max_qps / mw_max)
+		num_qps_mw = nt->max_qps / mw_max + 1;
 	else
-		num_qps_mw = nt->max_qps / NTB_NUM_MW;
+		num_qps_mw = nt->max_qps / mw_max;
 
 	tx_size = (unsigned int) ntb_get_mw_size(qp->ntb, mw_num) / num_qps_mw;
 	qp->rx_info = (struct ntb_rx_info *)
 	    ((char *)ntb_get_mw_vbase(qp->ntb, mw_num) +
-	    (qp_num / NTB_NUM_MW * tx_size));
+	    (qp_num / mw_max * tx_size));
 	tx_size -= sizeof(struct ntb_rx_info);
 
 	qp->tx_mw = qp->rx_info + 1;
@@ -1048,10 +1055,7 @@ ntb_transport_link_work(void *arg)
 	uint32_t val, i, num_mw;
 	int rc;
 
-	if (ntb_has_feature(ntb, NTB_REGS_THRU_MW))
-		num_mw = NTB_NUM_MW - 1;
-	else
-		num_mw = NTB_NUM_MW;
+	num_mw = ntb_get_max_mw(ntb);
 
 	/* send the local info, in the opposite order of the way we read it */
 	for (i = 0; i < num_mw; i++) {
@@ -1136,7 +1140,7 @@ ntb_transport_link_work(void *arg)
 	return;
 
 free_mws:
-	for (i = 0; i < NTB_NUM_MW; i++)
+	for (i = 0; i < NTB_MAX_NUM_MW; i++)
 		ntb_free_mw(nt, i);
 out:
 	if (ntb_query_link_status(ntb))
@@ -1207,17 +1211,20 @@ ntb_transport_setup_qp_mw(struct ntb_netdev *nt, unsigned int qp_num)
 	struct ntb_transport_qp *qp = &nt->qps[qp_num];
 	void *offset;
 	unsigned int rx_size, num_qps_mw;
-	uint8_t mw_num = QP_TO_MW(qp_num);
+	uint8_t mw_num, mw_max;
 	unsigned int i;
 
-	if (nt->max_qps % NTB_NUM_MW && mw_num + 1 < nt->max_qps / NTB_NUM_MW)
-		num_qps_mw = nt->max_qps / NTB_NUM_MW + 1;
+	mw_max = ntb_get_max_mw(nt->ntb);
+	mw_num = QP_TO_MW(nt->ntb, qp_num);
+
+	if (nt->max_qps % mw_max && mw_num + 1 < nt->max_qps / mw_max)
+		num_qps_mw = nt->max_qps / mw_max + 1;
 	else
-		num_qps_mw = nt->max_qps / NTB_NUM_MW;
+		num_qps_mw = nt->max_qps / mw_max;
 
 	rx_size = (unsigned int) nt->mw[mw_num].size / num_qps_mw;
 	qp->remote_rx_info = (void *)((uint8_t *)nt->mw[mw_num].virt_addr +
-			     (qp_num / NTB_NUM_MW * rx_size));
+			     (qp_num / mw_max * rx_size));
 	rx_size -= sizeof(struct ntb_rx_info);
 
 	qp->rx_buff = qp->remote_rx_info + 1;
diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c
index 58ff611ff7a0..c91a1e2ad43e 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.c
@@ -127,9 +127,11 @@ struct ntb_softc {
 
 	void			*ntb_transport;
 	ntb_event_callback	event_cb;
-	struct ntb_db_cb 	*db_cb;
+	struct ntb_db_cb	*db_cb;
+	uint8_t			max_cbs;
 
 	struct {
+		uint8_t max_mw;
 		uint8_t max_spads;
 		uint8_t max_db_bits;
 		uint8_t msix_cnt;
@@ -321,6 +323,8 @@ ntb_attach(device_t device)
 	if (error)
 		goto out;
 
+	ntb->limits.max_mw = NTB_MAX_NUM_MW;
+
 	error = ntb_map_pci_bars(ntb);
 	if (error)
 		goto out;
@@ -533,6 +537,7 @@ ntb_setup_xeon_msix(struct ntb_softc *ntb, uint32_t num_vectors)
 	 * slot, from which they will never be called back.
 	 */
 	ntb->db_cb[num_vectors - 1].reserved = true;
+	ntb->max_cbs--;
 	return (0);
 }
 
@@ -649,16 +654,32 @@ ntb_setup_interrupts(struct ntb_softc *ntb)
 	} else
 		num_vectors = 1;
 
+	/*
+	 * If allocating MSI-X interrupts succeeds, limit callbacks to the
+	 * number of MSI-X slots available.
+	 */
 	ntb_create_callbacks(ntb, num_vectors);
 
 	if (ntb->type == NTB_XEON)
 		rc = ntb_setup_xeon_msix(ntb, num_vectors);
 	else
 		rc = ntb_setup_soc_msix(ntb, num_vectors);
-	if (rc != 0)
+	if (rc != 0) {
 		device_printf(ntb->device,
 		    "Error allocating MSI-X interrupts: %d\n", rc);
 
+		/*
+		 * If allocating MSI-X interrupts failed and we're forced to
+		 * use legacy INTx anyway, the only limit on individual
+		 * callbacks is the number of doorbell bits.
+		 *
+		 * CEM: This seems odd to me but matches the behavior of the
+		 * Linux driver ca. September 2013
+		 */
+		ntb_free_callbacks(ntb);
+		ntb_create_callbacks(ntb, ntb->limits.max_db_bits);
+	}
+
 	if (ntb->type == NTB_XEON && rc == ENOSPC)
 		rc = ntb_setup_legacy_interrupt(ntb);
 
@@ -842,6 +863,7 @@ ntb_create_callbacks(struct ntb_softc *ntb, uint32_t num_vectors)
 {
 	uint32_t i;
 
+	ntb->max_cbs = num_vectors;
 	ntb->db_cb = malloc(num_vectors * sizeof(*ntb->db_cb), M_NTB,
 	    M_ZERO | M_WAITOK);
 	for (i = 0; i < num_vectors; i++) {
@@ -857,10 +879,11 @@ ntb_free_callbacks(struct ntb_softc *ntb)
 {
 	uint8_t i;
 
-	for (i = 0; i < ntb->limits.max_db_bits; i++)
+	for (i = 0; i < ntb->max_cbs; i++)
 		ntb_unregister_db_callback(ntb, i);
 
 	free(ntb->db_cb, M_NTB);
+	ntb->max_cbs = 0;
 }
 
 static struct ntb_hw_info *
@@ -989,14 +1012,16 @@ ntb_setup_xeon(struct ntb_softc *ntb)
 	 * This should already be the case based on the driver defaults, but
 	 * write the limit registers first just in case.
 	 */
-	if (HAS_FEATURE(NTB_REGS_THRU_MW))
+	if (HAS_FEATURE(NTB_REGS_THRU_MW)) {
+		/* Reserve the last MW for mapping remote spad */
+		ntb->limits.max_mw--;
 		/*
 		 * Set the Limit register to 4k, the minimum size, to prevent
 		 * an illegal access.
 		 */
 		ntb_reg_write(8, XEON_PBAR4LMT_OFFSET,
 		    ntb_get_mw_size(ntb, 1) + 0x1000);
-	else
+	} else
 		/*
 		 * Disable the limit register, just in case it is set to
 		 * something silly.
@@ -1432,8 +1457,7 @@ ntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data,
 {
 	struct ntb_db_cb *db_cb = &ntb->db_cb[idx];
 
-	if (idx >= ntb->allocated_interrupts || db_cb->callback ||
-	    db_cb->reserved) {
+	if (idx >= ntb->max_cbs || db_cb->callback != NULL || db_cb->reserved) {
 		device_printf(ntb->device, "Invalid Index.\n");
 		return (EINVAL);
 	}
@@ -1459,7 +1483,7 @@ void
 ntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx)
 {
 
-	if (idx >= ntb->allocated_interrupts || !ntb->db_cb[idx].callback)
+	if (idx >= ntb->max_cbs || ntb->db_cb[idx].callback == NULL)
 		return;
 
 	mask_ldb_interrupt(ntb, idx);
@@ -1518,12 +1542,12 @@ ntb_register_transport(struct ntb_softc *ntb, void *transport)
 void
 ntb_unregister_transport(struct ntb_softc *ntb)
 {
-	int i;
+	uint8_t i;
 
 	if (ntb->ntb_transport == NULL)
 		return;
 
-	for (i = 0; i < ntb->allocated_interrupts; i++)
+	for (i = 0; i < ntb->max_cbs; i++)
 		ntb_unregister_db_callback(ntb, i);
 
 	ntb_unregister_event_callback(ntb);
@@ -1546,6 +1570,20 @@ ntb_get_max_spads(struct ntb_softc *ntb)
 	return (ntb->limits.max_spads);
 }
 
+uint8_t
+ntb_get_max_cbs(struct ntb_softc *ntb)
+{
+
+	return (ntb->max_cbs);
+}
+
+uint8_t
+ntb_get_max_mw(struct ntb_softc *ntb)
+{
+
+	return (ntb->limits.max_mw);
+}
+
 /**
  * ntb_write_local_spad() - write to the secondary scratchpad register
  * @ntb: pointer to ntb_softc instance
@@ -1658,7 +1696,7 @@ void *
 ntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw)
 {
 
-	if (mw >= NTB_NUM_MW)
+	if (mw >= ntb_get_max_mw(ntb))
 		return (NULL);
 
 	return (ntb->bar_info[NTB_MW_TO_BAR(mw)].vbase);
@@ -1668,7 +1706,7 @@ vm_paddr_t
 ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw)
 {
 
-	if (mw >= NTB_NUM_MW)
+	if (mw >= ntb_get_max_mw(ntb))
 		return (0);
 
 	return (ntb->bar_info[NTB_MW_TO_BAR(mw)].pbase);
@@ -1687,7 +1725,7 @@ u_long
 ntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw)
 {
 
-	if (mw >= NTB_NUM_MW)
+	if (mw >= ntb_get_max_mw(ntb))
 		return (0);
 
 	return (ntb->bar_info[NTB_MW_TO_BAR(mw)].size);
@@ -1707,7 +1745,7 @@ void
 ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr)
 {
 
-	if (mw >= NTB_NUM_MW)
+	if (mw >= ntb_get_max_mw(ntb))
 		return;
 
 	switch (NTB_MW_TO_BAR(mw)) {
diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.h b/sys/dev/ntb/ntb_hw/ntb_hw.h
index 7abd10c54dbf..b6b85a02448e 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.h
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.h
@@ -31,7 +31,7 @@
 
 struct ntb_softc;
 
-#define NTB_NUM_MW	2
+#define NTB_MAX_NUM_MW	2
 
 enum ntb_link_event {
 	NTB_LINK_DOWN = 0,
@@ -61,6 +61,8 @@ void *ntb_find_transport(struct ntb_softc *ntb);
 struct ntb_softc *ntb_register_transport(struct ntb_softc *ntb,
     void *transport);
 void ntb_unregister_transport(struct ntb_softc *ntb);
+uint8_t ntb_get_max_cbs(struct ntb_softc *ntb);
+uint8_t ntb_get_max_mw(struct ntb_softc *ntb);
 uint8_t ntb_get_max_spads(struct ntb_softc *ntb);
 int ntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val);
 int ntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val);

From 9b0d40d85e3fdd2f9d2d73d39d54a84366496e9d Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Thu, 15 Oct 2015 23:46:07 +0000
Subject: [PATCH 268/380] NTB: MFV ab760a0c: Add split BAR support for Haswell

On the Haswell platform, a split BAR option to allow creation of 2 32bit
BARs (4 and 5) from the 64bit BAR 4. Adding support for this new option.

Authored by:	Dave Jiang
Obtained from:	Linux (Dual BSD/GPL driver)
Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/ntb_hw/ntb_hw.c   | 132 ++++++++++++++++++++++++++--------
 sys/dev/ntb/ntb_hw/ntb_hw.h   |   5 +-
 sys/dev/ntb/ntb_hw/ntb_regs.h |  38 ++++++----
 3 files changed, 132 insertions(+), 43 deletions(-)

diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c
index c91a1e2ad43e..7a6d0abf47d7 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.c
@@ -62,7 +62,8 @@ __FBSDID("$FreeBSD$");
 #define NTB_CONFIG_BAR	0
 #define NTB_B2B_BAR_1	1
 #define NTB_B2B_BAR_2	2
-#define NTB_MAX_BARS	3
+#define NTB_B2B_BAR_3	3
+#define NTB_MAX_BARS	4
 #define NTB_MW_TO_BAR(mw) ((mw) + 1)
 
 #define MAX_MSIX_INTERRUPTS MAX(XEON_MAX_DB_BITS, SOC_MAX_DB_BITS)
@@ -85,7 +86,7 @@ struct ntb_hw_info {
 	uint32_t		device_id;
 	const char		*desc;
 	enum ntb_device_type	type;
-	uint64_t		features;
+	uint32_t		features;
 };
 
 struct ntb_pci_bar_info {
@@ -142,6 +143,7 @@ struct ntb_softc {
 		uint32_t rdb;
 		uint32_t bar2_xlat;
 		uint32_t bar4_xlat;
+		uint32_t bar5_xlat;
 		uint32_t spad_remote;
 		uint32_t spad_local;
 		uint32_t lnk_cntl;
@@ -186,9 +188,11 @@ bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle,
 #define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset)
 #define ntb_reg_write(SIZE, offset, val) \
 	    ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val)
-#define ntb_mw_read(SIZE, offset) ntb_bar_read(SIZE, NTB_B2B_BAR_2, offset)
+#define ntb_mw_read(SIZE, offset) \
+	    ntb_bar_read(SIZE, NTB_MW_TO_BAR(ntb->limits.max_mw), offset)
 #define ntb_mw_write(SIZE, offset, val) \
-	    ntb_bar_write(SIZE, NTB_B2B_BAR_2, offset, val)
+	    ntb_bar_write(SIZE, NTB_MW_TO_BAR(ntb->limits.max_mw), \
+		offset, val)
 
 typedef int (*bar_map_strategy)(struct ntb_softc *ntb,
     struct ntb_pci_bar_info *bar);
@@ -221,6 +225,7 @@ static void unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx);
 static int ntb_create_callbacks(struct ntb_softc *ntb, uint32_t num_vectors);
 static void ntb_free_callbacks(struct ntb_softc *ntb);
 static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id);
+static void ntb_detect_max_mw(struct ntb_softc *ntb);
 static int ntb_detect_xeon(struct ntb_softc *ntb);
 static int ntb_detect_soc(struct ntb_softc *ntb);
 static int ntb_setup_xeon(struct ntb_softc *ntb);
@@ -323,7 +328,7 @@ ntb_attach(device_t device)
 	if (error)
 		goto out;
 
-	ntb->limits.max_mw = NTB_MAX_NUM_MW;
+	ntb_detect_max_mw(ntb);
 
 	error = ntb_map_pci_bars(ntb);
 	if (error)
@@ -357,6 +362,12 @@ ntb_detach(device_t device)
 	if (ntb->type == NTB_XEON)
 		ntb_teardown_xeon(ntb);
 	ntb_teardown_interrupts(ntb);
+
+	/*
+	 * Redetect total MWs so we unmap properly -- in case we lowered the
+	 * maximum to work around Xeon errata.
+	 */
+	ntb_detect_max_mw(ntb);
 	ntb_unmap_pci_bar(ntb);
 
 	return (0);
@@ -379,12 +390,22 @@ ntb_map_pci_bars(struct ntb_softc *ntb)
 		return (rc);
 
 	ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4);
-	if (HAS_FEATURE(NTB_REGS_THRU_MW))
+	if (HAS_FEATURE(NTB_REGS_THRU_MW) && !HAS_FEATURE(NTB_SPLIT_BAR))
 		rc = map_pci_bar(ntb, map_mmr_bar,
 		    &ntb->bar_info[NTB_B2B_BAR_2]);
 	else
 		rc = map_pci_bar(ntb, map_memory_window_bar,
 		    &ntb->bar_info[NTB_B2B_BAR_2]);
+	if (!HAS_FEATURE(NTB_SPLIT_BAR))
+		return (rc);
+
+	ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5);
+	if (HAS_FEATURE(NTB_REGS_THRU_MW))
+		rc = map_pci_bar(ntb, map_mmr_bar,
+		    &ntb->bar_info[NTB_B2B_BAR_3]);
+	else
+		rc = map_pci_bar(ntb, map_memory_window_bar,
+		    &ntb->bar_info[NTB_B2B_BAR_3]);
 	return (rc);
 }
 
@@ -484,7 +505,7 @@ ntb_unmap_pci_bar(struct ntb_softc *ntb)
 	struct ntb_pci_bar_info *current_bar;
 	int i;
 
-	for (i = 0; i< NTB_MAX_BARS; i++) {
+	for (i = 0; i < NTB_MAX_BARS; i++) {
 		current_bar = &ntb->bar_info[i];
 		if (current_bar->pci_resource != NULL)
 			bus_release_resource(ntb->device, SYS_RES_MEMORY,
@@ -906,6 +927,21 @@ ntb_teardown_xeon(struct ntb_softc *ntb)
 	ntb_hw_link_down(ntb);
 }
 
+static void
+ntb_detect_max_mw(struct ntb_softc *ntb)
+{
+
+	if (ntb->type == NTB_SOC) {
+		ntb->limits.max_mw = SOC_MAX_MW;
+		return;
+	}
+
+	if (HAS_FEATURE(NTB_SPLIT_BAR))
+		ntb->limits.max_mw = XEON_HSXSPLIT_MAX_MW;
+	else
+		ntb->limits.max_mw = XEON_SNB_MAX_MW;
+}
+
 static int
 ntb_detect_xeon(struct ntb_softc *ntb)
 {
@@ -919,6 +955,9 @@ ntb_detect_xeon(struct ntb_softc *ntb)
 	else
 		ntb->dev_type = NTB_DEV_DSD;
 
+	if ((ppd & XEON_PPD_SPLIT_BAR) != 0)
+		ntb->features |= NTB_SPLIT_BAR;
+
 	conn_type = ppd & XEON_PPD_CONN_TYPE;
 	switch (conn_type) {
 	case NTB_CONN_B2B:
@@ -968,6 +1007,8 @@ ntb_setup_xeon(struct ntb_softc *ntb)
 	ntb->reg_ofs.spad_local	= XEON_SPAD_OFFSET;
 	ntb->reg_ofs.bar2_xlat	= XEON_SBAR2XLAT_OFFSET;
 	ntb->reg_ofs.bar4_xlat	= XEON_SBAR4XLAT_OFFSET;
+	if (HAS_FEATURE(NTB_SPLIT_BAR))
+		ntb->reg_ofs.bar5_xlat = XEON_SBAR5XLAT_OFFSET;
 
 	switch (ntb->conn_type) {
 	case NTB_CONN_B2B:
@@ -1013,22 +1054,24 @@ ntb_setup_xeon(struct ntb_softc *ntb)
 	 * write the limit registers first just in case.
 	 */
 	if (HAS_FEATURE(NTB_REGS_THRU_MW)) {
-		/* Reserve the last MW for mapping remote spad */
-		ntb->limits.max_mw--;
 		/*
 		 * Set the Limit register to 4k, the minimum size, to prevent
 		 * an illegal access.
+		 *
+		 * XXX: Should this be PBAR5LMT / get_mw_size(, max_mw - 1)?
 		 */
 		ntb_reg_write(8, XEON_PBAR4LMT_OFFSET,
 		    ntb_get_mw_size(ntb, 1) + 0x1000);
+		/* Reserve the last MW for mapping remote spad */
+		ntb->limits.max_mw--;
 	} else
 		/*
 		 * Disable the limit register, just in case it is set to
-		 * something silly.
+		 * something silly.  A 64-bit write will also clear PBAR5LMT in
+		 * split-bar mode, and this is desired.
 		 */
 		ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0);
 
-
 	ntb->reg_ofs.lnk_cntl	 = XEON_NTBCNTL_OFFSET;
 	ntb->reg_ofs.lnk_stat	 = XEON_LINK_STATUS_OFFSET;
 	ntb->reg_ofs.spci_cmd	 = XEON_PCICMD_OFFSET;
@@ -1112,15 +1155,15 @@ configure_soc_secondary_side_bars(struct ntb_softc *ntb)
 {
 
 	if (ntb->dev_type == NTB_DEV_USD) {
-		ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR);
-		ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_USD_ADDR);
+		ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, MBAR23_DSD_ADDR);
+		ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, MBAR4_DSD_ADDR);
 		ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_USD_ADDR);
-		ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_USD_ADDR);
+		ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR4_USD_ADDR);
 	} else {
-		ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR);
-		ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_DSD_ADDR);
+		ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, MBAR23_USD_ADDR);
+		ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, MBAR4_USD_ADDR);
 		ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_DSD_ADDR);
-		ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_DSD_ADDR);
+		ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR4_DSD_ADDR);
 	}
 }
 
@@ -1129,13 +1172,19 @@ configure_xeon_secondary_side_bars(struct ntb_softc *ntb)
 {
 
 	if (ntb->dev_type == NTB_DEV_USD) {
-		ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR);
+		ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, MBAR23_DSD_ADDR);
 		if (HAS_FEATURE(NTB_REGS_THRU_MW))
 			ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
 			    MBAR01_DSD_ADDR);
 		else {
-			ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
-			    PBAR4XLAT_USD_ADDR);
+			if (HAS_FEATURE(NTB_SPLIT_BAR)) {
+				ntb_reg_write(4, XEON_PBAR4XLAT_OFFSET,
+				    MBAR4_DSD_ADDR);
+				ntb_reg_write(4, XEON_PBAR5XLAT_OFFSET,
+				    MBAR5_DSD_ADDR);
+			} else
+				ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
+				    MBAR4_DSD_ADDR);
 			/*
 			 * B2B_XLAT_OFFSET is a 64-bit register but can only be
 			 * written 32 bits at a time.
@@ -1147,15 +1196,25 @@ configure_xeon_secondary_side_bars(struct ntb_softc *ntb)
 		}
 		ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_USD_ADDR);
 		ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_USD_ADDR);
-		ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_USD_ADDR);
+		if (HAS_FEATURE(NTB_SPLIT_BAR)) {
+			ntb_reg_write(4, XEON_SBAR4BASE_OFFSET, MBAR4_USD_ADDR);
+			ntb_reg_write(4, XEON_SBAR5BASE_OFFSET, MBAR5_USD_ADDR);
+		} else
+			ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR4_USD_ADDR);
 	} else {
-		ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR);
+		ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, MBAR23_USD_ADDR);
 		if (HAS_FEATURE(NTB_REGS_THRU_MW))
 			ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
 			    MBAR01_USD_ADDR);
 		else {
-			ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
-			    PBAR4XLAT_DSD_ADDR);
+			if (HAS_FEATURE(NTB_SPLIT_BAR)) {
+				ntb_reg_write(4, XEON_PBAR4XLAT_OFFSET,
+				    MBAR4_USD_ADDR);
+				ntb_reg_write(4, XEON_PBAR5XLAT_OFFSET,
+				    MBAR5_USD_ADDR);
+			} else
+				ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
+				    MBAR4_USD_ADDR);
 			/*
 			 * B2B_XLAT_OFFSET is a 64-bit register but can only be
 			 * written 32 bits at a time.
@@ -1167,7 +1226,14 @@ configure_xeon_secondary_side_bars(struct ntb_softc *ntb)
 		}
 		ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_DSD_ADDR);
 		ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_DSD_ADDR);
-		ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_DSD_ADDR);
+		if (HAS_FEATURE(NTB_SPLIT_BAR)) {
+			ntb_reg_write(4, XEON_SBAR4BASE_OFFSET,
+			    MBAR4_DSD_ADDR);
+			ntb_reg_write(4, XEON_SBAR5BASE_OFFSET,
+			    MBAR5_DSD_ADDR);
+		} else
+			ntb_reg_write(8, XEON_SBAR4BASE_OFFSET,
+			    MBAR4_DSD_ADDR);
 	}
 }
 
@@ -1288,7 +1354,9 @@ ntb_hw_link_up(struct ntb_softc *ntb)
 	cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl);
 	cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK);
 	cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP;
-	cntl |= NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP;
+	cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP;
+	if (HAS_FEATURE(NTB_SPLIT_BAR))
+		cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP;
 	ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, cntl);
 }
 
@@ -1304,7 +1372,9 @@ ntb_hw_link_down(struct ntb_softc *ntb)
 
 	cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl);
 	cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP);
-	cntl &= ~(NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP);
+	cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP);
+	if (HAS_FEATURE(NTB_SPLIT_BAR))
+		cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP);
 	cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK;
 	ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, cntl);
 }
@@ -1753,7 +1823,13 @@ ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr)
 		ntb_reg_write(8, ntb->reg_ofs.bar2_xlat, addr);
 		break;
 	case NTB_B2B_BAR_2:
-		ntb_reg_write(8, ntb->reg_ofs.bar4_xlat, addr);
+		if (HAS_FEATURE(NTB_SPLIT_BAR))
+			ntb_reg_write(4, ntb->reg_ofs.bar4_xlat, addr);
+		else
+			ntb_reg_write(8, ntb->reg_ofs.bar4_xlat, addr);
+		break;
+	case NTB_B2B_BAR_3:
+		ntb_reg_write(4, ntb->reg_ofs.bar5_xlat, addr);
 		break;
 	}
 }
diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.h b/sys/dev/ntb/ntb_hw/ntb_hw.h
index b6b85a02448e..e6d90add8b71 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.h
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.h
@@ -31,7 +31,7 @@
 
 struct ntb_softc;
 
-#define NTB_MAX_NUM_MW	2
+#define NTB_MAX_NUM_MW	3
 
 enum ntb_link_event {
 	NTB_LINK_DOWN = 0,
@@ -78,11 +78,14 @@ void ntb_ring_doorbell(struct ntb_softc *ntb, unsigned int db);
 bool ntb_query_link_status(struct ntb_softc *ntb);
 device_t ntb_get_device(struct ntb_softc *ntb);
 
+/* Hardware owns the low 32 bits of features. */
 #define NTB_BAR_SIZE_4K		(1 << 0)
 /* REGS_THRU_MW is the equivalent of Linux's NTB_HWERR_SDOORBELL_LOCKUP */
 #define NTB_REGS_THRU_MW	(1 << 1)
 #define NTB_SB01BASE_LOCKUP	(1 << 2)
 #define NTB_B2BDOORBELL_BIT14	(1 << 3)
+/* Software/configuration owns the top 32 bits. */
+#define NTB_SPLIT_BAR		(1ull << 32)
 bool ntb_has_feature(struct ntb_softc *, uint64_t);
 
 #endif /* _NTB_HW_H_ */
diff --git a/sys/dev/ntb/ntb_hw/ntb_regs.h b/sys/dev/ntb/ntb_hw/ntb_regs.h
index ad6a2a671126..6cb8098ee3cb 100644
--- a/sys/dev/ntb/ntb_hw/ntb_regs.h
+++ b/sys/dev/ntb/ntb_hw/ntb_regs.h
@@ -34,6 +34,8 @@
 #define NTB_LINK_WIDTH_MASK	0x03f0
 
 #define XEON_MSIX_CNT		4
+#define XEON_SNB_MAX_MW		2
+#define XEON_HSXSPLIT_MAX_MW	3
 #define XEON_MAX_SPADS		16
 #define XEON_MAX_COMPAT_SPADS	16
 /* Reserve the uppermost bit for link interrupt */
@@ -50,15 +52,20 @@
 
 #define XEON_PBAR2LMT_OFFSET	0x0000
 #define XEON_PBAR4LMT_OFFSET	0x0008
+#define XEON_PBAR5LMT_OFFSET	0x000c
 #define XEON_PBAR2XLAT_OFFSET	0x0010
 #define XEON_PBAR4XLAT_OFFSET	0x0018
+#define XEON_PBAR5XLAT_OFFSET	0x001c
 #define XEON_SBAR2LMT_OFFSET	0x0020
 #define XEON_SBAR4LMT_OFFSET	0x0028
+#define XEON_SBAR5LMT_OFFSET	0x002c
 #define XEON_SBAR2XLAT_OFFSET	0x0030
 #define XEON_SBAR4XLAT_OFFSET	0x0038
+#define XEON_SBAR5XLAT_OFFSET	0x003c
 #define XEON_SBAR0BASE_OFFSET	0x0040
 #define XEON_SBAR2BASE_OFFSET	0x0048
 #define XEON_SBAR4BASE_OFFSET	0x0050
+#define XEON_SBAR5BASE_OFFSET	0x0054
 #define XEON_NTBCNTL_OFFSET	0x0058
 #define XEON_SBDF_OFFSET	0x005c
 #define XEON_PDOORBELL_OFFSET	0x0060
@@ -75,6 +82,7 @@
 #define XEON_B2B_XLAT_OFFSETU	0x0148
 
 #define SOC_MSIX_CNT		34
+#define SOC_MAX_MW		2
 #define SOC_MAX_SPADS		16
 #define SOC_MAX_DB_BITS		34
 #define SOC_DB_BITS_PER_VEC	1
@@ -111,17 +119,19 @@
 #define SOC_LTSSMSTATEJMP_OFFSET	(SOC_IP_BASE + 0x3040)
 #define SOC_IBSTERRRCRVSTS0_OFFSET	(SOC_IP_BASE + 0x3324)
 
-#define SOC_DESKEWSTS_DBERR	(1 << 15)
+#define SOC_DESKEWSTS_DBERR		(1 << 15)
 #define SOC_LTSSMERRSTS0_UNEXPECTEDEI	(1 << 20)
 #define SOC_LTSSMSTATEJMP_FORCEDETECT	(1 << 2)
-#define SOC_IBIST_ERR_OFLOW	0x7fff7fff
+#define SOC_IBIST_ERR_OFLOW		0x7fff7fff
 
 #define NTB_CNTL_CFG_LOCK		(1 << 0)
 #define NTB_CNTL_LINK_DISABLE		(1 << 1)
 #define NTB_CNTL_S2P_BAR23_SNOOP	(1 << 2)
 #define NTB_CNTL_P2S_BAR23_SNOOP	(1 << 4)
-#define NTB_CNTL_S2P_BAR45_SNOOP	(1 << 6)
-#define NTB_CNTL_P2S_BAR45_SNOOP	(1 << 8)
+#define NTB_CNTL_S2P_BAR4_SNOOP		(1 << 6)
+#define NTB_CNTL_P2S_BAR4_SNOOP		(1 << 8)
+#define NTB_CNTL_S2P_BAR5_SNOOP		(1 << 12)
+#define NTB_CNTL_P2S_BAR5_SNOOP		(1 << 14)
 #define SOC_CNTL_LINK_DOWN		(1 << 16)
 
 #define XEON_PBAR23SZ_OFFSET	0x00d0
@@ -129,6 +139,7 @@
 #define NTB_PPD_OFFSET		0x00d4
 #define XEON_PPD_CONN_TYPE	0x0003
 #define XEON_PPD_DEV_TYPE	0x0010
+#define XEON_PPD_SPLIT_BAR	0x0040
 #define SOC_PPD_INIT_LINK	0x0008
 #define SOC_PPD_CONN_TYPE	0x0300
 #define SOC_PPD_DEV_TYPE	0x1000
@@ -140,16 +151,15 @@
 #define NTB_DEV_DSD	1
 #define NTB_DEV_USD	0
 
-#define PBAR2XLAT_USD_ADDR	0x0000004000000000ull
-#define PBAR4XLAT_USD_ADDR	0x0000008000000000ull
-#define MBAR01_USD_ADDR		0x000000210000000cull
-#define MBAR23_USD_ADDR		0x000000410000000cull
-#define MBAR45_USD_ADDR		0x000000810000000cull
-#define PBAR2XLAT_DSD_ADDR	0x0000004100000000ull
-#define PBAR4XLAT_DSD_ADDR	0x0000008100000000ull
-#define MBAR01_DSD_ADDR		0x000000200000000cull
-#define MBAR23_DSD_ADDR		0x000000400000000cull
-#define MBAR45_DSD_ADDR		0x000000800000000cull
+/* All addresses are in low 32-bit space so 32-bit BARs can function */
+#define MBAR01_USD_ADDR		0x2100000cull
+#define MBAR23_USD_ADDR		0x4100000cull
+#define MBAR4_USD_ADDR		0x8100000cull
+#define MBAR5_USD_ADDR		0xa100000cull
+#define MBAR01_DSD_ADDR		0x2000000cull
+#define MBAR23_DSD_ADDR		0x4000000cull
+#define MBAR4_DSD_ADDR		0x8000000cull
+#define MBAR5_DSD_ADDR		0xa000000cull
 
 /* XEON Shadowed MMIO Space */
 #define XEON_SHADOW_PDOORBELL_OFFSET	0x60

From 7eb387b320613d2b6a50fec923d4001cc2443b8c Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Thu, 15 Oct 2015 23:49:05 +0000
Subject: [PATCH 269/380] Sort new entry intel_iwn.

Submitted by:	ngie
MFC after:	3 weeks
X-MFC-With:	r289391
Sponsored by:	EMC / Isilon Storage Division
---
 etc/mtree/BSD.usr.dist | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/etc/mtree/BSD.usr.dist b/etc/mtree/BSD.usr.dist
index 59cfa514f952..e4b921f3c00b 100644
--- a/etc/mtree/BSD.usr.dist
+++ b/etc/mtree/BSD.usr.dist
@@ -192,10 +192,10 @@
             legal
                 intel_ipw
                 ..
-                intel_iwn
-                ..
                 intel_iwi
                 ..
+                intel_iwn
+                ..
                 intel_wpi
                 ..
                 realtek

From e1f3daf5e38583f09f21948be0d9cd90fe06e7a4 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Fri, 16 Oct 2015 00:38:05 +0000
Subject: [PATCH 270/380] Rename the /usr/share/doc/legal files to
 driver.LICENSE to work around bug of installing 'realtek' and 'intel_iwn' as
 files rather then as a 'LICENSE' file in their directories.

Also add obsolete entries for the older names and names that existed in head
for a period of time.

Suggested by:	jmg
X-MFC-With:	r289391
MFC after:	3 weeks
Sponsored by:	EMC / Isilon Storage Division
---
 ObsoleteFiles.inc                  | 13 +++++++++++++
 etc/mtree/BSD.usr.dist             | 10 ----------
 share/doc/legal/intel_ipw/Makefile |  3 ++-
 share/doc/legal/intel_iwi/Makefile |  3 ++-
 share/doc/legal/intel_iwn/Makefile |  3 ++-
 share/doc/legal/intel_wpi/Makefile |  3 ++-
 share/doc/legal/realtek/Makefile   |  3 ++-
 share/man/man4/ipw.4               |  8 ++++----
 share/man/man4/ipwfw.4             |  8 ++++----
 share/man/man4/iwi.4               |  8 ++++----
 share/man/man4/iwifw.4             |  8 ++++----
 share/man/man4/rsu.4               |  4 ++--
 share/man/man4/urtwn.4             |  8 ++++----
 share/man/man4/urtwnfw.4           |  8 ++++----
 share/man/man4/wpi.4               |  6 +++---
 sys/tools/fw_stub.awk              |  2 +-
 16 files changed, 53 insertions(+), 45 deletions(-)

diff --git a/ObsoleteFiles.inc b/ObsoleteFiles.inc
index 50cefe379322..d7775fe30e67 100644
--- a/ObsoleteFiles.inc
+++ b/ObsoleteFiles.inc
@@ -38,6 +38,19 @@
 #   xargs -n1 | sort | uniq -d;
 # done
 
+# 20151015: Rename files due to file-installed-as-dir bug
+OLD_FILES+=usr/share/doc/legal/realtek
+OLD_FILES+=usr/share/doc/legal/realtek/LICENSE
+OLD_DIRS+=usr/share/doc/legal/realtek
+OLD_DIRS+=usr/share/doc/legal/intel_ipw
+OLD_FILES+=usr/share/doc/legal/intel_ipw/LICENSE
+OLD_FILES+=usr/share/doc/legal/intel_iwn
+OLD_FILES+=usr/share/doc/legal/intel_iwn/LICENSE
+OLD_DIRS+=usr/share/doc/legal/intel_iwn
+OLD_DIRS+=usr/share/doc/legal/intel_iwi
+OLD_FILES+=usr/share/doc/legal/intel_iwi/LICENSE
+OLD_DIRS+=usr/share/doc/legal/intel_wpi
+OLD_FILES+=usr/share/doc/legal/intel_wpi/LICENSE
 # 20151006: new libc++ import
 OLD_FILES+=usr/include/c++/__tuple_03
 # 20151006: new clang import which bumps version from 3.6.1 to 3.7.0.
diff --git a/etc/mtree/BSD.usr.dist b/etc/mtree/BSD.usr.dist
index e4b921f3c00b..d9cdcd11abb9 100644
--- a/etc/mtree/BSD.usr.dist
+++ b/etc/mtree/BSD.usr.dist
@@ -190,16 +190,6 @@
             atm
             ..
             legal
-                intel_ipw
-                ..
-                intel_iwi
-                ..
-                intel_iwn
-                ..
-                intel_wpi
-                ..
-                realtek
-                ..
             ..
             llvm
                 clang
diff --git a/share/doc/legal/intel_ipw/Makefile b/share/doc/legal/intel_ipw/Makefile
index 18a2c1272191..e52d8bbd0f3b 100644
--- a/share/doc/legal/intel_ipw/Makefile
+++ b/share/doc/legal/intel_ipw/Makefile
@@ -1,6 +1,7 @@
 # $FreeBSD$
 
 FILES=		${.CURDIR}/../../../../sys/contrib/dev/ipw/LICENSE
-FILESDIR=	${SHAREDIR}/doc/legal/intel_ipw
+FILESDIR=	${SHAREDIR}/doc/legal
+FILESNAME=	intel_ipw.LICENSE
 
 .include <bsd.prog.mk>
diff --git a/share/doc/legal/intel_iwi/Makefile b/share/doc/legal/intel_iwi/Makefile
index 6f0e6be15615..012ce7a4a22b 100644
--- a/share/doc/legal/intel_iwi/Makefile
+++ b/share/doc/legal/intel_iwi/Makefile
@@ -1,6 +1,7 @@
 # $FreeBSD$
 
 FILES=		${.CURDIR}/../../../../sys/contrib/dev/iwi/LICENSE
-FILESDIR=	${SHAREDIR}/doc/legal/intel_iwi
+FILESDIR=	${SHAREDIR}/doc/legal
+FILESNAME=	intel_iwi.LICENSE
 
 .include <bsd.prog.mk>
diff --git a/share/doc/legal/intel_iwn/Makefile b/share/doc/legal/intel_iwn/Makefile
index d2e04c9ba6fa..dd5772bb288d 100644
--- a/share/doc/legal/intel_iwn/Makefile
+++ b/share/doc/legal/intel_iwn/Makefile
@@ -1,6 +1,7 @@
 # $FreeBSD$
 
 FILES=		${.CURDIR}/../../../../sys/contrib/dev/iwn/LICENSE
-FILESDIR=	${SHAREDIR}/doc/legal/intel_iwn
+FILESDIR=	${SHAREDIR}/doc/legal
+FILESNAME=	intel_iwn.LICENSE
 
 .include <bsd.prog.mk>
diff --git a/share/doc/legal/intel_wpi/Makefile b/share/doc/legal/intel_wpi/Makefile
index 06b486354eea..61f6d5dea936 100644
--- a/share/doc/legal/intel_wpi/Makefile
+++ b/share/doc/legal/intel_wpi/Makefile
@@ -1,7 +1,8 @@
 # $FreeBSD$
 
 FILES=          ${.CURDIR}/../../../../sys/contrib/dev/wpi/LICENSE
-FILESDIR=       ${SHAREDIR}/doc/legal/intel_wpi
+FILESDIR=       ${SHAREDIR}/doc/legal
+FILESNAME=	intel_wpi.LICENSE
 
 .include <bsd.prog.mk>
 
diff --git a/share/doc/legal/realtek/Makefile b/share/doc/legal/realtek/Makefile
index 2d96c422957b..e168a18290df 100644
--- a/share/doc/legal/realtek/Makefile
+++ b/share/doc/legal/realtek/Makefile
@@ -1,6 +1,7 @@
 # $FreeBSD$
 
 FILES=		${.CURDIR}/../../../../sys/contrib/dev/urtwn/LICENSE
-FILESDIR=	${SHAREDIR}/doc/legal/realtek
+FILESDIR=	${SHAREDIR}/doc/legal
+FILESNAME=	realtek.LICENSE
 
 .include <bsd.prog.mk>
diff --git a/share/man/man4/ipw.4 b/share/man/man4/ipw.4
index 9dfbd6dce525..71ce5752836f 100644
--- a/share/man/man4/ipw.4
+++ b/share/man/man4/ipw.4
@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd April 13, 2008
+.Dd October 15, 2015
 .Dt IPW 4
 .Os
 .Sh NAME
@@ -77,14 +77,14 @@ This driver requires the firmware built with the
 .Nm ipwfw
 module to work.
 For the loaded firmware to be enabled for use the license at
-.Pa /usr/share/doc/legal/intel_ipw/LICENSE
+.Pa /usr/share/doc/legal/intel_ipw.LICENSE
 must be agreed by adding the following line to
 .Xr loader.conf 5 :
 .Pp
 .Dl "legal.intel_ipw.license_ack=1"
 .Sh FILES
-.Bl -tag -width ".Pa /usr/share/doc/legal/intel_ipw/LICENSE" -compact
-.It Pa /usr/share/doc/legal/intel_ipw/LICENSE
+.Bl -tag -width ".Pa /usr/share/doc/legal/intel_ipw.LICENSE" -compact
+.It Pa /usr/share/doc/legal/intel_ipw.LICENSE
 .Nm
 firmware license
 .El
diff --git a/share/man/man4/ipwfw.4 b/share/man/man4/ipwfw.4
index b901de8c0ee3..810fe2a32383 100644
--- a/share/man/man4/ipwfw.4
+++ b/share/man/man4/ipwfw.4
@@ -22,7 +22,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd January 14, 2010
+.Dd October 15, 2015
 .Dt IPWFW 4
 .Os
 .Sh NAME
@@ -59,14 +59,14 @@ Intel PRO/Wireless 2100 series of IEEE 802.11 adapters.
 It may be statically linked into the kernel, or loaded as a module.
 .Pp
 For the loaded firmware to be enabled for use the license at
-.Pa /usr/share/doc/legal/intel_ipw/LICENSE
+.Pa /usr/share/doc/legal/intel_ipw.LICENSE
 must be agreed to by adding the following line to
 .Xr loader.conf 5 :
 .Pp
 .Dl "legal.intel_ipw.license_ack=1"
 .Sh FILES
-.Bl -tag -width ".Pa /usr/share/doc/legal/intel_ipw/LICENSE" -compact
-.It Pa /usr/share/doc/legal/intel_ipw/LICENSE
+.Bl -tag -width ".Pa /usr/share/doc/legal/intel_ipw.LICENSE" -compact
+.It Pa /usr/share/doc/legal/intel_ipw.LICENSE
 .Nm
 firmware license
 .El
diff --git a/share/man/man4/iwi.4 b/share/man/man4/iwi.4
index 911e54ce5250..5d6b8f54247e 100644
--- a/share/man/man4/iwi.4
+++ b/share/man/man4/iwi.4
@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd April 13, 2008
+.Dd October 15, 2015
 .Dt IWI 4
 .Os
 .Sh NAME
@@ -77,14 +77,14 @@ This driver requires the firmware built with the
 .Nm iwifw
 module to work.
 For the loaded firmware to be enabled for use the license at
-.Pa /usr/share/doc/legal/intel_iwi/LICENSE
+.Pa /usr/share/doc/legal/intel_iwi.LICENSE
 must be agreed by adding the following line to
 .Xr loader.conf 5 :
 .Pp
 .Dl "legal.intel_iwi.license_ack=1"
 .Sh FILES
-.Bl -tag -width ".Pa /usr/share/doc/legal/intel_iwi/LICENSE" -compact
-.It Pa /usr/share/doc/legal/intel_iwi/LICENSE
+.Bl -tag -width ".Pa /usr/share/doc/legal/intel_iwi.LICENSE" -compact
+.It Pa /usr/share/doc/legal/intel_iwi.LICENSE
 .Nm
 firmware license
 .El
diff --git a/share/man/man4/iwifw.4 b/share/man/man4/iwifw.4
index 7effd3db7170..52f90e3ea6f2 100644
--- a/share/man/man4/iwifw.4
+++ b/share/man/man4/iwifw.4
@@ -22,7 +22,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd January 14, 2010
+.Dd October 15, 2015
 .Dt IWIFW 4
 .Os
 .Sh NAME
@@ -59,14 +59,14 @@ Intel PRO/Wireless 2200BG/2225BG/2915ABG series of IEEE 802.11 adapters.
 It may be statically linked into the kernel, or loaded as a module.
 .Pp
 For the loaded firmware to be enabled for use the license at
-.Pa /usr/share/doc/legal/intel_iwi/LICENSE
+.Pa /usr/share/doc/legal/intel_iwi.LICENSE
 must be agreed to by adding the following line to
 .Xr loader.conf 5 :
 .Pp
 .Dl "legal.intel_iwi.license_ack=1"
 .Sh FILES
-.Bl -tag -width ".Pa /usr/share/doc/legal/intel_iwi/LICENSE" -compact
-.It Pa /usr/share/doc/legal/intel_iwi/LICENSE
+.Bl -tag -width ".Pa /usr/share/doc/legal/intel_iwi.LICENSE" -compact
+.It Pa /usr/share/doc/legal/intel_iwi.LICENSE
 .Nm
 firmware license
 .El
diff --git a/share/man/man4/rsu.4 b/share/man/man4/rsu.4
index dceb55d0f11c..a9fe2a2f6414 100644
--- a/share/man/man4/rsu.4
+++ b/share/man/man4/rsu.4
@@ -15,7 +15,7 @@
 .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 .\"
-.Dd October 13, 2015
+.Dd October 15, 2015
 .Dt RSU 4
 .Os
 .Sh NAME
@@ -41,7 +41,7 @@ if_rsu_load="YES"
 .Ed
 .Pp
 After you have read the license in
-.Pa /usr/share/doc/legal/realtek
+.Pa /usr/share/doc/legal/realtek.LICENSE
 you will want to add the following lines to
 .Xr loader.conf 5 :
 .Bd -literal -offset indent
diff --git a/share/man/man4/urtwn.4 b/share/man/man4/urtwn.4
index 5a58200a81fe..a4c1b9f10a43 100644
--- a/share/man/man4/urtwn.4
+++ b/share/man/man4/urtwn.4
@@ -14,7 +14,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd October 24, 2014
+.Dd October 15, 2015
 .Dt URTWN 4
 .Os
 .Sh NAME
@@ -68,14 +68,14 @@ This driver requires the firmware built with the
 .Nm urtwnfw
 module to work.
 For the loaded firmware to be enabled for use the license at
-.Pa /usr/share/doc/legal/realtek
+.Pa /usr/share/doc/legal/realtek.LICENSE
 must be agreed by adding the following line to
 .Xr loader.conf 5 :
 .Pp
 .Dl "legal.realtek.license_ack=1"
 .Sh FILES
-.Bl -tag -width ".Pa /usr/share/doc/legal/realtek" -compact
-.It Pa /usr/share/doc/legal/realtek
+.Bl -tag -width ".Pa /usr/share/doc/legal/realtek.LICENSE" -compact
+.It Pa /usr/share/doc/legal/realtek.LICENSE
 .Nm
 firmware license
 .El
diff --git a/share/man/man4/urtwnfw.4 b/share/man/man4/urtwnfw.4
index c25f944f1b5a..70e6e67f20ed 100644
--- a/share/man/man4/urtwnfw.4
+++ b/share/man/man4/urtwnfw.4
@@ -22,7 +22,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd April 25, 2014
+.Dd October 15, 2015
 .Dt URTWNFW 4
 .Os
 .Sh NAME
@@ -61,14 +61,14 @@ It may be
 statically linked into the kernel, or loaded as a module.
 .Pp
 For the loaded firmware to be enabled for use the license at
-.Pa /usr/share/doc/legal/realtek
+.Pa /usr/share/doc/legal/realtek.LICENSE
 must be agreed to by adding the following line to
 .Xr loader.conf 5 :
 .Pp
 .Dl "legal.realtek.license_ack=1"
 .Sh FILES
-.Bl -tag -width ".Pa /usr/share/doc/legal/realtek" -compact
-.It Pa /usr/share/doc/legal/realtek
+.Bl -tag -width ".Pa /usr/share/doc/legal/realtek.LICENSE" -compact
+.It Pa /usr/share/doc/legal/realtek.LICENSE
 .Nm
 firmware license
 .El
diff --git a/share/man/man4/wpi.4 b/share/man/man4/wpi.4
index 4cf33c410e70..04539f7328a6 100644
--- a/share/man/man4/wpi.4
+++ b/share/man/man4/wpi.4
@@ -26,7 +26,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd October 5, 2008
+.Dd October 15, 2015
 .Dt WPI 4
 .Os
 .Sh NAME
@@ -72,8 +72,8 @@ This driver requires the firmware built with the
 .Nm wpifw
 module to work.
 .Sh FILES
-.Bl -tag -width ".Pa /usr/share/doc/legal/intel_wpi/LICENSE" -compact
-.It Pa /usr/share/doc/legal/intel_wpi/LICENSE
+.Bl -tag -width ".Pa /usr/share/doc/legal/intel_wpi.LICENSE" -compact
+.It Pa /usr/share/doc/legal/intel_wpi.LICENSE
 .Nm
 firmware license
 .El
diff --git a/sys/tools/fw_stub.awk b/sys/tools/fw_stub.awk
index 0c7d5676a3f0..4a82c0b2441b 100644
--- a/sys/tools/fw_stub.awk
+++ b/sys/tools/fw_stub.awk
@@ -156,7 +156,7 @@ if (opt_l) {
 		printc("\
 		TUNABLE_LONG_FETCH(\"legal." opt_l ".license_ack\", &" opt_l "_license_ack);\
 		if (!" opt_l "_license_ack) {\
-			printf(\"" opt_m ": You need to read the LICENSE file in /usr/share/doc/legal/" opt_l "/.\\n\");\
+			printf(\"" opt_m ": You need to read the LICENSE file in /usr/share/doc/legal/" opt_l ".LICENSE.\\n\");\
 			printf(\"" opt_m ": If you agree with the license, set legal." opt_l ".license_ack=1 in /boot/loader.conf.\\n\");\
 			return(EPERM);\
 		}\n");

From 30ae8a4cadb63310ab8b1cddab4ff0db29ab5443 Mon Sep 17 00:00:00 2001
From: hrs <hrs@FreeBSD.org>
Date: Fri, 16 Oct 2015 01:16:01 +0000
Subject: [PATCH 271/380] Fix a panic when destroying a lagg interface.

Differential Revision:	https://reviews.freebsd.org/D3883
---
 sys/net/if_lagg.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sys/net/if_lagg.c b/sys/net/if_lagg.c
index 0f286717e07a..095f87066a9c 100644
--- a/sys/net/if_lagg.c
+++ b/sys/net/if_lagg.c
@@ -916,7 +916,8 @@ lagg_port_destroy(struct lagg_port *lp, int rundelport)
 		 * iflladdr_event.
 		 */
 		sc->sc_primary = lp0;
-		lagg_port_lladdr(lp0, lladdr);
+		if (lp0 != NULL)
+			lagg_port_lladdr(lp0, lladdr);
 	}
 
 	/* Remove any pending lladdr changes from the queue */

From f39349df451605ba8516f1c7c4bd1f2c7826b5ac Mon Sep 17 00:00:00 2001
From: np <np@FreeBSD.org>
Date: Fri, 16 Oct 2015 01:19:55 +0000
Subject: [PATCH 272/380] cxgbe(4): support for the kernel RSS option.

You need PCBGROUP and RSS in the kernel config to use this.

Relnotes:	Yes
Sponsored by:	Chelsio Communications
---
 sys/dev/cxgbe/t4_main.c             | 185 +++++++++++++++++++++++++++-
 sys/modules/cxgbe/if_cxgbe/Makefile |   1 +
 2 files changed, 181 insertions(+), 5 deletions(-)

diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
index 53fcfbb44d92..a5c7608310bd 100644
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -30,6 +30,7 @@ __FBSDID("$FreeBSD$");
 
 #include "opt_inet.h"
 #include "opt_inet6.h"
+#include "opt_rss.h"
 
 #include <sys/param.h>
 #include <sys/conf.h>
@@ -55,6 +56,9 @@ __FBSDID("$FreeBSD$");
 #include <net/if_types.h>
 #include <net/if_dl.h>
 #include <net/if_vlan_var.h>
+#ifdef RSS
+#include <net/rss_config.h>
+#endif
 #if defined(__i386__) || defined(__amd64__)
 #include <vm/vm.h>
 #include <vm/pmap.h>
@@ -3464,6 +3468,71 @@ adapter_full_uninit(struct adapter *sc)
 	return (0);
 }
 
+#ifdef RSS
+#define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \
+    RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \
+    RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \
+    RSS_HASHTYPE_RSS_UDP_IPV6)
+
+/* Translates kernel hash types to hardware. */
+static int
+hashconfig_to_hashen(int hashconfig)
+{
+	int hashen = 0;
+
+	if (hashconfig & RSS_HASHTYPE_RSS_IPV4)
+		hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN;
+	if (hashconfig & RSS_HASHTYPE_RSS_IPV6)
+		hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN;
+	if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) {
+		hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN |
+		    F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
+	}
+	if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) {
+		hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN |
+		    F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
+	}
+	if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4)
+		hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
+	if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6)
+		hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
+
+	return (hashen);
+}
+
+/* Translates hardware hash types to kernel. */
+static int
+hashen_to_hashconfig(int hashen)
+{
+	int hashconfig = 0;
+
+	if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) {
+		/*
+		 * If UDP hashing was enabled it must have been enabled for
+		 * either IPv4 or IPv6 (inclusive or).  Enabling UDP without
+		 * enabling any 4-tuple hash is nonsense configuration.
+		 */
+		MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
+		    F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN));
+
+		if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
+			hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4;
+		if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
+			hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6;
+	}
+	if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
+		hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4;
+	if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
+		hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6;
+	if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
+		hashconfig |= RSS_HASHTYPE_RSS_IPV4;
+	if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
+		hashconfig |= RSS_HASHTYPE_RSS_IPV6;
+
+	return (hashconfig);
+}
+#endif
+
 int
 port_full_init(struct port_info *pi)
 {
@@ -3471,7 +3540,14 @@ port_full_init(struct port_info *pi)
 	struct ifnet *ifp = pi->ifp;
 	uint16_t *rss;
 	struct sge_rxq *rxq;
-	int rc, i, j;
+	int rc, i, j, hashen;
+#ifdef RSS
+	int nbuckets = rss_getnumbuckets();
+	int hashconfig = rss_gethashconfig();
+	int extra;
+	uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
+	uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
+#endif
 
 	ASSERT_SYNCHRONIZED_OP(sc);
 	KASSERT((pi->flags & PORT_INIT_DONE) == 0,
@@ -3490,13 +3566,42 @@ port_full_init(struct port_info *pi)
 	/*
 	 * Setup RSS for this port.  Save a copy of the RSS table for later use.
 	 */
+	if (pi->nrxq > pi->rss_size) {
+		if_printf(ifp, "nrxq (%d) > hw RSS table size (%d); "
+		    "some queues will never receive traffic.\n", pi->nrxq,
+		    pi->rss_size);
+	} else if (pi->rss_size % pi->nrxq) {
+		if_printf(ifp, "nrxq (%d), hw RSS table size (%d); "
+		    "expect uneven traffic distribution.\n", pi->nrxq,
+		    pi->rss_size);
+	}
+#ifdef RSS
+	MPASS(RSS_KEYSIZE == 40);
+	if (pi->nrxq != nbuckets) {
+		if_printf(ifp, "nrxq (%d) != kernel RSS buckets (%d);"
+		    "performance will be impacted.\n", pi->nrxq, nbuckets);
+	}
+
+	rss_getkey((void *)&raw_rss_key[0]);
+	for (i = 0; i < nitems(rss_key); i++) {
+		rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]);
+	}
+	t4_write_rss_key(sc, (void *)&rss_key[0], -1);
+#endif
 	rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
 	for (i = 0; i < pi->rss_size;) {
+#ifdef RSS
+		j = rss_get_indirection_to_bucket(i);
+		j %= pi->nrxq;
+		rxq = &sc->sge.rxq[pi->first_rxq + j];
+		rss[i++] = rxq->iq.abs_id;
+#else
 		for_each_rxq(pi, j, rxq) {
 			rss[i++] = rxq->iq.abs_id;
 			if (i == pi->rss_size)
 				break;
 		}
+#endif
 	}
 
 	rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
@@ -3506,6 +3611,54 @@ port_full_init(struct port_info *pi)
 		goto done;
 	}
 
+#ifdef RSS
+	hashen = hashconfig_to_hashen(hashconfig);
+
+	/*
+	 * We may have had to enable some hashes even though the global config
+	 * wants them disabled.  This is a potential problem that must be
+	 * reported to the user.
+	 */
+	extra = hashen_to_hashconfig(hashen) ^ hashconfig;
+
+	/*
+	 * If we consider only the supported hash types, then the enabled hashes
+	 * are a superset of the requested hashes.  In other words, there cannot
+	 * be any supported hash that was requested but not enabled, but there
+	 * can be hashes that were not requested but had to be enabled.
+	 */
+	extra &= SUPPORTED_RSS_HASHTYPES;
+	MPASS((extra & hashconfig) == 0);
+
+	if (extra) {
+		if_printf(ifp,
+		    "global RSS config (0x%x) cannot be accomodated.\n",
+		    hashconfig);
+	}
+	if (extra & RSS_HASHTYPE_RSS_IPV4)
+		if_printf(ifp, "IPv4 2-tuple hashing forced on.\n");
+	if (extra & RSS_HASHTYPE_RSS_TCP_IPV4)
+		if_printf(ifp, "TCP/IPv4 4-tuple hashing forced on.\n");
+	if (extra & RSS_HASHTYPE_RSS_IPV6)
+		if_printf(ifp, "IPv6 2-tuple hashing forced on.\n");
+	if (extra & RSS_HASHTYPE_RSS_TCP_IPV6)
+		if_printf(ifp, "TCP/IPv6 4-tuple hashing forced on.\n");
+	if (extra & RSS_HASHTYPE_RSS_UDP_IPV4)
+		if_printf(ifp, "UDP/IPv4 4-tuple hashing forced on.\n");
+	if (extra & RSS_HASHTYPE_RSS_UDP_IPV6)
+		if_printf(ifp, "UDP/IPv6 4-tuple hashing forced on.\n");
+#else
+	hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN |
+	    F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN |
+	    F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
+	    F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN;
+#endif
+	rc = -t4_config_vi_rss(sc, sc->mbox, pi->viid, hashen, rss[0]);
+	if (rc != 0) {
+		if_printf(ifp, "rss hash/defaultq config failed: %d\n", rc);
+		goto done;
+	}
+
 	pi->rss = rss;
 	pi->flags |= PORT_INIT_DONE;
 done:
@@ -8380,17 +8533,39 @@ tweak_tunables(void)
 {
 	int nc = mp_ncpus;	/* our snapshot of the number of CPUs */
 
-	if (t4_ntxq10g < 1)
+	if (t4_ntxq10g < 1) {
+#ifdef RSS
+		t4_ntxq10g = rss_getnumbuckets();
+#else
 		t4_ntxq10g = min(nc, NTXQ_10G);
+#endif
+	}
 
-	if (t4_ntxq1g < 1)
+	if (t4_ntxq1g < 1) {
+#ifdef RSS
+		/* XXX: way too many for 1GbE? */
+		t4_ntxq1g = rss_getnumbuckets();
+#else
 		t4_ntxq1g = min(nc, NTXQ_1G);
+#endif
+	}
 
-	if (t4_nrxq10g < 1)
+	if (t4_nrxq10g < 1) {
+#ifdef RSS
+		t4_nrxq10g = rss_getnumbuckets();
+#else
 		t4_nrxq10g = min(nc, NRXQ_10G);
+#endif
+	}
 
-	if (t4_nrxq1g < 1)
+	if (t4_nrxq1g < 1) {
+#ifdef RSS
+		/* XXX: way too many for 1GbE? */
+		t4_nrxq1g = rss_getnumbuckets();
+#else
 		t4_nrxq1g = min(nc, NRXQ_1G);
+#endif
+	}
 
 #ifdef TCP_OFFLOAD
 	if (t4_nofldtxq10g < 1)
diff --git a/sys/modules/cxgbe/if_cxgbe/Makefile b/sys/modules/cxgbe/if_cxgbe/Makefile
index a66e45a045f4..df343b27ea73 100644
--- a/sys/modules/cxgbe/if_cxgbe/Makefile
+++ b/sys/modules/cxgbe/if_cxgbe/Makefile
@@ -11,6 +11,7 @@ SRCS+=	device_if.h
 SRCS+=	opt_inet.h
 SRCS+=	opt_inet6.h
 SRCS+=	opt_ofed.h
+SRCS+=	opt_rss.h
 SRCS+=	pci_if.h
 SRCS+=	t4_hw.c
 SRCS+=	t4_l2t.c

From fb9846240a79d4b543da5de2f53d6a00ca0716dc Mon Sep 17 00:00:00 2001
From: imp <imp@FreeBSD.org>
Date: Fri, 16 Oct 2015 03:06:02 +0000
Subject: [PATCH 273/380] Do not relocate extents to make them contiguous if
 the underlying drive can do deletions. Ability to do deletions is a strong
 indication that this optimization will not help performance. It will only
 generate extra write traffic. These devices are typically flash based and
 have a limited number of write cycles. In addition, making the file
 contiguous in LBA space doesn't improve the access times from flash devices
 because they have no seek time.

Reviewed by: mckusick@
---
 sys/ufs/ffs/ffs_alloc.c | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/sys/ufs/ffs/ffs_alloc.c b/sys/ufs/ffs/ffs_alloc.c
index c587dfb3bbe5..19c00de6e1ba 100644
--- a/sys/ufs/ffs/ffs_alloc.c
+++ b/sys/ufs/ffs/ffs_alloc.c
@@ -481,9 +481,19 @@ ffs_reallocblks(ap)
 		struct cluster_save *a_buflist;
 	} */ *ap;
 {
+	struct ufsmount *ump;
 
-	if (doreallocblks == 0)
+	/*
+	 * If the underlying device can do deletes, then skip reallocating
+	 * the blocks of this file into contiguous sequences. Devices that
+	 * benefit from BIO_DELETE also benefit from not moving the data.
+	 * These devices are flash and therefore work less well with this
+	 * optimization. Also skip if reallocblks has been disabled globally.
+	 */
+	ump = VTOI(ap->a_vp)->i_ump;
+	if (ump->um_candelete || doreallocblks == 0)
 		return (ENOSPC);
+
 	/*
 	 * We can't wait in softdep prealloc as it may fsync and recurse
 	 * here.  Instead we simply fail to reallocate blocks if this
@@ -492,7 +502,7 @@ ffs_reallocblks(ap)
 	if (DOINGSOFTDEP(ap->a_vp))
 		if (softdep_prealloc(ap->a_vp, MNT_NOWAIT) != 0)
 			return (ENOSPC);
-	if (VTOI(ap->a_vp)->i_ump->um_fstype == UFS1)
+	if (ump->um_fstype == UFS1)
 		return (ffs_reallocblks_ufs1(ap));
 	return (ffs_reallocblks_ufs2(ap));
 }

From fb91ce0f235cb82bde4703ee6865d80ff9bee816 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Fri, 16 Oct 2015 04:07:27 +0000
Subject: [PATCH 274/380] Fix delete-old and check-old-files not removing old
 debug symbols.

This was handled for libraries in r256842 but for some reason was missed
for files (bsd.prog.mk).

MFC after:	1 week
Sponsored by:	EMC / Isilon Storage Division
Relnotes:	yes
---
 Makefile.inc1 | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/Makefile.inc1 b/Makefile.inc1
index 32be188ccb26..53c407fa24f9 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -1996,6 +1996,13 @@ delete-old-files:
 			chflags noschg "${DESTDIR}/$${file}" 2>/dev/null || true; \
 			rm ${RM_I} "${DESTDIR}/$${file}" <&3; \
 		fi; \
+		for ext in debug symbols; do \
+		  if ! [ -e "${DESTDIR}/$${file}" ] && [ -f \
+		      "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}" ]; then \
+			  rm ${RM_I} "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}" \
+			      <&3; \
+		  fi; \
+		done; \
 	done
 # Remove catpages without corresponding manpages.
 	@exec 3<&0; \
@@ -2018,6 +2025,11 @@ check-old-files:
 		if [ -f "${DESTDIR}/$${file}" -o -L "${DESTDIR}/$${file}" ]; then \
 		 	echo "${DESTDIR}/$${file}"; \
 		fi; \
+		for ext in debug symbols; do \
+		  if [ -f "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}" ]; then \
+			  echo "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}"; \
+		  fi; \
+		done; \
 	done
 # Check for catpages without corresponding manpages.
 	@find ${DESTDIR}/usr/share/man/cat* ! -type d | \

From 3e0a363d0d108468daa01e94af8e301fc54d03c3 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Fri, 16 Oct 2015 05:06:43 +0000
Subject: [PATCH 275/380] Similar to r289355, /usr/tests is within the base
 system so put the symbols into /usr/lib/debug.

This covers some missing files:
  /usr/tests/libexec/rtld-elf/.debug
  /usr/tests/libexec/rtld-elf/.debug/libpythagoras.so.0.debug
  /usr/tests/lib/libc/tls/.debug
  /usr/tests/lib/libc/tls/.debug/libh_tls_dynamic.so.1.debug
  /usr/tests/lib/libc/tls/.debug/h_tls_dlopen.so.debug
  /usr/tests/lib/libthr/dlopen/.debug
  /usr/tests/lib/libthr/dlopen/.debug/h_pthread_dlopen.so.1.debug
  /usr/tests/lib/libxo/.debug
  /usr/tests/lib/libxo/.debug/libenc_test.so.debug

Sponsored by:	EMC / Isilon Storage Division
---
 share/mk/bsd.lib.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/share/mk/bsd.lib.mk b/share/mk/bsd.lib.mk
index 8c8a0141e186..e906ab7e83a6 100644
--- a/share/mk/bsd.lib.mk
+++ b/share/mk/bsd.lib.mk
@@ -140,7 +140,7 @@ SHLIB_NAME_FULL=${SHLIB_NAME}.full
 # Use ${DEBUGDIR} for base system debug files, else .debug subdirectory
 .if ${_SHLIBDIR} == "/boot" ||\
     ${SHLIBDIR:C%/lib(/.*)?$%/lib%} == "/lib" ||\
-    ${SHLIBDIR:C%/usr/lib(32)?(/.*)?%/usr/lib%} == "/usr/lib"
+    ${SHLIBDIR:C%/usr/(tests/)?lib(32|exec)?(/.*)?%/usr/lib%} == "/usr/lib"
 DEBUGFILEDIR=${DEBUGDIR}${_SHLIBDIR}
 .else
 DEBUGFILEDIR=${_SHLIBDIR}/.debug

From b6f4b8daca17166c435559fc24c069201b36ec81 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Fri, 16 Oct 2015 05:13:21 +0000
Subject: [PATCH 276/380] Add entries for moved test symbols for r289355 and
 r289330.

This list is likely not complete.

Sponsored by:	EMC / Isilon Storage Division
---
 ObsoleteFiles.inc | 391 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 391 insertions(+)

diff --git a/ObsoleteFiles.inc b/ObsoleteFiles.inc
index d7775fe30e67..37bc3af022b1 100644
--- a/ObsoleteFiles.inc
+++ b/ObsoleteFiles.inc
@@ -38,6 +38,397 @@
 #   xargs -n1 | sort | uniq -d;
 # done
 
+# 20151015: test symbols moved to /usr/lib/debug
+OLD_DIRS+=usr/tests/lib/atf/libatf-c++/.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c++/.debug/atf_c++_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c++/.debug/build_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c++/.debug/check_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c++/.debug/config_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c++/.debug/macros_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c++/.debug/tests_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c++/.debug/utils_test.debug
+OLD_DIRS+=usr/tests/lib/atf/libatf-c++/detail/.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/application_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/env_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/exceptions_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/fs_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/process_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/sanity_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/text_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/version_helper.debug
+OLD_DIRS+=usr/tests/lib/atf/libatf-c/.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/atf_c_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/build_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/check_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/config_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/error_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/macros_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/tc_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/tp_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/utils_test.debug
+OLD_DIRS+=usr/tests/lib/atf/libatf-c/detail/.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/dynstr_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/env_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/fs_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/list_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/map_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/process_helpers.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/process_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/sanity_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/text_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/user_test.debug
+OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/version_helper.debug
+OLD_DIRS+=usr/tests/lib/atf/test-programs/.debug
+OLD_FILES+=usr/tests/lib/atf/test-programs/.debug/c_helpers.debug
+OLD_FILES+=usr/tests/lib/atf/test-programs/.debug/cpp_helpers.debug
+OLD_DIRS+=usr/tests/lib/libc/c063/.debug
+OLD_FILES+=usr/tests/lib/libc/c063/.debug/faccessat.debug
+OLD_FILES+=usr/tests/lib/libc/c063/.debug/fchmodat.debug
+OLD_FILES+=usr/tests/lib/libc/c063/.debug/fchownat.debug
+OLD_FILES+=usr/tests/lib/libc/c063/.debug/fexecve.debug
+OLD_FILES+=usr/tests/lib/libc/c063/.debug/fstatat.debug
+OLD_FILES+=usr/tests/lib/libc/c063/.debug/linkat.debug
+OLD_FILES+=usr/tests/lib/libc/c063/.debug/mkdirat.debug
+OLD_FILES+=usr/tests/lib/libc/c063/.debug/mkfifoat.debug
+OLD_FILES+=usr/tests/lib/libc/c063/.debug/mknodat.debug
+OLD_FILES+=usr/tests/lib/libc/c063/.debug/openat.debug
+OLD_FILES+=usr/tests/lib/libc/c063/.debug/readlinkat.debug
+OLD_FILES+=usr/tests/lib/libc/c063/.debug/renameat.debug
+OLD_FILES+=usr/tests/lib/libc/c063/.debug/symlinkat.debug
+OLD_FILES+=usr/tests/lib/libc/c063/.debug/unlinkat.debug
+OLD_FILES+=usr/tests/lib/libc/c063/.debug/utimensat.debug
+OLD_DIRS+=usr/tests/lib/libc/db/.debug
+OLD_FILES+=usr/tests/lib/libc/db/.debug/h_db.debug
+OLD_DIRS+=usr/tests/lib/libc/gen/.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/alarm_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/arc4random_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/assert_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/basedirname_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/dir_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/floatunditf_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/fnmatch_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/fpclassify2_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/fpclassify_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/fpsetmask_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/fpsetround_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/ftok_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/getcwd_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/getgrent_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/glob_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/humanize_number_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/isnan_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/nice_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/pause_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/raise_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/realpath_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/setdomainname_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/sethostname_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/sleep_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/syslog_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/time_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/ttyname_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/.debug/vis_test.debug
+OLD_DIRS+=usr/tests/lib/libc/gen/execve/.debug
+OLD_FILES+=usr/tests/lib/libc/gen/execve/.debug/execve_test.debug
+OLD_DIRS+=usr/tests/lib/libc/gen/posix_spawn/.debug
+OLD_FILES+=usr/tests/lib/libc/gen/posix_spawn/.debug/fileactions_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/posix_spawn/.debug/h_fileactions.debug
+OLD_FILES+=usr/tests/lib/libc/gen/posix_spawn/.debug/h_spawn.debug
+OLD_FILES+=usr/tests/lib/libc/gen/posix_spawn/.debug/h_spawnattr.debug
+OLD_FILES+=usr/tests/lib/libc/gen/posix_spawn/.debug/spawn_test.debug
+OLD_FILES+=usr/tests/lib/libc/gen/posix_spawn/.debug/spawnattr_test.debug
+OLD_DIRS+=usr/tests/lib/libc/hash/.debug
+OLD_FILES+=usr/tests/lib/libc/hash/.debug/h_hash.debug
+OLD_FILES+=usr/tests/lib/libc/hash/.debug/sha2_test.debug
+OLD_DIRS+=usr/tests/lib/libc/inet/.debug
+OLD_FILES+=usr/tests/lib/libc/inet/.debug/inet_network_test.debug
+OLD_DIRS+=usr/tests/lib/libc/locale/.debug
+OLD_FILES+=usr/tests/lib/libc/locale/.debug/io_test.debug
+OLD_FILES+=usr/tests/lib/libc/locale/.debug/mbrtowc_test.debug
+OLD_FILES+=usr/tests/lib/libc/locale/.debug/mbsnrtowcs_test.debug
+OLD_FILES+=usr/tests/lib/libc/locale/.debug/mbstowcs_test.debug
+OLD_FILES+=usr/tests/lib/libc/locale/.debug/mbtowc_test.debug
+OLD_FILES+=usr/tests/lib/libc/locale/.debug/wcscspn_test.debug
+OLD_FILES+=usr/tests/lib/libc/locale/.debug/wcspbrk_test.debug
+OLD_FILES+=usr/tests/lib/libc/locale/.debug/wcsspn_test.debug
+OLD_FILES+=usr/tests/lib/libc/locale/.debug/wcstod_test.debug
+OLD_FILES+=usr/tests/lib/libc/locale/.debug/wctomb_test.debug
+OLD_DIRS+=usr/tests/lib/libc/net/.debug
+OLD_FILES+=usr/tests/lib/libc/net/.debug/ether_aton_test.debug
+OLD_FILES+=usr/tests/lib/libc/net/.debug/getprotoent_test.debug
+OLD_FILES+=usr/tests/lib/libc/net/.debug/h_dns_server.debug
+OLD_FILES+=usr/tests/lib/libc/net/.debug/h_nsd_recurse.debug
+OLD_FILES+=usr/tests/lib/libc/net/.debug/h_protoent.debug
+OLD_FILES+=usr/tests/lib/libc/net/.debug/h_servent.debug
+OLD_DIRS+=usr/tests/lib/libc/regex/.debug
+OLD_FILES+=usr/tests/lib/libc/regex/.debug/exhaust_test.debug
+OLD_FILES+=usr/tests/lib/libc/regex/.debug/h_regex.debug
+OLD_FILES+=usr/tests/lib/libc/regex/.debug/regex_att_test.debug
+OLD_DIRS+=usr/tests/lib/libc/ssp/.debug
+OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_fgets.debug
+OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_getcwd.debug
+OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_gets.debug
+OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_memcpy.debug
+OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_memmove.debug
+OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_memset.debug
+OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_raw.debug
+OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_read.debug
+OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_readlink.debug
+OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_snprintf.debug
+OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_sprintf.debug
+OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_stpcpy.debug
+OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_stpncpy.debug
+OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_strcat.debug
+OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_strcpy.debug
+OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_strncat.debug
+OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_strncpy.debug
+OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_vsnprintf.debug
+OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_vsprintf.debug
+OLD_DIRS+=usr/tests/lib/libc/stdio/.debug
+OLD_FILES+=usr/tests/lib/libc/stdio/.debug/clearerr_test.debug
+OLD_FILES+=usr/tests/lib/libc/stdio/.debug/fflush_test.debug
+OLD_FILES+=usr/tests/lib/libc/stdio/.debug/fmemopen2_test.debug
+OLD_FILES+=usr/tests/lib/libc/stdio/.debug/fmemopen_test.debug
+OLD_FILES+=usr/tests/lib/libc/stdio/.debug/fopen_test.debug
+OLD_FILES+=usr/tests/lib/libc/stdio/.debug/fputc_test.debug
+OLD_FILES+=usr/tests/lib/libc/stdio/.debug/mktemp_test.debug
+OLD_FILES+=usr/tests/lib/libc/stdio/.debug/popen_test.debug
+OLD_FILES+=usr/tests/lib/libc/stdio/.debug/printf_test.debug
+OLD_FILES+=usr/tests/lib/libc/stdio/.debug/scanf_test.debug
+OLD_DIRS+=usr/tests/lib/libc/stdlib/.debug
+OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/abs_test.debug
+OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/atoi_test.debug
+OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/div_test.debug
+OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/exit_test.debug
+OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/getenv_test.debug
+OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/h_getopt.debug
+OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/h_getopt_long.debug
+OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/hsearch_test.debug
+OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/posix_memalign_test.debug
+OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/random_test.debug
+OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/strtod_test.debug
+OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/strtol_test.debug
+OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/system_test.debug
+OLD_DIRS+=usr/tests/lib/libc/string/.debug
+OLD_FILES+=usr/tests/lib/libc/string/.debug/memchr.debug
+OLD_FILES+=usr/tests/lib/libc/string/.debug/memcpy.debug
+OLD_FILES+=usr/tests/lib/libc/string/.debug/memmem.debug
+OLD_FILES+=usr/tests/lib/libc/string/.debug/memset.debug
+OLD_FILES+=usr/tests/lib/libc/string/.debug/strcat.debug
+OLD_FILES+=usr/tests/lib/libc/string/.debug/strchr.debug
+OLD_FILES+=usr/tests/lib/libc/string/.debug/strcmp.debug
+OLD_FILES+=usr/tests/lib/libc/string/.debug/strcpy.debug
+OLD_FILES+=usr/tests/lib/libc/string/.debug/strcspn.debug
+OLD_FILES+=usr/tests/lib/libc/string/.debug/strerror.debug
+OLD_FILES+=usr/tests/lib/libc/string/.debug/strlen.debug
+OLD_FILES+=usr/tests/lib/libc/string/.debug/strpbrk.debug
+OLD_FILES+=usr/tests/lib/libc/string/.debug/strrchr.debug
+OLD_FILES+=usr/tests/lib/libc/string/.debug/strspn.debug
+OLD_FILES+=usr/tests/lib/libc/string/.debug/swab.debug
+OLD_DIRS+=usr/tests/lib/libc/sys/.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/access_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/chroot_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/clock_gettime_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/connect_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/dup_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/fsync_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/getcontext_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/getgroups_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/getitimer_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/getlogin_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/getpid_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/getrusage_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/getsid_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/gettimeofday_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/issetugid_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/kevent_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/kill_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/link_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/listen_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/mincore_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/mkdir_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/mkfifo_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/mknod_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/mlock_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/mmap_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/mprotect_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/msgctl_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/msgget_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/msgrcv_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/msgsnd_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/msync_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/nanosleep_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/pipe2_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/pipe_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/poll_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/revoke_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/select_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/setrlimit_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/setuid_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/sigaction_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/sigqueue_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/sigtimedwait_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/socketpair_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/stat_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/timer_create_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/truncate_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/ucontext_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/umask_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/unlink_test.debug
+OLD_FILES+=usr/tests/lib/libc/sys/.debug/write_test.debug
+OLD_DIRS+=usr/tests/lib/libc/termios/.debug
+OLD_FILES+=usr/tests/lib/libc/termios/.debug/tcsetpgrp_test.debug
+OLD_DIRS+=usr/tests/lib/libc/tls/.debug
+OLD_FILES+=usr/tests/lib/libc/tls/.debug/h_tls_dlopen.so.debug
+OLD_FILES+=usr/tests/lib/libc/tls/.debug/libh_tls_dynamic.so.1.debug
+OLD_FILES+=usr/tests/lib/libc/tls/.debug/tls_dlopen_test.debug
+OLD_FILES+=usr/tests/lib/libc/tls/.debug/tls_dynamic_test.debug
+OLD_DIRS+=usr/tests/lib/libc/ttyio/.debug
+OLD_FILES+=usr/tests/lib/libc/ttyio/.debug/ttyio_test.debug
+OLD_DIRS+=usr/tests/lib/libcrypt/.debug
+OLD_FILES+=usr/tests/lib/libcrypt/.debug/crypt_tests.debug
+OLD_DIRS+=usr/tests/lib/libmp/.debug
+OLD_FILES+=usr/tests/lib/libmp/.debug/legacy_test.debug
+OLD_DIRS+=usr/tests/lib/libnv/.debug
+OLD_FILES+=usr/tests/lib/libnv/.debug/dnv_tests.debug
+OLD_FILES+=usr/tests/lib/libnv/.debug/nv_array_tests.debug
+OLD_FILES+=usr/tests/lib/libnv/.debug/nv_tests.debug
+OLD_FILES+=usr/tests/lib/libnv/.debug/nvlist_add_test.debug
+OLD_FILES+=usr/tests/lib/libnv/.debug/nvlist_exists_test.debug
+OLD_FILES+=usr/tests/lib/libnv/.debug/nvlist_free_test.debug
+OLD_FILES+=usr/tests/lib/libnv/.debug/nvlist_get_test.debug
+OLD_FILES+=usr/tests/lib/libnv/.debug/nvlist_move_test.debug
+OLD_FILES+=usr/tests/lib/libnv/.debug/nvlist_send_recv_test.debug
+OLD_DIRS+=usr/tests/lib/libpam/.debug
+OLD_FILES+=usr/tests/lib/libpam/.debug/t_openpam_ctype.debug
+OLD_FILES+=usr/tests/lib/libpam/.debug/t_openpam_readlinev.debug
+OLD_FILES+=usr/tests/lib/libpam/.debug/t_openpam_readword.debug
+OLD_DIRS+=usr/tests/lib/libproc/.debug
+OLD_FILES+=usr/tests/lib/libproc/.debug/proc_test.debug
+OLD_FILES+=usr/tests/lib/libproc/.debug/target_prog.debug
+OLD_DIRS+=usr/tests/lib/librt/.debug
+OLD_FILES+=usr/tests/lib/librt/.debug/sched_test.debug
+OLD_FILES+=usr/tests/lib/librt/.debug/sem_test.debug
+OLD_DIRS+=usr/tests/lib/libthr/.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/barrier_test.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/cond_test.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/condwait_test.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/detach_test.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/equal_test.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/fork_test.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/fpu_test.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/h_atexit.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/h_cancel.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/h_exit.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/h_resolv.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/join_test.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/kill_test.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/mutex_test.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/once_test.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/preempt_test.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/rwlock_test.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/sem_test.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/siglongjmp_test.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/sigmask_test.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/sigsuspend_test.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/sleep_test.debug
+OLD_FILES+=usr/tests/lib/libthr/.debug/swapcontext_test.debug
+OLD_DIRS+=usr/tests/lib/libthr/dlopen/.debug
+OLD_FILES+=usr/tests/lib/libthr/dlopen/.debug/dlopen_test.debug
+OLD_FILES+=usr/tests/lib/libthr/dlopen/.debug/h_pthread_dlopen.so.1.debug
+OLD_FILES+=usr/tests/lib/libthr/dlopen/.debug/main_pthread_create_test.debug
+OLD_DIRS+=usr/tests/lib/libutil/.debug
+OLD_FILES+=usr/tests/lib/libutil/.debug/flopen_test.debug
+OLD_FILES+=usr/tests/lib/libutil/.debug/grp_test.debug
+OLD_FILES+=usr/tests/lib/libutil/.debug/humanize_number_test.debug
+OLD_FILES+=usr/tests/lib/libutil/.debug/pidfile_test.debug
+OLD_FILES+=usr/tests/lib/libutil/.debug/trimdomain-nodomain_test.debug
+OLD_FILES+=usr/tests/lib/libutil/.debug/trimdomain_test.debug
+OLD_DIRS+=usr/tests/lib/libxo/.debug
+OLD_FILES+=usr/tests/lib/libxo/.debug/libenc_test.so.debug
+OLD_FILES+=usr/tests/lib/libxo/.debug/test_01.debug
+OLD_FILES+=usr/tests/lib/libxo/.debug/test_02.debug
+OLD_FILES+=usr/tests/lib/libxo/.debug/test_03.debug
+OLD_FILES+=usr/tests/lib/libxo/.debug/test_04.debug
+OLD_FILES+=usr/tests/lib/libxo/.debug/test_05.debug
+OLD_FILES+=usr/tests/lib/libxo/.debug/test_06.debug
+OLD_FILES+=usr/tests/lib/libxo/.debug/test_07.debug
+OLD_FILES+=usr/tests/lib/libxo/.debug/test_08.debug
+OLD_FILES+=usr/tests/lib/libxo/.debug/test_09.debug
+OLD_FILES+=usr/tests/lib/libxo/.debug/test_10.debug
+OLD_FILES+=usr/tests/lib/libxo/.debug/test_11.debug
+OLD_DIRS+=usr/tests/lib/msun/.debug
+OLD_FILES+=usr/tests/lib/msun/.debug/acos_test.debug
+OLD_FILES+=usr/tests/lib/msun/.debug/asin_test.debug
+OLD_FILES+=usr/tests/lib/msun/.debug/atan_test.debug
+OLD_FILES+=usr/tests/lib/msun/.debug/cbrt_test.debug
+OLD_FILES+=usr/tests/lib/msun/.debug/ceil_test.debug
+OLD_FILES+=usr/tests/lib/msun/.debug/cos_test.debug
+OLD_FILES+=usr/tests/lib/msun/.debug/cosh_test.debug
+OLD_FILES+=usr/tests/lib/msun/.debug/erf_test.debug
+OLD_FILES+=usr/tests/lib/msun/.debug/exp_test.debug
+OLD_FILES+=usr/tests/lib/msun/.debug/fmod_test.debug
+OLD_FILES+=usr/tests/lib/msun/.debug/infinity_test.debug
+OLD_FILES+=usr/tests/lib/msun/.debug/ldexp_test.debug
+OLD_FILES+=usr/tests/lib/msun/.debug/log_test.debug
+OLD_FILES+=usr/tests/lib/msun/.debug/pow_test.debug
+OLD_FILES+=usr/tests/lib/msun/.debug/precision_test.debug
+OLD_FILES+=usr/tests/lib/msun/.debug/round_test.debug
+OLD_FILES+=usr/tests/lib/msun/.debug/scalbn_test.debug
+OLD_FILES+=usr/tests/lib/msun/.debug/sin_test.debug
+OLD_FILES+=usr/tests/lib/msun/.debug/sinh_test.debug
+OLD_FILES+=usr/tests/lib/msun/.debug/sqrt_test.debug
+OLD_FILES+=usr/tests/lib/msun/.debug/tan_test.debug
+OLD_FILES+=usr/tests/lib/msun/.debug/tanh_test.debug
+OLD_DIRS+=usr/tests/libexec/rtld-elf/.debug
+OLD_FILES+=usr/tests/libexec/rtld-elf/.debug/ld_library_pathfds.debug
+OLD_FILES+=usr/tests/libexec/rtld-elf/.debug/libpythagoras.so.0.debug
+OLD_FILES+=usr/tests/libexec/rtld-elf/.debug/target.debug
+OLD_DIRS+=usr/tests/sbin/devd/.debug
+OLD_FILES+=usr/tests/sbin/devd/.debug/client_test.debug
+OLD_DIRS+=usr/tests/sbin/dhclient/.debug
+OLD_FILES+=usr/tests/sbin/dhclient/.debug/option-domain-search_test.debug
+OLD_DIRS+=usr/tests/share/examples/tests/atf/.debug
+OLD_FILES+=usr/tests/share/examples/tests/atf/.debug/printf_test.debug
+OLD_DIRS+=usr/tests/share/examples/tests/plain/.debug
+OLD_FILES+=usr/tests/share/examples/tests/plain/.debug/printf_test.debug
+OLD_DIRS+=usr/tests/sys/aio/.debug
+OLD_FILES+=usr/tests/sys/aio/.debug/aio_kqueue_test.debug
+OLD_FILES+=usr/tests/sys/aio/.debug/aio_test.debug
+OLD_FILES+=usr/tests/sys/aio/.debug/lio_kqueue_test.debug
+OLD_DIRS+=usr/tests/sys/fifo/.debug
+OLD_FILES+=usr/tests/sys/fifo/.debug/fifo_create.debug
+OLD_FILES+=usr/tests/sys/fifo/.debug/fifo_io.debug
+OLD_FILES+=usr/tests/sys/fifo/.debug/fifo_misc.debug
+OLD_FILES+=usr/tests/sys/fifo/.debug/fifo_open.debug
+OLD_DIRS+=usr/tests/sys/file/.debug
+OLD_FILES+=usr/tests/sys/file/.debug/closefrom_test.debug
+OLD_FILES+=usr/tests/sys/file/.debug/dup_test.debug
+OLD_FILES+=usr/tests/sys/file/.debug/fcntlflags_test.debug
+OLD_FILES+=usr/tests/sys/file/.debug/flock_helper.debug
+OLD_FILES+=usr/tests/sys/file/.debug/ftruncate_test.debug
+OLD_FILES+=usr/tests/sys/file/.debug/newfileops_on_fork_test.debug
+OLD_DIRS+=usr/tests/sys/kern/.debug
+OLD_FILES+=usr/tests/sys/kern/.debug/kern_descrip_test.debug
+OLD_FILES+=usr/tests/sys/kern/.debug/ptrace_test.debug
+OLD_FILES+=usr/tests/sys/kern/.debug/unix_seqpacket_test.debug
+OLD_DIRS+=usr/tests/sys/kern/execve/.debug
+OLD_FILES+=usr/tests/sys/kern/execve/.debug/execve_helper.debug
+OLD_FILES+=usr/tests/sys/kern/execve/.debug/good_aout.debug
+OLD_DIRS+=usr/tests/sys/kqueue/.debug
+OLD_FILES+=usr/tests/sys/kqueue/.debug/kqtest.debug
+OLD_DIRS+=usr/tests/sys/mqueue/.debug
+OLD_FILES+=usr/tests/sys/mqueue/.debug/mqtest1.debug
+OLD_FILES+=usr/tests/sys/mqueue/.debug/mqtest2.debug
+OLD_FILES+=usr/tests/sys/mqueue/.debug/mqtest3.debug
+OLD_FILES+=usr/tests/sys/mqueue/.debug/mqtest4.debug
+OLD_FILES+=usr/tests/sys/mqueue/.debug/mqtest5.debug
+OLD_DIRS+=usr/tests/sys/netinet/.debug
+OLD_FILES+=usr/tests/sys/netinet/.debug/udp_dontroute.debug
+OLD_DIRS+=usr/tests/sys/pjdfstest/.debug
+OLD_FILES+=usr/tests/sys/pjdfstest/.debug/pjdfstest.debug
+OLD_DIRS+=usr/tests/sys/vm/.debug
+OLD_FILES+=usr/tests/sys/vm/.debug/mmap_test.debug
 # 20151015: Rename files due to file-installed-as-dir bug
 OLD_FILES+=usr/share/doc/legal/realtek
 OLD_FILES+=usr/share/doc/legal/realtek/LICENSE

From b9b0c1d2483a1067011b80e1f661bf56a2a887c5 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Fri, 16 Oct 2015 05:54:41 +0000
Subject: [PATCH 277/380] Avoid warning race with creating 'ldscripts'
 directory during build.

In r204548 the 'rm -f ldscripts' was added likely due to reading the
conditional as 'else it is a file'.  That seems unlikely though and
the more likely case is just that the directory hasn't been created yet.

Because this races with ,ssother scripts, use 'mkdir -p' which is a minimal
modification vs upstream to avoid the warning of 'File exists' if another
script creates it first.  This could replace the 'test -d' as well but
then it's more unneeded change to the upstream script.

Sponsored by:	EMC / Isilon Storage Division
---
 gnu/usr.bin/binutils/ld/genscripts.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/gnu/usr.bin/binutils/ld/genscripts.sh b/gnu/usr.bin/binutils/ld/genscripts.sh
index 5090cd3f8779..238263eb75a1 100755
--- a/gnu/usr.bin/binutils/ld/genscripts.sh
+++ b/gnu/usr.bin/binutils/ld/genscripts.sh
@@ -50,8 +50,7 @@ fi
 if test -d ldscripts; then
   true
 else
-  rm -f ldscripts
-  mkdir ldscripts
+  mkdir -p ldscripts
 fi
 
 # Set some flags for the emultempl scripts.  USE_LIBPATH will

From 76d7758adbacf4718934aa900fbabc2b970d228e Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Fri, 16 Oct 2015 05:59:59 +0000
Subject: [PATCH 278/380] Tweak the default target to not suggest 'all' since
 it really doesn't do anything useful for most users.

MFC after:	2 weeks
Sponsored by:	EMC / Isilon Storage Division
---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index aa9c05736ebb..e22d8bcdf59f 100644
--- a/Makefile
+++ b/Makefile
@@ -259,9 +259,9 @@ ${TGTS}: .PHONY
 # if they want the historic behavior.
 .MAIN:	_guard
 
-_guard:
+_guard: .PHONY
 	@echo
-	@echo "Explicit target required (use \"all\" for historic behavior)"
+	@echo "Explicit target required.  Likely \"buildworld\" is wanted.  See build(7)."
 	@echo
 	@false
 

From 09ad05077aec0f7f4ce8cab9cdb012caf6ab9807 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Fri, 16 Oct 2015 08:57:11 +0000
Subject: [PATCH 279/380] Bump version and add notice about incompatibility
 introduced by resumable send/receive support in ZFS.

---
 UPDATING        | 6 ++++++
 sys/sys/param.h | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/UPDATING b/UPDATING
index baff1f13605b..12631e190841 100644
--- a/UPDATING
+++ b/UPDATING
@@ -31,6 +31,12 @@ NOTE TO PEOPLE WHO THINK THAT FreeBSD 11.x IS SLOW:
 	disable the most expensive debugging functionality run
 	"ln -s 'abort:false,junk:false' /etc/malloc.conf".)
 
+20151015:
+	Added ZFS support for resumable send/receive changed respective
+	IOCTL API/ABI.  Compatibility ABI shims were provided for other
+	functionality, while receive require version match between world
+	and kernel.
+
 20151012:
 	If you specify SENDMAIL_MC or SENDMAIL_CF in make.conf, mergemaster
 	and etcupdate will now use this file. A custom sendmail.cf is now
diff --git a/sys/sys/param.h b/sys/sys/param.h
index 67c525f7a348..25870c321c9a 100644
--- a/sys/sys/param.h
+++ b/sys/sys/param.h
@@ -58,7 +58,7 @@
  *		in the range 5 to 9.
  */
 #undef __FreeBSD_version
-#define __FreeBSD_version 1100080	/* Master, propagated to newvers */
+#define __FreeBSD_version 1100081	/* Master, propagated to newvers */
 
 /*
  * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,

From 7f5fdd9fd0fe05175bb9505518308a90bbe9674c Mon Sep 17 00:00:00 2001
From: ed <ed@FreeBSD.org>
Date: Fri, 16 Oct 2015 10:26:15 +0000
Subject: [PATCH 280/380] Use the right variable name.

MACHINE_CPUARCH expands to aarch64 for arm64, whereas MACHINE always
corresponds to the directory name under sys/ that contains the sources
for that architecture.
---
 sys/modules/cloudabi64/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/modules/cloudabi64/Makefile b/sys/modules/cloudabi64/Makefile
index 1e1a058acc71..a340e50439c9 100644
--- a/sys/modules/cloudabi64/Makefile
+++ b/sys/modules/cloudabi64/Makefile
@@ -1,7 +1,7 @@
 # $FreeBSD$
 
 .PATH: ${.CURDIR}/../../compat/cloudabi64
-.PATH: ${.CURDIR}/../../${MACHINE_CPUARCH}/cloudabi64
+.PATH: ${.CURDIR}/../../${MACHINE}/cloudabi64
 
 KMOD=	cloudabi64
 SRCS=	cloudabi64_fd.c cloudabi64_poll.c cloudabi64_sock.c \

From 7cd58b9488563c5becfc657f8fdc56f141ae9aec Mon Sep 17 00:00:00 2001
From: des <des@FreeBSD.org>
Date: Fri, 16 Oct 2015 12:21:44 +0000
Subject: [PATCH 281/380] Fix two bugs in HTTPS tunnelling:  - If the proxy
 returns a non-200 result, set the error code accordingly    so the caller /
 user gets a somewhat meaningful error message.  - Consume and discard any
 HTTP response header following the result line.

PR:		194483
Tested by:	Fabian Keil <fk@fabiankeil.de>
MFC after:	1 week
---
 lib/libfetch/http.c | 36 ++++++++++++++++++++++++++++++++----
 1 file changed, 32 insertions(+), 4 deletions(-)

diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c
index 91ca6a399739..f6a74540c8fc 100644
--- a/lib/libfetch/http.c
+++ b/lib/libfetch/http.c
@@ -1376,8 +1376,12 @@ http_connect(struct url *URL, struct url *purl, const char *flags)
 {
 	struct url *curl;
 	conn_t *conn;
+	hdr_t h;
+	http_headerbuf_t headerbuf;
+	const char *p;
 	int verbose;
 	int af, val;
+	int serrno;
 
 #ifdef INET6
 	af = AF_UNSPEC;
@@ -1398,6 +1402,7 @@ http_connect(struct url *URL, struct url *purl, const char *flags)
 	if ((conn = fetch_connect(curl->host, curl->port, af, verbose)) == NULL)
 		/* fetch_connect() has already set an error code */
 		return (NULL);
+	init_http_headerbuf(&headerbuf);
 	if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 && purl) {
 		http_cmd(conn, "CONNECT %s:%d HTTP/1.1",
 		    URL->host, URL->port);
@@ -1405,10 +1410,26 @@ http_connect(struct url *URL, struct url *purl, const char *flags)
 		    URL->host, URL->port);
 		http_cmd(conn, "");
 		if (http_get_reply(conn) != HTTP_OK) {
-			fetch_close(conn);
-			return (NULL);
+			http_seterr(conn->err);
+			goto ouch;
 		}
-		http_get_reply(conn);
+		/* Read and discard the rest of the proxy response */
+		if (fetch_getln(conn) < 0) {
+			fetch_syserr();
+			goto ouch;
+		}
+		do {
+			switch ((h = http_next_header(conn, &headerbuf, &p))) {
+			case hdr_syserror:
+				fetch_syserr();
+				goto ouch;
+			case hdr_error:
+				http_seterr(HTTP_PROTOCOL_ERROR);
+				goto ouch;
+			default:
+				/* ignore */ ;
+			}
+		} while (h < hdr_end);
 	}
 	if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 &&
 	    fetch_ssl(conn, URL, verbose) == -1) {
@@ -1416,13 +1437,20 @@ http_connect(struct url *URL, struct url *purl, const char *flags)
 		/* grrr */
 		errno = EAUTH;
 		fetch_syserr();
-		return (NULL);
+		goto ouch;
 	}
 
 	val = 1;
 	setsockopt(conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val, sizeof(val));
 
+	clean_http_headerbuf(&headerbuf);
 	return (conn);
+ouch:
+	serrno = errno;
+	clean_http_headerbuf(&headerbuf);
+	fetch_close(conn);
+	errno = serrno;
+	return (NULL);
 }
 
 static struct url *

From 095b2710edf0a3e95923ce849fcd9e3b5a1b5d63 Mon Sep 17 00:00:00 2001
From: des <des@FreeBSD.org>
Date: Fri, 16 Oct 2015 12:53:22 +0000
Subject: [PATCH 282/380] Use fopen()'s newfangled "e" flag instead of explicit
 fcntl() calls.

PR:		199801
Submitted by:	Jukka Ukkonen <jau@iki.fi>
MFC after:	1 week
---
 lib/libfetch/file.c | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/lib/libfetch/file.c b/lib/libfetch/file.c
index 8c1d4042dcc6..7b6462f5b69c 100644
--- a/lib/libfetch/file.c
+++ b/lib/libfetch/file.c
@@ -48,7 +48,7 @@ fetchXGetFile(struct url *u, struct url_stat *us, const char *flags)
 	if (us && fetchStatFile(u, us, flags) == -1)
 		return (NULL);
 
-	f = fopen(u->doc, "r");
+	f = fopen(u->doc, "re");
 
 	if (f == NULL) {
 		fetch_syserr();
@@ -61,7 +61,6 @@ fetchXGetFile(struct url *u, struct url_stat *us, const char *flags)
 		return (NULL);
 	}
 
-	fcntl(fileno(f), F_SETFD, FD_CLOEXEC);
 	return (f);
 }
 
@@ -77,9 +76,9 @@ fetchPutFile(struct url *u, const char *flags)
 	FILE *f;
 
 	if (CHECK_FLAG('a'))
-		f = fopen(u->doc, "a");
+		f = fopen(u->doc, "ae");
 	else
-		f = fopen(u->doc, "w+");
+		f = fopen(u->doc, "w+e");
 
 	if (f == NULL) {
 		fetch_syserr();
@@ -92,7 +91,6 @@ fetchPutFile(struct url *u, const char *flags)
 		return (NULL);
 	}
 
-	fcntl(fileno(f), F_SETFD, FD_CLOEXEC);
 	return (f);
 }
 

From f8f954a38eb13db0944f3eb5b8cc3493f1d544ff Mon Sep 17 00:00:00 2001
From: cy <cy@FreeBSD.org>
Date: Fri, 16 Oct 2015 14:04:16 +0000
Subject: [PATCH 283/380] Add default leap-seconds file. This should help ntp
 networks get the leap second date correct

Updates to the file can be obtained from ftp://time.nist.gov/pub/ or
ftp://tycho.usno.navy.mil/pub/ntp/.

Suggested by:	dwmalone
Reviewed by:	roberto, dwmalone, delphij
Approved by:	roberto
MFC after:	1 week
---
 etc/Makefile           |   3 ++
 etc/mtree/BSD.var.dist |   2 +
 etc/ntp.conf           |   5 ++
 etc/ntp/Makefile       |  10 ++++
 etc/ntp/leap-seconds   | 119 +++++++++++++++++++++++++++++++++++++++++
 5 files changed, 139 insertions(+)
 create mode 100644 etc/ntp/Makefile
 create mode 100644 etc/ntp/leap-seconds

diff --git a/etc/Makefile b/etc/Makefile
index 81b9a1973f2b..24c83d1eb2a9 100644
--- a/etc/Makefile
+++ b/etc/Makefile
@@ -240,6 +240,9 @@ distribution:
 	${_+_}cd ${.CURDIR}/defaults; ${MAKE} install
 	${_+_}cd ${.CURDIR}/devd; ${MAKE} install
 	${_+_}cd ${.CURDIR}/gss; ${MAKE} install
+.if ${MK_NTP} != "no"
+	${_+_}cd ${.CURDIR}/ntp; ${MAKE} install
+.endif
 	${_+_}cd ${.CURDIR}/periodic; ${MAKE} install
 .if ${MK_PKGBOOTSTRAP} != "no"
 	${_+_}cd ${.CURDIR}/pkg; ${MAKE} install
diff --git a/etc/mtree/BSD.var.dist b/etc/mtree/BSD.var.dist
index 7d4af0be5ab6..3a5852905f32 100644
--- a/etc/mtree/BSD.var.dist
+++ b/etc/mtree/BSD.var.dist
@@ -46,6 +46,8 @@
         ..
         ipf             mode=0700
         ..
+        ntp             mode=0700
+        ..
         pkg
         ..
         ports
diff --git a/etc/ntp.conf b/etc/ntp.conf
index ea3987766058..c4ad0aa1bde3 100644
--- a/etc/ntp.conf
+++ b/etc/ntp.conf
@@ -77,3 +77,8 @@ restrict 127.127.1.0
 #
 #server 127.127.1.0
 #fudge 127.127.1.0 stratum 10
+
+# See http://support.ntp.org/bin/view/Support/ConfiguringNTP#Section_6.14.
+# for documentation regarding leapfile. Updates to the file can be obtained
+# from ftp://time.nist.gov/pub/ or ftp://tycho.usno.navy.mil/pub/ntp/.
+leapfile "/etc/ntp/leap-seconds"
diff --git a/etc/ntp/Makefile b/etc/ntp/Makefile
new file mode 100644
index 000000000000..f1aff4f9006b
--- /dev/null
+++ b/etc/ntp/Makefile
@@ -0,0 +1,10 @@
+# $FreeBSD$
+
+NO_OBJ=
+
+FILES=	leap-seconds
+
+FILESDIR=	/etc/ntp
+FILESMODE=	644
+
+.include <bsd.prog.mk>
diff --git a/etc/ntp/leap-seconds b/etc/ntp/leap-seconds
new file mode 100644
index 000000000000..b8b41f2d1179
--- /dev/null
+++ b/etc/ntp/leap-seconds
@@ -0,0 +1,119 @@
+#
+# $FreeBSD$
+#
+#	ATOMIC TIME.
+#	The Coordinated Universal Time (UTC) is the reference time scale derived 
+#	from The "Temps Atomique International" (TAI) calculated by the Bureau 
+#	International des Poids et Mesures (BIPM) using a worldwide network of atomic 
+#	clocks. UTC differs from TAI by an integer number of seconds; it is the basis 
+#	of all activities in the world. 
+#
+#
+#	ASTRONOMICAL TIME (UT1) is the time scale based on the rate of rotation of the earth. 
+#	It is now mainly derived from Very Long Baseline Interferometry (VLBI). The various 
+#	irregular fluctuations progressively detected in the rotation rate of the Earth lead 
+#	in 1972 to the replacement of UT1 by UTC as the reference time scale. 
+#
+#
+#	LEAP SECOND
+#	Atomic clocks are more stable than the rate of the earth rotatiob since the later 
+#	undergoes a full range of geophysical perturbations at various time scales (lunisolar 
+#	and core-mantle torques,atmospheric and oceanic effetcs, ...)
+#	Leap seconds are needed to keep the two time scales in agreement, i.e. UT1-UTC smaller 
+#	than 0.9 second. So, when necessary a "leap second" is introduced in UTC.
+#	Since the adoption of this system in 1972 it has been necessary to add 26 seconds to UTC, 
+#	firstly due to the initial choice of the value of the second (1/86400 mean solar day of 
+#	the year 1820) and secondly to the general slowing down of the Earth's rotation. It is 
+#	theorically possible to have a negative leap second (a second removed from UTC), but so far, 
+#	all leap seconds have been positive (a second has been added to UTC). Based on what we know about the earth's rotation, 
+#	it is unlikely that we will ever have a negative leap second.
+#
+#
+#	HISTORY
+#	The first leap second was added on June 30, 1972. Until 2000, it was necessary in average to add a leap second at a rate 
+#	of 1 to 2 years. Since 2000, due to the fact that the earth rate of rotation is accelerating, leap seconds are introduced 
+#	with an average  frequency of 3 to 4 years.
+#
+#
+#	RESPONSABILITY OF THE DECISION TO INTRODUCE A LEAP SECOND IN UTC
+#	The decision to introduce a leap second in UTC is the responsibility of the Earth Orientation Center of 
+#	the International Earth Rotation and reference System Service (IERS). This center is located at Paris 
+#	Observatory. According to international agreements, leap second date have to occur at fixed date :
+#	first preference is given to the end of December and June, and second preference at the end of March 
+#	and September. Since the system was introduced in 1972, only dates in June and December were used.
+#
+#		Questions or comments to:
+#			Daniel Gambis, daniel.gambis@obspm.fr
+#			Christian Bizouard:  christian.bizouard@obspm.fr
+#			Earth orientation Center of the IERS
+#			Paris Observatory, France	
+#			
+#
+#
+#	VALIDITY OF THE FILE
+#	It is important to express the validity of the file. These next two dates are
+#	given in units of seconds since 1900.0.
+#
+#	1) Last update of the file. 
+#
+#	Updated through IERS Bulletin C (ftp://hpiers.obspm.fr/iers/bul/bulc/bulletinc.dat)
+#
+#	The following line shows the last update of this file in NTP timestamp: 
+#
+#$	3645216000 
+#	 
+#	2) Expiration date of the file given on a semi-annual basis: last June or last December
+#
+#	File expires on 28 December 2015
+#
+#	Expire date in NTP timestamp: 
+#
+#@	3660249600
+#
+#
+#	LIST OF LEAP SECONDS
+#	NTP timestamp (X parameter) is the number of seconds since 1900.0
+#
+#	MJD: The Modified Julian Day number. MJD = X/86400 + 15020
+#
+#	DTAI: The difference DTAI= TAI-UTC in units of seconds
+#	It is the quantity to add to UTC to get the time in TAI
+#
+#	Day Month Year : epoch in clear
+#
+#NTP Time      DTAI    Day Month Year
+#
+2272060800      10      # 1 Jan 1972
+2287785600      11      # 1 Jul 1972
+2303683200      12      # 1 Jan 1973
+2335219200      13      # 1 Jan 1974
+2366755200      14      # 1 Jan 1975
+2398291200      15      # 1 Jan 1976
+2429913600      16      # 1 Jan 1977
+2461449600      17      # 1 Jan 1978
+2492985600      18      # 1 Jan 1979
+2524521600      19      # 1 Jan 1980
+2571782400      20      # 1 Jul 1981
+2603318400      21      # 1 Jul 1982
+2634854400      22      # 1 Jul 1983
+2698012800      23      # 1 Jul 1985
+2776982400      24      # 1 Jan 1988
+2840140800      25      # 1 Jan 1990
+2871676800      26      # 1 Jan 1991
+2918937600      27      # 1 Jul 1992
+2950473600      28      # 1 Jul 1993
+2982009600      29      # 1 Jul 1994
+3029443200      30      # 1 Jan 1996
+3076704000      31      # 1 Jul 1997
+3124137600      32      # 1 Jan 1999
+3345062400      33      # 1 Jan 2006
+3439756800      34      # 1 Jan 2009
+3550089600      35      # 1 Jul 2012
+3644697600      36      # 1 Jul 2015
+#
+#	In order to verify the integrity of this file, a hash code 
+#	has been generated. For more information how to use
+#	this hash code, please consult the README file under the 
+#	'sha' repertory.
+#
+#h	620ba8af 37900668 95ac09ba d77640f9 6fd75493

From 0fdc52ff38d78fb152119b65b7b28259562e8a74 Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Fri, 16 Oct 2015 14:55:39 +0000
Subject: [PATCH 284/380] If falloc_caps() failed, cleanup needs to be
 performed.  This is a bug in r289026.

Sponsored by:	The FreeBSD Foundation
---
 sys/kern/kern_fork.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
index 0e28072f010f..bb35b9b25d8f 100644
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -848,7 +848,7 @@ fork1(struct thread *td, int flags, int pages, struct proc **procp,
 	if (flags & RFPROCDESC) {
 		error = falloc_caps(td, &fp_procdesc, procdescp, 0, fcaps);
 		if (error != 0)
-			return (error);
+			goto fail2;
 	}
 
 	mem_charged = 0;

From 8dec90b4ee4b279723902115d3f897aa0b51f4f1 Mon Sep 17 00:00:00 2001
From: emaste <emaste@FreeBSD.org>
Date: Fri, 16 Oct 2015 15:19:36 +0000
Subject: [PATCH 285/380] newfs_msdos: prefer snprintf to sprintf

Obtained from:	NetBSD
Sponsored by:	The FreeBSD Foundation
---
 sbin/newfs_msdos/mkfs_msdos.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sbin/newfs_msdos/mkfs_msdos.c b/sbin/newfs_msdos/mkfs_msdos.c
index f4222700365b..282803599742 100644
--- a/sbin/newfs_msdos/mkfs_msdos.c
+++ b/sbin/newfs_msdos/mkfs_msdos.c
@@ -585,7 +585,7 @@ int mkfs_msdos(const char *fname, const char *dtype,
 			  (u_int)tm->tm_min));
 		mk4(bsx->exVolumeID, x);
 		mklabel(bsx->exVolumeLabel, o.volume_label ? o.volume_label : "NO NAME");
-		sprintf(buf, "FAT%u", fat);
+		snprintf(buf, sizeof(buf), "FAT%u", fat);
 		setstr(bsx->exFileSysType, buf, sizeof(bsx->exFileSysType));
 		if (!o.bootstrap) {
 		    x1 += sizeof(struct bsx);

From c010511ad0603785af0d2aa799f3086742984702 Mon Sep 17 00:00:00 2001
From: jkim <jkim@FreeBSD.org>
Date: Fri, 16 Oct 2015 17:50:36 +0000
Subject: [PATCH 286/380] Add support for ARM EABI.

MFC after:	1 week
---
 usr.bin/ldd/ldd.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/usr.bin/ldd/ldd.c b/usr.bin/ldd/ldd.c
index 2f48e04a911d..c8e0be3d4552 100644
--- a/usr.bin/ldd/ldd.c
+++ b/usr.bin/ldd/ldd.c
@@ -386,9 +386,20 @@ is_executable(const char *fname, int fd, int *is_shlib, int *type)
 			return (0);
 		}
 		if (hdr.elf.e_type == ET_DYN) {
-			if (hdr.elf.e_ident[EI_OSABI] == ELFOSABI_FREEBSD) {
+			switch (hdr.elf.e_ident[EI_OSABI]) {
+			case ELFOSABI_FREEBSD:
 				*is_shlib = 1;
 				return (1);
+#ifdef __ARM_EABI__
+			case ELFOSABI_NONE:
+				if (hdr.elf.e_machine != EM_ARM)
+					break;
+				if (EF_ARM_EABI_VERSION(hdr.elf.e_flags) <
+				    EF_ARM_EABI_FREEBSD_MIN)
+					break;
+				*is_shlib = 1;
+				return (1);
+#endif
 			}
 			warnx("%s: not a FreeBSD ELF shared object", fname);
 			return (0);

From dfffba1e9869f54427035292cfa8093ad6156747 Mon Sep 17 00:00:00 2001
From: scottl <scottl@FreeBSD.org>
Date: Fri, 16 Oct 2015 17:56:43 +0000
Subject: [PATCH 287/380] Remove _FreeBSD_version check for something that was
 only an issue with 9-CURRENT.

Obtained from:  Netlfix, Inc
MFC after:      3 days
---
 sys/dev/mpr/mpr.c     | 2 --
 sys/dev/mpr/mprvar.h  | 7 +------
 sys/dev/mps/mps.c     | 2 --
 sys/dev/mps/mps_sas.h | 1 -
 sys/dev/mps/mpsvar.h  | 7 +------
 5 files changed, 2 insertions(+), 17 deletions(-)

diff --git a/sys/dev/mpr/mpr.c b/sys/dev/mpr/mpr.c
index 07e73c54e8be..ea8249b79888 100644
--- a/sys/dev/mpr/mpr.c
+++ b/sys/dev/mpr/mpr.c
@@ -1471,11 +1471,9 @@ mpr_setup_sysctl(struct mpr_softc *sc)
 	    OID_AUTO, "enable_ssu", CTLFLAG_RW, &sc->enable_ssu, 0,
 	    "enable SSU to SATA SSD/HDD at shutdown");
 
-#if __FreeBSD_version >= 900030
 	SYSCTL_ADD_UQUAD(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
 	    OID_AUTO, "chain_alloc_fail", CTLFLAG_RD,
 	    &sc->chain_alloc_fail, "chain allocation failures");
-#endif //FreeBSD_version >= 900030
 
 	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
 	    OID_AUTO, "spinup_wait_time", CTLFLAG_RD,
diff --git a/sys/dev/mpr/mprvar.h b/sys/dev/mpr/mprvar.h
index acac44c5baec..306324b674a6 100644
--- a/sys/dev/mpr/mprvar.h
+++ b/sys/dev/mpr/mprvar.h
@@ -265,9 +265,7 @@ struct mpr_softc {
 	int				chain_free_lowwater;
 	u_int				enable_ssu;
 	int				spinup_wait_time;
-#if __FreeBSD_version >= 900030
 	uint64_t			chain_alloc_fail;
-#endif
 	struct sysctl_ctx_list		sysctl_ctx;
 	struct sysctl_oid		*sysctl_tree;
 	char                            fw_version[16];
@@ -451,11 +449,8 @@ mpr_alloc_chain(struct mpr_softc *sc)
 		sc->chain_free--;
 		if (sc->chain_free < sc->chain_free_lowwater)
 			sc->chain_free_lowwater = sc->chain_free;
-	}
-#if __FreeBSD_version >= 900030
-	else
+	} else
 		sc->chain_alloc_fail++;
-#endif
 	return (chain);
 }
 
diff --git a/sys/dev/mps/mps.c b/sys/dev/mps/mps.c
index 7f73162830a9..4080fc678fc1 100644
--- a/sys/dev/mps/mps.c
+++ b/sys/dev/mps/mps.c
@@ -1468,11 +1468,9 @@ mps_setup_sysctl(struct mps_softc *sc)
 	    OID_AUTO, "enable_ssu", CTLFLAG_RW, &sc->enable_ssu, 0,
 	    "enable SSU to SATA SSD/HDD at shutdown");
 
-#if __FreeBSD_version >= 900030
 	SYSCTL_ADD_UQUAD(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
 	    OID_AUTO, "chain_alloc_fail", CTLFLAG_RD,
 	    &sc->chain_alloc_fail, "chain allocation failures");
-#endif //FreeBSD_version >= 900030
 
 	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
 	    OID_AUTO, "spinup_wait_time", CTLFLAG_RD,
diff --git a/sys/dev/mps/mps_sas.h b/sys/dev/mps/mps_sas.h
index d03258d9a7c9..825574574edc 100644
--- a/sys/dev/mps/mps_sas.h
+++ b/sys/dev/mps/mps_sas.h
@@ -58,7 +58,6 @@ struct mpssas_target {
 
 	uint16_t	tid;
 	SLIST_HEAD(, mpssas_lun) luns;
-	TAILQ_HEAD(, mps_command) commands;
 	struct mps_command *tm;
 	TAILQ_HEAD(, mps_command) timedout_commands;
 	uint16_t        exp_dev_handle;
diff --git a/sys/dev/mps/mpsvar.h b/sys/dev/mps/mpsvar.h
index 7a43e62aa12b..96bb4e1cfa64 100644
--- a/sys/dev/mps/mpsvar.h
+++ b/sys/dev/mps/mpsvar.h
@@ -283,9 +283,7 @@ struct mps_softc {
 	int				chain_free_lowwater;
 	u_int				enable_ssu;
 	int				spinup_wait_time;
-#if __FreeBSD_version >= 900030
 	uint64_t			chain_alloc_fail;
-#endif
 	struct sysctl_ctx_list		sysctl_ctx;
 	struct sysctl_oid		*sysctl_tree;
 	char                            fw_version[16];
@@ -483,11 +481,8 @@ mps_alloc_chain(struct mps_softc *sc)
 		sc->chain_free--;
 		if (sc->chain_free < sc->chain_free_lowwater)
 			sc->chain_free_lowwater = sc->chain_free;
-	}
-#if __FreeBSD_version >= 900030
-	else
+	} else
 		sc->chain_alloc_fail++;
-#endif
 	return (chain);
 }
 

From edede50e854e2ed52e55491c060b8bef667e3c0b Mon Sep 17 00:00:00 2001
From: scottl <scottl@FreeBSD.org>
Date: Fri, 16 Oct 2015 20:18:12 +0000
Subject: [PATCH 288/380] Revert an extra hunk that crept into the last commit.

Submitted by:	emax
Obtained from:	Netflix, Inc.
MFC after:	3 days
---
 sys/dev/mps/mps_sas.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sys/dev/mps/mps_sas.h b/sys/dev/mps/mps_sas.h
index 825574574edc..d03258d9a7c9 100644
--- a/sys/dev/mps/mps_sas.h
+++ b/sys/dev/mps/mps_sas.h
@@ -58,6 +58,7 @@ struct mpssas_target {
 
 	uint16_t	tid;
 	SLIST_HEAD(, mpssas_lun) luns;
+	TAILQ_HEAD(, mps_command) commands;
 	struct mps_command *tm;
 	TAILQ_HEAD(, mps_command) timedout_commands;
 	uint16_t        exp_dev_handle;

From a6fb14b1eb3656f76357b97ba97f1e0ac23d7907 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Fri, 16 Oct 2015 20:23:04 +0000
Subject: [PATCH 289/380] Remove .MAKE from targets that do more than just run
 sub-makes, such as calling rm or mtree.

MFC after:	1 week
Sponsored by:	EMC / Isilon Storage Division
---
 Makefile.inc1 | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Makefile.inc1 b/Makefile.inc1
index 53c407fa24f9..256a6f81a137 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -1063,7 +1063,7 @@ INSTALLKERNEL= ${_kernel}
 .endif
 .endfor
 
-buildkernel ${WMAKE_TGTS:N_worldtmp} ${.ALLTARGETS:M_*:N_worldtmp}: .MAKE .PHONY
+buildkernel ${WMAKE_TGTS:N_worldtmp:Nbuild32} ${.ALLTARGETS:M_*:N_worldtmp}: .MAKE .PHONY
 
 #
 # buildkernel
@@ -1493,7 +1493,7 @@ build-tools: .MAKE
 #
 # kernel-tools: Build kernel-building tools
 #
-kernel-tools: .MAKE
+kernel-tools:
 	mkdir -p ${MAKEOBJDIRPREFIX}/usr
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.usr.dist \
 	    -p ${MAKEOBJDIRPREFIX}/usr >/dev/null
@@ -1593,7 +1593,7 @@ NXBMAKE=	${NXBENV} ${MAKE} \
 # For non-clang enabled targets that are still using the in tree gcc
 # we must build a gperf binary for one instance of its Makefiles.  On
 # clang-enabled systems, the gperf binary is obsolete.
-native-xtools: .MAKE
+native-xtools:
 .if ${MK_GCC_BOOTSTRAP} != "no"
 	mkdir -p ${OBJTREE}/gperf_for_gcc/usr/bin
 	${_+_}@${ECHODIR} "===> ${_gperf} (obj,depend,all,install)"; \

From 57164fab2d336d9e041f19e41305834a27a05657 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Fri, 16 Oct 2015 20:59:03 +0000
Subject: [PATCH 290/380] Tweak the guard more to suggest 'all' if
 SUBDIR_OVERRIDE is specified.  In that case 'all' does make sense.

MFC after:	2 weeks
X-MFC-With:	r289411
Sponsored by:	EMC / Isilon Storage Division
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index e22d8bcdf59f..c3326b7ddc43 100644
--- a/Makefile
+++ b/Makefile
@@ -261,7 +261,7 @@ ${TGTS}: .PHONY
 
 _guard: .PHONY
 	@echo
-	@echo "Explicit target required.  Likely \"buildworld\" is wanted.  See build(7)."
+	@echo "Explicit target required.  Likely \"${SUBDIR_OVERRIDE:Dall:Ubuildworld}\" is wanted.  See build(7)."
 	@echo
 	@false
 

From f156796180ffc21e4e6afc8c9e4dd0337454681e Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Fri, 16 Oct 2015 21:09:15 +0000
Subject: [PATCH 291/380] Correct a bitrotted comment about installworld order
 requirements.

The case of make(1) using a new /bin/sh issue was fixed in r173219 when ITOOLS
was introduced.

There are still issues with mid-install errors leaving a system unusable that
are currently non-trivial to solve.  The safest ordering requires installing
rtld, libc and libthr (in that order) before anything else.  We don't do that
now though.  Much improvement is needed here still.

Discussed with:	kip and kan (rtld/library ordering)
MFC after:	1 week
Sponsored by:	EMC / Isilon Storage Division
---
 Makefile.inc1 | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/Makefile.inc1 b/Makefile.inc1
index 256a6f81a137..8b54edf4c457 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -50,11 +50,14 @@
 .include <bsd.arch.inc.mk>
 .include <bsd.compiler.mk>
 
-# We must do lib/ and libexec/ before bin/, because if installworld
-# installs a new /bin/sh, the 'make' command will *immediately*
-# use that new version.  And the new (dynamically-linked) /bin/sh
-# will expect to find appropriate libraries in /lib and /libexec.
-#
+# We must do lib/ and libexec/ before bin/ in case of a mid-install error to
+# keep the users system reasonably usable.  For static->dynamic root upgrades,
+# we don't want to install a dynamic binary without rtld and the needed
+# libraries.  More commonly, for dynamic root, we don't want to install a
+# binary that requires a newer library version that hasn't been installed yet.
+# This ordering is not a guarantee though.  The only guarantee of a working
+# system here would require fine-grained ordering of all components based
+# on their dependencies.
 SRCDIR?=	${.CURDIR}
 .if defined(SUBDIR_OVERRIDE)
 SUBDIR=	${SUBDIR_OVERRIDE}

From 8c5d7c6344268b268b4993b99ef0c9b7371b81ef Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Fri, 16 Oct 2015 22:41:31 +0000
Subject: [PATCH 292/380] Remove lockf as an ITOOL.

It was added in r152006 to handle serializing access of info/dir when
installing INFO files.  We no longer support INFO files since r276551
though.

Sponsored by:	EMC / Isilon Storage Division
---
 Makefile.inc1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile.inc1 b/Makefile.inc1
index 8b54edf4c457..a6ea0858a5ad 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -823,7 +823,7 @@ _zoneinfo=	zic tzsetup
 
 ITOOLS=	[ awk cap_mkdb cat chflags chmod chown cmp cp \
 	date echo egrep find grep id install ${_install-info} \
-	ln lockf make mkdir mtree mv pwd_mkdb \
+	ln make mkdir mtree mv pwd_mkdb \
 	rm sed services_mkdb sh strip sysctl test true uname wc ${_zoneinfo} \
 	${LOCAL_ITOOLS}
 

From 76795b74c93093a4d93530c380d2de4544e5b66c Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Fri, 16 Oct 2015 23:53:37 +0000
Subject: [PATCH 293/380] Fix adding manpages installed by LOCAL_DIRS to whatis
 file.

The ordering of 'etc' in the install has a long history dating back to the
first time it was realized it needed to be "last" in r4486.  That commit
still left it before LOCAL_DIRS though.  By having it before LOCAL_DIRS
any manpages they install were not being added to the whatis database in the
install image.  They would likely show up in the file after a periodic
rebuild of the file though.

Currently the whatis file is built by an 'afterinstall' hook in etc/Makefile
that calls share/man's 'makedb' target.

MFC after:	2 weeks
Sponsored by:	EMC / Isilon Storage Division
---
 Makefile.inc1 | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/Makefile.inc1 b/Makefile.inc1
index a6ea0858a5ad..2cfea9e422a7 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -88,10 +88,6 @@ SUBDIR+=	tests
 .if ${MK_OFED} != "no"
 SUBDIR+=contrib/ofed
 .endif
-#
-# We must do etc/ last for install/distribute to work.
-#
-SUBDIR+=etc
 
 # Local directories are last, since it is nice to at least get the base
 # system rebuilt before you do them.
@@ -115,6 +111,12 @@ SUBDIR+=	${_DIR}
 .endfor
 .endif
 
+# We must do etc/ last as it hooks into building the man whatis file
+# by calling 'makedb' in share/man.  This is only relevant for
+# install/distribute so they build the whatis file after every manpage is
+# installed.
+SUBDIR+=etc
+
 .if defined(NOCLEAN)
 NO_CLEAN=	${NOCLEAN}
 .endif

From 514260b6e74432abb7deca6b56c7c033ae57b669 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sat, 17 Oct 2015 03:13:22 +0000
Subject: [PATCH 294/380] Unify posixshm.c and shm_test.c

Convert both testcases over to ATF

Don't use hardcoded paths to /tmp; use mkstemp to generate
temporary paths for non-SHM_ANON created shm objects.
---
 tests/sys/posixshm/Makefile        |   6 +-
 tests/sys/posixshm/posixshm.c      | 627 ------------------------
 tests/sys/posixshm/posixshm_test.c | 739 +++++++++++++++++++++++++++++
 tests/sys/posixshm/shm_test.c      | 164 -------
 tests/sys/posixshm/test.c          | 128 -----
 tests/sys/posixshm/test.h          |  59 ---
 6 files changed, 740 insertions(+), 983 deletions(-)
 delete mode 100644 tests/sys/posixshm/posixshm.c
 create mode 100644 tests/sys/posixshm/posixshm_test.c
 delete mode 100644 tests/sys/posixshm/shm_test.c
 delete mode 100644 tests/sys/posixshm/test.c
 delete mode 100644 tests/sys/posixshm/test.h

diff --git a/tests/sys/posixshm/Makefile b/tests/sys/posixshm/Makefile
index 7ddd7f0e87fd..2acdfa4f64b5 100644
--- a/tests/sys/posixshm/Makefile
+++ b/tests/sys/posixshm/Makefile
@@ -2,10 +2,6 @@
 
 TESTSDIR=	${TESTSBASE}/sys/posixshm
 
-TAP_TESTS_C+=	posixshm_test
-PLAIN_TESTS_C+=	posixshm_test2
-
-SRCS.posixshm_test=	posixshm.c test.c
-SRCS.posixshm_test2=	shm_test.c
+ATF_TESTS_C+=	posixshm_test
 
 .include <bsd.test.mk>
diff --git a/tests/sys/posixshm/posixshm.c b/tests/sys/posixshm/posixshm.c
deleted file mode 100644
index 6f4c3069397f..000000000000
--- a/tests/sys/posixshm/posixshm.c
+++ /dev/null
@@ -1,627 +0,0 @@
-/*-
- * Copyright (c) 2006 Robert N. M. Watson
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/mman.h>
-#include <sys/resource.h>
-#include <sys/stat.h>
-#include <sys/syscall.h>
-#include <sys/wait.h>
-
-#include <errno.h>
-#include <fcntl.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-
-#include "test.h"
-
-#define	TEST_PATH	"/tmp/posixshm_regression_test"
-
-/*
- * Attempt a shm_open() that should fail with an expected error of 'error'.
- */
-static void
-shm_open_should_fail(const char *path, int flags, mode_t mode, int error)
-{
-	int fd;
-
-	fd = shm_open(path, flags, mode);
-	if (fd >= 0) {
-		fail_err("shm_open() didn't fail");
-		close(fd);
-		return;
-	}
-	if (errno != error) {
-		fail_errno("shm_open");
-		return;
-	}
-	pass();
-}
-
-/*
- * Attempt a shm_unlink() that should fail with an expected error of 'error'.
- */
-static void
-shm_unlink_should_fail(const char *path, int error)
-{
-
-	if (shm_unlink(path) >= 0) {
-		fail_err("shm_unlink() didn't fail");
-		return;
-	}
-	if (errno != error) {
-		fail_errno("shm_unlink");
-		return;
-	}
-	pass();
-}
-
-/*
- * Open the test object and write '1' to the first byte.  Returns valid fd
- * on success and -1 on failure.
- */
-static int
-scribble_object(void)
-{
-	char *page;
-	int fd;
-
-	fd = shm_open(TEST_PATH, O_CREAT | O_EXCL | O_RDWR, 0777);
-	if (fd < 0 && errno == EEXIST) {
-		if (shm_unlink(TEST_PATH) < 0) {
-			fail_errno("shm_unlink");
-			return (-1);
-		}
-		fd = shm_open(TEST_PATH, O_CREAT | O_EXCL | O_RDWR, 0777);
-	}
-	if (fd < 0) {
-		fail_errno("shm_open");
-		return (-1);
-	}
-	if (ftruncate(fd, getpagesize()) < 0) {
-		fail_errno("ftruncate");
-		close(fd);
-		shm_unlink(TEST_PATH);
-		return (-1);
-	}
-
-	page = mmap(0, getpagesize(), PROT_READ | PROT_WRITE, MAP_SHARED, fd,
-	    0);
-	if (page == MAP_FAILED) {
-		fail_errno("mmap");
-		close(fd);
-		shm_unlink(TEST_PATH);
-		return (-1);
-	}
-
-	page[0] = '1';
-
-	if (munmap(page, getpagesize()) < 0) {
-		fail_errno("munmap");
-		close(fd);
-		shm_unlink(TEST_PATH);
-		return (-1);
-	}
-
-	return (fd);
-}
-
-static void
-remap_object(void)
-{
-	char *page;
-	int fd;
-
-	fd = scribble_object();
-	if (fd < 0)
-		return;
-
-	page = mmap(0, getpagesize(), PROT_READ | PROT_WRITE, MAP_SHARED, fd,
-	    0);
-	if (page == MAP_FAILED) {		
-		fail_errno("mmap(2)");
-		close(fd);
-		shm_unlink(TEST_PATH);
-		return;
-	}
-
-	if (page[0] != '1') {		
-		fail_err("missing data");
-		close(fd);
-		shm_unlink(TEST_PATH);
-		return;
-	}
-
-	close(fd);
-	if (munmap(page, getpagesize()) < 0) {
-		fail_errno("munmap");
-		shm_unlink(TEST_PATH);
-		return;
-	}
-
-	if (shm_unlink(TEST_PATH) < 0) {
-		fail_errno("shm_unlink");
-		return;
-	}
-
-	pass();
-}
-TEST(remap_object, "remap object");
-
-static void
-reopen_object(void)
-{
-	char *page;
-	int fd;
-
-	fd = scribble_object();
-	if (fd < 0)
-		return;
-	close(fd);
-
-	fd = shm_open(TEST_PATH, O_RDONLY, 0777);
-	if (fd < 0) {
-		fail_errno("shm_open(2)");
-		shm_unlink(TEST_PATH);
-		return;
-	}
-	page = mmap(0, getpagesize(), PROT_READ, MAP_SHARED, fd, 0);
-	if (page == MAP_FAILED) {
-		fail_errno("mmap(2)");
-		close(fd);
-		shm_unlink(TEST_PATH);
-		return;
-	}
-
-	if (page[0] != '1') {
-		fail_err("missing data");
-		munmap(page, getpagesize());
-		close(fd);
-		shm_unlink(TEST_PATH);
-		return;
-	}
-
-	munmap(page, getpagesize());
-	close(fd);
-	shm_unlink(TEST_PATH);
-	pass();
-}
-TEST(reopen_object, "reopen object");
-
-static void
-readonly_mmap_write(void)
-{
-	char *page;
-	int fd;
-
-	fd = shm_open(TEST_PATH, O_RDONLY | O_CREAT, 0777);
-	if (fd < 0) {
-		fail_errno("shm_open");
-		return;
-	}
-
-	/* PROT_WRITE should fail with EACCES. */
-	page = mmap(0, getpagesize(), PROT_READ | PROT_WRITE, MAP_SHARED, fd,
-	    0);
-	if (page != MAP_FAILED) {		
-		fail_err("mmap(PROT_WRITE) succeeded");
-		munmap(page, getpagesize());
-		close(fd);
-		shm_unlink(TEST_PATH);
-		return;
-	}
-	if (errno != EACCES) {
-		fail_errno("mmap");
-		close(fd);
-		shm_unlink(TEST_PATH);
-		return;
-	}
-
-	close(fd);
-	shm_unlink(TEST_PATH);
-	pass();
-}
-TEST(readonly_mmap_write, "RDONLY object");
-
-static void
-open_after_unlink(void)
-{
-	int fd;
-
-	fd = shm_open(TEST_PATH, O_RDONLY | O_CREAT, 0777);
-	if (fd < 0) {
-		fail_errno("shm_open(1)");
-		return;
-	}
-	close(fd);
-
-	if (shm_unlink(TEST_PATH) < 0) {
-		fail_errno("shm_unlink");
-		return;
-	}
-
-	shm_open_should_fail(TEST_PATH, O_RDONLY, 0777, ENOENT);
-}
-TEST(open_after_unlink, "open after unlink");
-
-static void
-open_invalid_path(void)
-{
-
-	shm_open_should_fail("blah", O_RDONLY, 0777, EINVAL);
-}
-TEST(open_invalid_path, "open invalid path");
-
-static void
-open_write_only(void)
-{
-
-	shm_open_should_fail(TEST_PATH, O_WRONLY, 0777, EINVAL);
-}
-TEST(open_write_only, "open with O_WRONLY");
-
-static void
-open_extra_flags(void)
-{
-
-	shm_open_should_fail(TEST_PATH, O_RDONLY | O_DIRECT, 0777, EINVAL);
-}
-TEST(open_extra_flags, "open with extra flags");
-
-static void
-open_anon(void)
-{
-	int fd;
-
-	fd = shm_open(SHM_ANON, O_RDWR, 0777);
-	if (fd < 0) {
-		fail_errno("shm_open");
-		return;
-	}
-	close(fd);
-	pass();
-}
-TEST(open_anon, "open anonymous object");
-
-static void
-open_anon_readonly(void)
-{
-
-	shm_open_should_fail(SHM_ANON, O_RDONLY, 0777, EINVAL);
-}
-TEST(open_anon_readonly, "open SHM_ANON with O_RDONLY");
-
-static void
-open_bad_path_pointer(void)
-{
-
-	shm_open_should_fail((char *)1024, O_RDONLY, 0777, EFAULT);
-}
-TEST(open_bad_path_pointer, "open bad path pointer");
-
-static void
-open_path_too_long(void)
-{
-	char *page;
-
-	page = malloc(MAXPATHLEN + 1);
-	memset(page, 'a', MAXPATHLEN);
-	page[MAXPATHLEN] = '\0';
-	shm_open_should_fail(page, O_RDONLY, 0777, ENAMETOOLONG);
-	free(page);
-}
-TEST(open_path_too_long, "open pathname too long");
-
-static void
-open_nonexisting_object(void)
-{
-
-	shm_open_should_fail("/notreallythere", O_RDONLY, 0777, ENOENT);
-}
-TEST(open_nonexisting_object, "open nonexistent object");
-
-static void
-exclusive_create_existing_object(void)
-{
-	int fd;
-
-	fd = shm_open("/tmp/notreallythere", O_RDONLY | O_CREAT, 0777);
-	if (fd < 0) {
-		fail_errno("shm_open(O_CREAT)");
-		return;
-	}
-	close(fd);
-
-	shm_open_should_fail("/tmp/notreallythere", O_RDONLY | O_CREAT | O_EXCL,
-	    0777, EEXIST);
-
-	shm_unlink("/tmp/notreallythere");
-}
-TEST(exclusive_create_existing_object, "O_EXCL of existing object");
-
-static void
-trunc_resets_object(void)
-{
-	struct stat sb;
-	int fd;
-
-	/* Create object and set size to 1024. */
-	fd = shm_open(TEST_PATH, O_RDWR | O_CREAT, 0777);
-	if (fd < 0) {
-		fail_errno("shm_open(1)");
-		return;
-	}
-	if (ftruncate(fd, 1024) < 0) {
-		fail_errno("ftruncate");
-		close(fd);
-		return;
-	}
-	if (fstat(fd, &sb) < 0) {		
-		fail_errno("fstat(1)");
-		close(fd);
-		return;
-	}
-	if (sb.st_size != 1024) {
-		fail_err("size %d != 1024", (int)sb.st_size);
-		close(fd);
-		return;
-	}
-	close(fd);
-
-	/* Open with O_TRUNC which should reset size to 0. */
-	fd = shm_open(TEST_PATH, O_RDWR | O_TRUNC, 0777);
-	if (fd < 0) {
-		fail_errno("shm_open(2)");
-		return;
-	}
-	if (fstat(fd, &sb) < 0) {
-		fail_errno("fstat(2)");
-		close(fd);
-		return;
-	}
-	if (sb.st_size != 0) {
-		fail_err("size after O_TRUNC %d != 0", (int)sb.st_size);
-		close(fd);
-		return;
-	}
-	close(fd);
-	if (shm_unlink(TEST_PATH) < 0) {
-		fail_errno("shm_unlink");
-		return;
-	}
-	pass();
-}
-TEST(trunc_resets_object, "O_TRUNC resets size");
-
-static void
-unlink_bad_path_pointer(void)
-{
-
-	shm_unlink_should_fail((char *)1024, EFAULT);
-}
-TEST(unlink_bad_path_pointer, "unlink bad path pointer");
-
-static void
-unlink_path_too_long(void)
-{
-	char *page;
-
-	page = malloc(MAXPATHLEN + 1);
-	memset(page, 'a', MAXPATHLEN);
-	page[MAXPATHLEN] = '\0';
-	shm_unlink_should_fail(page, ENAMETOOLONG);
-	free(page);
-}
-TEST(unlink_path_too_long, "unlink pathname too long");
-
-static void
-test_object_resize(void)
-{
-	pid_t pid;
-	struct stat sb;
-	char *page;
-	int fd, status;
-
-	/* Start off with a size of a single page. */
-	fd = shm_open(SHM_ANON, O_CREAT | O_RDWR, 0777);
-	if (fd < 0) {
-		fail_errno("shm_open");
-		return;
-	}
-	if (ftruncate(fd, getpagesize()) < 0) {
-		fail_errno("ftruncate(1)");
-		close(fd);
-		return;
-	}
-	if (fstat(fd, &sb) < 0) {
-		fail_errno("fstat(1)");
-		close(fd);
-		return;
-	}
-	if (sb.st_size != getpagesize()) {
-		fail_err("first resize failed");
-		close(fd);
-		return;
-	}
-
-	/* Write a '1' to the first byte. */
-	page = mmap(0, getpagesize(), PROT_READ | PROT_WRITE, MAP_SHARED, fd,
-	    0);
-	if (page == MAP_FAILED) {
-		fail_errno("mmap(1)");
-		close(fd);
-		return;
-	}
-
-	page[0] = '1';
-
-	if (munmap(page, getpagesize()) < 0) {
-		fail_errno("munmap(1)");
-		close(fd);
-		return;
-	}
-
-	/* Grow the object to 2 pages. */
-	if (ftruncate(fd, getpagesize() * 2) < 0) {
-		fail_errno("ftruncate(2)");
-		close(fd);
-		return;
-	}
-	if (fstat(fd, &sb) < 0) {
-		fail_errno("fstat(2)");
-		close(fd);
-		return;
-	}
-	if (sb.st_size != getpagesize() * 2) {
-		fail_err("second resize failed");
-		close(fd);
-		return;
-	}
-
-	/* Check for '1' at the first byte. */
-	page = mmap(0, getpagesize() * 2, PROT_READ | PROT_WRITE, MAP_SHARED,
-	    fd, 0);
-	if (page == MAP_FAILED) {
-		fail_errno("mmap(2)");
-		close(fd);
-		return;
-	}
-
-	if (page[0] != '1') {
-		fail_err("missing data at 0");
-		close(fd);
-		return;
-	}
-
-	/* Write a '2' at the start of the second page. */
-	page[getpagesize()] = '2';
-
-	/* Shrink the object back to 1 page. */
-	if (ftruncate(fd, getpagesize()) < 0) {
-		fail_errno("ftruncate(3)");
-		close(fd);
-		return;
-	}
-	if (fstat(fd, &sb) < 0) {
-		fail_errno("fstat(3)");
-		close(fd);
-		return;
-	}
-	if (sb.st_size != getpagesize()) {
-		fail_err("third resize failed");
-		close(fd);
-		return;
-	}
-
-	/*
-	 * Fork a child process to make sure the second page is no
-	 * longer valid.
-	 */
-	pid = fork();
-	if (pid < 0) {
-		fail_errno("fork");
-		close(fd);
-		return;
-	}
-
-	if (pid == 0) {
-		struct rlimit lim;
-		char c;
-
-		/* Don't generate a core dump. */
-		getrlimit(RLIMIT_CORE, &lim);
-		lim.rlim_cur = 0;
-		setrlimit(RLIMIT_CORE, &lim);
-
-		/*
-		 * The previous ftruncate(2) shrunk the backing object
-		 * so that this address is no longer valid, so reading
-		 * from it should trigger a SIGSEGV.
-		 */
-		c = page[getpagesize()];
-		fprintf(stderr, "child: page 1: '%c'\n", c);
-		exit(0);
-	}
-	if (wait(&status) < 0) {
-		fail_errno("wait");
-		close(fd);
-		return;
-	}
-	if (!WIFSIGNALED(status) || WTERMSIG(status) != SIGSEGV) {
-		fail_err("child terminated with status %x", status);
-		close(fd);
-		return;
-	}
-
-	/* Grow the object back to 2 pages. */
-	if (ftruncate(fd, getpagesize() * 2) < 0) {
-		fail_errno("ftruncate(4)");
-		close(fd);
-		return;
-	}
-	if (fstat(fd, &sb) < 0) {
-		fail_errno("fstat(4)");
-		close(fd);
-		return;
-	}
-	if (sb.st_size != getpagesize() * 2) {
-		fail_err("second resize failed");
-		close(fd);
-		return;
-	}
-
-	/*
-	 * Note that the mapping at 'page' for the second page is
-	 * still valid, and now that the shm object has been grown
-	 * back up to 2 pages, there is now memory backing this page
-	 * so the read will work.  However, the data should be zero
-	 * rather than '2' as the old data was thrown away when the
-	 * object was shrunk and the new pages when an object are
-	 * grown are zero-filled.
-	 */
-	if (page[getpagesize()] != 0) {
-		fail_err("invalid data at %d", getpagesize());
-		close(fd);
-		return;
-	}
-
-	close(fd);
-	pass();
-}
-TEST(test_object_resize, "object resize");
-
-int
-main(int argc, char *argv[])
-{
-
-	run_tests();
-	return (0);
-}
diff --git a/tests/sys/posixshm/posixshm_test.c b/tests/sys/posixshm/posixshm_test.c
new file mode 100644
index 000000000000..d7c32a8e7be3
--- /dev/null
+++ b/tests/sys/posixshm/posixshm_test.c
@@ -0,0 +1,739 @@
+/*-
+ * Copyright (c) 2006 Robert N. M. Watson
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/mman.h>
+#include <sys/resource.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/wait.h>
+
+#include <errno.h>
+#include <fcntl.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <atf-c.h>
+
+#define	TEST_PATH_LEN	256
+static char test_path[TEST_PATH_LEN];
+
+static void
+gen_test_path(void)
+{
+	char *tmpdir = getenv("TMPDIR");
+
+	if (tmpdir == NULL)
+		tmpdir = "/tmp";
+
+	snprintf(test_path, sizeof(test_path), "%s/tmp.XXXXXX", tmpdir);
+	test_path[sizeof(test_path) - 1] = '\0';
+	ATF_REQUIRE_MSG(mkstemp(test_path) != -1,
+	    "mkstemp failed; errno=%d", errno);
+	ATF_REQUIRE_MSG(unlink(test_path) == 0,
+	    "unlink failed; errno=%d", errno);
+}
+
+/*
+ * Attempt a shm_open() that should fail with an expected error of 'error'.
+ */
+static void
+shm_open_should_fail(const char *path, int flags, mode_t mode, int error)
+{
+	int fd;
+
+	fd = shm_open(path, flags, mode);
+	ATF_CHECK_MSG(fd == -1, "shm_open didn't fail");
+	ATF_CHECK_MSG(error == errno,
+	    "shm_open didn't fail with expected errno; errno=%d; expected "
+	    "errno=%d", errno, error);
+}
+
+/*
+ * Attempt a shm_unlink() that should fail with an expected error of 'error'.
+ */
+static void
+shm_unlink_should_fail(const char *path, int error)
+{
+
+	ATF_CHECK_MSG(shm_unlink(path) == -1, "shm_unlink didn't fail");
+	ATF_CHECK_MSG(error == errno,
+	    "shm_unlink didn't fail with expected errno; errno=%d; expected "
+	    "errno=%d", errno, error);
+}
+
+/*
+ * Open the test object and write '1' to the first byte.  Returns valid fd
+ * on success and -1 on failure.
+ */
+static int
+scribble_object(void)
+{
+	char *page;
+	int fd;
+
+	gen_test_path();
+
+	fd = shm_open(test_path, O_CREAT|O_EXCL|O_RDWR, 0777);
+	if (fd < 0 && errno == EEXIST) {
+		if (shm_unlink(test_path) < 0)
+			atf_tc_fail("shm_unlink");
+		fd = shm_open(test_path, O_CREAT | O_EXCL | O_RDWR, 0777);
+	}
+	if (fd < 0)
+		atf_tc_fail("shm_open");
+	if (ftruncate(fd, getpagesize()) < 0)
+		atf_tc_fail("ftruncate");
+
+	page = mmap(0, getpagesize(), PROT_READ | PROT_WRITE, MAP_SHARED, fd,
+	    0);
+	if (page == MAP_FAILED)
+		atf_tc_fail("mmap");
+
+	page[0] = '1';
+	if (munmap(page, getpagesize()) < 0)
+		atf_tc_fail("munmap");
+
+	return (fd);
+}
+
+ATF_TC_WITHOUT_HEAD(remap_object);
+ATF_TC_BODY(remap_object, tc)
+{
+	char *page;
+	int fd;
+
+	fd = scribble_object();
+
+	page = mmap(0, getpagesize(), PROT_READ | PROT_WRITE, MAP_SHARED, fd,
+	    0);
+	if (page == MAP_FAILED) {		
+		atf_tc_fail("mmap(2)");
+		close(fd);
+		shm_unlink(test_path);
+		return;
+	}
+
+	if (page[0] != '1') {		
+		atf_tc_fail("missing data");
+		close(fd);
+		shm_unlink(test_path);
+		return;
+	}
+
+	close(fd);
+	if (munmap(page, getpagesize()) < 0) {
+		atf_tc_fail("munmap");
+		shm_unlink(test_path);
+		return;
+	}
+
+	if (shm_unlink(test_path) < 0) {
+		atf_tc_fail("shm_unlink");
+		return;
+	}
+
+}
+
+ATF_TC_WITHOUT_HEAD(reopen_object);
+ATF_TC_BODY(reopen_object, tc)
+{
+	char *page;
+	int fd;
+
+	fd = scribble_object();
+	close(fd);
+
+	fd = shm_open(test_path, O_RDONLY, 0777);
+	if (fd < 0) {
+		atf_tc_fail("shm_open(2)");
+		shm_unlink(test_path);
+		return;
+	}
+	page = mmap(0, getpagesize(), PROT_READ, MAP_SHARED, fd, 0);
+	if (page == MAP_FAILED) {
+		atf_tc_fail("mmap(2)");
+		close(fd);
+		shm_unlink(test_path);
+		return;
+	}
+
+	if (page[0] != '1') {
+		atf_tc_fail("missing data");
+		munmap(page, getpagesize());
+		close(fd);
+		shm_unlink(test_path);
+		return;
+	}
+
+	munmap(page, getpagesize());
+	close(fd);
+	shm_unlink(test_path);
+}
+
+ATF_TC_WITHOUT_HEAD(readonly_mmap_write);
+ATF_TC_BODY(readonly_mmap_write, tc)
+{
+	char *page;
+	int fd;
+
+	gen_test_path();
+
+	fd = shm_open(test_path, O_RDONLY | O_CREAT, 0777);
+	if (fd < 0) {
+		atf_tc_fail("shm_open");
+		return;
+	}
+
+	/* PROT_WRITE should fail with EACCES. */
+	page = mmap(0, getpagesize(), PROT_READ | PROT_WRITE, MAP_SHARED, fd,
+	    0);
+	if (page != MAP_FAILED) {		
+		atf_tc_fail("mmap(PROT_WRITE) succeeded");
+		munmap(page, getpagesize());
+		close(fd);
+		shm_unlink(test_path);
+		return;
+	}
+	if (errno != EACCES) {
+		atf_tc_fail("mmap");
+		close(fd);
+		shm_unlink(test_path);
+		return;
+	}
+
+	close(fd);
+	shm_unlink(test_path);
+}
+
+ATF_TC_WITHOUT_HEAD(open_after_link);
+ATF_TC_BODY(open_after_link, tc)
+{
+	int fd;
+
+	gen_test_path();
+
+	fd = shm_open(test_path, O_RDONLY | O_CREAT, 0777);
+	if (fd < 0) {
+		atf_tc_fail("shm_open(1)");
+		return;
+	}
+	close(fd);
+
+	if (shm_unlink(test_path) < 0) {
+		atf_tc_fail("shm_unlink");
+		return;
+	}
+
+	shm_open_should_fail(test_path, O_RDONLY, 0777, ENOENT);
+}
+
+ATF_TC_WITHOUT_HEAD(open_invalid_path);
+ATF_TC_BODY(open_invalid_path, tc)
+{
+
+	shm_open_should_fail("blah", O_RDONLY, 0777, EINVAL);
+}
+
+ATF_TC_WITHOUT_HEAD(open_write_only);
+ATF_TC_BODY(open_write_only, tc)
+{
+
+	gen_test_path();
+
+	shm_open_should_fail(test_path, O_WRONLY, 0777, EINVAL);
+}
+
+ATF_TC_WITHOUT_HEAD(open_extra_flags);
+ATF_TC_BODY(open_extra_flags, tc)
+{
+
+	gen_test_path();
+
+	shm_open_should_fail(test_path, O_RDONLY | O_DIRECT, 0777, EINVAL);
+}
+
+ATF_TC_WITHOUT_HEAD(open_anon);
+ATF_TC_BODY(open_anon, tc)
+{
+	int fd;
+
+	fd = shm_open(SHM_ANON, O_RDWR, 0777);
+	if (fd < 0) {
+		atf_tc_fail("shm_open");
+		return;
+	}
+	close(fd);
+}
+
+ATF_TC_WITHOUT_HEAD(open_anon_readonly);
+ATF_TC_BODY(open_anon_readonly, tc)
+{
+
+	shm_open_should_fail(SHM_ANON, O_RDONLY, 0777, EINVAL);
+}
+
+ATF_TC_WITHOUT_HEAD(open_bad_path_pointer);
+ATF_TC_BODY(open_bad_path_pointer, tc)
+{
+
+	shm_open_should_fail((char *)1024, O_RDONLY, 0777, EFAULT);
+}
+
+ATF_TC_WITHOUT_HEAD(open_path_too_long);
+ATF_TC_BODY(open_path_too_long, tc)
+{
+	char *page;
+
+	page = malloc(MAXPATHLEN + 1);
+	memset(page, 'a', MAXPATHLEN);
+	page[MAXPATHLEN] = '\0';
+	shm_open_should_fail(page, O_RDONLY, 0777, ENAMETOOLONG);
+	free(page);
+}
+
+ATF_TC_WITHOUT_HEAD(open_nonexisting_object);
+ATF_TC_BODY(open_nonexisting_object, tc)
+{
+
+	shm_open_should_fail("/notreallythere", O_RDONLY, 0777, ENOENT);
+}
+
+ATF_TC_WITHOUT_HEAD(open_create_existing_object);
+ATF_TC_BODY(open_create_existing_object, tc)
+{
+	int fd;
+
+	gen_test_path();
+
+	fd = shm_open(test_path, O_RDONLY|O_CREAT, 0777);
+	ATF_REQUIRE_MSG(fd != -1, "shm_open(O_CREAT) failed; errno=%d", errno);
+	close(fd);
+
+	shm_open_should_fail(test_path, O_RDONLY|O_CREAT|O_EXCL,
+	    0777, EEXIST);
+
+	shm_unlink("shm_object");
+}
+
+ATF_TC_WITHOUT_HEAD(trunc_resets_object);
+ATF_TC_BODY(trunc_resets_object, tc)
+{
+	struct stat sb;
+	int fd;
+
+	gen_test_path();
+
+	/* Create object and set size to 1024. */
+	fd = shm_open(test_path, O_RDWR | O_CREAT, 0777);
+	if (fd < 0) {
+		atf_tc_fail("shm_open(1)");
+		return;
+	}
+	if (ftruncate(fd, 1024) < 0) {
+		atf_tc_fail("ftruncate");
+		close(fd);
+		return;
+	}
+	if (fstat(fd, &sb) < 0) {		
+		atf_tc_fail("fstat(1)");
+		close(fd);
+		return;
+	}
+	if (sb.st_size != 1024) {
+		atf_tc_fail("size %d != 1024", (int)sb.st_size);
+		close(fd);
+		return;
+	}
+	close(fd);
+
+	/* Open with O_TRUNC which should reset size to 0. */
+	fd = shm_open(test_path, O_RDWR | O_TRUNC, 0777);
+	if (fd < 0) {
+		atf_tc_fail("shm_open(2)");
+		return;
+	}
+	if (fstat(fd, &sb) < 0) {
+		atf_tc_fail("fstat(2)");
+		close(fd);
+		return;
+	}
+	if (sb.st_size != 0) {
+		atf_tc_fail("size after O_TRUNC %d != 0", (int)sb.st_size);
+		close(fd);
+		return;
+	}
+	close(fd);
+	if (shm_unlink(test_path) < 0) {
+		atf_tc_fail("shm_unlink");
+		return;
+	}
+}
+
+ATF_TC_WITHOUT_HEAD(unlink_bad_path_pointer);
+ATF_TC_BODY(unlink_bad_path_pointer, tc)
+{
+
+	shm_unlink_should_fail((char *)1024, EFAULT);
+}
+
+ATF_TC_WITHOUT_HEAD(unlink_path_too_long);
+ATF_TC_BODY(unlink_path_too_long, tc)
+{
+	char *page;
+
+	page = malloc(MAXPATHLEN + 1);
+	memset(page, 'a', MAXPATHLEN);
+	page[MAXPATHLEN] = '\0';
+	shm_unlink_should_fail(page, ENAMETOOLONG);
+	free(page);
+}
+
+ATF_TC_WITHOUT_HEAD(object_resize);
+ATF_TC_BODY(object_resize, tc)
+{
+	pid_t pid;
+	struct stat sb;
+	char *page;
+	int fd, status;
+
+	/* Start off with a size of a single page. */
+	fd = shm_open(SHM_ANON, O_CREAT | O_RDWR, 0777);
+	if (fd < 0) {
+		atf_tc_fail("shm_open");
+		return;
+	}
+	if (ftruncate(fd, getpagesize()) < 0) {
+		atf_tc_fail("ftruncate(1)");
+		close(fd);
+		return;
+	}
+	if (fstat(fd, &sb) < 0) {
+		atf_tc_fail("fstat(1)");
+		close(fd);
+		return;
+	}
+	if (sb.st_size != getpagesize()) {
+		atf_tc_fail("first resize failed");
+		close(fd);
+		return;
+	}
+
+	/* Write a '1' to the first byte. */
+	page = mmap(0, getpagesize(), PROT_READ | PROT_WRITE, MAP_SHARED, fd,
+	    0);
+	if (page == MAP_FAILED) {
+		atf_tc_fail("mmap(1)");
+		close(fd);
+		return;
+	}
+
+	page[0] = '1';
+
+	if (munmap(page, getpagesize()) < 0) {
+		atf_tc_fail("munmap(1)");
+		close(fd);
+		return;
+	}
+
+	/* Grow the object to 2 pages. */
+	if (ftruncate(fd, getpagesize() * 2) < 0) {
+		atf_tc_fail("ftruncate(2)");
+		close(fd);
+		return;
+	}
+	if (fstat(fd, &sb) < 0) {
+		atf_tc_fail("fstat(2)");
+		close(fd);
+		return;
+	}
+	if (sb.st_size != getpagesize() * 2) {
+		atf_tc_fail("second resize failed");
+		close(fd);
+		return;
+	}
+
+	/* Check for '1' at the first byte. */
+	page = mmap(0, getpagesize() * 2, PROT_READ | PROT_WRITE, MAP_SHARED,
+	    fd, 0);
+	if (page == MAP_FAILED) {
+		atf_tc_fail("mmap(2)");
+		close(fd);
+		return;
+	}
+
+	if (page[0] != '1') {
+		atf_tc_fail("missing data at 0");
+		close(fd);
+		return;
+	}
+
+	/* Write a '2' at the start of the second page. */
+	page[getpagesize()] = '2';
+
+	/* Shrink the object back to 1 page. */
+	if (ftruncate(fd, getpagesize()) < 0) {
+		atf_tc_fail("ftruncate(3)");
+		close(fd);
+		return;
+	}
+	if (fstat(fd, &sb) < 0) {
+		atf_tc_fail("fstat(3)");
+		close(fd);
+		return;
+	}
+	if (sb.st_size != getpagesize()) {
+		atf_tc_fail("third resize failed");
+		close(fd);
+		return;
+	}
+
+	/*
+	 * Fork a child process to make sure the second page is no
+	 * longer valid.
+	 */
+	pid = fork();
+	if (pid < 0) {
+		atf_tc_fail("fork");
+		close(fd);
+		return;
+	}
+
+	if (pid == 0) {
+		struct rlimit lim;
+		char c;
+
+		/* Don't generate a core dump. */
+		getrlimit(RLIMIT_CORE, &lim);
+		lim.rlim_cur = 0;
+		setrlimit(RLIMIT_CORE, &lim);
+
+		/*
+		 * The previous ftruncate(2) shrunk the backing object
+		 * so that this address is no longer valid, so reading
+		 * from it should trigger a SIGSEGV.
+		 */
+		c = page[getpagesize()];
+		fprintf(stderr, "child: page 1: '%c'\n", c);
+		exit(0);
+	}
+	if (wait(&status) < 0) {
+		atf_tc_fail("wait");
+		close(fd);
+		return;
+	}
+	if (!WIFSIGNALED(status) || WTERMSIG(status) != SIGSEGV) {
+		atf_tc_fail("child terminated with status %x", status);
+		close(fd);
+		return;
+	}
+
+	/* Grow the object back to 2 pages. */
+	if (ftruncate(fd, getpagesize() * 2) < 0) {
+		atf_tc_fail("ftruncate(4)");
+		close(fd);
+		return;
+	}
+	if (fstat(fd, &sb) < 0) {
+		atf_tc_fail("fstat(4)");
+		close(fd);
+		return;
+	}
+	if (sb.st_size != getpagesize() * 2) {
+		atf_tc_fail("second resize failed");
+		close(fd);
+		return;
+	}
+
+	/*
+	 * Note that the mapping at 'page' for the second page is
+	 * still valid, and now that the shm object has been grown
+	 * back up to 2 pages, there is now memory backing this page
+	 * so the read will work.  However, the data should be zero
+	 * rather than '2' as the old data was thrown away when the
+	 * object was shrunk and the new pages when an object are
+	 * grown are zero-filled.
+	 */
+	if (page[getpagesize()] != 0) {
+		atf_tc_fail("invalid data at %d", getpagesize());
+		close(fd);
+		return;
+	}
+
+	close(fd);
+}
+
+/* Signal handler which does nothing. */
+static void
+ignoreit(int sig __unused)
+{
+	;
+}
+
+ATF_TC_WITHOUT_HEAD(shm_functionality_across_fork);
+ATF_TC_BODY(shm_functionality_across_fork, tc)
+{
+	char *cp, c;
+	int error, desc, rv;
+	long scval;
+	sigset_t ss;
+	struct sigaction sa;
+	void *region;
+	size_t i, psize;
+
+#ifndef _POSIX_SHARED_MEMORY_OBJECTS
+	printf("_POSIX_SHARED_MEMORY_OBJECTS is undefined\n");
+#else
+	printf("_POSIX_SHARED_MEMORY_OBJECTS is defined as %ld\n", 
+	       (long)_POSIX_SHARED_MEMORY_OBJECTS - 0);
+	if (_POSIX_SHARED_MEMORY_OBJECTS - 0 == -1)
+		printf("***Indicates this feature may be unsupported!\n");
+#endif
+	errno = 0;
+	scval = sysconf(_SC_SHARED_MEMORY_OBJECTS);
+	if (scval == -1 && errno != 0) {
+		atf_tc_fail("sysconf(_SC_SHARED_MEMORY_OBJECTS) failed; "
+		    "errno=%d", errno);
+	} else {
+		printf("sysconf(_SC_SHARED_MEMORY_OBJECTS) returns %ld\n",
+		       scval);
+		if (scval == -1)
+			printf("***Indicates this feature is unsupported!\n");
+	}
+
+	errno = 0;
+	scval = sysconf(_SC_PAGESIZE);
+	if (scval == -1 && errno != 0) {
+		atf_tc_fail("sysconf(_SC_PAGESIZE) failed; errno=%d", errno);
+	} else if (scval <= 0 || (size_t)psize != psize) {
+		fprintf(stderr, "bogus return from sysconf(_SC_PAGESIZE): %ld",
+		    scval);
+		psize = 4096;
+	} else {
+		printf("sysconf(_SC_PAGESIZE) returns %ld\n", scval);
+		psize = scval;
+	}
+
+	gen_test_path();
+	desc = shm_open(test_path, O_EXCL | O_CREAT | O_RDWR, 0600);
+
+	ATF_REQUIRE_MSG(desc >= 0, "shm_open failed; errno=%d", errno);
+	ATF_REQUIRE_MSG(shm_unlink(test_path) == 0,
+	    "shm_unlink failed; errno=%d", errno);
+	ATF_REQUIRE_MSG(ftruncate(desc, (off_t)psize) != -1,
+	    "ftruncate failed; errno=%d", errno);
+
+	region = mmap((void *)0, psize, PROT_READ | PROT_WRITE, MAP_SHARED,
+		      desc, (off_t)0);
+	ATF_REQUIRE_MSG(region != MAP_FAILED, "mmap failed; errno=%d", errno);
+	memset(region, '\377', psize);
+
+	sa.sa_flags = 0;
+	sa.sa_handler = ignoreit;
+	sigemptyset(&sa.sa_mask);
+	ATF_REQUIRE_MSG(sigaction(SIGUSR1, &sa, (struct sigaction *)0) == 0,
+	    "sigaction failed; errno=%d", errno);
+
+	sigemptyset(&ss);
+	sigaddset(&ss, SIGUSR1);
+	ATF_REQUIRE_MSG(sigprocmask(SIG_BLOCK, &ss, (sigset_t *)0) == 0,
+	    "sigprocmask failed; errno=%d", errno);
+
+	rv = fork();
+	ATF_REQUIRE_MSG(rv != -1, "fork failed; errno=%d", errno);
+	if (rv == 0) {
+		sigemptyset(&ss);
+		sigsuspend(&ss);
+
+		for (cp = region; cp < (char *)region + psize; cp++) {
+			if (*cp != '\151')
+				_exit(1);
+		}
+		if (lseek(desc, 0, SEEK_SET) == -1)
+			_exit(1);
+		for (i = 0; i < psize; i++) {
+			error = read(desc, &c, 1);
+			if (c != '\151')
+				_exit(1);
+		}
+		_exit(0);
+	} else {
+		int status;
+
+		memset(region, '\151', psize - 2);
+		error = pwrite(desc, region, 2, psize - 2);
+		if (error != 2) {
+			if (error >= 0)
+				atf_tc_fail("short write; %d bytes written",
+				    error);
+			else
+				atf_tc_fail("shmfd write");
+		}
+		kill(rv, SIGUSR1);
+		waitpid(rv, &status, 0);
+
+		if (WIFEXITED(status) && WEXITSTATUS(status) == 0) {
+			printf("Functionality test successful\n");
+		} else if (WIFEXITED(status)) {
+			atf_tc_fail("Child process exited with status %d",
+			    WEXITSTATUS(status));
+		} else {
+			atf_tc_fail("Child process terminated with %s",
+			    strsignal(WTERMSIG(status)));
+		}
+	}
+}
+
+ATF_TP_ADD_TCS(tp)
+{
+
+	ATF_TP_ADD_TC(tp, remap_object);
+	ATF_TP_ADD_TC(tp, reopen_object);
+	ATF_TP_ADD_TC(tp, readonly_mmap_write);
+	ATF_TP_ADD_TC(tp, open_after_link);
+	ATF_TP_ADD_TC(tp, open_invalid_path);
+	ATF_TP_ADD_TC(tp, open_write_only);
+	ATF_TP_ADD_TC(tp, open_extra_flags);
+	ATF_TP_ADD_TC(tp, open_anon);
+	ATF_TP_ADD_TC(tp, open_anon_readonly);
+	ATF_TP_ADD_TC(tp, open_bad_path_pointer);
+	ATF_TP_ADD_TC(tp, open_path_too_long);
+	ATF_TP_ADD_TC(tp, open_nonexisting_object);
+	ATF_TP_ADD_TC(tp, open_create_existing_object);
+	ATF_TP_ADD_TC(tp, shm_functionality_across_fork);
+	ATF_TP_ADD_TC(tp, trunc_resets_object);
+	ATF_TP_ADD_TC(tp, unlink_bad_path_pointer);
+	ATF_TP_ADD_TC(tp, unlink_path_too_long);
+	ATF_TP_ADD_TC(tp, object_resize);
+
+	return (atf_no_error());
+}
diff --git a/tests/sys/posixshm/shm_test.c b/tests/sys/posixshm/shm_test.c
deleted file mode 100644
index a46fc0ba1e26..000000000000
--- a/tests/sys/posixshm/shm_test.c
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Test the POSIX shared-memory API.
- * Dedicated to the public domain by Garrett A. Wollman, 2000.
- * $FreeBSD$
- */
-
-#include <sys/types.h>
-#include <sys/mman.h>
-#include <sys/wait.h>
-
-#include <err.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <signal.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-
-/*
- * Signal handler which does nothing.
- */
-static void 
-ignoreit(int sig __unused)
-{
-	;
-}
-
-int
-main(int argc, char **argv)
-{
-	char buf[1024], *cp, c;
-	int error, desc, rv;
-	long scval;
-	sigset_t ss;
-	struct sigaction sa;
-	void *region;
-	size_t i, psize;
-
-#ifndef _POSIX_SHARED_MEMORY_OBJECTS
-	printf("_POSIX_SHARED_MEMORY_OBJECTS is undefined\n");
-#else
-	printf("_POSIX_SHARED_MEMORY_OBJECTS is defined as %ld\n", 
-	       (long)_POSIX_SHARED_MEMORY_OBJECTS - 0);
-	if (_POSIX_SHARED_MEMORY_OBJECTS - 0 == -1)
-		printf("***Indicates this feature may be unsupported!\n");
-#endif
-	errno = 0;
-	scval = sysconf(_SC_SHARED_MEMORY_OBJECTS);
-	if (scval == -1 && errno != 0) {
-		err(1, "sysconf(_SC_SHARED_MEMORY_OBJECTS)");
-	} else {
-		printf("sysconf(_SC_SHARED_MEMORY_OBJECTS) returns %ld\n",
-		       scval);
-		if (scval == -1)
-			printf("***Indicates this feature is unsupported!\n");
-	}
-
-	errno = 0;
-	scval = sysconf(_SC_PAGESIZE);
-	if (scval == -1 && errno != 0) {
-		err(1, "sysconf(_SC_PAGESIZE)");
-	} else if (scval <= 0 || (size_t)psize != psize) {
-		warnx("bogus return from sysconf(_SC_PAGESIZE): %ld",
-		      scval);
-		psize = 4096;
-	} else {
-		printf("sysconf(_SC_PAGESIZE) returns %ld\n", scval);
-		psize = scval;
-	}
-
-	argc--, argv++;
-
-	if (*argv) {
-		strncat(buf, *argv, (sizeof buf) - 1);
-		desc = shm_open(buf, O_EXCL | O_CREAT | O_RDWR, 0600);
-	} else {
-		do {
-			/*
-			 * Can't use mkstemp for obvious reasons...
-			 */
-			char *tmpdir = getenv("TMPDIR");
-			if (tmpdir == NULL)
-				tmpdir = "/tmp";
-			snprintf(buf, sizeof(buf) - 1,
-			    "%s/shmtest.XXXXXXXXXXXX", tmpdir);
-			buf[sizeof(buf) - 1] = '\0';
-			mktemp(buf);
-			desc = shm_open(buf, O_EXCL | O_CREAT | O_RDWR, 0600);
-		} while (desc < 0 && errno == EEXIST);
-	}
-
-	if (desc < 0)
-		err(1, "shm_open");
-
-	if (shm_unlink(buf) < 0)
-		err(1, "shm_unlink");
-
-	if (ftruncate(desc, (off_t)psize) < 0)
-		err(1, "ftruncate");
-
-	region = mmap((void *)0, psize, PROT_READ | PROT_WRITE, MAP_SHARED,
-		      desc, (off_t)0);
-	if (region == MAP_FAILED)
-		err(1, "mmap");
-	memset(region, '\377', psize);
-
-	sa.sa_flags = 0;
-	sa.sa_handler = ignoreit;
-	sigemptyset(&sa.sa_mask);
-	if (sigaction(SIGUSR1, &sa, (struct sigaction *)0) < 0)
-		err(1, "sigaction");
-
-	sigemptyset(&ss);
-	sigaddset(&ss, SIGUSR1);
-	if (sigprocmask(SIG_BLOCK, &ss, (sigset_t *)0) < 0)
-		err(1, "sigprocmask");
-
-	rv = fork();
-	if (rv < 0) {
-		err(1, "fork");
-	} else if (rv == 0) {
-		sigemptyset(&ss);
-		sigsuspend(&ss);
-
-		for (cp = region; cp < (char *)region + psize; cp++) {
-			if (*cp != '\151')
-				_exit(1);
-		}
-		if (lseek(desc, 0, SEEK_SET) == -1)
-			_exit(1);
-		for (i = 0; i < psize; i++) {
-			error = read(desc, &c, 1);
-			if (c != '\151')
-				_exit(1);
-		}
-		_exit(0);
-	} else {
-		int status;
-
-		memset(region, '\151', psize - 2);
-		error = pwrite(desc, region, 2, psize - 2);
-		if (error != 2) {
-			if (error >= 0)
-				errx(1, "short write %d", error);
-			else
-				err(1, "shmfd write");
-		}
-		kill(rv, SIGUSR1);
-		waitpid(rv, &status, 0);
-
-		if (WIFEXITED(status) && WEXITSTATUS(status) == 0) {
-			printf("Functionality test successful\n");
-			exit(0);
-		} else if (WIFEXITED(status)) {
-			printf("Child process exited with status %d\n",
-			       WEXITSTATUS(status));
-		} else {
-			printf("Child process terminated with %s\n",
-			       strsignal(WTERMSIG(status)));
-		}
-	}
-	exit(1);
-}
diff --git a/tests/sys/posixshm/test.c b/tests/sys/posixshm/test.c
deleted file mode 100644
index 8583a0dbbac6..000000000000
--- a/tests/sys/posixshm/test.c
+++ /dev/null
@@ -1,128 +0,0 @@
-/*-
- * Copyright (c) 2008 Yahoo!, Inc.
- * All rights reserved.
- * Written by: John Baldwin <jhb@FreeBSD.org>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. Neither the name of the author nor the names of any co-contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <stdarg.h>
-#include <stdio.h>
-
-#include "test.h"
-
-static int test_index;
-static struct regression_test *test;
-static int test_acknowleged;
-
-SET_DECLARE(regression_tests_set, struct regression_test);
-
-/*
- * Outputs a test summary of the following:
- *
- * <status> <test #> [name] [# <fmt> [fmt args]]
- */
-static void
-vprint_status(const char *status, const char *fmt, va_list ap)
-{
-
-	printf("%s %d", status, test_index);
-	if (test->rt_name)
-		printf(" - %s", test->rt_name);
-	if (fmt) {
-		printf(" # ");
-		vprintf(fmt, ap);
-	}
-	printf("\n");
-	test_acknowleged = 1;
-}
-
-static void
-print_status(const char *status, const char *fmt, ...)
-{
-	va_list ap;
-
-	va_start(ap, fmt);
-	vprint_status(status, fmt, ap);
-	va_end(ap);
-}
-
-void
-pass(void)
-{
-
-	print_status("ok", NULL);
-}
-
-void
-fail(void)
-{
-
-	print_status("not ok", NULL);
-}
-
-void
-fail_err(const char *fmt, ...)
-{
-	va_list ap;
-
-	va_start(ap, fmt);
-	vprint_status("not ok", fmt, ap);
-	va_end(ap);	
-}
-
-void
-skip(const char *reason)
-{
-
-	print_status("ok", "skip %s", reason);
-}
-
-void
-todo(const char *reason)
-{
-
-	print_status("not ok", "TODO %s", reason);
-}
-
-void
-run_tests(void)
-{
-	struct regression_test **testp;
-
-	printf("1..%td\n", SET_COUNT(regression_tests_set));
-	test_index = 1;
-	SET_FOREACH(testp, regression_tests_set) {
-		test_acknowleged = 0;
-		test = *testp;
-		test->rt_function();
-		if (!test_acknowleged)
-			print_status("not ok", "unknown status");
-		test_index++;
-	}
-}
diff --git a/tests/sys/posixshm/test.h b/tests/sys/posixshm/test.h
deleted file mode 100644
index 505679b1fdb0..000000000000
--- a/tests/sys/posixshm/test.h
+++ /dev/null
@@ -1,59 +0,0 @@
-/*-
- * Copyright (c) 2008 Yahoo!, Inc.
- * All rights reserved.
- * Written by: John Baldwin <jhb@FreeBSD.org>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. Neither the name of the author nor the names of any co-contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#ifndef __TEST_H__
-#define	__TEST_H__
-
-#include <sys/linker_set.h>
-
-struct regression_test {
-	void	(*rt_function)(void);
-	const char *rt_name;
-};
-
-#define	TEST(function, name)						\
-	static struct regression_test _regtest_##function = {		\
-		(function),						\
-		(name)							\
-	};								\
-	DATA_SET(regression_tests_set, _regtest_##function)
-
-void	fail(void);
-void	fail_err(const char *fmt, ...);
-void	pass(void);
-void	run_tests(void);
-void	skip(const char *reason);
-void	todo(const char *reason);
-
-#define	fail_errno(tag)		fail_err("%s: %s", (tag), strerror(errno))
-
-#endif /* !__TEST_H__ */

From 303afd501b2fc4c24ba43ea7149f10159dba5808 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Sat, 17 Oct 2015 03:51:50 +0000
Subject: [PATCH 295/380] Rework the world subdir build targets to use the
 standard SUBDIR_PARALLEL mechanism.

Back in r30113, the 'par-*' targets were added to parallelize portions of
the build in a very similar fashion as the SUBDIR_PARALLEL feature used in
r263778.  Calling a target without 'par-' (for 'parallel') resulted in the
standard bsd.subdir.mk handling without parallelization.  Given we have
SUBDIR_PARALLEL now there is no reason to duplicate the handling here.

In build logs this will result in the ${dir}.${target}__D targets now showing
as the normal ${target}_subdir_${dir} targets.

I audited all of the uses of Makefile.inc1 and Makefile's targets that use
bsd.subdir.mk and found that all but 'all' and 'install' were fine to use
as always parallel.
  - For 'install' (from installworld -j) the ordering of lib/ and libexec/
    before the rest of the system (described in r289433), and etc/ being last
    (described in r289435), is all that matters.  So now a .WAIT is added in
    the proper places when invoking any 'install*' target.  A parallel
    installworld does work and took 46% of the time a non-parallel
    install would take on my system with -j15 to ZFS.
  - For 'all' I left the default handling for this to not run in parallel.  A
    'par-all' target is still used by the 'everything' stage of buildworld
    to continue building in parallel as it already has been.  This works
    because most of the dependencies are handled by the early bootstrap
    phases as well as 'libraries' and 'includes' phases.  This lets
    all of the SUBDIR build in parallel fine, such as bin/ and lib/.  This
    will not work if the user invokes 'all' though as we have dependencies
    spread all over the system with no way to depend between them (except
    for the dirdeps feature in the META_MODE build).  Calling 'make all'
    from the top-level is still useful at least when using SUBDIR_OVERRIDE.

MFC after:	2 weeks
Sponsored by:	EMC / Isilon Storage Division
---
 Makefile.inc1 | 47 +++++++++++++++++++++++++----------------------
 1 file changed, 25 insertions(+), 22 deletions(-)

diff --git a/Makefile.inc1 b/Makefile.inc1
index 2cfea9e422a7..e491053e2f1d 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -63,6 +63,10 @@ SRCDIR?=	${.CURDIR}
 SUBDIR=	${SUBDIR_OVERRIDE}
 .else
 SUBDIR=	lib libexec
+.if make(install*)
+# Ensure libraries are installed before progressing.
+SUBDIR+=.WAIT
+.endif
 SUBDIR+=bin
 .if ${MK_CDDL} != "no"
 SUBDIR+=cddl
@@ -115,6 +119,9 @@ SUBDIR+=	${_DIR}
 # by calling 'makedb' in share/man.  This is only relevant for
 # install/distribute so they build the whatis file after every manpage is
 # installed.
+.if make(install*)
+SUBDIR+=.WAIT
+.endif
 SUBDIR+=etc
 
 .if defined(NOCLEAN)
@@ -589,9 +596,9 @@ _cleanobj:
 	@echo "--------------------------------------------------------------"
 	@echo ">>> stage 2.1: cleaning up the object tree"
 	@echo "--------------------------------------------------------------"
-	${_+_}cd ${.CURDIR}; ${WMAKE} ${CLEANDIR:S/^/par-/}
+	${_+_}cd ${.CURDIR}; ${WMAKE} ${CLEANDIR}
 .if defined(LIB32TMP)
-	${_+_}cd ${.CURDIR}; ${LIB32WMAKE} -f Makefile.inc1 ${CLEANDIR:S/^/par-/}
+	${_+_}cd ${.CURDIR}; ${LIB32WMAKE} -f Makefile.inc1 ${CLEANDIR}
 .endif
 .endif
 _obj:
@@ -599,7 +606,7 @@ _obj:
 	@echo "--------------------------------------------------------------"
 	@echo ">>> stage 2.2: rebuilding the object tree"
 	@echo "--------------------------------------------------------------"
-	${_+_}cd ${.CURDIR}; ${WMAKE} par-obj
+	${_+_}cd ${.CURDIR}; ${WMAKE} obj
 _build-tools:
 	@echo
 	@echo "--------------------------------------------------------------"
@@ -618,7 +625,7 @@ _includes:
 	@echo "--------------------------------------------------------------"
 	@echo ">>> stage 4.1: building includes"
 	@echo "--------------------------------------------------------------"
-	${_+_}cd ${.CURDIR}; ${WMAKE} SHARED=symlinks par-includes
+	${_+_}cd ${.CURDIR}; ${WMAKE} SHARED=symlinks includes
 _libraries:
 	@echo
 	@echo "--------------------------------------------------------------"
@@ -632,7 +639,7 @@ _depend:
 	@echo "--------------------------------------------------------------"
 	@echo ">>> stage 4.3: make dependencies"
 	@echo "--------------------------------------------------------------"
-	${_+_}cd ${.CURDIR}; ${WMAKE} par-depend
+	${_+_}cd ${.CURDIR}; ${WMAKE} depend
 everything:
 	@echo
 	@echo "--------------------------------------------------------------"
@@ -1944,22 +1951,18 @@ _startup_libs: ${_startup_libs:S/$/__L/}
 _prebuild_libs: ${_prebuild_libs:S/$/__L/}
 _generic_libs: ${_generic_libs:S/$/__L/}
 
-.for __target in all clean cleandepend cleandir depend includes obj
-.for entry in ${SUBDIR}
-${entry}.${__target}__D: .PHONY .MAKE
-	${_+_}@set -e; if test -d ${.CURDIR}/${entry}.${MACHINE_ARCH}; then \
-		${ECHODIR} "===> ${DIRPRFX}${entry}.${MACHINE_ARCH} (${__target})"; \
-		edir=${entry}.${MACHINE_ARCH}; \
-		cd ${.CURDIR}/$${edir}; \
-	else \
-		${ECHODIR} "===> ${DIRPRFX}${entry} (${__target})"; \
-		edir=${entry}; \
-		cd ${.CURDIR}/$${edir}; \
-	fi; \
-	${MAKE} ${__target} DIRPRFX=${DIRPRFX}$${edir}/
-.endfor
-par-${__target}: ${SUBDIR:S/$/.${__target}__D/}
-.endfor
+# Enable SUBDIR_PARALLEL when not calling 'make all', unless called as
+# 'par-all'.  This is because it is unlikely that running 'make all' from
+# the top-level, especially with a SUBDIR_OVERRIDE or LOCAL_DIRS set,
+# will have a reliable build if SUBDIRs are built in parallel.  This is
+# safe for the world stage of buildworld though since it has already
+# built libraries in a proper order and installed includes into WORLDTMP.
+# Special handling is done for SUBDIR ordering for 'install*' to avoid
+# trashing a system if it crashes mid-install.
+par-all: all .PHONY
+.if !make(all)
+SUBDIR_PARALLEL=
+.endif
 
 .include <bsd.subdir.mk>
 
@@ -2267,7 +2270,7 @@ _xi-cross-tools:
 .endfor
 
 _xi-includes:
-	${_+_}cd ${.CURDIR}; ${CD2MAKE} -f Makefile.inc1 par-includes \
+	${_+_}cd ${.CURDIR}; ${CD2MAKE} -f Makefile.inc1 includes \
 		DESTDIR=${XDDESTDIR}
 
 _xi-libraries:

From 6a86a0a266596cfe281141e8f9b3f054984565a0 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Sat, 17 Oct 2015 04:03:53 +0000
Subject: [PATCH 296/380] Export OSRELDATE so sub-makes don't look it up again.

We pass BOOTSTRAPPING=${OSRELDATE} to some of the sub-makes.  Rather than
chase every ${MAKE} invokation, just export it as bmake lets us.

Sponsored by:	EMC / Isilon Storage Division
---
 Makefile.inc1 | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile.inc1 b/Makefile.inc1
index e491053e2f1d..541858529949 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -148,6 +148,7 @@ OSRELDATE!=	awk '/^\#define[[:space:]]*__FreeBSD_version/ { print $$3 }' \
 .else
 OSRELDATE=	0
 .endif
+.export OSRELDATE
 .endif
 
 .if !defined(VERSION)

From aeb7087308a5f4fcd0b616d5b2b137f2571f73a7 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Sat, 17 Oct 2015 05:49:07 +0000
Subject: [PATCH 297/380] Do as r289391 did for share/mk and make installing to
 a non-existent directory an error.

Most of these do a 'mkdir -p' or 'install -d' before installing, but add
the trailing / here for consistency with the userland install.

MFC after:	2 weeks
X-MFC-With:	r289391
Sponsored by:	EMC / Isilon Storage Division
---
 sys/conf/dtb.mk       |  2 +-
 sys/conf/kern.post.mk | 10 +++++-----
 sys/conf/kmod.mk      |  4 ++--
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/sys/conf/dtb.mk b/sys/conf/dtb.mk
index 41bc9bb20d8d..df6af54eae7d 100644
--- a/sys/conf/dtb.mk
+++ b/sys/conf/dtb.mk
@@ -69,7 +69,7 @@ _dtbinstall:
 	test -d ${DESTDIR}${DTBDIR} || ${INSTALL} -d -o ${DTBOWN} -g ${DTBGRP} ${DESTDIR}${DTBDIR}
 .for _dtb in ${DTB}
 	${INSTALL} -o ${DTBOWN} -g ${DTBGRP} -m ${DTBMODE} \
-	    ${_INSTALLFLAGS} ${_dtb} ${DESTDIR}${DTBDIR}
+	    ${_INSTALLFLAGS} ${_dtb} ${DESTDIR}${DTBDIR}/
 .endfor
 .endif # !target(realinstall)
 .endif # !target(install)
diff --git a/sys/conf/kern.post.mk b/sys/conf/kern.post.mk
index 55b4fd7a21ea..ddf828ed3177 100644
--- a/sys/conf/kern.post.mk
+++ b/sys/conf/kern.post.mk
@@ -270,22 +270,22 @@ kernel-install:
 	fi
 .endif
 	mkdir -p ${DESTDIR}${KODIR}
-	${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO} ${DESTDIR}${KODIR}
+	${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO} ${DESTDIR}${KODIR}/
 .if defined(DEBUG) && !defined(INSTALL_NODEBUG) && ${MK_KERNEL_SYMBOLS} != "no"
 	mkdir -p ${DESTDIR}${KERN_DEBUGDIR}${KODIR}
-	${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO}.debug ${DESTDIR}${KERN_DEBUGDIR}${KODIR}
+	${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO}.debug ${DESTDIR}${KERN_DEBUGDIR}${KODIR}/
 .endif
 .if defined(KERNEL_EXTRA_INSTALL)
-	${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_EXTRA_INSTALL} ${DESTDIR}${KODIR}
+	${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_EXTRA_INSTALL} ${DESTDIR}${KODIR}/
 .endif
 
 
 
 kernel-reinstall:
 	@-chflags -R noschg ${DESTDIR}${KODIR}
-	${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO} ${DESTDIR}${KODIR}
+	${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO} ${DESTDIR}${KODIR}/
 .if defined(DEBUG) && !defined(INSTALL_NODEBUG) && ${MK_KERNEL_SYMBOLS} != "no"
-	${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO}.debug ${DESTDIR}${KERN_DEBUGDIR}${KODIR}
+	${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO}.debug ${DESTDIR}${KERN_DEBUGDIR}${KODIR}/
 .endif
 
 config.o env.o hints.o vers.o vnode_if.o:
diff --git a/sys/conf/kmod.mk b/sys/conf/kmod.mk
index 06e4cfcdc1f4..8bbcfcb767d0 100644
--- a/sys/conf/kmod.mk
+++ b/sys/conf/kmod.mk
@@ -296,10 +296,10 @@ realinstall: _kmodinstall
 .ORDER: beforeinstall _kmodinstall
 _kmodinstall:
 	${INSTALL} -o ${KMODOWN} -g ${KMODGRP} -m ${KMODMODE} \
-	    ${_INSTALLFLAGS} ${PROG} ${DESTDIR}${KMODDIR}
+	    ${_INSTALLFLAGS} ${PROG} ${DESTDIR}${KMODDIR}/
 .if defined(DEBUG_FLAGS) && !defined(INSTALL_NODEBUG) && ${MK_KERNEL_SYMBOLS} != "no"
 	${INSTALL} -o ${KMODOWN} -g ${KMODGRP} -m ${KMODMODE} \
-	    ${_INSTALLFLAGS} ${PROG}.debug ${DESTDIR}${KERN_DEBUGDIR}${KMODDIR}
+	    ${_INSTALLFLAGS} ${PROG}.debug ${DESTDIR}${KERN_DEBUGDIR}${KMODDIR}/
 .endif
 
 .include <bsd.links.mk>

From 5de21407923790e3d30d3be8549792335cdffa3c Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Sat, 17 Oct 2015 05:55:45 +0000
Subject: [PATCH 298/380] Always export VERSION to the environment to avoid
 looking it up again in sub-makes.

Some of the world phases that used plain '${MAKE} -f Makefile.inc1' were not
passing this variable along which caused them to look it up again.  By
using bmake's .export we can remove it from all of the other environment
lines.

Add a comment about the usage for VERSION for ctfmerge.

Sponsored by:	EMC / Isilon Storage Division
---
 Makefile.inc1 | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/Makefile.inc1 b/Makefile.inc1
index 541858529949..9ab06edd1fb3 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -151,12 +151,14 @@ OSRELDATE=	0
 .export OSRELDATE
 .endif
 
+# Set VERSION for CTFMERGE to use via the default CTFFLAGS=-L VERSION.
 .if !defined(VERSION)
 REVISION!=	${MAKE} -C ${SRCDIR}/release -V REVISION
 BRANCH!=	${MAKE} -C ${SRCDIR}/release -V BRANCH
 SRCRELDATE!=	awk '/^\#define[[:space:]]*__FreeBSD_version/ { print $$3 }' \
 		${SRCDIR}/sys/sys/param.h
 VERSION=	FreeBSD ${REVISION}-${BRANCH:C/-p[0-9]+$//} ${TARGET_ARCH} ${SRCRELDATE}
+.export VERSION
 .endif
 
 KNOWN_ARCHES?=	aarch64/arm64 amd64 arm armeb/arm armv6/arm armv6hf/arm i386 i386/pc98 mips mipsel/mips mips64el/mips mips64/mips mipsn32el/mips mipsn32/mips powerpc powerpc64/powerpc sparc64
@@ -265,7 +267,6 @@ CROSSENV+=	${TARGET_CFLAGS}
 BMAKEENV=	INSTALL="sh ${.CURDIR}/tools/install.sh" \
 		PATH=${BPATH}:${PATH} \
 		WORLDTMP=${WORLDTMP} \
-		VERSION="${VERSION}" \
 		MAKEFLAGS="-m ${.CURDIR}/tools/build/mk ${.MAKEFLAGS}"
 # need to keep this in sync with targets/pseudo/bootstrap-tools/Makefile
 BSARGS= 	DESTDIR= \
@@ -302,8 +303,7 @@ XMAKE=		TOOLS_PREFIX=${WORLDTMP} ${BMAKE} \
 # kernel-tools stage
 KTMAKEENV=	INSTALL="sh ${.CURDIR}/tools/install.sh" \
 		PATH=${BPATH}:${PATH} \
-		WORLDTMP=${WORLDTMP} \
-		VERSION="${VERSION}"
+		WORLDTMP=${WORLDTMP}
 KTMAKE=		TOOLS_PREFIX=${WORLDTMP} MAKEOBJDIRPREFIX=${WORLDTMP} \
 		${KTMAKEENV} ${MAKE} ${WORLD_FLAGS} -f Makefile.inc1 \
 		DESTDIR= \
@@ -316,7 +316,6 @@ KTMAKE=		TOOLS_PREFIX=${WORLDTMP} MAKEOBJDIRPREFIX=${WORLDTMP} \
 # world stage
 WMAKEENV=	${CROSSENV} \
 		_LDSCRIPTROOT= \
-		VERSION="${VERSION}" \
 		INSTALL="sh ${.CURDIR}/tools/install.sh" \
 		PATH=${TMPPATH}
 
@@ -459,7 +458,6 @@ LIB32FLAGS+=		--sysroot=${WORLDTMP}
 # Yes, the flags are redundant.
 LIB32WMAKEENV+=	MAKEOBJDIRPREFIX=${LIB32_OBJTREE} \
 		_LDSCRIPTROOT=${LIB32TMP} \
-		VERSION="${VERSION}" \
 		INSTALL="sh ${.CURDIR}/tools/install.sh" \
 		PATH=${TMPPATH} \
 		LIBDIR=/usr/lib32 \
@@ -1587,7 +1585,6 @@ cross-tools: .MAKE
 
 NXBENV=		MAKEOBJDIRPREFIX=${OBJTREE}/nxb \
 		INSTALL="sh ${.CURDIR}/tools/install.sh" \
-		VERSION="${VERSION}" \
 		PATH=${PATH}:${OBJTREE}/gperf_for_gcc/usr/bin
 NXBMAKE=	${NXBENV} ${MAKE} \
 		TBLGEN=${OBJTREE}/nxb-bin/usr/bin/tblgen \

From 0b6ec825282f8bf6db82e1f9b2a489698bea0dd8 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Sat, 17 Oct 2015 05:57:29 +0000
Subject: [PATCH 299/380] For 'buildenvvars' show any .exported variables as
 well to cover recent exporting of OSRELDATE and VERSION.  These already do
 export to 'buildenv' fine.

Sponsored by:	EMC / Isilon Storage Division
---
 Makefile.inc1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile.inc1 b/Makefile.inc1
index 9ab06edd1fb3..c0ef145d2e97 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -747,7 +747,7 @@ buildworld_epilogue:
 # to quote multiword values.
 #
 buildenvvars:
-	@echo ${WMAKEENV:Q}
+	@echo ${WMAKEENV:Q} ${.MAKE.EXPORTED:@v@$v=\"${$v}\"@}
 
 .if ${.TARGETS:Mbuildenv}
 .if ${.MAKEFLAGS:M-j}

From 33618c2e43bba49d08a709c4ae2249b043123798 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Sat, 17 Oct 2015 07:32:46 +0000
Subject: [PATCH 300/380] Fix ZFS ABI compat shims for `zfs receive` after
 r289362.

Difference appeared much less drammatic then seemed originally.
---
 UPDATING                                               |  6 ------
 .../contrib/opensolaris/common/zfs/zfs_ioctl_compat.c  | 10 ++++++++++
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/UPDATING b/UPDATING
index 12631e190841..baff1f13605b 100644
--- a/UPDATING
+++ b/UPDATING
@@ -31,12 +31,6 @@ NOTE TO PEOPLE WHO THINK THAT FreeBSD 11.x IS SLOW:
 	disable the most expensive debugging functionality run
 	"ln -s 'abort:false,junk:false' /etc/malloc.conf".)
 
-20151015:
-	Added ZFS support for resumable send/receive changed respective
-	IOCTL API/ABI.  Compatibility ABI shims were provided for other
-	functionality, while receive require version match between world
-	and kernel.
-
 20151012:
 	If you specify SENDMAIL_MC or SENDMAIL_CF in make.conf, mergemaster
 	and etcupdate will now use this file. A custom sendmail.cf is now
diff --git a/sys/cddl/contrib/opensolaris/common/zfs/zfs_ioctl_compat.c b/sys/cddl/contrib/opensolaris/common/zfs/zfs_ioctl_compat.c
index 4cb1804aad66..39ad7ce5f104 100644
--- a/sys/cddl/contrib/opensolaris/common/zfs/zfs_ioctl_compat.c
+++ b/sys/cddl/contrib/opensolaris/common/zfs/zfs_ioctl_compat.c
@@ -84,6 +84,7 @@ zfs_cmd_compat_get(zfs_cmd_t *zc, caddr_t addr, const int cflag)
 		ZCMD_COPY(zc_share);
 		ZCMD_COPY(zc_jailid);
 		ZCMD_COPY(zc_objset_stats);
+		zc->zc_begin_record.drr_u.drr_begin = edbp_c->zc_begin_record;
 		ZCMD_COPY(zc_inject_record);
 		ZCMD_COPY(zc_defer_destroy);
 		ZCMD_COPY(zc_flags);
@@ -126,6 +127,7 @@ zfs_cmd_compat_get(zfs_cmd_t *zc, caddr_t addr, const int cflag)
 		ZCMD_COPY(zc_share);
 		ZCMD_COPY(zc_jailid);
 		ZCMD_COPY(zc_objset_stats);
+		zc->zc_begin_record.drr_u.drr_begin = zcmd_c->zc_begin_record;
 		ZCMD_COPY(zc_inject_record);
 
 		/* boolean_t -> uint32_t */
@@ -168,6 +170,7 @@ zfs_cmd_compat_get(zfs_cmd_t *zc, caddr_t addr, const int cflag)
 		zc->zc_share = zcdm_c->zc_share;
 		zc->zc_jailid = zcdm_c->zc_jailid;
 		zc->zc_objset_stats = zcdm_c->zc_objset_stats;
+		zc->zc_begin_record.drr_u.drr_begin = zcdm_c->zc_begin_record;
 		zc->zc_defer_destroy = zcdm_c->zc_defer_destroy;
 		(void)zcdm_c->zc_temphold;
 		zc->zc_action_handle = zcdm_c->zc_action_handle;
@@ -211,6 +214,7 @@ zfs_cmd_compat_get(zfs_cmd_t *zc, caddr_t addr, const int cflag)
 		zc->zc_share = zc28_c->zc_share;
 		zc->zc_jailid = zc28_c->zc_jailid;
 		zc->zc_objset_stats = zc28_c->zc_objset_stats;
+		zc->zc_begin_record.drr_u.drr_begin = zc28_c->zc_begin_record;
 		zc->zc_defer_destroy = zc28_c->zc_defer_destroy;
 		(void)zc28_c->zc_temphold;
 		zc->zc_action_handle = zc28_c->zc_action_handle;
@@ -279,6 +283,7 @@ zfs_cmd_compat_get(zfs_cmd_t *zc, caddr_t addr, const int cflag)
 		zc->zc_share = zc_c->zc_share;
 		zc->zc_jailid = zc_c->zc_jailid;
 		zc->zc_objset_stats = zc_c->zc_objset_stats;
+		zc->zc_begin_record.drr_u.drr_begin = zc_c->zc_begin_record;
 
 		/* zc->zc_inject_record */
 		zc->zc_inject_record.zi_objset =
@@ -343,6 +348,7 @@ zfs_cmd_compat_put(zfs_cmd_t *zc, caddr_t addr, const int request,
 		ZCMD_COPY(zc_share);
 		ZCMD_COPY(zc_jailid);
 		ZCMD_COPY(zc_objset_stats);
+		edbp_c->zc_begin_record = zc->zc_begin_record.drr_u.drr_begin;
 		ZCMD_COPY(zc_inject_record);
 		ZCMD_COPY(zc_defer_destroy);
 		ZCMD_COPY(zc_flags);
@@ -384,6 +390,7 @@ zfs_cmd_compat_put(zfs_cmd_t *zc, caddr_t addr, const int request,
 		ZCMD_COPY(zc_share);
 		ZCMD_COPY(zc_jailid);
 		ZCMD_COPY(zc_objset_stats);
+		zcmd_c->zc_begin_record = zc->zc_begin_record.drr_u.drr_begin;
 		ZCMD_COPY(zc_inject_record);
 
 		/* boolean_t -> uint32_t */
@@ -425,6 +432,7 @@ zfs_cmd_compat_put(zfs_cmd_t *zc, caddr_t addr, const int request,
 		zcdm_c->zc_share = zc->zc_share;
 		zcdm_c->zc_jailid = zc->zc_jailid;
 		zcdm_c->zc_objset_stats = zc->zc_objset_stats;
+		zcdm_c->zc_begin_record = zc->zc_begin_record.drr_u.drr_begin;
 		zcdm_c->zc_defer_destroy = zc->zc_defer_destroy;
 		zcdm_c->zc_temphold = 0;
 		zcdm_c->zc_action_handle = zc->zc_action_handle;
@@ -469,6 +477,7 @@ zfs_cmd_compat_put(zfs_cmd_t *zc, caddr_t addr, const int request,
 		zc28_c->zc_share = zc->zc_share;
 		zc28_c->zc_jailid = zc->zc_jailid;
 		zc28_c->zc_objset_stats = zc->zc_objset_stats;
+		zc28_c->zc_begin_record = zc->zc_begin_record.drr_u.drr_begin;
 		zc28_c->zc_defer_destroy = zc->zc_defer_destroy;
 		zc28_c->zc_temphold = 0;
 		zc28_c->zc_action_handle = zc->zc_action_handle;
@@ -539,6 +548,7 @@ zfs_cmd_compat_put(zfs_cmd_t *zc, caddr_t addr, const int request,
 		zc_c->zc_share = zc->zc_share;
 		zc_c->zc_jailid = zc->zc_jailid;
 		zc_c->zc_objset_stats = zc->zc_objset_stats;
+		zc_c->zc_begin_record = zc->zc_begin_record.drr_u.drr_begin;
 
 		/* zc_inject_record */
 		zc_c->zc_inject_record.zi_objset =

From 42e86731a7458e60ee9aa9d82a479ebb1d2611e3 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sat, 17 Oct 2015 08:51:10 +0000
Subject: [PATCH 301/380] Replace /dev/acd0 with /dev/cd1

atapicd(4) has been removed since r249083, and if a system has more than one
optical drive, it will likely be /dev/cd1

Update mount.conf(8) to reflect the change in behavior

MFC after: never
Sponsored by: EMC / Isilon Storage Division
---
 sbin/mount/mount.conf.8  | 6 +++---
 sys/kern/vfs_mountroot.c | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/sbin/mount/mount.conf.8 b/sbin/mount/mount.conf.8
index 45b8257ebae9..48af514dff64 100644
--- a/sbin/mount/mount.conf.8
+++ b/sbin/mount/mount.conf.8
@@ -26,7 +26,7 @@
 .\" $FreeBSD$
 .\"
 .\"
-.Dd July 7, 2013
+.Dd October 17, 2013
 .Dt MOUNT.CONF 8
 .Os
 .Sh NAME
@@ -154,7 +154,7 @@ will direct the kernel to try mounting the root file system
 first as an ISO CD9660 file system on
 .Pa /dev/cd0 ,
 then if that does not work, as an ISO CD9660 file system on
-.Pa /dev/acd0 ,
+.Pa /dev/cd1 ,
 and then if that does not work, as a UFS file system on
 .Pa /dev/ada0s1a .
 If that does not work, a
@@ -167,7 +167,7 @@ Finally if that does not work, the kernel will panic.
 .Li .timeout 3
 cd9660:/dev/cd0 ro
 .Li .timeout 0
-cd9660:/dev/acd0 ro
+cd9660:/dev/cd1 ro
 .Li .timeout 3
 ufs:/dev/ada0s1a
 .Li .ask
diff --git a/sys/kern/vfs_mountroot.c b/sys/kern/vfs_mountroot.c
index 73ed5b652b2e..aecddbda2a63 100644
--- a/sys/kern/vfs_mountroot.c
+++ b/sys/kern/vfs_mountroot.c
@@ -468,9 +468,9 @@ parse_dir_ask(char **conf)
 	printf("\n");
 	printf("    eg. ufs:/dev/da0s1a\n");
 	printf("        zfs:tank\n");
-	printf("        cd9660:/dev/acd0 ro\n");
+	printf("        cd9660:/dev/cd0 ro\n");
 	printf("          (which is equivalent to: ");
-	printf("mount -t cd9660 -o ro /dev/acd0 /)\n");
+	printf("mount -t cd9660 -o ro /dev/cd0 /)\n");
 	printf("\n");
 	printf("  ?               List valid disk boot devices\n");
 	printf("  .               Yield 1 second (for background tasks)\n");
@@ -837,7 +837,7 @@ vfs_mountroot_conf0(struct sbuf *sb)
 	if (boothowto & RB_CDROM) {
 		sbuf_printf(sb, "cd9660:/dev/cd0 ro\n");
 		sbuf_printf(sb, ".timeout 0\n");
-		sbuf_printf(sb, "cd9660:/dev/acd0 ro\n");
+		sbuf_printf(sb, "cd9660:/dev/cd1 ro\n");
 		sbuf_printf(sb, ".timeout %d\n", root_mount_timeout);
 	}
 	s = kern_getenv("vfs.root.mountfrom");

From 8274c9ca5f94449626c9a69121df86e5d6ffeb1e Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sat, 17 Oct 2015 09:07:53 +0000
Subject: [PATCH 302/380] Set dev->fd to -1 when calling cam_close_spec_device
 with a valid dev->fd descriptor to avoid trashing valid file descriptors that
 access dev->fd at a later point in time

PR: 192671
Submitted by: Scott Ferris <scott.ferris@isilon.com>
MFC after: 1 week
Sponsored by: EMC / Isilon Storage Division
---
 lib/libcam/camlib.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/libcam/camlib.c b/lib/libcam/camlib.c
index f3202470a0e2..b7024a6d6097 100644
--- a/lib/libcam/camlib.c
+++ b/lib/libcam/camlib.c
@@ -676,8 +676,10 @@ cam_close_spec_device(struct cam_device *dev)
 	if (dev == NULL)
 		return;
 
-	if (dev->fd >= 0)
+	if (dev->fd >= 0) {
 		close(dev->fd);
+		dev->fd = -1;
+	}
 }
 
 char *

From 0a77d5c6e12f3fcaf03e8f81d67113b78418dd47 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sat, 17 Oct 2015 09:26:16 +0000
Subject: [PATCH 303/380] Install share/zoneinfo in a deterministic way by
 sorting the results from find

This helps produce deterministic METALOG output

PR: 200674
Submitted by: Fabian Keil <fk@fabiankeil.de>
Reviewed by: emaste
MFC after: 1 week
Obtained from: ElectroBSD
---
 share/zoneinfo/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/share/zoneinfo/Makefile b/share/zoneinfo/Makefile
index ebb26c7c7ad1..b3d968dc4b4e 100644
--- a/share/zoneinfo/Makefile
+++ b/share/zoneinfo/Makefile
@@ -79,7 +79,7 @@ zoneinfo: yearistype ${TDATA}
 
 beforeinstall:
 	cd ${TZBUILDDIR} && \
-	    find * -type f -print -exec ${INSTALL} \
+	    find -s * -type f -print -exec ${INSTALL} \
 	    -o ${BINOWN} -g ${BINGRP} -m ${NOBINMODE} \
 	    \{} ${DESTDIR}/usr/share/zoneinfo/\{} \;
 	${INSTALL} -o ${BINOWN} -g ${BINGRP} -m ${NOBINMODE} \

From 2bdfbb5b0ff243d6f130a88cfeb394ae3d150af8 Mon Sep 17 00:00:00 2001
From: trasz <trasz@FreeBSD.org>
Date: Sat, 17 Oct 2015 13:00:34 +0000
Subject: [PATCH 304/380] Add iscsictl(8) and iscsid(8) to rescue(8).  The
 point is to make it easier to build md_root images from rescue(8), to use
 with iSCSI boot.

The change increases the size of rescue by 62kB, from 8728kB to 8790kB.

Reviewed by:	bapt@
MFC after:	1 month
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D3865
---
 rescue/rescue/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/rescue/rescue/Makefile b/rescue/rescue/Makefile
index 8e682bd1f2ed..4d8b63d0f825 100644
--- a/rescue/rescue/Makefile
+++ b/rescue/rescue/Makefile
@@ -179,7 +179,7 @@ CRUNCH_BUILDOPTS_dhclient= -DRELEASE_CRUNCH -Dlint
 #
 CRUNCH_SRCDIRS+= usr.bin
 
-CRUNCH_PROGS_usr.bin= head mt nc sed tail tee
+CRUNCH_PROGS_usr.bin= head iscsictl mt nc sed tail tee
 
 CRUNCH_PROGS_usr.bin+= gzip
 CRUNCH_ALIAS_gzip= gunzip gzcat zcat
@@ -214,7 +214,7 @@ CRUNCH_ALIAS_id= groups whoami
 #
 CRUNCH_SRCDIRS+= usr.sbin
 
-CRUNCH_PROGS_usr.sbin+= chroot
+CRUNCH_PROGS_usr.sbin+= chroot iscsid
 
 CRUNCH_PROGS_usr.sbin+= chown
 CRUNCH_ALIAS_chown= chgrp

From 16294b3699f26c533525aac9a852c8259e5a5f2c Mon Sep 17 00:00:00 2001
From: trasz <trasz@FreeBSD.org>
Date: Sat, 17 Oct 2015 13:06:52 +0000
Subject: [PATCH 305/380] Add -w flag to iscsictl(8) utility, to make it wait
 for successfull session establishment.  Scripting is kind of hard without it.

Reviewed by:	mav@
MFC after:	1 month
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D3872
---
 usr.bin/iscsictl/iscsictl.8 | 17 ++++++--
 usr.bin/iscsictl/iscsictl.c | 79 +++++++++++++++++++++++++++++++++++--
 2 files changed, 89 insertions(+), 7 deletions(-)

diff --git a/usr.bin/iscsictl/iscsictl.8 b/usr.bin/iscsictl/iscsictl.8
index 93bf7ed18f87..ffa2e15022ef 100644
--- a/usr.bin/iscsictl/iscsictl.8
+++ b/usr.bin/iscsictl/iscsictl.8
@@ -27,7 +27,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd September 12, 2014
+.Dd October 17, 2015
 .Dt ISCSICTL 8
 .Os
 .Sh NAME
@@ -36,7 +36,9 @@
 .Sh SYNOPSIS
 .Nm
 .Fl A
-.Fl p Ar portal Fl t Ar target Op Fl u Ar user Fl s Ar secret
+.Fl p Ar portal Fl t Ar target
+.Op Fl u Ar user Fl s Ar secret
+.Op Fl w Ar timeout
 .Nm
 .Fl A
 .Fl d Ar discovery-host Op Fl u Ar user Fl s Ar secret
@@ -70,6 +72,7 @@
 .Nm
 .Fl L
 .Op Fl v
+.Op Fl w Ar timeout
 .Sh DESCRIPTION
 The
 .Nm
@@ -113,6 +116,10 @@ Target name.
 CHAP login.
 .It Fl v
 Verbose mode.
+.It Fl w
+Instead of returning immediately, wait up to
+.Ar timeout
+seconds until all configured sessions are successfully established.
 .El
 .Pp
 Certain parameters are necessary when adding a session.
@@ -132,9 +139,11 @@ via configuration file.
 .Pp
 Since connecting to the target is performed in background, non-zero
 exit status does not mean that the session was successfully established.
-Use
+Use either
 .Nm Fl L
-to check the connection status.
+to check the connection status, or the
+.Fl w
+flag to wait for session establishment.
 .Pp
 Note that in order for the iSCSI initiator to be able to connect to a target,
 the
diff --git a/usr.bin/iscsictl/iscsictl.c b/usr.bin/iscsictl/iscsictl.c
index ac83b490ecab..4eb019b51f89 100644
--- a/usr.bin/iscsictl/iscsictl.c
+++ b/usr.bin/iscsictl/iscsictl.c
@@ -592,12 +592,70 @@ kernel_list(int iscsi_fd, const struct target *targ __unused,
 	return (0);
 }
 
+static int
+kernel_wait(int iscsi_fd, int timeout)
+{
+	struct iscsi_session_state *states = NULL;
+	const struct iscsi_session_state *state;
+	const struct iscsi_session_conf *conf;
+	struct iscsi_session_list isl;
+	unsigned int i, nentries = 1;
+	bool all_connected;
+	int error;
+
+	for (;;) {
+		for (;;) {
+			states = realloc(states,
+			    nentries * sizeof(struct iscsi_session_state));
+			if (states == NULL)
+				xo_err(1, "realloc");
+
+			memset(&isl, 0, sizeof(isl));
+			isl.isl_nentries = nentries;
+			isl.isl_pstates = states;
+
+			error = ioctl(iscsi_fd, ISCSISLIST, &isl);
+			if (error != 0 && errno == EMSGSIZE) {
+				nentries *= 4;
+				continue;
+			}
+			break;
+		}
+		if (error != 0) {
+			xo_warn("ISCSISLIST");
+			return (error);
+		}
+
+		all_connected = true;
+		for (i = 0; i < isl.isl_nentries; i++) {
+			state = &states[i];
+			conf = &state->iss_conf;
+
+			if (!state->iss_connected) {
+				all_connected = false;
+				break;
+			}
+		}
+
+		if (all_connected)
+			return (0);
+
+		sleep(1);
+
+		if (timeout > 0) {
+			timeout--;
+			if (timeout == 0)
+				return (1);
+		}
+	}
+}
+
 static void
 usage(void)
 {
 
 	fprintf(stderr, "usage: iscsictl -A -p portal -t target "
-	    "[-u user -s secret]\n");
+	    "[-u user -s secret] [-w timeout]\n");
 	fprintf(stderr, "       iscsictl -A -d discovery-host "
 	    "[-u user -s secret]\n");
 	fprintf(stderr, "       iscsictl -A -a [-c path]\n");
@@ -609,7 +667,7 @@ usage(void)
 	fprintf(stderr, "       iscsictl -R [-p portal] [-t target]\n");
 	fprintf(stderr, "       iscsictl -R -a\n");
 	fprintf(stderr, "       iscsictl -R -n nickname [-c path]\n");
-	fprintf(stderr, "       iscsictl -L [-v]\n");
+	fprintf(stderr, "       iscsictl -L [-v] [-w timeout]\n");
 	exit(1);
 }
 
@@ -631,6 +689,7 @@ main(int argc, char **argv)
 	const char *conf_path = DEFAULT_CONFIG_PATH;
 	char *nickname = NULL, *discovery_host = NULL, *portal = NULL,
 	     *target = NULL, *user = NULL, *secret = NULL;
+	int timeout = -1;
 	long long session_id = -1;
 	char *end;
 	int ch, error, iscsi_fd, retval, saved_errno;
@@ -641,7 +700,7 @@ main(int argc, char **argv)
 	argc = xo_parse_args(argc, argv);
 	xo_open_container("iscsictl");
 
-	while ((ch = getopt(argc, argv, "AMRLac:d:i:n:p:t:u:s:v")) != -1) {
+	while ((ch = getopt(argc, argv, "AMRLac:d:i:n:p:t:u:s:vw:")) != -1) {
 		switch (ch) {
 		case 'A':
 			Aflag = 1;
@@ -692,6 +751,13 @@ main(int argc, char **argv)
 		case 'v':
 			vflag = 1;
 			break;
+		case 'w':
+			timeout = strtol(optarg, &end, 10);
+			if ((size_t)(end - optarg) != strlen(optarg))
+				xo_errx(1, "trailing characters after timeout");
+			if (timeout < 0)
+				xo_errx(1, "timeout cannot be negative");
+			break;
 		case '?':
 		default:
 			usage();
@@ -782,6 +848,8 @@ main(int argc, char **argv)
 
 		if (vflag != 0)
 			xo_errx(1, "-v cannot be used with -M");
+		if (timeout != -1)
+			xo_errx(1, "-w cannot be used with -M");
 
 	} else if (Rflag != 0) {
 		if (user != NULL)
@@ -811,6 +879,8 @@ main(int argc, char **argv)
 			xo_errx(1, "-i cannot be used with -R");
 		if (vflag != 0)
 			xo_errx(1, "-v cannot be used with -R");
+		if (timeout != -1)
+			xo_errx(1, "-w cannot be used with -R");
 
 	} else {
 		assert(Lflag != 0);
@@ -896,6 +966,9 @@ main(int argc, char **argv)
 			failed += kernel_list(iscsi_fd, targ, vflag);
 	}
 
+	if (timeout != -1)
+		failed += kernel_wait(iscsi_fd, timeout);
+
 	error = close(iscsi_fd);
 	if (error != 0)
 		xo_err(1, "close");

From a1cbced296301ad40506e3a0fc6ae3bfa505ed91 Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Sat, 17 Oct 2015 13:20:42 +0000
Subject: [PATCH 306/380] Add checks for kernel VA accesses to the copyin(9)
 and related functions on arm64.

Reviewed by:	andrew
Sponsored by:	The FreeBSD Foundation
Differential revision:	https://reviews.freebsd.org/D3907
---
 sys/arm64/arm64/copyinout.S | 14 +++++++++++++-
 sys/arm64/arm64/genassym.c  |  2 ++
 sys/arm64/arm64/support.S   | 37 +++++++++++++++++++++++++++++++++++++
 3 files changed, 52 insertions(+), 1 deletion(-)

diff --git a/sys/arm64/arm64/copyinout.S b/sys/arm64/arm64/copyinout.S
index 0982c70fc7a7..1ba81068e439 100644
--- a/sys/arm64/arm64/copyinout.S
+++ b/sys/arm64/arm64/copyinout.S
@@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$");
  */
 ENTRY(copyio_fault)
 	SET_FAULT_HANDLER(xzr, x1) /* Clear the handler */
+copyio_fault_nopcb:
 	mov	x0, #EFAULT
 	ret
 END(copyio_fault)
@@ -51,6 +52,10 @@ END(copyio_fault)
  */
 ENTRY(copyout)
 	cbz	x2, 2f		/* If len == 0 then skip loop */
+	add	x3, x1, x2
+	ldr	x4, =VM_MAXUSER_ADDRESS
+	cmp	x3, x4
+	b.hi	copyio_fault_nopcb
 
 	adr	x6, copyio_fault /* Get the handler address */
 	SET_FAULT_HANDLER(x6, x7) /* Set the handler */
@@ -73,6 +78,10 @@ END(copyout)
  */
 ENTRY(copyin)
 	cbz	x2, 2f		/* If len == 0 then skip loop */
+	add	x3, x0, x2
+	ldr	x4, =VM_MAXUSER_ADDRESS
+	cmp	x3, x4
+	b.hi	copyio_fault_nopcb
 
 	adr	x6, copyio_fault /* Get the handler address */
 	SET_FAULT_HANDLER(x6, x7) /* Set the handler */
@@ -97,11 +106,14 @@ ENTRY(copyinstr)
 	mov	x5, xzr		/* count = 0 */
 	mov	w4, #1		/* If zero return faulure */
 	cbz	x2, 3f		/* If len == 0 then skip loop */
+	ldr	x7, =VM_MAXUSER_ADDRESS
 
 	adr	x6, copyio_fault /* Get the handler address */
 	SET_FAULT_HANDLER(x6, x7) /* Set the handler */
 
-1:	ldrb	w4, [x0], #1	/* Load from uaddr */
+1:	cmp	x0, x7
+	b.cs	copyio_fault
+	ldrb	w4, [x0], #1	/* Load from uaddr */
 	strb	w4, [x1], #1	/* Store in kaddr */
 	add	x5, x5, #1	/* count++ */
 	cbz	w4, 2f		/* Break when NUL-terminated */
diff --git a/sys/arm64/arm64/genassym.c b/sys/arm64/arm64/genassym.c
index 33b204216663..f8dfedab83a5 100644
--- a/sys/arm64/arm64/genassym.c
+++ b/sys/arm64/arm64/genassym.c
@@ -38,6 +38,8 @@ __FBSDID("$FreeBSD$");
 #include <machine/vmparam.h>
 
 ASSYM(KERNBASE, KERNBASE);
+ASSYM(VM_MAXUSER_ADDRESS, VM_MAXUSER_ADDRESS);
+
 ASSYM(TDF_ASTPENDING, TDF_ASTPENDING);
 ASSYM(TDF_NEEDRESCHED, TDF_NEEDRESCHED);
 
diff --git a/sys/arm64/arm64/support.S b/sys/arm64/arm64/support.S
index 914a6e3145e9..749fc6e70269 100644
--- a/sys/arm64/arm64/support.S
+++ b/sys/arm64/arm64/support.S
@@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$");
  */
 ENTRY(fsu_fault)
 	SET_FAULT_HANDLER(xzr, x1)	/* Reset the handler function */
+fsu_fault_nopcb:
 	mov	x0, #-1
 	ret
 END(fsu_fault)
@@ -49,6 +50,9 @@ END(fsu_fault)
  * int casueword32(volatile uint32_t *, uint32_t, uint32_t *, uint32_t)
  */
 ENTRY(casueword32)
+	ldr	x4, =(VM_MAXUSER_ADDRESS-3)
+	cmp	x0, x4
+	b.cs	fsu_fault_nopcb
 	adr	x6, fsu_fault		/* Load the fault handler */
 	SET_FAULT_HANDLER(x6, x4)	/* And set it */
 1:	ldxr	w4, [x0]		/* Load-exclusive the data */
@@ -67,6 +71,9 @@ END(casueword32)
  * int casueword(volatile u_long *, u_long, u_long *, u_long)
  */
 ENTRY(casueword)
+	ldr	x4, =(VM_MAXUSER_ADDRESS-7)
+	cmp	x0, x4
+	b.cs	fsu_fault_nopcb
 	adr	x6, fsu_fault		/* Load the fault handler */
 	SET_FAULT_HANDLER(x6, x4)	/* And set it */
 1:	ldxr	x4, [x0]		/* Load-exclusive the data */
@@ -85,6 +92,9 @@ END(casueword)
  * int fubyte(volatile const void *)
  */
 ENTRY(fubyte)
+	ldr	x1, =VM_MAXUSER_ADDRESS
+	cmp	x0, x1
+	b.cs	fsu_fault_nopcb
 	adr	x6, fsu_fault		/* Load the fault handler */
 	SET_FAULT_HANDLER(x6, x1)	/* And set it */
 	ldrb	w0, [x0]		/* Try loading the data */
@@ -96,6 +106,9 @@ END(fubyte)
  * int fuword(volatile const void *)
  */
 ENTRY(fuword16)
+	ldr	x1, =(VM_MAXUSER_ADDRESS-1)
+	cmp	x0, x1
+	b.cs	fsu_fault_nopcb
 	adr	x6, fsu_fault		/* Load the fault handler */
 	SET_FAULT_HANDLER(x6, x1)	/* And set it */
 	ldrh	w0, [x0]		/* Try loading the data */
@@ -107,6 +120,9 @@ END(fuword16)
  * int32_t fueword32(volatile const void *, int32_t *)
  */
 ENTRY(fueword32)
+	ldr	x2, =(VM_MAXUSER_ADDRESS-3)
+	cmp	x0, x2
+	b.cs	fsu_fault_nopcb
 	adr	x6, fsu_fault		/* Load the fault handler */
 	SET_FAULT_HANDLER(x6, x2)	/* And set it */
 	ldr	w0, [x0]		/* Try loading the data */
@@ -122,6 +138,9 @@ END(fueword32)
  */
 ENTRY(fueword)
 EENTRY(fueword64)
+	ldr	x2, =(VM_MAXUSER_ADDRESS-7)
+	cmp	x0, x2
+	b.cs	fsu_fault_nopcb
 	adr	x6, fsu_fault		/* Load the fault handler */
 	SET_FAULT_HANDLER(x6, x2)	/* And set it */
 	ldr	x0, [x0]		/* Try loading the data */
@@ -136,6 +155,9 @@ END(fueword)
  * int subyte(volatile void *, int)
  */
 ENTRY(subyte)
+	ldr	x2, =VM_MAXUSER_ADDRESS
+	cmp	x0, x2
+	b.cs	fsu_fault_nopcb
 	adr	x6, fsu_fault		/* Load the fault handler */
 	SET_FAULT_HANDLER(x6, x2)	/* And set it */
 	strb	w1, [x0]		/* Try storing the data */
@@ -148,6 +170,9 @@ END(subyte)
  * int suword16(volatile void *, int)
  */
 ENTRY(suword16)
+	ldr	x2, =(VM_MAXUSER_ADDRESS-1)
+	cmp	x0, x2
+	b.cs	fsu_fault_nopcb
 	adr	x6, fsu_fault		/* Load the fault handler */
 	SET_FAULT_HANDLER(x6, x2)	/* And set it */
 	strh	w1, [x0]		/* Try storing the data */
@@ -160,6 +185,9 @@ END(suword16)
  * int suword32(volatile void *, int)
  */
 ENTRY(suword32)
+	ldr	x2, =(VM_MAXUSER_ADDRESS-3)
+	cmp	x0, x2
+	b.cs	fsu_fault_nopcb
 	adr	x6, fsu_fault		/* Load the fault handler */
 	SET_FAULT_HANDLER(x6, x2)	/* And set it */
 	str	w1, [x0]		/* Try storing the data */
@@ -173,6 +201,9 @@ END(suword32)
  */
 ENTRY(suword)
 EENTRY(suword64)
+	ldr	x2, =(VM_MAXUSER_ADDRESS-7)
+	cmp	x0, x2
+	b.cs	fsu_fault_nopcb
 	adr	x6, fsu_fault		/* Load the fault handler */
 	SET_FAULT_HANDLER(x6, x2)	/* And set it */
 	str	x1, [x0]		/* Try storing the data */
@@ -201,6 +232,9 @@ END(fsu_fault)
  * int fuswintr(void *)
  */
 ENTRY(fuswintr)
+	ldr	x1, =(VM_MAXUSER_ADDRESS-3)
+	cmp	x0, x1
+	b.cs	fsu_fault_nopcb
 	adr	x6, fsu_intr_fault	/* Load the fault handler */
 	SET_FAULT_HANDLER(x6, x1)	/* And set it */
 	ldr	w0, [x0]		/* Try loading the data */
@@ -212,6 +246,9 @@ END(fuswintr)
  * int suswintr(void *base, int word)
  */
 ENTRY(suswintr)
+	ldr	x2, =(VM_MAXUSER_ADDRESS-3)
+	cmp	x0, x2
+	b.cs	fsu_fault_nopcb
 	adr	x6, fsu_intr_fault	/* Load the fault handler */
 	SET_FAULT_HANDLER(x6, x2)	/* And set it */
 	str	w1, [x0]		/* Try storing the data */

From 9361fca5c29379875f8417bf2b3fb8b6e29e45e3 Mon Sep 17 00:00:00 2001
From: andrew <andrew@FreeBSD.org>
Date: Sat, 17 Oct 2015 14:07:47 +0000
Subject: [PATCH 307/380] Rename build_block_pagetable to
 build_l2_block_pagetable in preperation for adding support for 4 levels of
 page tables.

Obtained from:	Patrick Wildt <patrick@bitrig.org>
---
 sys/arm64/arm64/locore.S | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sys/arm64/arm64/locore.S b/sys/arm64/arm64/locore.S
index c5da907118d3..7226d9d39c4a 100644
--- a/sys/arm64/arm64/locore.S
+++ b/sys/arm64/arm64/locore.S
@@ -357,7 +357,7 @@ create_pagetables:
 	mov	x7, #NORMAL_MEM
 	mov	x8, #(KERNBASE & L2_BLOCK_MASK)
 	mov	x9, x28
-	bl	build_block_pagetable
+	bl	build_l2_block_pagetable
 
 	/* Move to the l1 table */
 	add	x26, x26, #PAGE_SIZE
@@ -427,7 +427,7 @@ build_section_pagetable:
  * Builds an L1 -> L2 table descriptor
  *
  * This is a link for a 1GiB block of memory with up to 2MiB regions mapped
- * within it by build_block_pagetable.
+ * within it by build_l1_block_pagetable.
  *
  *  x6  = L1 table
  *  x8  = Virtual Address
@@ -463,7 +463,7 @@ link_l1_pagetable:
  *  x10 = Entry count (TODO)
  *  x11, x12 and x13 are trashed
  */
-build_block_pagetable:
+build_l2_block_pagetable:
 	/*
 	 * Build the L2 table entry.
 	 */

From 8fbeca388aeeec742a6758d69f01c3d29265bb16 Mon Sep 17 00:00:00 2001
From: dumbbell <dumbbell@FreeBSD.org>
Date: Sat, 17 Oct 2015 14:48:39 +0000
Subject: [PATCH 308/380] drm/i915: Reduce diff with Linux 3.8

There is no functional change. The goal is to ease the future update to
Linux 3.8's i915 driver.

MFC after:	2 months
---
 sys/dev/drm2/i915/intel_dp.c  |  33 ++-
 sys/dev/drm2/i915/intel_iic.c | 378 +++++++++++++++-------------------
 2 files changed, 179 insertions(+), 232 deletions(-)

diff --git a/sys/dev/drm2/i915/intel_dp.c b/sys/dev/drm2/i915/intel_dp.c
index 5791caf319c5..eba5fe3206ee 100644
--- a/sys/dev/drm2/i915/intel_dp.c
+++ b/sys/dev/drm2/i915/intel_dp.c
@@ -555,12 +555,12 @@ intel_dp_aux_native_read(struct intel_dp *intel_dp,
 }
 
 static int
-intel_dp_i2c_aux_ch(device_t idev, int mode, uint8_t write_byte,
-    uint8_t *read_byte)
+intel_dp_i2c_aux_ch(device_t adapter, int mode,
+		    uint8_t write_byte, uint8_t *read_byte)
 {
-	struct iic_dp_aux_data *data;
-	struct intel_dp *intel_dp;
-	uint16_t address;
+	struct iic_dp_aux_data *data = device_get_softc(adapter);
+	struct intel_dp *intel_dp = data->priv;
+	uint16_t address = data->address;
 	uint8_t msg[5];
 	uint8_t reply[2];
 	unsigned retry;
@@ -568,10 +568,6 @@ intel_dp_i2c_aux_ch(device_t idev, int mode, uint8_t write_byte,
 	int reply_bytes;
 	int ret;
 
-	data = device_get_softc(idev);
-	intel_dp = data->priv;
-	address = data->address;
-
 	intel_dp_check_edp(intel_dp);
 	/* Set up the command byte */
 	if (mode & MODE_I2C_READ)
@@ -609,7 +605,7 @@ intel_dp_i2c_aux_ch(device_t idev, int mode, uint8_t write_byte,
 				      reply, reply_bytes);
 		if (ret < 0) {
 			DRM_DEBUG_KMS("aux_ch failed %d\n", ret);
-			return (ret);
+			return ret;
 		}
 
 		switch (reply[0] & AUX_NATIVE_REPLY_MASK) {
@@ -620,14 +616,14 @@ intel_dp_i2c_aux_ch(device_t idev, int mode, uint8_t write_byte,
 			break;
 		case AUX_NATIVE_REPLY_NACK:
 			DRM_DEBUG_KMS("aux_ch native nack\n");
-			return (-EREMOTEIO);
+			return -EREMOTEIO;
 		case AUX_NATIVE_REPLY_DEFER:
 			DELAY(100);
 			continue;
 		default:
 			DRM_ERROR("aux_ch invalid native reply 0x%02x\n",
 				  reply[0]);
-			return (-EREMOTEIO);
+			return -EREMOTEIO;
 		}
 
 		switch (reply[0] & AUX_I2C_REPLY_MASK) {
@@ -638,19 +634,19 @@ intel_dp_i2c_aux_ch(device_t idev, int mode, uint8_t write_byte,
 			return (0/*reply_bytes - 1*/);
 		case AUX_I2C_REPLY_NACK:
 			DRM_DEBUG_KMS("aux_i2c nack\n");
-			return (-EREMOTEIO);
+			return -EREMOTEIO;
 		case AUX_I2C_REPLY_DEFER:
 			DRM_DEBUG_KMS("aux_i2c defer\n");
 			DELAY(100);
 			break;
 		default:
 			DRM_ERROR("aux_i2c invalid reply 0x%02x\n", reply[0]);
-			return (-EREMOTEIO);
+			return -EREMOTEIO;
 		}
 	}
 
 	DRM_ERROR("too many retries, giving up\n");
-	return (-EREMOTEIO);
+	return -EREMOTEIO;
 }
 
 static void ironlake_edp_panel_vdd_on(struct intel_dp *intel_dp);
@@ -660,7 +656,7 @@ static int
 intel_dp_i2c_init(struct intel_dp *intel_dp,
 		  struct intel_connector *intel_connector, const char *name)
 {
-	int ret;
+	int	ret;
 
 	DRM_DEBUG_KMS("i2c_init %s\n", name);
 
@@ -669,7 +665,7 @@ intel_dp_i2c_init(struct intel_dp *intel_dp,
 	    intel_dp_i2c_aux_ch, intel_dp, &intel_dp->dp_iic_bus,
 	    &intel_dp->adapter);
 	ironlake_edp_panel_vdd_off(intel_dp, false);
-	return (ret);
+	return ret;
 }
 
 static bool
@@ -956,8 +952,7 @@ static void ironlake_wait_panel_status(struct intel_dp *intel_dp,
 		      I915_READ(PCH_PP_STATUS),
 		      I915_READ(PCH_PP_CONTROL));
 
-	if (_intel_wait_for(dev,
-	    (I915_READ(PCH_PP_STATUS) & mask) == value, 5000, 10, "915iwp")) {
+	if (_intel_wait_for(dev, (I915_READ(PCH_PP_STATUS) & mask) == value, 5000, 10, "915iwp")) {
 		DRM_ERROR("Panel status timeout: status %08x control %08x\n",
 			  I915_READ(PCH_PP_STATUS),
 			  I915_READ(PCH_PP_CONTROL));
diff --git a/sys/dev/drm2/i915/intel_iic.c b/sys/dev/drm2/i915/intel_iic.c
index 36a5b9e5661b..c04b27bf2422 100644
--- a/sys/dev/drm2/i915/intel_iic.c
+++ b/sys/dev/drm2/i915/intel_iic.c
@@ -96,51 +96,10 @@ struct intel_iic_softc {
 	uint32_t reg0;
 };
 
-static void
-intel_iic_quirk_set(struct drm_i915_private *dev_priv, bool enable)
-{
-	u32 val;
-
-	/* When using bit bashing for I2C, this bit needs to be set to 1 */
-	if (!IS_PINEVIEW(dev_priv->dev))
-		return;
-
-	val = I915_READ(DSPCLK_GATE_D);
-	if (enable)
-		val |= DPCUNIT_CLOCK_GATE_DISABLE;
-	else
-		val &= ~DPCUNIT_CLOCK_GATE_DISABLE;
-	I915_WRITE(DSPCLK_GATE_D, val);
-}
-
-static u32
-intel_iic_get_reserved(device_t idev)
-{
-	struct intel_iic_softc *sc;
-	struct drm_device *dev;
-	struct drm_i915_private *dev_priv;
-	u32 reserved;
-
-	sc = device_get_softc(idev);
-	dev = sc->drm_dev;
-	dev_priv = dev->dev_private;
-
-	if (!IS_I830(dev) && !IS_845G(dev)) {
-		reserved = I915_READ_NOTRACE(sc->reg) &
-		    (GPIO_DATA_PULLUP_DISABLE | GPIO_CLOCK_PULLUP_DISABLE);
-	} else {
-		reserved = 0;
-	}
-
-	return (reserved);
-}
-
 void
 intel_iic_reset(struct drm_device *dev)
 {
-	struct drm_i915_private *dev_priv;
-
-	dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = dev->dev_private;
 	I915_WRITE(dev_priv->gpio_mmio_base + GMBUS0, 0);
 }
 
@@ -157,86 +116,119 @@ intel_iicbus_reset(device_t idev, u_char speed, u_char addr, u_char *oldaddr)
 	return (0);
 }
 
-static void
-intel_iicbb_setsda(device_t idev, int val)
+static void intel_i2c_quirk_set(struct drm_i915_private *dev_priv, bool enable)
 {
-	struct intel_iic_softc *sc;
-	struct drm_i915_private *dev_priv;
-	u32 reserved;
-	u32 data_bits;
+	u32 val;
 
-	sc = device_get_softc(idev);
-	dev_priv = sc->drm_dev->dev_private;
+	/* When using bit bashing for I2C, this bit needs to be set to 1 */
+	if (!IS_PINEVIEW(dev_priv->dev))
+		return;
 
-	reserved = intel_iic_get_reserved(idev);
-	if (val)
-		data_bits = GPIO_DATA_DIR_IN | GPIO_DATA_DIR_MASK;
+	val = I915_READ(DSPCLK_GATE_D);
+	if (enable)
+		val |= DPCUNIT_CLOCK_GATE_DISABLE;
 	else
-		data_bits = GPIO_DATA_DIR_OUT | GPIO_DATA_DIR_MASK |
-		    GPIO_DATA_VAL_MASK;
-
-	I915_WRITE_NOTRACE(sc->reg, reserved | data_bits);
-	POSTING_READ(sc->reg);
+		val &= ~DPCUNIT_CLOCK_GATE_DISABLE;
+	I915_WRITE(DSPCLK_GATE_D, val);
 }
 
-static void
-intel_iicbb_setscl(device_t idev, int val)
+static u32 get_reserved(device_t idev)
 {
-	struct intel_iic_softc *sc;
-	struct drm_i915_private *dev_priv;
-	u32 clock_bits, reserved;
+	struct intel_iic_softc *sc = device_get_softc(idev);
+	struct drm_device *dev = sc->drm_dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 reserved = 0;
 
-	sc = device_get_softc(idev);
-	dev_priv = sc->drm_dev->dev_private;
+	if (!IS_I830(dev) && !IS_845G(dev))
+		reserved = I915_READ_NOTRACE(sc->reg) &
+					     (GPIO_DATA_PULLUP_DISABLE |
+					      GPIO_CLOCK_PULLUP_DISABLE);
 
-	reserved = intel_iic_get_reserved(idev);
-	if (val)
-		clock_bits = GPIO_CLOCK_DIR_IN | GPIO_CLOCK_DIR_MASK;
-	else
-		clock_bits = GPIO_CLOCK_DIR_OUT | GPIO_CLOCK_DIR_MASK |
-		    GPIO_CLOCK_VAL_MASK;
-
-	I915_WRITE_NOTRACE(sc->reg, reserved | clock_bits);
-	POSTING_READ(sc->reg);
+	return reserved;
 }
 
-static int
-intel_iicbb_getsda(device_t idev)
+static int get_clock(device_t adapter)
 {
-	struct intel_iic_softc *sc;
-	struct drm_i915_private *dev_priv;
-	u32 reserved;
-
-	sc = device_get_softc(idev);
-	dev_priv = sc->drm_dev->dev_private;
-
-	reserved = intel_iic_get_reserved(idev);
-
-	I915_WRITE_NOTRACE(sc->reg, reserved | GPIO_DATA_DIR_MASK);
-	I915_WRITE_NOTRACE(sc->reg, reserved);
-	return ((I915_READ_NOTRACE(sc->reg) & GPIO_DATA_VAL_IN) != 0);
-}
-
-static int
-intel_iicbb_getscl(device_t idev)
-{
-	struct intel_iic_softc *sc;
-	struct drm_i915_private *dev_priv;
-	u32 reserved;
-
-	sc = device_get_softc(idev);
-	dev_priv = sc->drm_dev->dev_private;
-
-	reserved = intel_iic_get_reserved(idev);
-
+	struct intel_iic_softc *sc = device_get_softc(adapter);
+	struct drm_i915_private *dev_priv = sc->drm_dev->dev_private;
+	u32 reserved = get_reserved(adapter);
 	I915_WRITE_NOTRACE(sc->reg, reserved | GPIO_CLOCK_DIR_MASK);
 	I915_WRITE_NOTRACE(sc->reg, reserved);
 	return ((I915_READ_NOTRACE(sc->reg) & GPIO_CLOCK_VAL_IN) != 0);
 }
 
+static int get_data(device_t adapter)
+{
+	struct intel_iic_softc *sc = device_get_softc(adapter);
+	struct drm_i915_private *dev_priv = sc->drm_dev->dev_private;
+	u32 reserved = get_reserved(adapter);
+	I915_WRITE_NOTRACE(sc->reg, reserved | GPIO_DATA_DIR_MASK);
+	I915_WRITE_NOTRACE(sc->reg, reserved);
+	return ((I915_READ_NOTRACE(sc->reg) & GPIO_DATA_VAL_IN) != 0);
+}
+
+static void set_clock(device_t adapter, int state_high)
+{
+	struct intel_iic_softc *sc = device_get_softc(adapter);
+	struct drm_i915_private *dev_priv = sc->drm_dev->dev_private;
+	u32 reserved = get_reserved(adapter);
+	u32 clock_bits;
+
+	if (state_high)
+		clock_bits = GPIO_CLOCK_DIR_IN | GPIO_CLOCK_DIR_MASK;
+	else
+		clock_bits = GPIO_CLOCK_DIR_OUT | GPIO_CLOCK_DIR_MASK |
+			GPIO_CLOCK_VAL_MASK;
+
+	I915_WRITE_NOTRACE(sc->reg, reserved | clock_bits);
+	POSTING_READ(sc->reg);
+}
+
+static void set_data(device_t adapter, int state_high)
+{
+	struct intel_iic_softc *sc = device_get_softc(adapter);
+	struct drm_i915_private *dev_priv = sc->drm_dev->dev_private;
+	u32 reserved = get_reserved(adapter);
+	u32 data_bits;
+
+	if (state_high)
+		data_bits = GPIO_DATA_DIR_IN | GPIO_DATA_DIR_MASK;
+	else
+		data_bits = GPIO_DATA_DIR_OUT | GPIO_DATA_DIR_MASK |
+			GPIO_DATA_VAL_MASK;
+
+	I915_WRITE_NOTRACE(sc->reg, reserved | data_bits);
+	POSTING_READ(sc->reg);
+}
+
+static int
+intel_gpio_pre_xfer(device_t adapter)
+{
+	struct intel_iic_softc *sc = device_get_softc(adapter);
+	struct drm_i915_private *dev_priv = sc->drm_dev->dev_private;
+
+	intel_iic_reset(sc->drm_dev);
+	intel_i2c_quirk_set(dev_priv, true);
+	IICBB_SETSDA(adapter, 1);
+	IICBB_SETSCL(adapter, 1);
+	DELAY(I2C_RISEFALL_TIME);
+	return 0;
+}
+
+static void
+intel_gpio_post_xfer(device_t adapter)
+{
+	struct intel_iic_softc *sc = device_get_softc(adapter);
+	struct drm_i915_private *dev_priv = sc->drm_dev->dev_private;
+
+	IICBB_SETSDA(adapter, 1);
+	IICBB_SETSCL(adapter, 1);
+	intel_i2c_quirk_set(dev_priv, false);
+}
+
 static int
 gmbus_xfer_read(struct drm_i915_private *dev_priv, struct iic_msg *msg,
-    u32 gmbus1_index)
+		u32 gmbus1_index)
 {
 	int reg_offset = dev_priv->gpio_mmio_base;
 	u16 len = msg->len;
@@ -254,19 +246,19 @@ gmbus_xfer_read(struct drm_i915_private *dev_priv, struct iic_msg *msg,
 		u32 gmbus2;
 
 		ret = _intel_wait_for(sc->drm_dev,
-		    ((gmbus2 = I915_READ(GMBUS2 + reg_offset)) &
-		    (GMBUS_SATOER | GMBUS_HW_RDY)),
-		    50, 1, "915gbr");
+			       ((gmbus2 = I915_READ(GMBUS2 + reg_offset)) &
+			       (GMBUS_SATOER | GMBUS_HW_RDY)),
+			       50, 1, "915gbr");
 		if (ret)
-			return (-ETIMEDOUT);
+			return -ETIMEDOUT;
 		if (gmbus2 & GMBUS_SATOER)
-			return (-ENXIO);
+			return -ENXIO;
 
 		val = I915_READ(GMBUS3 + reg_offset);
 		do {
 			*buf++ = val & 0xff;
 			val >>= 8;
-		} while (--len != 0 && ++loop < 4);
+		} while (--len && ++loop < 4);
 	}
 
 	return 0;
@@ -299,18 +291,18 @@ gmbus_xfer_write(struct drm_i915_private *dev_priv, struct iic_msg *msg)
 		val = loop = 0;
 		do {
 			val |= *buf++ << (8 * loop);
-		} while (--len != 0 && ++loop < 4);
+		} while (--len && ++loop < 4);
 
 		I915_WRITE(GMBUS3 + reg_offset, val);
 
 		ret = _intel_wait_for(sc->drm_dev,
-		    ((gmbus2 = I915_READ(GMBUS2 + reg_offset)) &
-		    (GMBUS_SATOER | GMBUS_HW_RDY)),
-		    50, 1, "915gbw");
+			       ((gmbus2 = I915_READ(GMBUS2 + reg_offset)) &
+			       (GMBUS_SATOER | GMBUS_HW_RDY)),
+			       50, 1, "915gbw");
 		if (ret)
-			return (-ETIMEDOUT);
+			return -ETIMEDOUT;
 		if (gmbus2 & GMBUS_SATOER)
-			return (-ENXIO);
+			return -ENXIO;
 	}
 	return 0;
 }
@@ -356,20 +348,20 @@ gmbus_xfer_index_read(struct drm_i915_private *dev_priv, struct iic_msg *msgs)
 }
 
 static int
-intel_gmbus_transfer(device_t idev, struct iic_msg *msgs, uint32_t nmsgs)
+gmbus_xfer(device_t adapter,
+	   struct iic_msg *msgs,
+	   uint32_t num)
 {
-	struct intel_iic_softc *sc;
-	struct drm_i915_private *dev_priv;
+	struct intel_iic_softc *sc = device_get_softc(adapter);
+	struct drm_i915_private *dev_priv = sc->drm_dev->dev_private;
 	int error, i, ret, reg_offset, unit;
 
 	error = 0;
-	sc = device_get_softc(idev);
-	dev_priv = sc->drm_dev->dev_private;
-	unit = device_get_unit(idev);
+	unit = device_get_unit(adapter);
 
 	sx_xlock(&dev_priv->gmbus_sx);
 	if (sc->force_bit_dev) {
-		error = -IICBUS_TRANSFER(dev_priv->bbbus[unit], msgs, nmsgs);
+		error = -IICBUS_TRANSFER(dev_priv->bbbus[unit], msgs, num);
 		goto out;
 	}
 
@@ -377,10 +369,10 @@ intel_gmbus_transfer(device_t idev, struct iic_msg *msgs, uint32_t nmsgs)
 
 	I915_WRITE(GMBUS0 + reg_offset, sc->reg0);
 
-	for (i = 0; i < nmsgs; i++) {
+	for (i = 0; i < num; i++) {
 		u32 gmbus2;
 
-		if (gmbus_is_index_read(msgs, i, nmsgs)) {
+		if (gmbus_is_index_read(msgs, i, num)) {
 			error = gmbus_xfer_index_read(dev_priv, &msgs[i]);
 			i += 1;  /* set i to the index of the read xfer */
 		} else if (msgs[i].flags & IIC_M_RD) {
@@ -413,12 +405,12 @@ intel_gmbus_transfer(device_t idev, struct iic_msg *msgs, uint32_t nmsgs)
 	/* Mark the GMBUS interface as disabled after waiting for idle.
 	 * We will re-enable it at the start of the next xfer,
 	 * till then let it sleep.
- 	 */
+	 */
 	if (_intel_wait_for(dev,
 	    (I915_READ(GMBUS2 + reg_offset) & GMBUS_ACTIVE) == 0,
 	    10, 1, "915gbu")) {
 		DRM_DEBUG_KMS("GMBUS [%s] timed out waiting for idle\n",
-		    sc->name);
+			 sc->name);
 		error = -ETIMEDOUT;
 	}
 	I915_WRITE(GMBUS0 + reg_offset, 0);
@@ -459,81 +451,16 @@ clear_err:
 
 timeout:
 	DRM_INFO("GMBUS [%s] timed out, falling back to bit banging on pin %d\n",
-	    sc->name, sc->reg0 & 0xff);
+		 sc->name, sc->reg0 & 0xff);
 	I915_WRITE(GMBUS0 + reg_offset, 0);
 
-	/*
-	 * Hardware may not support GMBUS over these pins?
-	 * Try GPIO bitbanging instead.
-	 */
+	/* Hardware may not support GMBUS over these pins? Try GPIO bitbanging instead. */
 	sc->force_bit_dev = true;
-	error = -IICBUS_TRANSFER(idev, msgs, nmsgs);
-	goto out;
+	error = -IICBUS_TRANSFER(adapter, msgs, num);
 
 out:
 	sx_xunlock(&dev_priv->gmbus_sx);
-	return (-error);
-}
-
-device_t 
-intel_gmbus_get_adapter(struct drm_i915_private *dev_priv,
-    unsigned port)
-{
-
-	if (!intel_gmbus_is_port_valid(port))
-		DRM_ERROR("GMBUS get adapter %d: invalid port\n", port);
-	return (intel_gmbus_is_port_valid(port) ? dev_priv->gmbus[port - 1] :
-	    NULL);
-}
-
-void
-intel_gmbus_set_speed(device_t idev, int speed)
-{
-	struct intel_iic_softc *sc;
-
-	sc = device_get_softc(device_get_parent(idev));
-
-	sc->reg0 = (sc->reg0 & ~(0x3 << 8)) | speed;
-}
-
-void
-intel_gmbus_force_bit(device_t idev, bool force_bit)
-{
-	struct intel_iic_softc *sc;
-
-	sc = device_get_softc(device_get_parent(idev));
-	sc->force_bit_dev = force_bit;
-}
-
-static int
-intel_iicbb_pre_xfer(device_t idev)
-{
-	struct intel_iic_softc *sc;
-	struct drm_i915_private *dev_priv;
-
-	sc = device_get_softc(idev);
-	dev_priv = sc->drm_dev->dev_private;
-
-	intel_iic_reset(sc->drm_dev);
-	intel_iic_quirk_set(dev_priv, true);
-	IICBB_SETSDA(idev, 1);
-	IICBB_SETSCL(idev, 1);
-	DELAY(I2C_RISEFALL_TIME);
-	return (0);
-}
-
-static void
-intel_iicbb_post_xfer(device_t idev)
-{
-	struct intel_iic_softc *sc;
-	struct drm_i915_private *dev_priv;
-
-	sc = device_get_softc(idev);
-	dev_priv = sc->drm_dev->dev_private;
-
-	IICBB_SETSDA(idev, 1);
-	IICBB_SETSCL(idev, 1);
-	intel_iic_quirk_set(dev_priv, false);
+	return -error;
 }
 
 static int
@@ -663,7 +590,7 @@ static device_method_t intel_gmbus_methods[] = {
 	DEVMETHOD(device_attach,	intel_gmbus_attach),
 	DEVMETHOD(device_detach,	intel_gmbus_detach),
 	DEVMETHOD(iicbus_reset,		intel_iicbus_reset),
-	DEVMETHOD(iicbus_transfer,	intel_gmbus_transfer),
+	DEVMETHOD(iicbus_transfer,	gmbus_xfer),
 	DEVMETHOD_END
 };
 static driver_t intel_gmbus_driver = {
@@ -686,12 +613,12 @@ static device_method_t intel_iicbb_methods[] =	{
 
 	DEVMETHOD(iicbb_callback,	iicbus_null_callback),
 	DEVMETHOD(iicbb_reset,		intel_iicbus_reset),
-	DEVMETHOD(iicbb_setsda,		intel_iicbb_setsda),
-	DEVMETHOD(iicbb_setscl,		intel_iicbb_setscl),
-	DEVMETHOD(iicbb_getsda,		intel_iicbb_getsda),
-	DEVMETHOD(iicbb_getscl,		intel_iicbb_getscl),
-	DEVMETHOD(iicbb_pre_xfer,	intel_iicbb_pre_xfer),
-	DEVMETHOD(iicbb_post_xfer,	intel_iicbb_post_xfer),
+	DEVMETHOD(iicbb_setsda,		set_data),
+	DEVMETHOD(iicbb_setscl,		set_clock),
+	DEVMETHOD(iicbb_getsda,		get_data),
+	DEVMETHOD(iicbb_getscl,		get_clock),
+	DEVMETHOD(iicbb_pre_xfer,	intel_gpio_pre_xfer),
+	DEVMETHOD(iicbb_post_xfer,	intel_gpio_post_xfer),
 	DEVMETHOD_END
 };
 static driver_t intel_iicbb_driver = {
@@ -704,20 +631,19 @@ DRIVER_MODULE_ORDERED(intel_iicbb, drmn, intel_iicbb_driver,
     intel_iicbb_devclass, 0, 0, SI_ORDER_FIRST);
 DRIVER_MODULE(iicbb, intel_iicbb, iicbb_driver, iicbb_devclass, 0, 0);
 
-int
-intel_setup_gmbus(struct drm_device *dev)
+int intel_setup_gmbus(struct drm_device *dev)
 {
-	struct drm_i915_private *dev_priv;
+	struct drm_i915_private *dev_priv = dev->dev_private;
 	device_t iic_dev;
-	int i, ret;
+	int ret, i;
 
-	dev_priv = dev->dev_private;
-	sx_init(&dev_priv->gmbus_sx, "gmbus");
 	if (HAS_PCH_SPLIT(dev))
 		dev_priv->gpio_mmio_base = PCH_GPIOA - GPIOA;
 	else
 		dev_priv->gpio_mmio_base = 0;
 
+	sx_init(&dev_priv->gmbus_sx, "gmbus");
+
 	/*
 	 * The Giant there is recursed, most likely.  Normally, the
 	 * intel_setup_gmbus() is called from the attach method of the
@@ -786,14 +712,41 @@ intel_setup_gmbus(struct drm_device *dev)
 
 		intel_iic_reset(dev);
 	}
-
 	mtx_unlock(&Giant);
-	return (0);
+
+	return 0;
 
 err:
 	intel_teardown_gmbus_m(dev, i);
 	mtx_unlock(&Giant);
-	return (ret);
+	return ret;
+}
+
+device_t intel_gmbus_get_adapter(struct drm_i915_private *dev_priv,
+					    unsigned port)
+{
+
+	if (!intel_gmbus_is_port_valid(port))
+		DRM_ERROR("GMBUS get adapter %d: invalid port\n", port);
+	return (intel_gmbus_is_port_valid(port) ? dev_priv->gmbus[port - 1] :
+	    NULL);
+}
+
+void intel_gmbus_set_speed(device_t adapter, int speed)
+{
+	struct intel_iic_softc *sc;
+
+	sc = device_get_softc(device_get_parent(adapter));
+
+	sc->reg0 = (sc->reg0 & ~(0x3 << 8)) | speed;
+}
+
+void intel_gmbus_force_bit(device_t adapter, bool force_bit)
+{
+	struct intel_iic_softc *sc;
+
+	sc = device_get_softc(device_get_parent(adapter));
+	sc->force_bit_dev = force_bit;
 }
 
 static void
@@ -806,8 +759,7 @@ intel_teardown_gmbus_m(struct drm_device *dev, int m)
 	sx_destroy(&dev_priv->gmbus_sx);
 }
 
-void
-intel_teardown_gmbus(struct drm_device *dev)
+void intel_teardown_gmbus(struct drm_device *dev)
 {
 
 	mtx_lock(&Giant);

From a82fb7c8ee60cf1c00ba8e0ddc2b04a57c99db94 Mon Sep 17 00:00:00 2001
From: jah <jah@FreeBSD.org>
Date: Sat, 17 Oct 2015 14:58:55 +0000
Subject: [PATCH 309/380] Don't page-align the physical address when calling
 PHYS_TO_VM_PAGE().

M    busdma_bounce.c
---
 sys/x86/x86/busdma_bounce.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sys/x86/x86/busdma_bounce.c b/sys/x86/x86/busdma_bounce.c
index 48f9bb317878..cac6c7158caa 100644
--- a/sys/x86/x86/busdma_bounce.c
+++ b/sys/x86/x86/busdma_bounce.c
@@ -1006,7 +1006,8 @@ add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
 		bpage->busaddr |= addr & PAGE_MASK;
 	}
 	bpage->datavaddr = vaddr;
-	bpage->datapage = PHYS_TO_VM_PAGE(addr & ~PAGE_MASK);
+	/* PHYS_TO_VM_PAGE() will truncate unaligned addresses. */
+	bpage->datapage = PHYS_TO_VM_PAGE(addr);
 	bpage->dataoffs = addr & PAGE_MASK;
 	bpage->datacount = size;
 	STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);

From c8894251e10b362c36b70b0575a33d4624abfd1c Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Sat, 17 Oct 2015 15:50:52 +0000
Subject: [PATCH 310/380] Conditionalize the META_MODE tool handling on
 MK_META_MODE.

It was not being used outside of META_MODE but this should make it more clear
that it is only for META_MODE.

Sponsored by:	EMC / Isilon Storage Division
---
 lib/clang/clang.build.mk | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/clang/clang.build.mk b/lib/clang/clang.build.mk
index acb09f51c544..0c564a5fadf1 100644
--- a/lib/clang/clang.build.mk
+++ b/lib/clang/clang.build.mk
@@ -39,6 +39,7 @@ CXXFLAGS.clang+= -stdlib=libc++
 
 .PATH:	${LLVM_SRCS}/${SRCDIR}
 
+.if ${MK_META_MODE} == "yes"
 .if empty(TOOLSDIR) || !exists(${TOOLSDIR}/usr/bin/clang-tblgen)
 .if ${MACHINE} == "host" && defined(BOOTSTRAPPING_TOOLS)
 .if !empty(LEGACY_TOOLS) && exists(${LEGACY_TOOLS}/usr/bin/tblgen)
@@ -57,6 +58,7 @@ TOOLSDIR?=
 TBLGEN= ${TOOLSDIR}/usr/bin/tblgen
 CLANG_TBLGEN= ${TOOLSDIR}/usr/bin/clang-tblgen
 .endif
+.endif	# ${MK_META_MODE} == "yes"
 TBLGEN?=	tblgen
 CLANG_TBLGEN?=	clang-tblgen
 

From b5af77bf9f311f59fdf978fc5132901f1f3a4037 Mon Sep 17 00:00:00 2001
From: trasz <trasz@FreeBSD.org>
Date: Sat, 17 Oct 2015 16:05:42 +0000
Subject: [PATCH 311/380] Remove write-only variable.

Submitted by:	Dominic Marks
MFC after:	1 month
Sponsored by:	The FreeBSD Foundation
---
 usr.bin/iscsictl/iscsictl.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/usr.bin/iscsictl/iscsictl.c b/usr.bin/iscsictl/iscsictl.c
index 4eb019b51f89..12ede3c7948d 100644
--- a/usr.bin/iscsictl/iscsictl.c
+++ b/usr.bin/iscsictl/iscsictl.c
@@ -597,7 +597,6 @@ kernel_wait(int iscsi_fd, int timeout)
 {
 	struct iscsi_session_state *states = NULL;
 	const struct iscsi_session_state *state;
-	const struct iscsi_session_conf *conf;
 	struct iscsi_session_list isl;
 	unsigned int i, nentries = 1;
 	bool all_connected;
@@ -629,7 +628,6 @@ kernel_wait(int iscsi_fd, int timeout)
 		all_connected = true;
 		for (i = 0; i < isl.isl_nentries; i++) {
 			state = &states[i];
-			conf = &state->iss_conf;
 
 			if (!state->iss_connected) {
 				all_connected = false;

From d50682643925d5cfbafa2f2297f3283a11e758cb Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Sat, 17 Oct 2015 16:42:54 +0000
Subject: [PATCH 312/380] Rework the 'make -n -n' feature such that '-n'
 recurses and '-N' does not.

Bmake has a documented feature of '-N' to skip executing commands which is
specifically intended for debugging top-level builds and not recursing into
sub-directories.  This matches the older 'make -n' behavior we added which made
'-n -n' the recursing target and '-n' a non-recursing target.

Removing the '-n -n' feature allows the build to work as documented in
the bmake manpage with '-n' and '-N'.  The older '-n -n' feature was also
not documented anywhere that I could see.

Note that the ${_+_} var is still needed as currently bmake incorrectly
executes '+' commands when '-N' is specified.

The '-n' and '-n -n' features were broken for several reasons prior to this.
r251748 made '_+_' never expand with '-n -n' which resulted in many
sub-directories not being visited until fixed 2 years later in r288391, and
many targets were given .MAKE over the past few years which resulted in
non-sub-make commands, such as rm and ln and mtree, to be executed.

This should also allow removing some indirection hacks in bsd.subdir.mk and
other cases of .USE that have a .MAKE by using '+'.

Sponsored by:	EMC / Isilon Storage Division
Discussed on:	arch@ (mostly silence)
---
 Makefile        |  8 +-------
 UPDATING        |  6 ++++++
 share/mk/sys.mk | 13 ++++++-------
 3 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/Makefile b/Makefile
index c3326b7ddc43..130fdbf67200 100644
--- a/Makefile
+++ b/Makefile
@@ -243,14 +243,8 @@ cleanworld:
 # Handle the user-driven targets, using the source relative mk files.
 #
 
-.if !(!empty(.MAKEFLAGS:M-n) && ${.MAKEFLAGS:M-n} == "-n")
-# skip this for -n to avoid changing previous behavior of 
-# 'make -n buildworld' etc.  Using -n -n will run it.
-${TGTS}: .MAKE
 tinderbox toolchains kernel-toolchains: .MAKE
-.endif
-
-${TGTS}: .PHONY
+${TGTS}: .PHONY .MAKE
 	${_+_}@cd ${.CURDIR}; ${_MAKE} ${.TARGET}
 
 # The historic default "all" target creates files which may cause stale
diff --git a/UPDATING b/UPDATING
index baff1f13605b..1095ba6ea5d1 100644
--- a/UPDATING
+++ b/UPDATING
@@ -31,6 +31,12 @@ NOTE TO PEOPLE WHO THINK THAT FreeBSD 11.x IS SLOW:
 	disable the most expensive debugging functionality run
 	"ln -s 'abort:false,junk:false' /etc/malloc.conf".)
 
+20151017:
+	The build previously allowed using 'make -n' to not recurse into
+	sub-directories while showing what commands would be executed, and
+	'make -n -n' to recursively show commands.  Now 'make -n' will recurse
+	and 'make -N' will not.
+
 20151012:
 	If you specify SENDMAIL_MC or SENDMAIL_CF in make.conf, mergemaster
 	and etcupdate will now use this file. A custom sendmail.cf is now
diff --git a/share/mk/sys.mk b/share/mk/sys.mk
index e8f4892cd99d..b6c75b6f96a6 100644
--- a/share/mk/sys.mk
+++ b/share/mk/sys.mk
@@ -145,13 +145,12 @@ ECHODIR		?=	true
 .endif
 .endif
 
-.if defined(.PARSEDIR)
-# _+_ appears to be a workaround for the special src .MAKE not working.
-# setting it to + interferes with -N
-_+_		?=
-.elif !empty(.MAKEFLAGS:M-n) && ${.MAKEFLAGS:M-n} == "-n"
-# the check above matches only a single -n, so -n -n will result
-# in _+_ = +
+.if ${.MAKEFLAGS:M-N}
+# bmake -N is supposed to skip executing anything but it does not skip
+# exeucting '+' commands.  The '+' feature is used where .MAKE
+# is not safe for the entire target.  -N is intended to skip building sub-makes
+# so it executing '+' commands is not right.  Work around the bug by not
+# setting '+' when -N is used.
 _+_		?=
 .else
 _+_		?=	+

From 45d403ed233a66523203dfdc9aa62f85efc13787 Mon Sep 17 00:00:00 2001
From: melifaro <melifaro@FreeBSD.org>
Date: Sat, 17 Oct 2015 17:26:44 +0000
Subject: [PATCH 313/380] Remove several compat functions from pre-fib era.

---
 sys/net/route.c      |  7 -------
 sys/net/route.h      |  1 -
 sys/netinet/in_rmx.c | 20 --------------------
 sys/netinet/in_var.h |  2 --
 4 files changed, 30 deletions(-)

diff --git a/sys/net/route.c b/sys/net/route.c
index 4035b03db623..2bcd4ed87131 100644
--- a/sys/net/route.c
+++ b/sys/net/route.c
@@ -931,13 +931,6 @@ rt_flushifroutes(struct ifnet *ifp)
 #define	ifpaddr	info->rti_info[RTAX_IFP]
 #define	flags	info->rti_flags
 
-int
-rt_getifa(struct rt_addrinfo *info)
-{
-
-	return (rt_getifa_fib(info, RT_DEFAULT_FIB));
-}
-
 /*
  * Look up rt_addrinfo for a specific fib.  Note that if rti_ifa is defined,
  * it will be referenced so the caller must free it.
diff --git a/sys/net/route.h b/sys/net/route.h
index c363032ea233..e62aaa613e73 100644
--- a/sys/net/route.h
+++ b/sys/net/route.h
@@ -387,7 +387,6 @@ void	rt_flushifroutes(struct ifnet *ifp);
 
 /* XXX MRT COMPAT VERSIONS THAT SET UNIVERSE to 0 */
 /* Thes are used by old code not yet converted to use multiple FIBS */
-int	 rt_getifa(struct rt_addrinfo *);
 void	 rtalloc_ign(struct route *ro, u_long ignflags);
 void	 rtalloc(struct route *ro); /* XXX deprecated, use rtalloc_ign(ro, 0) */
 struct rtentry *rtalloc1(struct sockaddr *, int, u_long);
diff --git a/sys/netinet/in_rmx.c b/sys/netinet/in_rmx.c
index 0c50cddd6635..e17d36fa7edb 100644
--- a/sys/netinet/in_rmx.c
+++ b/sys/netinet/in_rmx.c
@@ -228,19 +228,6 @@ in_rtalloc_ign(struct route *ro, u_long ignflags, u_int fibnum)
 	rtalloc_ign_fib(ro, ignflags, fibnum);
 }
 
-int
-in_rtrequest( int req,
-	struct sockaddr *dst,
-	struct sockaddr *gateway,
-	struct sockaddr *netmask,
-	int flags,
-	struct rtentry **ret_nrt,
-	u_int fibnum)
-{
-	return (rtrequest_fib(req, dst, gateway, netmask, 
-	    flags, ret_nrt, fibnum));
-}
-
 struct rtentry *
 in_rtalloc1(struct sockaddr *dst, int report, u_long ignflags, u_int fibnum)
 {
@@ -264,10 +251,3 @@ in_rtalloc(struct route *ro, u_int fibnum)
 	rtalloc_ign_fib(ro, 0UL, fibnum);
 }
 
-#if 0
-int	 in_rt_getifa(struct rt_addrinfo *, u_int fibnum);
-int	 in_rtioctl(u_long, caddr_t, u_int);
-int	 in_rtrequest1(int, struct rt_addrinfo *, struct rtentry **, u_int);
-#endif
-
-
diff --git a/sys/netinet/in_var.h b/sys/netinet/in_var.h
index f9e98128bc1c..e491ba4dd5a9 100644
--- a/sys/netinet/in_var.h
+++ b/sys/netinet/in_var.h
@@ -391,8 +391,6 @@ void	 in_rtalloc(struct route *ro, u_int fibnum);
 struct rtentry *in_rtalloc1(struct sockaddr *, int, u_long, u_int);
 void	 in_rtredirect(struct sockaddr *, struct sockaddr *,
 	    struct sockaddr *, int, struct sockaddr *, u_int);
-int	 in_rtrequest(int, struct sockaddr *,
-	    struct sockaddr *, struct sockaddr *, int, struct rtentry **, u_int);
 #endif /* _KERNEL */
 
 /* INET6 stuff */

From c55ea073397f5dc06529bc3581b9dae9c0690139 Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Sat, 17 Oct 2015 18:22:18 +0000
Subject: [PATCH 314/380] Fix wrong PATH being set for world 'includes' stage
 after r289438.

The 'includes' target is currently a pseudo target in bsd.subdir.mk that
does 'cd ${.CURDIR} && ${MAKE} buildincludes && ${MAKE} installincludes',
versus all over targets that just recurse.

In Makefile.inc1 the older duplicated bsd.subdir.mk logic for calling
'includes' was being executed in each subdir directly, meaning 'cd lib && make
includes' became 'cd lib && make buildincludes && make installincludes'.  Now
that the bsd.subdir.mk logic is used it is calling 'make buildincludes && make
installincludes' from the top-level which pulls in the PATH=<default path>
from /Makefile.

The sub-make logic for 'includes' in bsd.subdir.mk was attempted to be removed
in r289282 but turned out to be wrong.  I have a working version now but
it is not yet ready for commit.  So for now in Makefile.inc1 split out
'includes' to 'buildincludes' and 'installincludes' which will avoid the
problem.

MFC after:	2 weeks
X-MFC-With:	r289438
Sponsored by:	EMC / Isilon Storage Division
---
 Makefile.inc1 | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/Makefile.inc1 b/Makefile.inc1
index c0ef145d2e97..44b1171cdfa7 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -624,7 +624,8 @@ _includes:
 	@echo "--------------------------------------------------------------"
 	@echo ">>> stage 4.1: building includes"
 	@echo "--------------------------------------------------------------"
-	${_+_}cd ${.CURDIR}; ${WMAKE} SHARED=symlinks includes
+	${_+_}cd ${.CURDIR}; ${WMAKE} SHARED=symlinks buildincludes
+	${_+_}cd ${.CURDIR}; ${WMAKE} SHARED=symlinks installincludes
 _libraries:
 	@echo
 	@echo "--------------------------------------------------------------"
@@ -2268,7 +2269,9 @@ _xi-cross-tools:
 .endfor
 
 _xi-includes:
-	${_+_}cd ${.CURDIR}; ${CD2MAKE} -f Makefile.inc1 includes \
+	${_+_}cd ${.CURDIR}; ${CD2MAKE} -f Makefile.inc1 buildincludes \
+		DESTDIR=${XDDESTDIR}
+	${_+_}cd ${.CURDIR}; ${CD2MAKE} -f Makefile.inc1 installincludes \
 		DESTDIR=${XDDESTDIR}
 
 _xi-libraries:

From ef80c05a84f47c786279fd01f430ab1ba90ef4aa Mon Sep 17 00:00:00 2001
From: bdrewery <bdrewery@FreeBSD.org>
Date: Sat, 17 Oct 2015 18:59:14 +0000
Subject: [PATCH 315/380] Remove unneeded MK_CTF=no when MK_CDDL=no.

This has been handled since r228158 made MK_CTF dependent on MK_CDDL
in share/mk/bsd.opts.mk.

MFC after:	2 weeks
Sponsored by:	EMC / Isilon Storage Division
---
 Makefile.inc1 | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/Makefile.inc1 b/Makefile.inc1
index 44b1171cdfa7..6056c37d5198 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -325,10 +325,6 @@ HMAKE=		PATH=${TMPPATH} ${MAKE} LOCAL_MTREE=${LOCAL_MTREE:Q}
 HMAKE+=		PATH=${TMPPATH} METALOG=${METALOG} -DNO_ROOT
 .endif
 
-.if ${MK_CDDL} == "no"
-WMAKEENV+=	MK_CTF=no
-.endif
-
 .if defined(CROSS_TOOLCHAIN)
 LOCALBASE?=	/usr/local
 .include "${LOCALBASE}/share/toolchains/${CROSS_TOOLCHAIN}.mk"

From 91fe36c7aa76c619a34df6f68612a9527a2d5ade Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sat, 17 Oct 2015 19:48:17 +0000
Subject: [PATCH 316/380] Only use -fstack-protector-strong with supported
 compilers

This includes clang 3.5.0+, gcc 4.2.1, gcc 4.8.4+

This allows me to do subdirectory makes again after setting
MAKESYSPATH on 10.2-RELEASE as it comes with clang 3.4.1.

As a sidenote: this isn't technically correct for all vintages
of gcc 4.2.1, but will be correct when gcc is rebuilt/reinstalled
after r286074, so this version check should be good enough.

X-MFC with: r288669
Differential Revision: https://reviews.freebsd.org/D3924
Reviewed by: emaste, pfg
---
 share/mk/bsd.sys.mk | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/share/mk/bsd.sys.mk b/share/mk/bsd.sys.mk
index d5a469676d41..68351a0d22a7 100644
--- a/share/mk/bsd.sys.mk
+++ b/share/mk/bsd.sys.mk
@@ -148,9 +148,13 @@ CXXFLAGS.clang+=	 -Wno-c++11-extensions
 
 .if ${MK_SSP} != "no" && \
     ${MACHINE_CPUARCH} != "arm" && ${MACHINE_CPUARCH} != "mips"
+.if (${COMPILER_TYPE} == "clang" && ${COMPILER_VERSION} >= 30500) || \
+    (${COMPILER_TYPE} == "gcc" && \
+     (${COMPILER_VERSION} == 40201 || ${COMPILER_VERSION} >= 40800))
 # Don't use -Wstack-protector as it breaks world with -Werror.
 SSP_CFLAGS?=	-fstack-protector-strong
 CFLAGS+=	${SSP_CFLAGS}
+.endif
 .endif # SSP && !ARM && !MIPS
 
 # Allow user-specified additional warning flags, plus compiler specific flag overrides.

From 033aea1f46af1feb86552e90de8a75392cb3c7e3 Mon Sep 17 00:00:00 2001
From: andrew <andrew@FreeBSD.org>
Date: Sat, 17 Oct 2015 19:52:17 +0000
Subject: [PATCH 317/380] Replace build_section_pagetable with
 build_l1_block_pagetable as it takes an extra argument to specify the number
 of 1GiB pages to map. This should be a nop as we are only mapping a single
 page, but when we move to use an extra level of page tables we will be able
 to map a second block, e.g. if the kernel was loaded over a 1GiB boundary.

---
 sys/arm64/arm64/locore.S | 82 ++++++++++++++++++++++++----------------
 1 file changed, 49 insertions(+), 33 deletions(-)

diff --git a/sys/arm64/arm64/locore.S b/sys/arm64/arm64/locore.S
index 7226d9d39c4a..204306d063c8 100644
--- a/sys/arm64/arm64/locore.S
+++ b/sys/arm64/arm64/locore.S
@@ -379,7 +379,8 @@ create_pagetables:
 	mov	x7, #DEVICE_MEM
 	mov	x8, #(SOCDEV_VA)	/* VA start */
 	mov	x9, #(SOCDEV_PA)	/* PA start */
-	bl	build_section_pagetable
+	mov	x10, #1
+	bl	build_l1_block_pagetable
 #endif
 
 	/* Create the VA = PA map */
@@ -387,47 +388,18 @@ create_pagetables:
 	mov	x7, #NORMAL_UNCACHED /* Uncached as it's only needed early on */
 	mov	x9, x27
 	mov	x8, x9		/* VA start (== PA start) */
-	bl	build_section_pagetable
+	mov	x10, #1
+	bl	build_l1_block_pagetable
 
 	/* Restore the Link register */
 	mov	x30, x5
 	ret
 
-/*
- * Builds a 1 GiB page table entry
- *  x6 = L1 table
- *  x7 = Type (0 = Device, 1 = Normal)
- *  x8 = VA start
- *  x9 = PA start (trashed)
- *  x11, x12 and x13 are trashed
- */
-build_section_pagetable:
-	/*
-	 * Build the L1 table entry.
-	 */
-	/* Find the table index */
-	lsr	x11, x8, #L1_SHIFT
-	and	x11, x11, #Ln_ADDR_MASK
-
-	/* Build the L1 block entry */
-	lsl	x12, x7, #2
-	orr	x12, x12, #L1_BLOCK
-	orr	x12, x12, #(ATTR_AF)
-
-	/* Only use the output address bits */
-	lsr	x9, x9, #L1_SHIFT
-	orr	x12, x12, x9, lsl #L1_SHIFT
-
-	/* Store the entry */
-	str	x12, [x6, x11, lsl #3]
-
-	ret
-
 /*
  * Builds an L1 -> L2 table descriptor
  *
  * This is a link for a 1GiB block of memory with up to 2MiB regions mapped
- * within it by build_l1_block_pagetable.
+ * within it by build_l2_block_pagetable.
  *
  *  x6  = L1 table
  *  x8  = Virtual Address
@@ -454,6 +426,50 @@ link_l1_pagetable:
 
 	ret
 
+/*
+ * Builds count 1 GiB page table entry
+ *  x6  = L1 table
+ *  x7  = Type (0 = Device, 1 = Normal)
+ *  x8  = VA start
+ *  x9  = PA start (trashed)
+ *  x10 = Entry count (TODO)
+ *  x11, x12 and x13 are trashed
+ */
+build_l1_block_pagetable:
+	/*
+	 * Build the L1 table entry.
+	 */
+	/* Find the table index */
+	lsr	x11, x8, #L1_SHIFT
+	and	x11, x11, #Ln_ADDR_MASK
+
+	/* Build the L1 block entry */
+	lsl	x12, x7, #2
+	orr	x12, x12, #L1_BLOCK
+	orr	x12, x12, #(ATTR_AF)
+#ifdef SMP
+	orr	x12, x12, ATTR_SH(ATTR_SH_IS)
+#endif
+
+	/* Only use the output address bits */
+	lsr	x9, x9, #L1_SHIFT
+
+	/* Set the physical address for this virtual address */
+1:	orr	x12, x12, x9, lsl #L1_SHIFT
+
+	/* Store the entry */
+	str	x12, [x6, x11, lsl #3]
+
+	/* Clear the address bits */
+	and	x12, x12, #ATTR_MASK_L
+
+	sub	x10, x10, #1
+	add	x11, x11, #1
+	add	x9, x9, #1
+	cbnz	x10, 1b
+
+2:	ret
+
 /*
  * Builds count 2 MiB page table entry
  *  x6  = L2 table

From bb2ba2b05822ddcfcb0bdbc2f911c03050029ccf Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Sat, 17 Oct 2015 19:55:58 +0000
Subject: [PATCH 318/380] Document bitset(9)

---
 lib/libc/string/ffs.3      |   5 +-
 share/man/man3/bitstring.3 |   5 +-
 share/man/man9/Makefile    |  27 +++
 share/man/man9/bitset.9    | 400 +++++++++++++++++++++++++++++++++++++
 4 files changed, 433 insertions(+), 4 deletions(-)
 create mode 100644 share/man/man9/bitset.9

diff --git a/lib/libc/string/ffs.3 b/lib/libc/string/ffs.3
index 192771ffaeea..8d2601d6efc7 100644
--- a/lib/libc/string/ffs.3
+++ b/lib/libc/string/ffs.3
@@ -30,7 +30,7 @@
 .\"     @(#)ffs.3	8.2 (Berkeley) 4/19/94
 .\" $FreeBSD$
 .\"
-.Dd September 29, 2012
+.Dd October 17, 2015
 .Dt FFS 3
 .Os
 .Sh NAME
@@ -81,7 +81,8 @@ Bits are numbered starting at 1, the least significant bit.
 A return value of zero from any of these functions means that the
 argument was zero.
 .Sh SEE ALSO
-.Xr bitstring 3
+.Xr bitstring 3 ,
+.Xr bitset 9
 .Sh HISTORY
 The
 .Fn ffs
diff --git a/share/man/man3/bitstring.3 b/share/man/man3/bitstring.3
index e7e942603492..89a9524bae03 100644
--- a/share/man/man3/bitstring.3
+++ b/share/man/man3/bitstring.3
@@ -30,7 +30,7 @@
 .\"     @(#)bitstring.3	8.1 (Berkeley) 7/19/93
 .\" $FreeBSD$
 .\"
-.Dd July 19, 1993
+.Dd October 17, 2015
 .Dt BITSTRING 3
 .Os
 .Sh NAME
@@ -178,7 +178,8 @@ make_lpr_available()
 }
 .Ed
 .Sh SEE ALSO
-.Xr malloc 3
+.Xr malloc 3 ,
+.Xr bitset 9
 .Sh HISTORY
 The
 .Nm bitstring
diff --git a/share/man/man9/Makefile b/share/man/man9/Makefile
index f26123df1343..a5293664e3b8 100644
--- a/share/man/man9/Makefile
+++ b/share/man/man9/Makefile
@@ -11,6 +11,7 @@ MAN=	accept_filter.9 \
 	altq.9 \
 	atomic.9 \
 	bios.9 \
+	bitset.9 \
 	boot.9 \
 	bpf.9 \
 	buf.9 \
@@ -429,6 +430,32 @@ MLINKS+=atomic.9 atomic_add.9 \
 	atomic.9 atomic_subtract.9 \
 	atomic.9 atomic_swap.9 \
 	atomic.9 atomic_testandset.9
+MLINKS+=bitset.9 BITSET_DEFINE.9 \
+	bitset.9 BITSET_T_INITIALIZER.9 \
+	bitset.9 BITSET_FSET.9 \
+	bitset.9 BIT_CLR.9 \
+	bitset.9 BIT_COPY.9 \
+	bitset.9 BIT_ISSET.9 \
+	bitset.9 BIT_SET.9 \
+	bitset.9 BIT_ZERO.9 \
+	bitset.9 BIT_FILL.9 \
+	bitset.9 BIT_SETOF.9 \
+	bitset.9 BIT_EMPTY.9 \
+	bitset.9 BIT_ISFULLSET.9 \
+	bitset.9 BIT_FFS.9 \
+	bitset.9 BIT_COUNT.9 \
+	bitset.9 BIT_SUBSET.9 \
+	bitset.9 BIT_OVERLAP.9 \
+	bitset.9 BIT_CMP.9 \
+	bitset.9 BIT_OR.9 \
+	bitset.9 BIT_AND.9 \
+	bitset.9 BIT_NAND.9 \
+	bitset.9 BIT_CLR_ATOMIC.9 \
+	bitset.9 BIT_SET_ATOMIC.9 \
+	bitset.9 BIT_SET_ATOMIC_ACQ.9 \
+	bitset.9 BIT_AND_ATOMIC.9 \
+	bitset.9 BIT_OR_ATOMIC.9 \
+	bitset.9 BIT_COPY_STORE_REL.9
 MLINKS+=bpf.9 bpfattach.9 \
 	bpf.9 bpfattach2.9 \
 	bpf.9 bpfdetach.9 \
diff --git a/share/man/man9/bitset.9 b/share/man/man9/bitset.9
new file mode 100644
index 000000000000..361b0e541c51
--- /dev/null
+++ b/share/man/man9/bitset.9
@@ -0,0 +1,400 @@
+.\" Copyright (c) 2015 Conrad Meyer <cem@FreeBSD.org>
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+.\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+.\" TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+.\" PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE
+.\" LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+.\" POSSIBILITY OF SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd October 17, 2015
+.Dt BITSET 9
+.Os
+.Sh NAME
+.Nm bitset(9)
+\(em
+.Nm BITSET_DEFINE ,
+.Nm BITSET_T_INITIALIZER ,
+.Nm BITSET_FSET ,
+.Nm BIT_CLR ,
+.Nm BIT_COPY ,
+.Nm BIT_ISSET ,
+.Nm BIT_SET ,
+.Nm BIT_ZERO ,
+.Nm BIT_FILL ,
+.Nm BIT_SETOF ,
+.Nm BIT_EMPTY ,
+.Nm BIT_ISFULLSET ,
+.Nm BIT_FFS ,
+.Nm BIT_COUNT ,
+.Nm BIT_SUBSET ,
+.Nm BIT_OVERLAP ,
+.Nm BIT_CMP ,
+.Nm BIT_OR ,
+.Nm BIT_AND ,
+.Nm BIT_NAND ,
+.Nm BIT_CLR_ATOMIC ,
+.Nm BIT_SET_ATOMIC ,
+.Nm BIT_SET_ATOMIC_ACQ ,
+.Nm BIT_AND_ATOMIC ,
+.Nm BIT_OR_ATOMIC ,
+.Nm BIT_COPY_STORE_REL
+.Nd bitset manipulation macros
+.Sh SYNOPSIS
+.In sys/_bitset.h
+.In sys/bitset.h
+.\"
+.Fn BITSET_DEFINE "STRUCTNAME" "const SETSIZE"
+.Fn BITSET_T_INITIALIZER "ARRAY_CONTENTS"
+.Fn BITSET_FSET "N_WORDS"
+.\"
+.Fn BIT_CLR "const SETSIZE" "size_t bit" "struct STRUCTNAME *bitset"
+.Fn BIT_COPY "const SETSIZE" "struct STRUCTNAME *from" "struct STRUCTNAME *to"
+.Ft bool
+.Fn BIT_ISSET "const SETSIZE" "size_t bit" "struct STRUCTNAME *bitset"
+.Fn BIT_SET "const SETSIZE" "size_t bit" "struct STRUCTNAME *bitset"
+.Fn BIT_ZERO "const SETSIZE" "struct STRUCTNAME *bitset"
+.Fn BIT_FILL "const SETSIZE" "struct STRUCTNAME *bitset"
+.Fn BIT_SETOF "const SETSIZE" "size_t bit" "struct STRUCTNAME *bitset"
+.Ft bool
+.Fn BIT_EMPTY "const SETSIZE" "struct STRUCTNAME *bitset"
+.Ft bool
+.Fn BIT_ISFULLSET "const SETSIZE" "struct STRUCTNAME *bitset"
+.Ft size_t
+.Fn BIT_FFS "const SETSIZE" "struct STRUCTNAME *bitset"
+.Ft size_t
+.Fn BIT_COUNT "const SETSIZE" "struct STRUCTNAME *bitset"
+.\"
+.Ft bool
+.Fo BIT_SUBSET
+.Fa "const SETSIZE" "struct STRUCTNAME *haystack" "struct STRUCTNAME *needle"
+.Fc
+.Ft bool
+.Fo BIT_OVERLAP
+.Fa "const SETSIZE" "struct STRUCTNAME *bitset1" "struct STRUCTNAME *bitset2"
+.Fc
+.Ft bool
+.Fo BIT_CMP
+.Fa "const SETSIZE" "struct STRUCTNAME *bitset1" "struct STRUCTNAME *bitset2"
+.Fc
+.Fn BIT_OR "const SETSIZE" "struct STRUCTNAME *dst" "struct STRUCTNAME *src"
+.Fn BIT_AND "const SETSIZE" "struct STRUCTNAME *dst" "struct STRUCTNAME *src"
+.Fn BIT_NAND "const SETSIZE" "struct STRUCTNAME *dst" "struct STRUCTNAME *src"
+.\"
+.Fn BIT_CLR_ATOMIC "const SETSIZE" "size_t bit" "struct STRUCTNAME *bitset"
+.Fn BIT_SET_ATOMIC "const SETSIZE" "size_t bit" "struct STRUCTNAME *bitset"
+.Fn BIT_SET_ATOMIC_ACQ "const SETSIZE" "size_t bit" "struct STRUCTNAME *bitset"
+.\"
+.Fo BIT_AND_ATOMIC
+.Fa "const SETSIZE" "struct STRUCTNAME *dst" "struct STRUCTNAME *src"
+.Fc
+.Fo BIT_OR_ATOMIC
+.Fa "const SETSIZE" "struct STRUCTNAME *dst" "struct STRUCTNAME *src"
+.Fc
+.Fo BIT_COPY_STORE_REL
+.Fa "const SETSIZE" "struct STRUCTNAME *from" "struct STRUCTNAME *to"
+.Fc
+.Sh DESCRIPTION
+The
+.Nm
+family of macros provide a flexible and efficient bitset implementation if the
+maximum size of the set is known at compilation.
+Throughout this manual page, the name
+.Fa SETSIZE
+refers to the size of the bitset in bits.
+Individual bits in bitsets are referenced with indices zero through
+.Fa SETSIZE - 1 .
+One example use of
+.In sys/bitset.h
+is
+.In sys/cpuset.h .
+.Pp
+The
+.Fn BITSET_DEFINE
+macro defines a bitset struct
+.Fa STRUCTNAME
+with room to represent
+.Fa SETSIZE
+bits.
+.Pp
+The
+.Fn BITSET_T_INITIALIZER
+macro allows one to initialize a bitset struct with a compile time literal
+value.
+.Pp
+The
+.Fn BITSET_FSET
+macro generates a compile time literal, usable by
+.Fn BITSET_T_INITIALIZER ,
+representing a full bitset (all bits set).
+For examples of
+.Fn BITSET_T_INITIALIZER
+and
+.Fn BITSET_FSET
+usage, see the
+.Sx BITSET_T_INITIALIZER EXAMPLE
+section.
+The
+.Fa N_WORDS
+parameter to
+.Fn BITSET_FSET
+should be:
+.Bd -literal -offset indent
+__bitset_words(SETSIZE)
+.Ed
+.Pp
+The
+.Fn BIT_CLR
+macro clears bit
+.Fa bit
+in the bitset pointed to by
+.Fa bitset .
+The
+.Fn BIT_CLR_ATOMIC
+macro is identical, but the bit is cleared atomically.
+.Pp
+The
+.Fn BIT_COPY
+macro copies the contents of the bitset
+.Fa from
+to the bitset
+.Fa to .
+.Fn BIT_COPY_STORE_REL
+is similar, but copies component machine words from
+.Fa from
+and writes them to
+.Fa to
+with atomic store with release semantics.
+(That is, if
+.Fa to
+is composed of multiple machine words,
+.Fn BIT_COPY_STORE_REL
+performs multiple individually atomic operations.)
+.Pp
+The
+.Fn BIT_SET
+macro sets bit
+.Fa bit
+in the bitset pointed to by
+.Fa bitset .
+The
+.Fn BIT_SET_ATOMIC
+macro is identical, but the bit is set atomically.
+The
+.Fn BIT_SET_ATOMIC_ACQ
+macro sets the bit with acquire semantics.
+.Pp
+The
+.Fn BIT_ZERO
+macro clears all bits in
+.Fa bitset .
+.Pp
+The
+.Fn BIT_FILL
+macro sets all bits in
+.Fa bitset .
+.Pp
+The
+.Fn BIT_SETOF
+macro clears all bits in
+.Fa bitset
+before setting only bit
+.Fa bit .
+.Pp
+The
+.Fn BIT_EMPTY
+macro returns
+.Dv true
+if
+.Fa bitset
+is empty.
+.Pp
+The
+.Fn BIT_ISFULLSET
+macro returns
+.Dv true
+if
+.Fa bitset
+is full (all bits set).
+.Pp
+The
+.Fn BIT_FFS
+macro returns the 1-index of the first (lowest) set bit in
+.Fa bitset ,
+or zero if
+.Fa bitset
+is empty.
+Like with
+.Xr ffs 3 ,
+to use the non-zero result of
+.Fn BIT_FFS
+as a
+.Fa bit
+index parameter to any other
+.Nm
+macro, you must subtract one from the result.
+.Pp
+The
+.Fn BIT_COUNT
+macro returns the total number of set bits in
+.Fa bitset .
+.Pp
+The
+.Fn BIT_SUBSET
+macro returns
+.Dv true
+if
+.Fa needle
+is a subset of
+.Fa haystack .
+.Pp
+The
+.Fn BIT_OVERLAP
+macro returns
+.Dv true
+if
+.Fa bitset1
+and
+.Fa bitset2
+have any common bits.
+(That is, if
+.Fa bitset1
+AND
+.Fa bitset2
+is not the empty set.)
+.Pp
+The
+.Fn BIT_CMP
+macro returns
+.Dv true
+if
+.Fa bitset1
+is NOT equal to
+.Fa bitset2 .
+.Pp
+The
+.Fn BIT_OR
+macro sets bits present in
+.Fa src
+in
+.Fa dst .
+(It is the
+.Nm
+equivalent of the scalar:
+.Fa dst
+|=
+.Fa src . )
+.Fn BIT_OR_ATOMIC
+is similar, but sets bits in the component machine words in
+.Fa dst
+atomically.
+(That is, if
+.Fa dst
+is composed of multiple machine words,
+.Fn BIT_OR_ATOMIC
+performs multiple individually atomic operations.)
+.Pp
+The
+.Fn BIT_AND
+macro clears bits absent from
+.Fa src
+from
+.Fa dst .
+(It is the
+.Nm
+equivalent of the scalar:
+.Fa dst
+&=
+.Fa src . )
+.Fn BIT_AND_ATOMIC
+is similar, with the same atomic semantics as
+.Fn BIT_OR_ATOMIC .
+.Pp
+The
+.Fn BIT_NAND
+macro clears bits set in
+.Fa src
+from
+.Fa dst .
+(It is the
+.Nm
+equivalent of the scalar:
+.Fa dst
+&=
+.Fa ~ src . )
+.Sh BITSET_T_INITIALIZER EXAMPLE
+.Bd -literal
+BITSET_DEFINE(_myset, MYSETSIZE);
+
+struct _myset myset;
+
+/* Initialize myset to filled (all bits set) */
+myset = BITSET_T_INITIALIZER(BITSET_FSET(__bitset_words(MYSETSIZE)));
+
+/* Initialize myset to only the lowest bit set */
+myset = BITSET_T_INITIALIZER(0x1);
+.Ed
+.Sh SEE ALSO
+The older
+.Xr bitstring 3 .
+.Sh HISTORY
+.In sys/cpuset.h
+first appeared in
+.Fx 7.1 ,
+released in January 2009, and in
+.Fx 8.0 ,
+released in November 2009 .
+.Pp
+The
+.Nm
+macros first appeared in
+.Fx 10.0
+in January 2014.
+They were MFCed to
+.Fx 9.3 ,
+released in July 2014.
+.Pp
+This manual page first appeared in
+.Fx 11.0 .
+.Sh AUTHORS
+.An -nosplit
+The
+.Nm
+macros were written for
+.In sys/cpuset.h
+by
+.An Jeff Roberson Aq Mt jeff@FreeBSD.org ;
+they were generalized and pulled out as
+.In sys/_bitset.h
+and
+.In sys/bitset.h
+by
+.An Attilio Rao Aq Mt attilio@FreeBSD.org .
+This manual page was written by
+.An Conrad Meyer Aq Mt cem@FreeBSD.org .
+.Sh CAVEATS
+The
+.Fa SETSIZE
+argument to all of these macros must match the value given to
+.Fn BITSET_DEFINE .
+.Pp
+Unlike every other reference to individual set members, which are zero-indexed,
+.Fn BIT_FFS
+returns a one-indexed result (or zero if the set is empty).

From 4cfd9e94877579cedf7caddb77ff4087bbaa6792 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sat, 17 Oct 2015 21:11:42 +0000
Subject: [PATCH 319/380] Make iscsictl and iscsid build if MK_ISCSI == yes

MFC after: 1 month
X-MFC with: r289452
---
 rescue/rescue/Makefile | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/rescue/rescue/Makefile b/rescue/rescue/Makefile
index 4d8b63d0f825..45a6cc211656 100644
--- a/rescue/rescue/Makefile
+++ b/rescue/rescue/Makefile
@@ -179,7 +179,7 @@ CRUNCH_BUILDOPTS_dhclient= -DRELEASE_CRUNCH -Dlint
 #
 CRUNCH_SRCDIRS+= usr.bin
 
-CRUNCH_PROGS_usr.bin= head iscsictl mt nc sed tail tee
+CRUNCH_PROGS_usr.bin= head mt nc sed tail tee
 
 CRUNCH_PROGS_usr.bin+= gzip
 CRUNCH_ALIAS_gzip= gunzip gzcat zcat
@@ -214,12 +214,17 @@ CRUNCH_ALIAS_id= groups whoami
 #
 CRUNCH_SRCDIRS+= usr.sbin
 
-CRUNCH_PROGS_usr.sbin+= chroot iscsid
+CRUNCH_PROGS_usr.sbin+= chroot
 
 CRUNCH_PROGS_usr.sbin+= chown
 CRUNCH_ALIAS_chown= chgrp
 ##################################################################
 CRUNCH_LIBS+= -lm
 
+.if ${MK_ISCSI} != "no"
+CRUNCH_PROGS_usr.bin+=	iscsictl
+CRUNCH_PROGS_usr.sbin+=	iscsid
+.endif
+
 .include <bsd.crunchgen.mk>
 .include <bsd.prog.mk>

From 88a4a1403b18f1c162ce780a04ea5884aa2c2afa Mon Sep 17 00:00:00 2001
From: adrian <adrian@FreeBSD.org>
Date: Sun, 18 Oct 2015 00:59:28 +0000
Subject: [PATCH 320/380] if_arge: fix up TX workaround; add TX/RX requirements
 for busdma; add stats

The early ethernet MACs (I think AR71xx and AR913x) require that both
TX and RX require 4-byte alignment for all packets.

The later MACs have started relaxing the requirements.

For now, the 1-byte TX and 1-byte RX alignment requirements are only for
the QCA955x SoCs.  I'll add in the relaxed requirements as I review the
datasheets and do testing.

* Add a hardware flags field and 1-byte / 4-byte TX/RX alignment.
* .. defaulting to 4-byte TX and 4-byte RX alignment.
* Only enforce the TX alignment fixup if the hardware requires a 4-byte
  TX alignment.  This avoids a call to m_defrag().
* Add counters for various situations for further debugging.
* Set the 1-byte and 4-byte busdma alignment requirement when
  the tag is created.

This improves the straight bridging performance from 130mbit/sec
to 180mbit/sec, purely by removing the need for TX path bounce buffers.

The main performance issue is the RX alignment requirement and any RX
bounce buffering that's occuring.  (In a local test, removing the RX
fixup path and just aligning buffers raises the performance to above
400mbit/sec.

In theory it's a no-op for SoCs before the QCA955x.

Tested:

* QCA9558 SoC in AP135 board, using software bridging between arge0/arge1.
---
 sys/mips/atheros/if_arge.c    | 113 +++++++++++++++++++++++++++-------
 sys/mips/atheros/if_argevar.h |  28 ++++++++-
 2 files changed, 119 insertions(+), 22 deletions(-)

diff --git a/sys/mips/atheros/if_arge.c b/sys/mips/atheros/if_arge.c
index d0676f2ea432..446cd64d3282 100644
--- a/sys/mips/atheros/if_arge.c
+++ b/sys/mips/atheros/if_arge.c
@@ -298,6 +298,29 @@ arge_attach_sysctl(device_t dev)
 		"tx_pkts_unaligned", CTLFLAG_RW, &sc->stats.tx_pkts_unaligned,
 		0, "number of TX unaligned packets");
 
+	SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+		"tx_pkts_unaligned_start", CTLFLAG_RW, &sc->stats.tx_pkts_unaligned_start,
+		0, "number of TX unaligned packets (start)");
+
+	SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+		"tx_pkts_unaligned_len", CTLFLAG_RW, &sc->stats.tx_pkts_unaligned_len,
+		0, "number of TX unaligned packets (len)");
+
+	SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+		"tx_pkts_nosegs", CTLFLAG_RW, &sc->stats.tx_pkts_nosegs,
+		0, "number of TX packets fail with no ring slots avail");
+
+	SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+		"intr_stray_filter", CTLFLAG_RW, &sc->stats.intr_stray,
+		0, "number of stray interrupts (filter)");
+
+	SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+		"intr_stray_intr", CTLFLAG_RW, &sc->stats.intr_stray2,
+		0, "number of stray interrupts (intr)");
+
+	SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+		"intr_ok", CTLFLAG_RW, &sc->stats.intr_ok,
+		0, "number of OK interrupts");
 #ifdef	ARGE_DEBUG
 	SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "tx_prod",
 	    CTLFLAG_RW, &sc->arge_cdata.arge_tx_prod, 0, "");
@@ -626,6 +649,22 @@ arge_attach(device_t dev)
 		local_macstr = NULL;
 	}
 
+	/*
+	 * Hardware workarounds.
+	 */
+	switch (ar71xx_soc) {
+	case AR71XX_SOC_QCA9556:
+	case AR71XX_SOC_QCA9558:
+		/* Arbitrary alignment */
+		sc->arge_hw_flags |= ARGE_HW_FLG_TX_DESC_ALIGN_1BYTE;
+		sc->arge_hw_flags |= ARGE_HW_FLG_RX_DESC_ALIGN_1BYTE;
+		break;
+	default:
+		sc->arge_hw_flags |= ARGE_HW_FLG_TX_DESC_ALIGN_4BYTE;
+		sc->arge_hw_flags |= ARGE_HW_FLG_RX_DESC_ALIGN_4BYTE;
+		break;
+	}
+
 	/*
 	 * Some units (eg the TP-Link WR-1043ND) do not have a convenient
 	 * EEPROM location to read the ethernet MAC address from.
@@ -825,6 +864,9 @@ arge_attach(device_t dev)
 	ARGE_WRITE(sc, AR71XX_MAC_FIFO_CFG0,
 	    FIFO_CFG0_ALL << FIFO_CFG0_ENABLE_SHIFT);
 
+	/*
+	 * SoC specific bits.
+	 */
 	switch (ar71xx_soc) {
 		case AR71XX_SOC_AR7240:
 		case AR71XX_SOC_AR7241:
@@ -1351,24 +1393,35 @@ arge_init_locked(struct arge_softc *sc)
  * Return whether the mbuf chain is correctly aligned
  * for the arge TX engine.
  *
- * The TX engine requires each fragment to be aligned to a
- * 4 byte boundary and the size of each fragment except
- * the last to be a multiple of 4 bytes.
+ * All the MACs have a length requirement: any non-final
+ * fragment (ie, descriptor with MORE bit set) needs to have
+ * a length divisible by 4.
  *
- * XXX TODO: I believe this is only a bug on the AR71xx and
- * AR913x MACs. The later MACs (AR724x and later) does not
- * need this workaround.
+ * The AR71xx, AR913x require the start address also be
+ * DWORD aligned.  The later MACs don't.
  */
 static int
-arge_mbuf_chain_is_tx_aligned(struct mbuf *m0)
+arge_mbuf_chain_is_tx_aligned(struct arge_softc *sc, struct mbuf *m0)
 {
 	struct mbuf *m;
 
 	for (m = m0; m != NULL; m = m->m_next) {
-		if((mtod(m, intptr_t) & 3) != 0)
+		/*
+		 * Only do this for chips that require it.
+		 */
+		if ((sc->arge_hw_flags & ARGE_HW_FLG_TX_DESC_ALIGN_4BYTE) &&
+		    (mtod(m, intptr_t) & 3) != 0) {
+			sc->stats.tx_pkts_unaligned_start++;
 			return 0;
-		if ((m->m_next != NULL) && ((m->m_len & 0x03) != 0))
+		}
+
+		/*
+		 * All chips have this requirement for length.
+		 */
+		if ((m->m_next != NULL) && ((m->m_len & 0x03) != 0)) {
+			sc->stats.tx_pkts_unaligned_len++;
 			return 0;
+		}
 	}
 	return 1;
 }
@@ -1389,15 +1442,10 @@ arge_encap(struct arge_softc *sc, struct mbuf **m_head)
 	ARGE_LOCK_ASSERT(sc);
 
 	/*
-	 * Fix mbuf chain, all fragments should be 4 bytes aligned and
-	 * even 4 bytes
-	 *
-	 * XXX TODO: I believe this is only a bug on the AR71xx and
-	 * AR913x MACs. The later MACs (AR724x and later) does not
-	 * need this workaround.
+	 * Fix mbuf chain based on hardware alignment constraints.
 	 */
 	m = *m_head;
-	if (! arge_mbuf_chain_is_tx_aligned(m)) {
+	if (! arge_mbuf_chain_is_tx_aligned(sc, m)) {
 		sc->stats.tx_pkts_unaligned++;
 		m = m_defrag(*m_head, M_NOWAIT);
 		if (m == NULL) {
@@ -1427,6 +1475,7 @@ arge_encap(struct arge_softc *sc, struct mbuf **m_head)
 	/* Check number of available descriptors. */
 	if (sc->arge_cdata.arge_tx_cnt + nsegs >= (ARGE_TX_RING_COUNT - 1)) {
 		bus_dmamap_unload(sc->arge_cdata.arge_tx_tag, txd->tx_dmamap);
+		sc->stats.tx_pkts_nosegs++;
 		return (ENOBUFS);
 	}
 
@@ -1444,7 +1493,9 @@ arge_encap(struct arge_softc *sc, struct mbuf **m_head)
 		desc = &sc->arge_rdata.arge_tx_ring[prod];
 		desc->packet_ctrl = ARGE_DMASIZE(txsegs[i].ds_len);
 
-		if (txsegs[i].ds_addr & 3)
+		/* XXX Note: only relevant for older MACs; but check length! */
+		if ((sc->arge_hw_flags & ARGE_HW_FLG_TX_DESC_ALIGN_4BYTE) &&
+		    (txsegs[i].ds_addr & 3))
 			panic("TX packet address unaligned\n");
 
 		desc->packet_addr = txsegs[i].ds_addr;
@@ -1715,6 +1766,16 @@ arge_dma_alloc(struct arge_softc *sc)
 	struct arge_txdesc	*txd;
 	struct arge_rxdesc	*rxd;
 	int			error, i;
+	int			arge_tx_align, arge_rx_align;
+
+	/* Assume 4 byte alignment by default */
+	arge_tx_align = 4;
+	arge_rx_align = 4;
+
+	if (sc->arge_hw_flags & ARGE_HW_FLG_TX_DESC_ALIGN_1BYTE)
+		arge_tx_align = 1;
+	if (sc->arge_hw_flags & ARGE_HW_FLG_RX_DESC_ALIGN_1BYTE)
+		arge_rx_align = 1;
 
 	/* Create parent DMA tag. */
 	error = bus_dma_tag_create(
@@ -1775,7 +1836,7 @@ arge_dma_alloc(struct arge_softc *sc)
 	/* Create tag for Tx buffers. */
 	error = bus_dma_tag_create(
 	    sc->arge_cdata.arge_parent_tag,	/* parent */
-	    sizeof(uint32_t), 0,	/* alignment, boundary */
+	    arge_tx_align, 0,		/* alignment, boundary */
 	    BUS_SPACE_MAXADDR,		/* lowaddr */
 	    BUS_SPACE_MAXADDR,		/* highaddr */
 	    NULL, NULL,			/* filter, filterarg */
@@ -1793,7 +1854,7 @@ arge_dma_alloc(struct arge_softc *sc)
 	/* Create tag for Rx buffers. */
 	error = bus_dma_tag_create(
 	    sc->arge_cdata.arge_parent_tag,	/* parent */
-	    ARGE_RX_ALIGN, 0,		/* alignment, boundary */
+	    arge_rx_align, 0,		/* alignment, boundary */
 	    BUS_SPACE_MAXADDR,		/* lowaddr */
 	    BUS_SPACE_MAXADDR,		/* highaddr */
 	    NULL, NULL,			/* filter, filterarg */
@@ -2108,6 +2169,11 @@ arge_newbuf(struct arge_softc *sc, int idx)
 	if (m == NULL)
 		return (ENOBUFS);
 	m->m_len = m->m_pkthdr.len = MCLBYTES;
+
+	/*
+	 * Add extra space to "adjust" (copy) the packet back to be aligned
+	 * for purposes of IPv4/IPv6 header contents.
+	 */
 	m_adj(m, sizeof(uint64_t));
 
 	if (bus_dmamap_load_mbuf_sg(sc->arge_cdata.arge_rx_tag,
@@ -2126,7 +2192,8 @@ arge_newbuf(struct arge_softc *sc, int idx)
 	sc->arge_cdata.arge_rx_sparemap = map;
 	rxd->rx_m = m;
 	desc = rxd->desc;
-	if (segs[0].ds_addr & 3)
+	if ((sc->arge_hw_flags & ARGE_HW_FLG_RX_DESC_ALIGN_4BYTE) &&
+	    segs[0].ds_addr & 3)
 		panic("RX packet address unaligned");
 	desc->packet_addr = segs[0].ds_addr;
 	desc->packet_ctrl = ARGE_DESC_EMPTY | ARGE_DMASIZE(segs[0].ds_len);
@@ -2331,10 +2398,12 @@ arge_intr_filter(void *arg)
 	if (status & DMA_INTR_ALL) {
 		sc->arge_intr_status |= status;
 		ARGE_WRITE(sc, AR71XX_DMA_INTR, 0);
+		sc->stats.intr_ok++;
 		return (FILTER_SCHEDULE_THREAD);
 	}
 
 	sc->arge_intr_status = 0;
+	sc->stats.intr_stray++;
 	return (FILTER_STRAY);
 }
 
@@ -2355,8 +2424,10 @@ arge_intr(void *arg)
 	/*
 	 * Is it our interrupt at all?
 	 */
-	if (status == 0)
+	if (status == 0) {
+		sc->stats.intr_stray2++;
 		return;
+	}
 
 	if (status & DMA_INTR_RX_BUS_ERROR) {
 		ARGE_WRITE(sc, AR71XX_DMA_RX_STATUS, DMA_RX_STATUS_BUS_ERROR);
diff --git a/sys/mips/atheros/if_argevar.h b/sys/mips/atheros/if_argevar.h
index 048c375ae7bf..ae94588b97d6 100644
--- a/sys/mips/atheros/if_argevar.h
+++ b/sys/mips/atheros/if_argevar.h
@@ -37,7 +37,10 @@
 #define	ARGE_TX_DMA_SIZE	ARGE_TX_RING_COUNT * sizeof(struct arge_desc)
 #define	ARGE_MAXFRAGS		8
 #define ARGE_RING_ALIGN		sizeof(struct arge_desc)
-#define ARGE_RX_ALIGN		sizeof(uint32_t)
+#define ARGE_RX_ALIGN_4BYTE	sizeof(uint32_t)
+#define ARGE_RX_ALIGN_1BYTE	sizeof(char)
+#define ARGE_TX_ALIGN_4BYTE	sizeof(uint32_t)
+#define ARGE_TX_ALIGN_1BYTE	sizeof(char)
 #define ARGE_MAXFRAGS		8
 #define	ARGE_TX_RING_ADDR(sc, i)	\
     ((sc)->arge_rdata.arge_tx_ring_paddr + sizeof(struct arge_desc) * (i))
@@ -149,6 +152,22 @@ struct arge_pll_data {
 	uint32_t pll_1000;
 };
 
+/*
+ * Hardware specific behaviours.
+ */
+
+/*
+ * Older chips support 4 byte only transmit and receive
+ * addresses.
+ *
+ * Later chips support arbitrary TX and later later,
+ * arbitrary RX addresses.
+ */
+#define	ARGE_HW_FLG_TX_DESC_ALIGN_4BYTE	0x00000001
+#define	ARGE_HW_FLG_RX_DESC_ALIGN_4BYTE	0x00000002
+#define	ARGE_HW_FLG_TX_DESC_ALIGN_1BYTE	0x00000004
+#define	ARGE_HW_FLG_RX_DESC_ALIGN_1BYTE	0x00000008
+
 struct arge_softc {
 	struct ifnet		*arge_ifp;	/* interface info */
 	device_t		arge_dev;
@@ -180,13 +199,20 @@ struct arge_softc {
 	uint32_t		arge_intr_status;
 	int			arge_mac_unit;
 	int			arge_if_flags;
+	uint32_t		arge_hw_flags;
 	uint32_t		arge_debug;
 	uint32_t		arge_mdiofreq;
 	struct {
 		uint32_t	tx_pkts_unaligned;
+		uint32_t	tx_pkts_unaligned_start;
+		uint32_t	tx_pkts_unaligned_len;
+		uint32_t	tx_pkts_nosegs;
 		uint32_t	tx_pkts_aligned;
 		uint32_t	rx_overflow;
 		uint32_t	tx_underflow;
+		uint32_t	intr_stray;
+		uint32_t	intr_stray2;
+		uint32_t	intr_ok;
 	} stats;
 };
 

From 522b95ca9c9cefc2ee62bf1650ed2f2909d42886 Mon Sep 17 00:00:00 2001
From: ian <ian@FreeBSD.org>
Date: Sun, 18 Oct 2015 01:03:43 +0000
Subject: [PATCH 321/380] Fix a strange macro re-definition compile error.  If
 the VM_MAXUSER_ADDRESS value is defined as a config option the definition is
 emitted into opt_global.h which is force-included into everything.  In
 addition, the symbol is emitted by the genassym mechanism, but that by its
 nature reduces the value to a 0xnnnnnnnn number.  When compiling a .S file
 you end up with two different definitions of the macro (they evaluate to the
 same number, but the text is different, upsetting the compiler).

Nothing has changed about this code for a while but the compile error is
new, so this must be fallout from the clang 3.7 update or something.
---
 sys/arm/mv/std-pj4b.mv | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/sys/arm/mv/std-pj4b.mv b/sys/arm/mv/std-pj4b.mv
index ed9fd00bab69..8a64fab7bc1e 100644
--- a/sys/arm/mv/std-pj4b.mv
+++ b/sys/arm/mv/std-pj4b.mv
@@ -5,4 +5,8 @@ cpu		CPU_MV_PJ4B
 machine 	arm	armv6
 makeoptions	CONF_CFLAGS="-march=armv7a"
 
-options		VM_MAXUSER_ADDRESS="(KERNBASE-(1024*1024*1024))"
+# This was originally defined as "(KERNBASE-(1024*1024*1024))" but that
+# (in opt_global.h) clashed with the value emitted by genassym which
+# reduces the original macro text to its numeric value.  The only way
+# to avoid that is to define it here as the numeric value genassym emits.
+options		VM_MAXUSER_ADDRESS="0x80000000"

From dbf17386daffa471de2b4e78349b38842eca6a13 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sun, 18 Oct 2015 01:21:29 +0000
Subject: [PATCH 322/380] Port over diff from ^/user/ngie/more-tests to replace
 `make test` with `make regress`

---
 share/mk/bsd.README    |  8 ++++----
 share/mk/bsd.test.mk   | 15 +++++----------
 share/mk/suite.test.mk | 40 +++++++---------------------------------
 3 files changed, 16 insertions(+), 47 deletions(-)

diff --git a/share/mk/bsd.README b/share/mk/bsd.README
index cb6e41f3cf07..a12d0a81e4c6 100644
--- a/share/mk/bsd.README
+++ b/share/mk/bsd.README
@@ -463,14 +463,14 @@ It has seven targets:
                 install target is executed.
 	lint:
 		run lint on the source files.
-	tags:
-		create a tags file for the source files.
-	test:
+	regress:
 		runs the test programs from the object directory; if the
 		Makefile does not itself define the target test, the
-		targets beforetest and aftertest may also be used to
+		targets beforeregress and afterregress may also be used to
 		cause actions immediately before and after the test
 		target is executed.
+	tags:
+		create a tags file for the source files.
 
 It sets/uses the following variables, among many others:
 
diff --git a/share/mk/bsd.test.mk b/share/mk/bsd.test.mk
index 86351cff3c6c..69ada898cb3b 100644
--- a/share/mk/bsd.test.mk
+++ b/share/mk/bsd.test.mk
@@ -76,19 +76,14 @@ PROGS_TARGETS+= install
 .include <suite.test.mk>
 .endif
 
-.if !target(realtest)
-realtest: .PHONY
+.if !target(realregress)
+realregress: .PHONY
 	@echo "$@ not defined; skipping"
 .endif
 
-test: .PHONY
-.ORDER: beforetest realtest
-test: beforetest realtest
-
-.if target(aftertest)
-.ORDER: realtest aftertest
-test: aftertest
-.endif
+beforeregress realregress afterregress regress: .PHONY
+.ORDER: beforeregress realregress afterregress
+regress: beforeregress realregress afterregress
 
 .ifdef PROG
 # we came here via bsd.progs.mk below
diff --git a/share/mk/suite.test.mk b/share/mk/suite.test.mk
index c447a171bf52..0b244c8b7660 100644
--- a/share/mk/suite.test.mk
+++ b/share/mk/suite.test.mk
@@ -51,7 +51,7 @@ KYUAFILE?= auto
 
 # Path to the prefix of the installed Kyua CLI, if any.
 #
-# If kyua is installed from ports, we automatically define a realtest target
+# If kyua is installed from ports, we automatically define a realregress target
 # below to run the tests using this tool.  The tools are searched for in the
 # hierarchy specified by this variable.
 KYUA_PREFIX?= /usr/local
@@ -92,9 +92,13 @@ Kyuafile.auto: Makefile
 	@mv Kyuafile.auto.tmp Kyuafile.auto
 .endif
 
+_kyuafile=	${DESTDIR}${TESTSDIR}/Kyuafile
+
 KYUA?= ${KYUA_PREFIX}/bin/kyua
+
+realregress: .PHONY
 .if exists(${KYUA})
-# Definition of the "make test" target and supporting variables.
+# Definition of the "make regress" target and supporting variables.
 #
 # This target, by necessity, can only work for native builds (i.e. a FreeBSD
 # host building a release for the same system).  The target runs Kyua, which is
@@ -103,35 +107,5 @@ KYUA?= ${KYUA_PREFIX}/bin/kyua
 # Due to the dependencies of the binaries built by the source tree and how they
 # are used by tests, it is highly possible for a execution of "make test" to
 # report bogus results unless the new binaries are put in place.
-realtest: .PHONY
-	@echo "*** WARNING: make test is experimental"
-	@echo "***"
-	@echo "*** Using this test does not preclude you from running the tests"
-	@echo "*** installed in ${TESTSBASE}.  This test run may raise false"
-	@echo "*** positives and/or false negatives."
-	@echo
-	@${KYUA} test -k ${DESTDIR}${TESTSDIR}/Kyuafile; \
-	result=0; \
-	echo; \
-	echo "*** Once again, note that "make test" is unsupported."; \
-	test $${result} -eq 0
+	@${KYUA} test -k ${DESTDIR}${TESTSDIR}/Kyuafile
 .endif
-
-beforetest: .PHONY
-.if defined(TESTSDIR)
-.if ${TESTSDIR} == ${TESTSBASE}
-# Forbid running from ${TESTSBASE}.  It can cause false positives/negatives and
-# it does not cover all the tests (e.g. it misses testing software in external).
-	@echo "*** Sorry, you cannot use make test from src/tests.  Install the"
-	@echo "*** tests into their final location and run them from ${TESTSBASE}"
-	@false
-.else
-	@echo "*** Using this test does not preclude you from running the tests"
-	@echo "*** installed in ${TESTSBASE}.  This test run may raise false"
-	@echo "*** positives and/or false negatives."
-.endif
-.else
-	@echo "*** No TESTSDIR defined; nothing to do."
-	@false
-.endif
-	@echo

From 2b1070cca99fcd6b56125e04c9b0076599fc2b39 Mon Sep 17 00:00:00 2001
From: cy <cy@FreeBSD.org>
Date: Sun, 18 Oct 2015 03:09:03 +0000
Subject: [PATCH 323/380] Really fix ipfilter bug 3600459.

Obtained from:	ipfilter cvs repo r1.48.2.25, r1.72 and NetBSD repo r1.4
MFC after:	3 days
---
 sys/contrib/ipfilter/netinet/ip_state.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/sys/contrib/ipfilter/netinet/ip_state.c b/sys/contrib/ipfilter/netinet/ip_state.c
index a3930ea69fc4..2ceec3262ec9 100644
--- a/sys/contrib/ipfilter/netinet/ip_state.c
+++ b/sys/contrib/ipfilter/netinet/ip_state.c
@@ -3650,7 +3650,6 @@ ipf_state_del(softc, is, why)
 	if (is->is_ref > 0) {
 		int refs;
 
-		is->is_ref--;
 		refs = is->is_ref;
 		MUTEX_EXIT(&is->is_lock);
 		if (!orphan)
@@ -3667,7 +3666,7 @@ ipf_state_del(softc, is, why)
 		}
 	}
 
-	is->is_ref = 0;
+	ASSERT(is->is_ref == 0);
 	MUTEX_EXIT(&is->is_lock);
 
 	if (is->is_tqehead[0] != NULL) {

From 49d57a74871534cd60670d00ac5817201bee6af1 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sun, 18 Oct 2015 04:07:40 +0000
Subject: [PATCH 324/380] Only enable -fstack-protector-strong on gcc 4.9+ and
 default to -fstack-protector when -fstack-protector-strong is not available,
 like it was implicitly before r288669

As noted by antoine@, devel/gcc (which is 4.8.5) lacks -fstack-protector-strong
support, whereas 4.8.4i (devel/gcc48) has the support.

Until a version is available which has -fstack-protector-strong support, be
conservative and only enable support with 4.9+.

Reviewed by: pfg
X-MFC with: r288669, r289465
Differential Revision: https://reviews.freebsd.org/D3924
---
 share/mk/bsd.sys.mk | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/share/mk/bsd.sys.mk b/share/mk/bsd.sys.mk
index 68351a0d22a7..cad2b2f79645 100644
--- a/share/mk/bsd.sys.mk
+++ b/share/mk/bsd.sys.mk
@@ -150,11 +150,13 @@ CXXFLAGS.clang+=	 -Wno-c++11-extensions
     ${MACHINE_CPUARCH} != "arm" && ${MACHINE_CPUARCH} != "mips"
 .if (${COMPILER_TYPE} == "clang" && ${COMPILER_VERSION} >= 30500) || \
     (${COMPILER_TYPE} == "gcc" && \
-     (${COMPILER_VERSION} == 40201 || ${COMPILER_VERSION} >= 40800))
+     (${COMPILER_VERSION} == 40201 || ${COMPILER_VERSION} >= 40900))
 # Don't use -Wstack-protector as it breaks world with -Werror.
 SSP_CFLAGS?=	-fstack-protector-strong
-CFLAGS+=	${SSP_CFLAGS}
+.else
+SSP_CFLAGS?=	-fstack-protector
 .endif
+CFLAGS+=	${SSP_CFLAGS}
 .endif # SSP && !ARM && !MIPS
 
 # Allow user-specified additional warning flags, plus compiler specific flag overrides.

From 5145e8ee6b08f444a75801e5448af27c3df60fbc Mon Sep 17 00:00:00 2001
From: sbruno <sbruno@FreeBSD.org>
Date: Sun, 18 Oct 2015 04:50:51 +0000
Subject: [PATCH 325/380] Correctly use the default values for location of MAC
 addrs of arge0, arge1, ath0.  woo!

Reviewed by:	adrian
---
 sys/mips/conf/TP-MR3020.hints | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sys/mips/conf/TP-MR3020.hints b/sys/mips/conf/TP-MR3020.hints
index 9e8946f2a0d7..1ec954746195 100644
--- a/sys/mips/conf/TP-MR3020.hints
+++ b/sys/mips/conf/TP-MR3020.hints
@@ -26,18 +26,18 @@ hint.arswitch.0.is_gmii=1	# arge1 <-> switch PHY is GMII
 # arge0 - MII, autoneg, phy(4)
 hint.arge.0.phymask=0x10	# PHY4
 hint.arge.0.mdio=mdioproxy1	# .. off of the switch mdiobus
+hint.arge.0.eeprommac=0x1fff0000
 
 # arge1 - GMII, 1000/full
 hint.arge.1.phymask=0x0		# No directly mapped PHYs
 hint.arge.1.media=1000
 hint.arge.1.fduplex=1
+hint.arge.1.eeprommac=0x1fff0006
 
 # Where the ART is - last 64k in the flash
 # 0x9fff1000 ?
-hint.ath.0.eepromaddr=0x1fff1000
+hint.ath.0.eepromaddr=0x1fff0000
 hint.ath.0.eepromsize=16384
- 
-hint.ar71xx.0.eeprom_mac_addr=0x1f01fc00
 
 # The board 4MiB flash layout in uboot env:
 #

From bae0f45cc488c4da58e0d394200527044d69681a Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sun, 18 Oct 2015 04:50:53 +0000
Subject: [PATCH 326/380] MFhead @ r289481

---
 share/mk/bsd.sys.mk                     |   6 +-
 sys/arm/mv/std-pj4b.mv                  |   6 +-
 sys/contrib/ipfilter/netinet/ip_state.c |   3 +-
 sys/mips/atheros/if_arge.c              | 113 +++++++++++++++++++-----
 sys/mips/atheros/if_argevar.h           |  28 +++++-
 5 files changed, 129 insertions(+), 27 deletions(-)

diff --git a/share/mk/bsd.sys.mk b/share/mk/bsd.sys.mk
index 68351a0d22a7..cad2b2f79645 100644
--- a/share/mk/bsd.sys.mk
+++ b/share/mk/bsd.sys.mk
@@ -150,11 +150,13 @@ CXXFLAGS.clang+=	 -Wno-c++11-extensions
     ${MACHINE_CPUARCH} != "arm" && ${MACHINE_CPUARCH} != "mips"
 .if (${COMPILER_TYPE} == "clang" && ${COMPILER_VERSION} >= 30500) || \
     (${COMPILER_TYPE} == "gcc" && \
-     (${COMPILER_VERSION} == 40201 || ${COMPILER_VERSION} >= 40800))
+     (${COMPILER_VERSION} == 40201 || ${COMPILER_VERSION} >= 40900))
 # Don't use -Wstack-protector as it breaks world with -Werror.
 SSP_CFLAGS?=	-fstack-protector-strong
-CFLAGS+=	${SSP_CFLAGS}
+.else
+SSP_CFLAGS?=	-fstack-protector
 .endif
+CFLAGS+=	${SSP_CFLAGS}
 .endif # SSP && !ARM && !MIPS
 
 # Allow user-specified additional warning flags, plus compiler specific flag overrides.
diff --git a/sys/arm/mv/std-pj4b.mv b/sys/arm/mv/std-pj4b.mv
index ed9fd00bab69..8a64fab7bc1e 100644
--- a/sys/arm/mv/std-pj4b.mv
+++ b/sys/arm/mv/std-pj4b.mv
@@ -5,4 +5,8 @@ cpu		CPU_MV_PJ4B
 machine 	arm	armv6
 makeoptions	CONF_CFLAGS="-march=armv7a"
 
-options		VM_MAXUSER_ADDRESS="(KERNBASE-(1024*1024*1024))"
+# This was originally defined as "(KERNBASE-(1024*1024*1024))" but that
+# (in opt_global.h) clashed with the value emitted by genassym which
+# reduces the original macro text to its numeric value.  The only way
+# to avoid that is to define it here as the numeric value genassym emits.
+options		VM_MAXUSER_ADDRESS="0x80000000"
diff --git a/sys/contrib/ipfilter/netinet/ip_state.c b/sys/contrib/ipfilter/netinet/ip_state.c
index a3930ea69fc4..2ceec3262ec9 100644
--- a/sys/contrib/ipfilter/netinet/ip_state.c
+++ b/sys/contrib/ipfilter/netinet/ip_state.c
@@ -3650,7 +3650,6 @@ ipf_state_del(softc, is, why)
 	if (is->is_ref > 0) {
 		int refs;
 
-		is->is_ref--;
 		refs = is->is_ref;
 		MUTEX_EXIT(&is->is_lock);
 		if (!orphan)
@@ -3667,7 +3666,7 @@ ipf_state_del(softc, is, why)
 		}
 	}
 
-	is->is_ref = 0;
+	ASSERT(is->is_ref == 0);
 	MUTEX_EXIT(&is->is_lock);
 
 	if (is->is_tqehead[0] != NULL) {
diff --git a/sys/mips/atheros/if_arge.c b/sys/mips/atheros/if_arge.c
index d0676f2ea432..446cd64d3282 100644
--- a/sys/mips/atheros/if_arge.c
+++ b/sys/mips/atheros/if_arge.c
@@ -298,6 +298,29 @@ arge_attach_sysctl(device_t dev)
 		"tx_pkts_unaligned", CTLFLAG_RW, &sc->stats.tx_pkts_unaligned,
 		0, "number of TX unaligned packets");
 
+	SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+		"tx_pkts_unaligned_start", CTLFLAG_RW, &sc->stats.tx_pkts_unaligned_start,
+		0, "number of TX unaligned packets (start)");
+
+	SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+		"tx_pkts_unaligned_len", CTLFLAG_RW, &sc->stats.tx_pkts_unaligned_len,
+		0, "number of TX unaligned packets (len)");
+
+	SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+		"tx_pkts_nosegs", CTLFLAG_RW, &sc->stats.tx_pkts_nosegs,
+		0, "number of TX packets fail with no ring slots avail");
+
+	SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+		"intr_stray_filter", CTLFLAG_RW, &sc->stats.intr_stray,
+		0, "number of stray interrupts (filter)");
+
+	SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+		"intr_stray_intr", CTLFLAG_RW, &sc->stats.intr_stray2,
+		0, "number of stray interrupts (intr)");
+
+	SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+		"intr_ok", CTLFLAG_RW, &sc->stats.intr_ok,
+		0, "number of OK interrupts");
 #ifdef	ARGE_DEBUG
 	SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "tx_prod",
 	    CTLFLAG_RW, &sc->arge_cdata.arge_tx_prod, 0, "");
@@ -626,6 +649,22 @@ arge_attach(device_t dev)
 		local_macstr = NULL;
 	}
 
+	/*
+	 * Hardware workarounds.
+	 */
+	switch (ar71xx_soc) {
+	case AR71XX_SOC_QCA9556:
+	case AR71XX_SOC_QCA9558:
+		/* Arbitrary alignment */
+		sc->arge_hw_flags |= ARGE_HW_FLG_TX_DESC_ALIGN_1BYTE;
+		sc->arge_hw_flags |= ARGE_HW_FLG_RX_DESC_ALIGN_1BYTE;
+		break;
+	default:
+		sc->arge_hw_flags |= ARGE_HW_FLG_TX_DESC_ALIGN_4BYTE;
+		sc->arge_hw_flags |= ARGE_HW_FLG_RX_DESC_ALIGN_4BYTE;
+		break;
+	}
+
 	/*
 	 * Some units (eg the TP-Link WR-1043ND) do not have a convenient
 	 * EEPROM location to read the ethernet MAC address from.
@@ -825,6 +864,9 @@ arge_attach(device_t dev)
 	ARGE_WRITE(sc, AR71XX_MAC_FIFO_CFG0,
 	    FIFO_CFG0_ALL << FIFO_CFG0_ENABLE_SHIFT);
 
+	/*
+	 * SoC specific bits.
+	 */
 	switch (ar71xx_soc) {
 		case AR71XX_SOC_AR7240:
 		case AR71XX_SOC_AR7241:
@@ -1351,24 +1393,35 @@ arge_init_locked(struct arge_softc *sc)
  * Return whether the mbuf chain is correctly aligned
  * for the arge TX engine.
  *
- * The TX engine requires each fragment to be aligned to a
- * 4 byte boundary and the size of each fragment except
- * the last to be a multiple of 4 bytes.
+ * All the MACs have a length requirement: any non-final
+ * fragment (ie, descriptor with MORE bit set) needs to have
+ * a length divisible by 4.
  *
- * XXX TODO: I believe this is only a bug on the AR71xx and
- * AR913x MACs. The later MACs (AR724x and later) does not
- * need this workaround.
+ * The AR71xx, AR913x require the start address also be
+ * DWORD aligned.  The later MACs don't.
  */
 static int
-arge_mbuf_chain_is_tx_aligned(struct mbuf *m0)
+arge_mbuf_chain_is_tx_aligned(struct arge_softc *sc, struct mbuf *m0)
 {
 	struct mbuf *m;
 
 	for (m = m0; m != NULL; m = m->m_next) {
-		if((mtod(m, intptr_t) & 3) != 0)
+		/*
+		 * Only do this for chips that require it.
+		 */
+		if ((sc->arge_hw_flags & ARGE_HW_FLG_TX_DESC_ALIGN_4BYTE) &&
+		    (mtod(m, intptr_t) & 3) != 0) {
+			sc->stats.tx_pkts_unaligned_start++;
 			return 0;
-		if ((m->m_next != NULL) && ((m->m_len & 0x03) != 0))
+		}
+
+		/*
+		 * All chips have this requirement for length.
+		 */
+		if ((m->m_next != NULL) && ((m->m_len & 0x03) != 0)) {
+			sc->stats.tx_pkts_unaligned_len++;
 			return 0;
+		}
 	}
 	return 1;
 }
@@ -1389,15 +1442,10 @@ arge_encap(struct arge_softc *sc, struct mbuf **m_head)
 	ARGE_LOCK_ASSERT(sc);
 
 	/*
-	 * Fix mbuf chain, all fragments should be 4 bytes aligned and
-	 * even 4 bytes
-	 *
-	 * XXX TODO: I believe this is only a bug on the AR71xx and
-	 * AR913x MACs. The later MACs (AR724x and later) does not
-	 * need this workaround.
+	 * Fix mbuf chain based on hardware alignment constraints.
 	 */
 	m = *m_head;
-	if (! arge_mbuf_chain_is_tx_aligned(m)) {
+	if (! arge_mbuf_chain_is_tx_aligned(sc, m)) {
 		sc->stats.tx_pkts_unaligned++;
 		m = m_defrag(*m_head, M_NOWAIT);
 		if (m == NULL) {
@@ -1427,6 +1475,7 @@ arge_encap(struct arge_softc *sc, struct mbuf **m_head)
 	/* Check number of available descriptors. */
 	if (sc->arge_cdata.arge_tx_cnt + nsegs >= (ARGE_TX_RING_COUNT - 1)) {
 		bus_dmamap_unload(sc->arge_cdata.arge_tx_tag, txd->tx_dmamap);
+		sc->stats.tx_pkts_nosegs++;
 		return (ENOBUFS);
 	}
 
@@ -1444,7 +1493,9 @@ arge_encap(struct arge_softc *sc, struct mbuf **m_head)
 		desc = &sc->arge_rdata.arge_tx_ring[prod];
 		desc->packet_ctrl = ARGE_DMASIZE(txsegs[i].ds_len);
 
-		if (txsegs[i].ds_addr & 3)
+		/* XXX Note: only relevant for older MACs; but check length! */
+		if ((sc->arge_hw_flags & ARGE_HW_FLG_TX_DESC_ALIGN_4BYTE) &&
+		    (txsegs[i].ds_addr & 3))
 			panic("TX packet address unaligned\n");
 
 		desc->packet_addr = txsegs[i].ds_addr;
@@ -1715,6 +1766,16 @@ arge_dma_alloc(struct arge_softc *sc)
 	struct arge_txdesc	*txd;
 	struct arge_rxdesc	*rxd;
 	int			error, i;
+	int			arge_tx_align, arge_rx_align;
+
+	/* Assume 4 byte alignment by default */
+	arge_tx_align = 4;
+	arge_rx_align = 4;
+
+	if (sc->arge_hw_flags & ARGE_HW_FLG_TX_DESC_ALIGN_1BYTE)
+		arge_tx_align = 1;
+	if (sc->arge_hw_flags & ARGE_HW_FLG_RX_DESC_ALIGN_1BYTE)
+		arge_rx_align = 1;
 
 	/* Create parent DMA tag. */
 	error = bus_dma_tag_create(
@@ -1775,7 +1836,7 @@ arge_dma_alloc(struct arge_softc *sc)
 	/* Create tag for Tx buffers. */
 	error = bus_dma_tag_create(
 	    sc->arge_cdata.arge_parent_tag,	/* parent */
-	    sizeof(uint32_t), 0,	/* alignment, boundary */
+	    arge_tx_align, 0,		/* alignment, boundary */
 	    BUS_SPACE_MAXADDR,		/* lowaddr */
 	    BUS_SPACE_MAXADDR,		/* highaddr */
 	    NULL, NULL,			/* filter, filterarg */
@@ -1793,7 +1854,7 @@ arge_dma_alloc(struct arge_softc *sc)
 	/* Create tag for Rx buffers. */
 	error = bus_dma_tag_create(
 	    sc->arge_cdata.arge_parent_tag,	/* parent */
-	    ARGE_RX_ALIGN, 0,		/* alignment, boundary */
+	    arge_rx_align, 0,		/* alignment, boundary */
 	    BUS_SPACE_MAXADDR,		/* lowaddr */
 	    BUS_SPACE_MAXADDR,		/* highaddr */
 	    NULL, NULL,			/* filter, filterarg */
@@ -2108,6 +2169,11 @@ arge_newbuf(struct arge_softc *sc, int idx)
 	if (m == NULL)
 		return (ENOBUFS);
 	m->m_len = m->m_pkthdr.len = MCLBYTES;
+
+	/*
+	 * Add extra space to "adjust" (copy) the packet back to be aligned
+	 * for purposes of IPv4/IPv6 header contents.
+	 */
 	m_adj(m, sizeof(uint64_t));
 
 	if (bus_dmamap_load_mbuf_sg(sc->arge_cdata.arge_rx_tag,
@@ -2126,7 +2192,8 @@ arge_newbuf(struct arge_softc *sc, int idx)
 	sc->arge_cdata.arge_rx_sparemap = map;
 	rxd->rx_m = m;
 	desc = rxd->desc;
-	if (segs[0].ds_addr & 3)
+	if ((sc->arge_hw_flags & ARGE_HW_FLG_RX_DESC_ALIGN_4BYTE) &&
+	    segs[0].ds_addr & 3)
 		panic("RX packet address unaligned");
 	desc->packet_addr = segs[0].ds_addr;
 	desc->packet_ctrl = ARGE_DESC_EMPTY | ARGE_DMASIZE(segs[0].ds_len);
@@ -2331,10 +2398,12 @@ arge_intr_filter(void *arg)
 	if (status & DMA_INTR_ALL) {
 		sc->arge_intr_status |= status;
 		ARGE_WRITE(sc, AR71XX_DMA_INTR, 0);
+		sc->stats.intr_ok++;
 		return (FILTER_SCHEDULE_THREAD);
 	}
 
 	sc->arge_intr_status = 0;
+	sc->stats.intr_stray++;
 	return (FILTER_STRAY);
 }
 
@@ -2355,8 +2424,10 @@ arge_intr(void *arg)
 	/*
 	 * Is it our interrupt at all?
 	 */
-	if (status == 0)
+	if (status == 0) {
+		sc->stats.intr_stray2++;
 		return;
+	}
 
 	if (status & DMA_INTR_RX_BUS_ERROR) {
 		ARGE_WRITE(sc, AR71XX_DMA_RX_STATUS, DMA_RX_STATUS_BUS_ERROR);
diff --git a/sys/mips/atheros/if_argevar.h b/sys/mips/atheros/if_argevar.h
index 048c375ae7bf..ae94588b97d6 100644
--- a/sys/mips/atheros/if_argevar.h
+++ b/sys/mips/atheros/if_argevar.h
@@ -37,7 +37,10 @@
 #define	ARGE_TX_DMA_SIZE	ARGE_TX_RING_COUNT * sizeof(struct arge_desc)
 #define	ARGE_MAXFRAGS		8
 #define ARGE_RING_ALIGN		sizeof(struct arge_desc)
-#define ARGE_RX_ALIGN		sizeof(uint32_t)
+#define ARGE_RX_ALIGN_4BYTE	sizeof(uint32_t)
+#define ARGE_RX_ALIGN_1BYTE	sizeof(char)
+#define ARGE_TX_ALIGN_4BYTE	sizeof(uint32_t)
+#define ARGE_TX_ALIGN_1BYTE	sizeof(char)
 #define ARGE_MAXFRAGS		8
 #define	ARGE_TX_RING_ADDR(sc, i)	\
     ((sc)->arge_rdata.arge_tx_ring_paddr + sizeof(struct arge_desc) * (i))
@@ -149,6 +152,22 @@ struct arge_pll_data {
 	uint32_t pll_1000;
 };
 
+/*
+ * Hardware specific behaviours.
+ */
+
+/*
+ * Older chips support 4 byte only transmit and receive
+ * addresses.
+ *
+ * Later chips support arbitrary TX and later later,
+ * arbitrary RX addresses.
+ */
+#define	ARGE_HW_FLG_TX_DESC_ALIGN_4BYTE	0x00000001
+#define	ARGE_HW_FLG_RX_DESC_ALIGN_4BYTE	0x00000002
+#define	ARGE_HW_FLG_TX_DESC_ALIGN_1BYTE	0x00000004
+#define	ARGE_HW_FLG_RX_DESC_ALIGN_1BYTE	0x00000008
+
 struct arge_softc {
 	struct ifnet		*arge_ifp;	/* interface info */
 	device_t		arge_dev;
@@ -180,13 +199,20 @@ struct arge_softc {
 	uint32_t		arge_intr_status;
 	int			arge_mac_unit;
 	int			arge_if_flags;
+	uint32_t		arge_hw_flags;
 	uint32_t		arge_debug;
 	uint32_t		arge_mdiofreq;
 	struct {
 		uint32_t	tx_pkts_unaligned;
+		uint32_t	tx_pkts_unaligned_start;
+		uint32_t	tx_pkts_unaligned_len;
+		uint32_t	tx_pkts_nosegs;
 		uint32_t	tx_pkts_aligned;
 		uint32_t	rx_overflow;
 		uint32_t	tx_underflow;
+		uint32_t	intr_stray;
+		uint32_t	intr_stray2;
+		uint32_t	intr_ok;
 	} stats;
 };
 

From 5df1a219f1a344ce0427fc63a47867b2ba37721f Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sun, 18 Oct 2015 05:25:51 +0000
Subject: [PATCH 327/380] Ensure fdescfs is mounted on /dev/fd/ in the length
 testcase

---
 contrib/netbsd-tests/bin/dd/t_dd.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/contrib/netbsd-tests/bin/dd/t_dd.sh b/contrib/netbsd-tests/bin/dd/t_dd.sh
index d713ad9cb246..62379c241484 100755
--- a/contrib/netbsd-tests/bin/dd/t_dd.sh
+++ b/contrib/netbsd-tests/bin/dd/t_dd.sh
@@ -44,6 +44,12 @@ length_head() {
 	                "the one expected to fail.  (NetBSD PR bin/8521)"
 }
 length_body() {
+	# Begin FreeBSD
+	if ! df /dev/fd | grep -q '^fdescfs'; then
+		atf_skip "fdescfs is not mounted on /dev/fd"
+	fi
+	# End FreeBSD
+
 	test_dd_length 512 \
 	    "dd if=/dev/zero of=/dev/fd/5 count=1 5>&1 >/dev/null 2>/dev/null"
 	test_dd_length 512 \

From e84cdeae2f06fd326af8175317ce7ef9a9ff562d Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sun, 18 Oct 2015 05:29:49 +0000
Subject: [PATCH 328/380] Integrate contrib/netbsd-tests/bin/dd into the
 FreeBSD test suite as bin/dd/tests

---
 bin/dd/Makefile          | 6 ++++++
 bin/dd/tests/Makefile    | 7 +++++++
 etc/mtree/BSD.tests.dist | 2 ++
 3 files changed, 15 insertions(+)
 create mode 100644 bin/dd/tests/Makefile

diff --git a/bin/dd/Makefile b/bin/dd/Makefile
index eb8ec8527b17..5f07dbc248ac 100644
--- a/bin/dd/Makefile
+++ b/bin/dd/Makefile
@@ -1,6 +1,8 @@
 #	@(#)Makefile	8.1 (Berkeley) 5/31/93
 # $FreeBSD$
 
+.include <src.opts.mk>
+
 PROG=	dd
 SRCS=	args.c conv.c conv_tab.c dd.c misc.c position.c
 
@@ -24,4 +26,8 @@ test: ${PROG} gen
 .endfor
 	@rm -f gen
 
+.if ${MK_TESTS} != "no"
+SUBDIR+=	tests
+.endif
+
 .include <bsd.prog.mk>
diff --git a/bin/dd/tests/Makefile b/bin/dd/tests/Makefile
new file mode 100644
index 000000000000..40fb114642cd
--- /dev/null
+++ b/bin/dd/tests/Makefile
@@ -0,0 +1,7 @@
+# $FreeBSD$
+
+NETBSD_ATF_TESTS_SH=	dd
+
+.include <netbsd-tests.test.mk>
+
+.include <bsd.test.mk>
diff --git a/etc/mtree/BSD.tests.dist b/etc/mtree/BSD.tests.dist
index c31eaaa97e33..8d293230f269 100644
--- a/etc/mtree/BSD.tests.dist
+++ b/etc/mtree/BSD.tests.dist
@@ -12,6 +12,8 @@
         ..
         date
         ..
+        dd
+        ..
         expr
         ..
         ls

From bac9b01be8903ec374282900a1218b1535072ab5 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sun, 18 Oct 2015 05:51:44 +0000
Subject: [PATCH 329/380] Clean up trailing whitespace

MFC after: 3 days
---
 bin/test/tests/legacy_test.sh | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/bin/test/tests/legacy_test.sh b/bin/test/tests/legacy_test.sh
index 922955129d9f..8dae88f0d7af 100644
--- a/bin/test/tests/legacy_test.sh
+++ b/bin/test/tests/legacy_test.sh
@@ -2,7 +2,7 @@
 
 #-
 # Copyright (c) June 1996 Wolfram Schneider <wosch@FreeBSD.org>. Berlin.
-# All rights reserved. 
+# All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -31,7 +31,7 @@
 # $FreeBSD$
 
 # force a specified test program, e.g. `env test=/bin/test sh regress.sh'
-: ${test=test}		
+: ${test=test}
 
 t ()
 {
@@ -54,13 +54,13 @@ t ()
 count=0
 echo "1..130"
 
-t 0 'b = b' 
-t 0 'b == b' 
-t 1 'b != b' 
-t 0 '\( b = b \)' 
-t 0 '\( b == b \)' 
-t 1 '! \( b = b \)' 
-t 1 '! \( b == b \)' 
+t 0 'b = b'
+t 0 'b == b'
+t 1 'b != b'
+t 0 '\( b = b \)'
+t 0 '\( b == b \)'
+t 1 '! \( b = b \)'
+t 1 '! \( b == b \)'
 t 1 '! -f /etc/passwd'
 
 t 0 '-h = -h'

From 5c0da1966cae498c05787547279f17707ffb1967 Mon Sep 17 00:00:00 2001
From: ngie <ngie@FreeBSD.org>
Date: Sun, 18 Oct 2015 07:30:50 +0000
Subject: [PATCH 330/380] Make libxo depend on libutil because it uses
 humanize_number after r287111

Remove overlinking in lib/libxo/tests, sbin/savecore, and
usr.bin/{iscsictl,wc,xo}

PR: 203673
Sponsored by: EMC / Isilon Storage Division
---
 bin/ps/Makefile                  | 2 +-
 lib/libxo/Makefile               | 2 ++
 lib/libxo/tests/Makefile         | 3 +--
 lib/libxo/tests/encoder/Makefile | 3 +--
 sbin/savecore/Makefile           | 2 +-
 share/mk/src.libnames.mk         | 1 +
 usr.bin/iscsictl/Makefile        | 2 +-
 usr.bin/wc/Makefile              | 2 +-
 usr.bin/xo/Makefile              | 2 +-
 9 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/bin/ps/Makefile b/bin/ps/Makefile
index 5177832131a5..79e9fc6cf0ec 100644
--- a/bin/ps/Makefile
+++ b/bin/ps/Makefile
@@ -11,6 +11,6 @@ SRCS=	fmt.c keyword.c nlist.c print.c ps.c
 # on large systems.
 #
 CFLAGS+=-DLAZY_PS
-LIBADD=	m kvm jail xo util
+LIBADD=	m kvm jail xo
 
 .include <bsd.prog.mk>
diff --git a/lib/libxo/Makefile b/lib/libxo/Makefile
index d854a4c1d3c2..f8453f405282 100644
--- a/lib/libxo/Makefile
+++ b/lib/libxo/Makefile
@@ -19,6 +19,8 @@ CFLAGS+=-DXO_ENCODERDIR=\"/usr/lib/libxo/encoder\"
 INCS=	xo.h xo_encoder.h
 INCSDIR=${INCLUDEDIR}/libxo
 
+LIBADD=	util
+
 WARNS?= 5
 
 MAN+=	libxo.3
diff --git a/lib/libxo/tests/Makefile b/lib/libxo/tests/Makefile
index 2a4a99cac1dc..1a0f7c69cfee 100644
--- a/lib/libxo/tests/Makefile
+++ b/lib/libxo/tests/Makefile
@@ -242,8 +242,7 @@ PROGS+= test_11
 
 CFLAGS+=	-I${LIBXOSRC}/libxo
 
-DPADD=		${LIBXO} ${LIBUTIL}
-LDADD=		-lxo -lutil
+LIBADD=		xo
 
 SUBDIR+=	encoder
 
diff --git a/lib/libxo/tests/encoder/Makefile b/lib/libxo/tests/encoder/Makefile
index 2cd86b917855..2fe42457f2e8 100644
--- a/lib/libxo/tests/encoder/Makefile
+++ b/lib/libxo/tests/encoder/Makefile
@@ -14,7 +14,6 @@ SRCS=		enc_test.c
 
 CFLAGS+=	-I${LIBXOSRC}/libxo
 
-DPADD+=		${LIBXO}
-LDADD+=		-lxo
+LIBADD=		xo
 
 .include <bsd.lib.mk>
diff --git a/sbin/savecore/Makefile b/sbin/savecore/Makefile
index e46496f49c98..5134e4c546a7 100644
--- a/sbin/savecore/Makefile
+++ b/sbin/savecore/Makefile
@@ -1,7 +1,7 @@
 # $FreeBSD$
 
 PROG=	savecore
-LIBADD=	z xo util
+LIBADD=	z xo
 MAN=	savecore.8
 
 .include <bsd.prog.mk>
diff --git a/share/mk/src.libnames.mk b/share/mk/src.libnames.mk
index e3d5baecaa29..06e451bdfa22 100644
--- a/share/mk/src.libnames.mk
+++ b/share/mk/src.libnames.mk
@@ -239,6 +239,7 @@ _DP_vmmapi=	util
 _DP_ctf=	z
 _DP_proc=	rtld_db util
 _DP_dtrace=	rtld_db pthread
+_DP_xo=		util
 
 # Define spacial cases
 LDADD_supcplusplus=	-lsupc++
diff --git a/usr.bin/iscsictl/Makefile b/usr.bin/iscsictl/Makefile
index 9b2e1f17587e..4437e2d1e3ca 100644
--- a/usr.bin/iscsictl/Makefile
+++ b/usr.bin/iscsictl/Makefile
@@ -6,7 +6,7 @@ CFLAGS+=	-I${.CURDIR}
 CFLAGS+=	-I${.CURDIR}/../../sys/dev/iscsi
 MAN=		iscsi.conf.5 iscsictl.8
 
-LIBADD=		xo util
+LIBADD=		xo
 
 YFLAGS+=	-v
 LFLAGS+=	-i
diff --git a/usr.bin/wc/Makefile b/usr.bin/wc/Makefile
index 6fa21fa69755..540e33d15c92 100644
--- a/usr.bin/wc/Makefile
+++ b/usr.bin/wc/Makefile
@@ -2,6 +2,6 @@
 # $FreeBSD$
 
 PROG=	wc
-LIBADD=	xo util
+LIBADD=	xo
 
 .include <bsd.prog.mk>
diff --git a/usr.bin/xo/Makefile b/usr.bin/xo/Makefile
index 92692e550cf8..0773987d6ab1 100644
--- a/usr.bin/xo/Makefile
+++ b/usr.bin/xo/Makefile
@@ -12,7 +12,7 @@ MAN=	xo.1
 # XXX For xoversion.h
 CFLAGS+=-I${LIBXOSRC}/libxo
 
-LIBADD=	xo util
+LIBADD=	xo
 
 .if ${MK_TESTS} != "no"
 SUBDIR+=	tests

From 6011fa6be42372357cde4f3ec4f2a29701f67138 Mon Sep 17 00:00:00 2001
From: jmg <jmg@FreeBSD.org>
Date: Sun, 18 Oct 2015 08:08:37 +0000
Subject: [PATCH 331/380] page sized is not spelled 4096 on all arches...

---
 sys/sys/mbuf.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/sys/mbuf.h b/sys/sys/mbuf.h
index 119bb6dd5ce1..8509a6ce49a2 100644
--- a/sys/sys/mbuf.h
+++ b/sys/sys/mbuf.h
@@ -344,7 +344,7 @@ struct mbuf {
  */
 #define	EXT_CLUSTER	1	/* mbuf cluster */
 #define	EXT_SFBUF	2	/* sendfile(2)'s sf_bufs */
-#define	EXT_JUMBOP	3	/* jumbo cluster 4096 bytes */
+#define	EXT_JUMBOP	3	/* jumbo cluster page sized */
 #define	EXT_JUMBO9	4	/* jumbo cluster 9216 bytes */
 #define	EXT_JUMBO16	5	/* jumbo cluster 16184 bytes */
 #define	EXT_PACKET	6	/* mbuf+cluster from packet zone */

From c1a65e2b3fddbf58c4e8adbdcc00d3b93585e8ee Mon Sep 17 00:00:00 2001
From: jmg <jmg@FreeBSD.org>
Date: Sun, 18 Oct 2015 08:13:51 +0000
Subject: [PATCH 332/380] drop a bunch of white space at end of lines and end
 of files... -x -wb apparently doesn't hide end of file white space changes..

This is to reduce the amount of diff for my PCIe HP changes..
---
 sys/dev/pci/pci.c     |  8 ++++----
 sys/dev/pci/pci_pci.c | 19 +++++++++----------
 sys/dev/pci/pcib_if.m |  3 +--
 3 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/sys/dev/pci/pci.c b/sys/dev/pci/pci.c
index 98b6b53a0724..39d311f65095 100644
--- a/sys/dev/pci/pci.c
+++ b/sys/dev/pci/pci.c
@@ -535,7 +535,7 @@ pci_romsize(uint64_t testval)
 	}
 	return (ln2size);
 }
-	
+
 /* return log2 of address range supported by map register */
 
 static int
@@ -1706,7 +1706,7 @@ pci_remap_msix_method(device_t dev, device_t child, int count,
 		free(used, M_DEVBUF);
 		return (EINVAL);
 	}
-	
+
 	/* Make sure none of the resources are allocated. */
 	for (i = 0; i < msix->msix_table_len; i++) {
 		if (msix->msix_table[i].mte_vector == 0)
@@ -2003,7 +2003,7 @@ pci_remap_intr_method(device_t bus, device_t dev, u_int irq)
 	struct msix_table_entry *mte;
 	struct msix_vector *mv;
 	uint64_t addr;
-	uint32_t data;	
+	uint32_t data;
 	int error, i, j;
 
 	/*
@@ -4830,7 +4830,7 @@ pci_deactivate_resource(device_t dev, device_t child, int type,
 	if (error)
 		return (error);
 
-	/* Disable decoding for device ROMs. */	
+	/* Disable decoding for device ROMs. */
 	if (device_get_parent(child) == dev) {
 		dinfo = device_get_ivars(child);
 		if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid))
diff --git a/sys/dev/pci/pci_pci.c b/sys/dev/pci/pci_pci.c
index 11a4bcac7950..f5a5b7415f0b 100644
--- a/sys/dev/pci/pci_pci.c
+++ b/sys/dev/pci/pci_pci.c
@@ -57,7 +57,7 @@ static int		pcib_resume(device_t dev);
 static int		pcib_power_for_sleep(device_t pcib, device_t dev,
 			    int *pstate);
 static uint16_t		pcib_ari_get_rid(device_t pcib, device_t dev);
-static uint32_t		pcib_read_config(device_t dev, u_int b, u_int s, 
+static uint32_t		pcib_read_config(device_t dev, u_int b, u_int s,
     u_int f, u_int reg, int width);
 static void		pcib_write_config(device_t dev, u_int b, u_int s,
     u_int f, u_int reg, uint32_t val, int width);
@@ -266,7 +266,7 @@ pcib_add_window_resources(struct pcib_window *w, struct resource **res,
 	free(w->res, M_DEVBUF);
 	w->res = newarray;
 	w->count += count;
-	
+
 	for (i = 0; i < count; i++) {
 		error = rman_manage_region(&w->rman, rman_get_start(res[i]),
 		    rman_get_end(res[i]));
@@ -783,7 +783,7 @@ pcib_get_mem_decode(struct pcib_softc *sc)
 		sc->pmembase = PCI_PPBMEMBASE(0, pmemlow);
 
 	pmemlow = pci_read_config(dev, PCIR_PMLIMITL_1, 2);
-	if ((pmemlow & PCIM_BRPM_MASK) == PCIM_BRPM_64)	
+	if ((pmemlow & PCIM_BRPM_MASK) == PCIM_BRPM_64)
 		sc->pmemlimit = PCI_PPBMEMLIMIT(
 		    pci_read_config(dev, PCIR_PMLIMITH_1, 4), pmemlow);
 	else
@@ -1126,7 +1126,7 @@ int
 pcib_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
 {
     struct pcib_softc	*sc = device_get_softc(dev);
-    
+
     switch (which) {
     case PCIB_IVAR_DOMAIN:
 	*result = sc->domain;
@@ -1243,9 +1243,9 @@ pcib_alloc_new_window(struct pcib_softc *sc, struct pcib_window *w, int type,
 				return (0);
 			}
 		}
-		return (ENOSPC);		
+		return (ENOSPC);
 	}
-	
+
 	wmask = (1ul << w->step) - 1;
 	if (RF_ALIGNMENT(flags) < w->step) {
 		flags &= ~RF_ALIGNMENT_MASK;
@@ -1337,7 +1337,7 @@ pcib_expand_window(struct pcib_softc *sc, struct pcib_window *w, int type,
 		KASSERT(w->base == rman_get_start(res),
 		    ("existing resource mismatch"));
 		force_64k_base = 0;
-	}	
+	}
 
 	error = bus_adjust_resource(sc->dev, type, res, force_64k_base ?
 	    rman_get_start(res) : base, limit);
@@ -1657,7 +1657,7 @@ pcib_release_resource(device_t dev, device_t child, int type, int rid,
  * is set up to, or capable of handling them.
  */
 struct resource *
-pcib_alloc_resource(device_t dev, device_t child, int type, int *rid, 
+pcib_alloc_resource(device_t dev, device_t child, int type, int *rid,
     u_long start, u_long end, u_long count, u_int flags)
 {
 	struct pcib_softc	*sc = device_get_softc(dev);
@@ -1928,7 +1928,7 @@ pcib_route_interrupt(device_t pcib, device_t dev, int pin)
     int		parent_intpin;
     int		intnum;
 
-    /*	
+    /*
      *
      * The PCI standard defines a swizzle of the child-side device/intpin to
      * the parent-side intpin as follows.
@@ -2116,4 +2116,3 @@ pcib_try_enable_ari(device_t pcib, device_t dev)
 
 	return (0);
 }
-
diff --git a/sys/dev/pci/pcib_if.m b/sys/dev/pci/pcib_if.m
index 7c23ddc6973f..6fdc0f42518e 100644
--- a/sys/dev/pci/pcib_if.m
+++ b/sys/dev/pci/pcib_if.m
@@ -97,7 +97,7 @@ METHOD void write_config {
 };
 
 #
-# Route an interrupt.  Returns a value suitable for stuffing into 
+# Route an interrupt.  Returns a value suitable for stuffing into
 # a device's interrupt register.
 #
 METHOD int route_interrupt {
@@ -206,4 +206,3 @@ METHOD void decode_rid {
 	int 		*slot;
 	int 		*func;
 } DEFAULT pcib_decode_rid;
-

From 9d188e9d5cda858010e5985995d7becd469456f8 Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Sun, 18 Oct 2015 09:33:28 +0000
Subject: [PATCH 333/380] Only marker is guaranteed to be present on the queue
 after the relock in vm_pageout_fallback_object_lock() and
 vm_pageout_page_lock().  The check for the m->queue == queue assumes that the
 page does belong to a queue.

Modify the 'unchanged' calculation bu dereferencing the marker tailq
pointers, which is known to belong to the queue.  Since for a page m
linked to the queue, m->queue must be equal to the queue index, assert
this instead of checking.

In collaboration with:	alc
Sponsored by:	The FreeBSD Foundation (kib)
MFC after:	2 weeks
---
 sys/vm/vm_pageout.c | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c
index e0d00f5172f3..188866efbec4 100644
--- a/sys/vm/vm_pageout.c
+++ b/sys/vm/vm_pageout.c
@@ -292,11 +292,21 @@ vm_pageout_fallback_object_lock(vm_page_t m, vm_page_t *next)
 	vm_page_lock(m);
 	vm_pagequeue_lock(pq);
 
-	/* Page queue might have changed. */
+	/*
+	 * The page's object might have changed, and/or the page might
+	 * have moved from its original position in the queue.  If the
+	 * page's object has changed, then the caller should abandon
+	 * processing the page because the wrong object lock was
+	 * acquired.  Use the marker's plinks.q, not the page's, to
+	 * determine if the page has been moved.  The state of the
+	 * page's plinks.q can be indeterminate; whereas, the marker's
+	 * plinks.q must be valid.
+	 */
 	*next = TAILQ_NEXT(&marker, plinks.q);
-	unchanged = (m->queue == queue &&
-		     m->object == object &&
-		     &marker == TAILQ_NEXT(m, plinks.q));
+	unchanged = m->object == object &&
+	    m == TAILQ_PREV(&marker, pglist, plinks.q);
+	KASSERT(!unchanged || m->queue == queue,
+	    ("page %p queue %d %d", m, queue, m->queue));
 	TAILQ_REMOVE(&pq->pq_pl, &marker, plinks.q);
 	return (unchanged);
 }
@@ -333,7 +343,9 @@ vm_pageout_page_lock(vm_page_t m, vm_page_t *next)
 
 	/* Page queue might have changed. */
 	*next = TAILQ_NEXT(&marker, plinks.q);
-	unchanged = (m->queue == queue && &marker == TAILQ_NEXT(m, plinks.q));
+	unchanged = m == TAILQ_PREV(&marker, pglist, plinks.q);
+	KASSERT(!unchanged || m->queue == queue,
+	    ("page %p queue %d %d", m, queue, m->queue));
 	TAILQ_REMOVE(&pq->pq_pl, &marker, plinks.q);
 	return (unchanged);
 }

From 4fb8e1965fbd0cdf69e826ebec0c93501cf890cf Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Sun, 18 Oct 2015 11:21:08 +0000
Subject: [PATCH 334/380] MFV r289493: 5745 zfs set allows only one dataset
 property to be set at a time

Reviewed by: Christopher Siden <christopher.siden@delphix.com>
Reviewed by: George Wilson <george@delphix.com>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Bayard Bell <buffer.g.overflow@gmail.com>
Reviewed by: Richard PALO <richard@NetBSD.org>
Reviewed by: Steven Hartland <killing@multiplay.co.uk>
Approved by: Rich Lowe <richlowe@richlowe.net>
Author: Chris Williamson <chris.williamson@delphix.com>

illumos/illumos-gate@30925561c223021e91d15899cbe75f80e54d8889
---
 cddl/contrib/opensolaris/cmd/zfs/zfs.8        |  14 +-
 cddl/contrib/opensolaris/cmd/zfs/zfs_main.c   |  76 ++++---
 .../opensolaris/lib/libzfs/common/libzfs.h    |   3 +-
 .../lib/libzfs/common/libzfs_dataset.c        | 196 ++++++++++++------
 .../lib/libzfs/common/libzfs_util.c           |  16 +-
 5 files changed, 200 insertions(+), 105 deletions(-)

diff --git a/cddl/contrib/opensolaris/cmd/zfs/zfs.8 b/cddl/contrib/opensolaris/cmd/zfs/zfs.8
index 9cfa4dec37f2..5eca00fe0080 100644
--- a/cddl/contrib/opensolaris/cmd/zfs/zfs.8
+++ b/cddl/contrib/opensolaris/cmd/zfs/zfs.8
@@ -18,7 +18,7 @@
 .\" information: Portions Copyright [yyyy] [name of copyright owner]
 .\"
 .\" Copyright (c) 2010, Sun Microsystems, Inc. All Rights Reserved.
-.\" Copyright (c) 2014 by Delphix. All rights reserved.
+.\" Copyright (c) 2011, 2014 by Delphix. All rights reserved.
 .\" Copyright (c) 2011, Pawel Jakub Dawidek <pjd@FreeBSD.org>
 .\" Copyright (c) 2012, Glen Barber <gjb@FreeBSD.org>
 .\" Copyright (c) 2012, Bryan Drewery <bdrewery@FreeBSD.org>
@@ -117,7 +117,7 @@
 .Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot
 .Nm
 .Cm set
-.Ar property Ns = Ns Ar value
+.Ar property Ns = Ns Ar value Oc ...
 .Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot Ns ...
 .Nm
 .Cm get
@@ -2121,14 +2121,14 @@ option, but sorts by property in descending order.
 .It Xo
 .Nm
 .Cm set
-.Ar property Ns = Ns Ar value
+.Ar property Ns = Ns Ar value Oc ...
 .Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot
 .Xc
 .Pp
-Sets the property to the given value for each dataset. Only some properties can
-be edited. See the "Properties" section for more information on what properties
-can be set and acceptable values. Numeric values can be specified as exact
-values, or in a human-readable form with a suffix of
+Sets the property or list of properties to the given value(s) for each dataset.
+Only some properties can be edited. See the "Properties" section for more
+information on what properties can be set and acceptable values. Numeric values
+can be specified as exact values, or in a human-readable form with a suffix of
 .Sy B , K , M , G , T , P , E , Z
 (for bytes, kilobytes, megabytes, gigabytes, terabytes, petabytes, exabytes, or
 zettabytes, respectively). User properties can be set on snapshots. For more
diff --git a/cddl/contrib/opensolaris/cmd/zfs/zfs_main.c b/cddl/contrib/opensolaris/cmd/zfs/zfs_main.c
index 9b123ced6671..e3ac1ca30d1a 100644
--- a/cddl/contrib/opensolaris/cmd/zfs/zfs_main.c
+++ b/cddl/contrib/opensolaris/cmd/zfs/zfs_main.c
@@ -283,7 +283,7 @@ get_usage(zfs_help_t idx)
 		    "<filesystem|volume|snapshot>\n"
 		    "\tsend [-nvPe] -t <receive_resume_token>\n"));
 	case HELP_SET:
-		return (gettext("\tset <property=value> "
+		return (gettext("\tset <property=value> ... "
 		    "<filesystem|volume|snapshot> ...\n"));
 	case HELP_SHARE:
 		return (gettext("\tshare <-a | filesystem>\n"));
@@ -498,6 +498,10 @@ usage(boolean_t requested)
 	exit(requested ? 0 : 2);
 }
 
+/*
+ * Take a property=value argument string and add it to the given nvlist.
+ * Modifies the argument inplace.
+ */
 static int
 parseprop(nvlist_t *props, char *propname)
 {
@@ -505,7 +509,7 @@ parseprop(nvlist_t *props, char *propname)
 
 	if ((propval = strchr(propname, '=')) == NULL) {
 		(void) fprintf(stderr, gettext("missing "
-		    "'=' for -o option\n"));
+		    "'=' for property=value argument\n"));
 		return (-1);
 	}
 	*propval = '\0';
@@ -632,7 +636,7 @@ zfs_do_clone(int argc, char **argv)
 	while ((c = getopt(argc, argv, "o:p")) != -1) {
 		switch (c) {
 		case 'o':
-			if (parseprop(props, optarg))
+			if (parseprop(props, optarg) != 0)
 				return (1);
 			break;
 		case 'p':
@@ -3532,21 +3536,17 @@ out:
 }
 
 /*
- * zfs set property=value { fs | snap | vol } ...
+ * zfs set property=value ... { fs | snap | vol } ...
  *
- * Sets the given property for all datasets specified on the command line.
+ * Sets the given properties for all datasets specified on the command line.
  */
-typedef struct set_cbdata {
-	char		*cb_propname;
-	char		*cb_value;
-} set_cbdata_t;
 
 static int
 set_callback(zfs_handle_t *zhp, void *data)
 {
-	set_cbdata_t *cbp = data;
+	nvlist_t *props = data;
 
-	if (zfs_prop_set(zhp, cbp->cb_propname, cbp->cb_value) != 0) {
+	if (zfs_prop_set_list(zhp, props) != 0) {
 		switch (libzfs_errno(g_zfs)) {
 		case EZFS_MOUNTFAILED:
 			(void) fprintf(stderr, gettext("property may be set "
@@ -3565,7 +3565,8 @@ set_callback(zfs_handle_t *zhp, void *data)
 static int
 zfs_do_set(int argc, char **argv)
 {
-	set_cbdata_t cb;
+	nvlist_t *props = NULL;
+	int ds_start = -1; /* argv idx of first dataset arg */
 	int ret = 0;
 
 	/* check for options */
@@ -3577,36 +3578,51 @@ zfs_do_set(int argc, char **argv)
 
 	/* check number of arguments */
 	if (argc < 2) {
-		(void) fprintf(stderr, gettext("missing property=value "
-		    "argument\n"));
+		(void) fprintf(stderr, gettext("missing arguments\n"));
 		usage(B_FALSE);
 	}
 	if (argc < 3) {
-		(void) fprintf(stderr, gettext("missing dataset name\n"));
+		if (strchr(argv[1], '=') == NULL) {
+			(void) fprintf(stderr, gettext("missing property=value "
+			    "argument(s)\n"));
+		} else {
+			(void) fprintf(stderr, gettext("missing dataset "
+			    "name(s)\n"));
+		}
 		usage(B_FALSE);
 	}
 
-	/* validate property=value argument */
-	cb.cb_propname = argv[1];
-	if (((cb.cb_value = strchr(cb.cb_propname, '=')) == NULL) ||
-	    (cb.cb_value[1] == '\0')) {
-		(void) fprintf(stderr, gettext("missing value in "
-		    "property=value argument\n"));
+	/* validate argument order:  prop=val args followed by dataset args */
+	for (int i = 1; i < argc; i++) {
+		if (strchr(argv[i], '=') != NULL) {
+			if (ds_start > 0) {
+				/* out-of-order prop=val argument */
+				(void) fprintf(stderr, gettext("invalid "
+				    "argument order\n"), i);
+				usage(B_FALSE);
+			}
+		} else if (ds_start < 0) {
+			ds_start = i;
+		}
+	}
+	if (ds_start < 0) {
+		(void) fprintf(stderr, gettext("missing dataset name(s)\n"));
 		usage(B_FALSE);
 	}
 
-	*cb.cb_value = '\0';
-	cb.cb_value++;
-
-	if (*cb.cb_propname == '\0') {
-		(void) fprintf(stderr,
-		    gettext("missing property in property=value argument\n"));
-		usage(B_FALSE);
+	/* Populate a list of property settings */
+	if (nvlist_alloc(&props, NV_UNIQUE_NAME, 0) != 0)
+		nomem();
+	for (int i = 1; i < ds_start; i++) {
+		if ((ret = parseprop(props, argv[i])) != 0)
+			goto error;
 	}
 
-	ret = zfs_for_each(argc - 2, argv + 2, 0,
-	    ZFS_TYPE_DATASET, NULL, NULL, 0, set_callback, &cb);
+	ret = zfs_for_each(argc - ds_start, argv + ds_start, 0,
+	    ZFS_TYPE_DATASET, NULL, NULL, 0, set_callback, props);
 
+error:
+	nvlist_free(props);
 	return (ret);
 }
 
diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs.h b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs.h
index 48410408d0f8..80a31c0dbcc1 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs.h
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs.h
@@ -23,7 +23,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011 Pawel Jakub Dawidek <pawel@dawidek.net>.
  * All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
  * Copyright (c) 2012 Martin Matuska <mm@FreeBSD.org>. All rights reserved.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
@@ -430,6 +430,7 @@ extern nvlist_t *zfs_valid_proplist(libzfs_handle_t *, zfs_type_t,
 
 extern const char *zfs_prop_to_name(zfs_prop_t);
 extern int zfs_prop_set(zfs_handle_t *, const char *, const char *);
+extern int zfs_prop_set_list(zfs_handle_t *, nvlist_t *);
 extern int zfs_prop_get(zfs_handle_t *, zfs_prop_t, char *, size_t,
     zprop_source_t *, char *, size_t, boolean_t);
 extern int zfs_prop_get_recvd(zfs_handle_t *, const char *, char *, size_t,
diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_dataset.c b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_dataset.c
index f6c86cfe1788..11b3df37bbe4 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_dataset.c
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_dataset.c
@@ -1534,15 +1534,10 @@ zfs_setprop_error(libzfs_handle_t *hdl, zfs_prop_t prop, int err,
 int
 zfs_prop_set(zfs_handle_t *zhp, const char *propname, const char *propval)
 {
-	zfs_cmd_t zc = { 0 };
 	int ret = -1;
-	prop_changelist_t *cl = NULL;
 	char errbuf[1024];
 	libzfs_handle_t *hdl = zhp->zfs_hdl;
-	nvlist_t *nvl = NULL, *realprops;
-	zfs_prop_t prop;
-	boolean_t do_prefix = B_TRUE;
-	int added_resv;
+	nvlist_t *nvl = NULL;
 
 	(void) snprintf(errbuf, sizeof (errbuf),
 	    dgettext(TEXT_DOMAIN, "cannot set property for '%s'"),
@@ -1554,79 +1549,148 @@ zfs_prop_set(zfs_handle_t *zhp, const char *propname, const char *propval)
 		goto error;
 	}
 
-	if ((realprops = zfs_valid_proplist(hdl, zhp->zfs_type, nvl,
+	ret = zfs_prop_set_list(zhp, nvl);
+
+error:
+	nvlist_free(nvl);
+	return (ret);
+}
+
+
+
+/*
+ * Given an nvlist of property names and values, set the properties for the
+ * given dataset.
+ */
+int
+zfs_prop_set_list(zfs_handle_t *zhp, nvlist_t *props)
+{
+	zfs_cmd_t zc = { 0 };
+	int ret = -1;
+	prop_changelist_t **cls = NULL;
+	int cl_idx;
+	char errbuf[1024];
+	libzfs_handle_t *hdl = zhp->zfs_hdl;
+	nvlist_t *nvl;
+	int nvl_len;
+	int added_resv;
+
+	(void) snprintf(errbuf, sizeof (errbuf),
+	    dgettext(TEXT_DOMAIN, "cannot set property for '%s'"),
+	    zhp->zfs_name);
+
+	if ((nvl = zfs_valid_proplist(hdl, zhp->zfs_type, props,
 	    zfs_prop_get_int(zhp, ZFS_PROP_ZONED), zhp, errbuf)) == NULL)
 		goto error;
 
-	nvlist_free(nvl);
-	nvl = realprops;
-
-	prop = zfs_name_to_prop(propname);
-
-	/* We don't support those properties on FreeBSD. */
-	switch (prop) {
-	case ZFS_PROP_DEVICES:
-	case ZFS_PROP_ISCSIOPTIONS:
-	case ZFS_PROP_XATTR:
-	case ZFS_PROP_VSCAN:
-	case ZFS_PROP_NBMAND:
-	case ZFS_PROP_MLSLABEL:
-		(void) snprintf(errbuf, sizeof (errbuf),
-		    "property '%s' not supported on FreeBSD", propname);
-		ret = zfs_error(hdl, EZFS_PERM, errbuf);
-		goto error;
-	}
-
-	if (prop == ZFS_PROP_VOLSIZE) {
-		if ((added_resv = zfs_add_synthetic_resv(zhp, nvl)) == -1)
-			goto error;
-	}
-
-	if ((cl = changelist_gather(zhp, prop, 0, 0)) == NULL)
-		goto error;
-
-	if (prop == ZFS_PROP_MOUNTPOINT && changelist_haszonedchild(cl)) {
-		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
-		    "child dataset with inherited mountpoint is used "
-		    "in a non-global zone"));
-		ret = zfs_error(hdl, EZFS_ZONED, errbuf);
-		goto error;
-	}
-
 	/*
-	 * We don't want to unmount & remount the dataset when changing
-	 * its canmount property to 'on' or 'noauto'.  We only use
-	 * the changelist logic to unmount when setting canmount=off.
+	 * We have to check for any extra properties which need to be added
+	 * before computing the length of the nvlist.
 	 */
-	if (prop == ZFS_PROP_CANMOUNT) {
-		uint64_t idx;
-		int err = zprop_string_to_index(prop, propval, &idx,
-		    ZFS_TYPE_DATASET);
-		if (err == 0 && idx != ZFS_CANMOUNT_OFF)
-			do_prefix = B_FALSE;
+	for (nvpair_t *elem = nvlist_next_nvpair(nvl, NULL);
+	    elem != NULL;
+	    elem = nvlist_next_nvpair(nvl, elem)) {
+		if (zfs_name_to_prop(nvpair_name(elem)) == ZFS_PROP_VOLSIZE &&
+		    (added_resv = zfs_add_synthetic_resv(zhp, nvl)) == -1) {
+			goto error;
+		}
 	}
-
-	if (do_prefix && (ret = changelist_prefix(cl)) != 0)
+	/*
+	 * Check how many properties we're setting and allocate an array to
+	 * store changelist pointers for postfix().
+	 */
+	nvl_len = 0;
+	for (nvpair_t *elem = nvlist_next_nvpair(nvl, NULL);
+	    elem != NULL;
+	    elem = nvlist_next_nvpair(nvl, elem))
+		nvl_len++;
+	if ((cls = calloc(nvl_len, sizeof (prop_changelist_t *))) == NULL)
 		goto error;
 
+	cl_idx = 0;
+	for (nvpair_t *elem = nvlist_next_nvpair(nvl, NULL);
+	    elem != NULL;
+	    elem = nvlist_next_nvpair(nvl, elem)) {
+
+		zfs_prop_t prop = zfs_name_to_prop(nvpair_name(elem));
+
+		assert(cl_idx < nvl_len);
+		/*
+		 * We don't want to unmount & remount the dataset when changing
+		 * its canmount property to 'on' or 'noauto'.  We only use
+		 * the changelist logic to unmount when setting canmount=off.
+		 */
+		if (!(prop == ZFS_PROP_CANMOUNT &&
+		    fnvpair_value_uint64(elem) != ZFS_CANMOUNT_OFF)) {
+			cls[cl_idx] = changelist_gather(zhp, prop, 0, 0);
+			if (cls[cl_idx] == NULL)
+				goto error;
+		}
+
+		if (prop == ZFS_PROP_MOUNTPOINT &&
+		    changelist_haszonedchild(cls[cl_idx])) {
+			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+			    "child dataset with inherited mountpoint is used "
+			    "in a non-global zone"));
+			ret = zfs_error(hdl, EZFS_ZONED, errbuf);
+			goto error;
+		}
+
+		/* We don't support those properties on FreeBSD. */
+		switch (prop) {
+		case ZFS_PROP_DEVICES:
+		case ZFS_PROP_ISCSIOPTIONS:
+		case ZFS_PROP_XATTR:
+		case ZFS_PROP_VSCAN:
+		case ZFS_PROP_NBMAND:
+		case ZFS_PROP_MLSLABEL:
+			(void) snprintf(errbuf, sizeof (errbuf),
+			    "property '%s' not supported on FreeBSD",
+			    nvpair_name(elem));
+			ret = zfs_error(hdl, EZFS_PERM, errbuf);
+			goto error;
+		}
+
+		if (cls[cl_idx] != NULL &&
+		    (ret = changelist_prefix(cls[cl_idx])) != 0)
+			goto error;
+
+		cl_idx++;
+	}
+	assert(cl_idx == nvl_len);
+
 	/*
-	 * Execute the corresponding ioctl() to set this property.
+	 * Execute the corresponding ioctl() to set this list of properties.
 	 */
 	(void) strlcpy(zc.zc_name, zhp->zfs_name, sizeof (zc.zc_name));
 
-	if (zcmd_write_src_nvlist(hdl, &zc, nvl) != 0)
+	if ((ret = zcmd_write_src_nvlist(hdl, &zc, nvl)) != 0 ||
+	    (ret = zcmd_alloc_dst_nvlist(hdl, &zc, 0)) != 0)
 		goto error;
 
 	ret = zfs_ioctl(hdl, ZFS_IOC_SET_PROP, &zc);
 
 	if (ret != 0) {
-		zfs_setprop_error(hdl, prop, errno, errbuf);
+		/* Get the list of unset properties back and report them. */
+		nvlist_t *errorprops = NULL;
+		if (zcmd_read_dst_nvlist(hdl, &zc, &errorprops) != 0)
+			goto error;
+		for (nvpair_t *elem = nvlist_next_nvpair(nvl, NULL);
+		    elem != NULL;
+		    elem = nvlist_next_nvpair(nvl, elem)) {
+			zfs_prop_t prop = zfs_name_to_prop(nvpair_name(elem));
+			zfs_setprop_error(hdl, prop, errno, errbuf);
+		}
+		nvlist_free(errorprops);
+
 		if (added_resv && errno == ENOSPC) {
 			/* clean up the volsize property we tried to set */
 			uint64_t old_volsize = zfs_prop_get_int(zhp,
 			    ZFS_PROP_VOLSIZE);
 			nvlist_free(nvl);
+			nvl = NULL;
 			zcmd_free_nvlists(&zc);
+
 			if (nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0) != 0)
 				goto error;
 			if (nvlist_add_uint64(nvl,
@@ -1638,8 +1702,13 @@ zfs_prop_set(zfs_handle_t *zhp, const char *propname, const char *propval)
 			(void) zfs_ioctl(hdl, ZFS_IOC_SET_PROP, &zc);
 		}
 	} else {
-		if (do_prefix)
-			ret = changelist_postfix(cl);
+		for (cl_idx = 0; cl_idx < nvl_len; cl_idx++) {
+			if (cls[cl_idx] != NULL) {
+				int clp_err = changelist_postfix(cls[cl_idx]);
+				if (clp_err != 0)
+					ret = clp_err;
+			}
+		}
 
 		/*
 		 * Refresh the statistics so the new property value
@@ -1652,8 +1721,13 @@ zfs_prop_set(zfs_handle_t *zhp, const char *propname, const char *propval)
 error:
 	nvlist_free(nvl);
 	zcmd_free_nvlists(&zc);
-	if (cl)
-		changelist_free(cl);
+	if (cls != NULL) {
+		for (cl_idx = 0; cl_idx < nvl_len; cl_idx++) {
+			if (cls[cl_idx] != NULL)
+				changelist_free(cls[cl_idx]);
+		}
+		free(cls);
+	}
 	return (ret);
 }
 
@@ -4203,7 +4277,7 @@ zfs_smb_acl_mgmt(libzfs_handle_t *hdl, char *dataset, char *path,
 	if (cmd == ZFS_SMB_ACL_RENAME) {
 		if (nvlist_alloc(&nvlist, NV_UNIQUE_NAME, 0) != 0) {
 			(void) no_memory(hdl);
-			return (NULL);
+			return (0);
 		}
 	}
 
diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_util.c b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_util.c
index ab8a9bffefba..79738882bdb1 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_util.c
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_util.c
@@ -22,7 +22,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
- * Copyright (c) 2012 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  */
 
 /*
@@ -782,8 +782,9 @@ zcmd_alloc_dst_nvlist(libzfs_handle_t *hdl, zfs_cmd_t *zc, size_t len)
 	if (len == 0)
 		len = 16 * 1024;
 	zc->zc_nvlist_dst_size = len;
-	if ((zc->zc_nvlist_dst = (uint64_t)(uintptr_t)
-	    zfs_alloc(hdl, zc->zc_nvlist_dst_size)) == 0)
+	zc->zc_nvlist_dst =
+	    (uint64_t)(uintptr_t)zfs_alloc(hdl, zc->zc_nvlist_dst_size);
+	if (zc->zc_nvlist_dst == 0)
 		return (-1);
 
 	return (0);
@@ -798,9 +799,9 @@ int
 zcmd_expand_dst_nvlist(libzfs_handle_t *hdl, zfs_cmd_t *zc)
 {
 	free((void *)(uintptr_t)zc->zc_nvlist_dst);
-	if ((zc->zc_nvlist_dst = (uint64_t)(uintptr_t)
-	    zfs_alloc(hdl, zc->zc_nvlist_dst_size))
-	    == 0)
+	zc->zc_nvlist_dst =
+	    (uint64_t)(uintptr_t)zfs_alloc(hdl, zc->zc_nvlist_dst_size);
+	if (zc->zc_nvlist_dst == 0)
 		return (-1);
 
 	return (0);
@@ -815,6 +816,9 @@ zcmd_free_nvlists(zfs_cmd_t *zc)
 	free((void *)(uintptr_t)zc->zc_nvlist_conf);
 	free((void *)(uintptr_t)zc->zc_nvlist_src);
 	free((void *)(uintptr_t)zc->zc_nvlist_dst);
+	zc->zc_nvlist_conf = NULL;
+	zc->zc_nvlist_src = NULL;
+	zc->zc_nvlist_dst = NULL;
 }
 
 static int

From a715e58b8c9563e168cabb7f06e58523d5ba33c1 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Sun, 18 Oct 2015 11:36:43 +0000
Subject: [PATCH 335/380] MFV r247180: Update vendor/illumos/dist and
 vendor-sys/illumos/dist to illumos-gate 13967:92bec6d87f59

Illumos ZFS issues:
  3557 dumpvp_size is not updated correctly when a dump zvol's size is
       changed
  3558 setting the volsize on a dump device does not return back ENOSPC
  3559 setting a volsize larger than the space available sometimes succeeds
---
 cddl/contrib/opensolaris/cmd/zfs/zfs_main.c   | 12 +++++++++-
 .../lib/libzfs/common/libzfs_dataset.c        | 22 ++++++++++++++-----
 2 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/cddl/contrib/opensolaris/cmd/zfs/zfs_main.c b/cddl/contrib/opensolaris/cmd/zfs/zfs_main.c
index e3ac1ca30d1a..ffc1a099d999 100644
--- a/cddl/contrib/opensolaris/cmd/zfs/zfs_main.c
+++ b/cddl/contrib/opensolaris/cmd/zfs/zfs_main.c
@@ -845,10 +845,12 @@ zfs_do_create(int argc, char **argv)
 
 	if (type == ZFS_TYPE_VOLUME && !noreserve) {
 		zpool_handle_t *zpool_handle;
+		nvlist_t *real_props;
 		uint64_t spa_version;
 		char *p;
 		zfs_prop_t resv_prop;
 		char *strval;
+		char msg[1024];
 
 		if (p = strchr(argv[0], '/'))
 			*p = '\0';
@@ -864,7 +866,15 @@ zfs_do_create(int argc, char **argv)
 			resv_prop = ZFS_PROP_REFRESERVATION;
 		else
 			resv_prop = ZFS_PROP_RESERVATION;
-		volsize = zvol_volsize_to_reservation(volsize, props);
+
+		(void) snprintf(msg, sizeof (msg),
+		    gettext("cannot create '%s'"), argv[0]);
+		if (props && (real_props = zfs_valid_proplist(g_zfs, type,
+		    props, 0, NULL, msg)) == NULL)
+			goto error;
+
+		volsize = zvol_volsize_to_reservation(volsize, real_props);
+		nvlist_free(real_props);
 
 		if (nvlist_lookup_string(props, zfs_prop_to_name(resv_prop),
 		    &strval) != 0) {
diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_dataset.c b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_dataset.c
index 11b3df37bbe4..74b0cb9d0bd2 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_dataset.c
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_dataset.c
@@ -1403,6 +1403,7 @@ zfs_add_synthetic_resv(zfs_handle_t *zhp, nvlist_t *nvl)
 	uint64_t old_reservation;
 	uint64_t new_reservation;
 	zfs_prop_t resv_prop;
+	nvlist_t *props;
 
 	/*
 	 * If this is an existing volume, and someone is setting the volsize,
@@ -1412,16 +1413,25 @@ zfs_add_synthetic_resv(zfs_handle_t *zhp, nvlist_t *nvl)
 	if (zfs_which_resv_prop(zhp, &resv_prop) < 0)
 		return (-1);
 	old_reservation = zfs_prop_get_int(zhp, resv_prop);
-	if ((zvol_volsize_to_reservation(old_volsize, zhp->zfs_props) !=
-	    old_reservation) || nvlist_lookup_uint64(nvl,
-	    zfs_prop_to_name(resv_prop), &new_reservation) != ENOENT) {
+
+	props = fnvlist_alloc();
+	fnvlist_add_uint64(props, zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE),
+	    zfs_prop_get_int(zhp, ZFS_PROP_VOLBLOCKSIZE));
+
+	if ((zvol_volsize_to_reservation(old_volsize, props) !=
+	    old_reservation) || nvlist_exists(nvl,
+	    zfs_prop_to_name(resv_prop))) {
+		fnvlist_free(props);
 		return (0);
 	}
 	if (nvlist_lookup_uint64(nvl, zfs_prop_to_name(ZFS_PROP_VOLSIZE),
-	    &new_volsize) != 0)
+	    &new_volsize) != 0) {
+		fnvlist_free(props);
 		return (-1);
-	new_reservation = zvol_volsize_to_reservation(new_volsize,
-	    zhp->zfs_props);
+	}
+	new_reservation = zvol_volsize_to_reservation(new_volsize, props);
+	fnvlist_free(props);
+
 	if (nvlist_add_uint64(nvl, zfs_prop_to_name(resv_prop),
 	    new_reservation) != 0) {
 		(void) no_memory(zhp->zfs_hdl);

From 97500960c515969155e2b3bc490ce191e40254ee Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Sun, 18 Oct 2015 11:44:31 +0000
Subject: [PATCH 336/380] MFC r289498: 6298 zfs_create_008_neg and
 zpool_create_023_neg need to be updated for large block support.

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: John Kennedy <john.kennedy@delphix.com>
Approved by: Robert Mustacchi <rm@joyent.com>
Author: Joe Stein <joe.stein@delphix.com>

illumos/illumos-gate@e9316f7696401f3e5e263a5939031cb8d5641a88
---
 cddl/contrib/opensolaris/cmd/zfs/zfs_main.c   |  8 +--
 .../opensolaris/lib/libzfs/common/libzfs.h    |  4 +-
 .../lib/libzfs/common/libzfs_dataset.c        | 51 +++++++++++++------
 .../lib/libzfs/common/libzfs_pool.c           | 19 ++++++-
 4 files changed, 60 insertions(+), 22 deletions(-)

diff --git a/cddl/contrib/opensolaris/cmd/zfs/zfs_main.c b/cddl/contrib/opensolaris/cmd/zfs/zfs_main.c
index ffc1a099d999..b8466fe24e17 100644
--- a/cddl/contrib/opensolaris/cmd/zfs/zfs_main.c
+++ b/cddl/contrib/opensolaris/cmd/zfs/zfs_main.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  * Copyright 2012 Milan Jurik. All rights reserved.
  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
  * Copyright (c) 2011-2012 Pawel Jakub Dawidek <pawel@dawidek.net>.
@@ -861,7 +861,6 @@ zfs_do_create(int argc, char **argv)
 			goto error;
 		spa_version = zpool_get_prop_int(zpool_handle,
 		    ZPOOL_PROP_VERSION, NULL);
-		zpool_close(zpool_handle);
 		if (spa_version >= SPA_VERSION_REFRESERVATION)
 			resv_prop = ZFS_PROP_REFRESERVATION;
 		else
@@ -870,8 +869,11 @@ zfs_do_create(int argc, char **argv)
 		(void) snprintf(msg, sizeof (msg),
 		    gettext("cannot create '%s'"), argv[0]);
 		if (props && (real_props = zfs_valid_proplist(g_zfs, type,
-		    props, 0, NULL, msg)) == NULL)
+		    props, 0, NULL, zpool_handle, msg)) == NULL) {
+			zpool_close(zpool_handle);
 			goto error;
+		}
+		zpool_close(zpool_handle);
 
 		volsize = zvol_volsize_to_reservation(volsize, real_props);
 		nvlist_free(real_props);
diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs.h b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs.h
index 80a31c0dbcc1..8ab15461c8ee 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs.h
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs.h
@@ -23,7 +23,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011 Pawel Jakub Dawidek <pawel@dawidek.net>.
  * All rights reserved.
- * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
  * Copyright (c) 2012 Martin Matuska <mm@FreeBSD.org>. All rights reserved.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
@@ -426,7 +426,7 @@ extern const char *zfs_prop_column_name(zfs_prop_t);
 extern boolean_t zfs_prop_align_right(zfs_prop_t);
 
 extern nvlist_t *zfs_valid_proplist(libzfs_handle_t *, zfs_type_t,
-    nvlist_t *, uint64_t, zfs_handle_t *, const char *);
+    nvlist_t *, uint64_t, zfs_handle_t *, zpool_handle_t *, const char *);
 
 extern const char *zfs_prop_to_name(zfs_prop_t);
 extern int zfs_prop_set(zfs_handle_t *, const char *, const char *);
diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_dataset.c b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_dataset.c
index 74b0cb9d0bd2..234d2cd5bf97 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_dataset.c
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_dataset.c
@@ -22,7 +22,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
- * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2012 DEY Storage Systems, Inc.  All rights reserved.
  * Copyright (c) 2011-2012 Pawel Jakub Dawidek <pawel@dawidek.net>.
  * All rights reserved.
@@ -890,7 +890,8 @@ zfs_which_resv_prop(zfs_handle_t *zhp, zfs_prop_t *resv_prop)
  */
 nvlist_t *
 zfs_valid_proplist(libzfs_handle_t *hdl, zfs_type_t type, nvlist_t *nvl,
-    uint64_t zoned, zfs_handle_t *zhp, const char *errbuf)
+    uint64_t zoned, zfs_handle_t *zhp, zpool_handle_t *zpool_hdl,
+    const char *errbuf)
 {
 	nvpair_t *elem;
 	uint64_t intval;
@@ -1084,8 +1085,8 @@ zfs_valid_proplist(libzfs_handle_t *hdl, zfs_type_t type, nvlist_t *nvl,
 		case ZFS_PROP_RECORDSIZE:
 		{
 			int maxbs = SPA_MAXBLOCKSIZE;
-			if (zhp != NULL) {
-				maxbs = zpool_get_prop_int(zhp->zpool_hdl,
+			if (zpool_hdl != NULL) {
+				maxbs = zpool_get_prop_int(zpool_hdl,
 				    ZPOOL_PROP_MAXBLOCKSIZE, NULL);
 			}
 			/*
@@ -1590,7 +1591,8 @@ zfs_prop_set_list(zfs_handle_t *zhp, nvlist_t *props)
 	    zhp->zfs_name);
 
 	if ((nvl = zfs_valid_proplist(hdl, zhp->zfs_type, props,
-	    zfs_prop_get_int(zhp, ZFS_PROP_ZONED), zhp, errbuf)) == NULL)
+	    zfs_prop_get_int(zhp, ZFS_PROP_ZONED), zhp, zhp->zpool_hdl,
+	    errbuf)) == NULL)
 		goto error;
 
 	/*
@@ -3254,9 +3256,23 @@ zfs_create(libzfs_handle_t *hdl, const char *path, zfs_type_t type,
 	else
 		ost = DMU_OST_ZFS;
 
+	/* open zpool handle for prop validation */
+	char pool_path[MAXNAMELEN];
+	(void) strlcpy(pool_path, path, sizeof (pool_path));
+
+	/* truncate pool_path at first slash */
+	char *p = strchr(pool_path, '/');
+	if (p != NULL)
+		*p = '\0';
+
+	zpool_handle_t *zpool_handle = zpool_open(hdl, pool_path);
+
 	if (props && (props = zfs_valid_proplist(hdl, type, props,
-	    zoned, NULL, errbuf)) == 0)
+	    zoned, NULL, zpool_handle, errbuf)) == 0) {
+		zpool_close(zpool_handle);
 		return (-1);
+	}
+	zpool_close(zpool_handle);
 
 	if (type == ZFS_TYPE_VOLUME) {
 		/*
@@ -3324,13 +3340,6 @@ zfs_create(libzfs_handle_t *hdl, const char *path, zfs_type_t type,
 			    "parent '%s' is not a filesystem"), parent);
 			return (zfs_error(hdl, EZFS_BADTYPE, errbuf));
 
-		case EDOM:
-			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
-			    "volume block size must be power of 2 from "
-			    "512B to 128KB"));
-
-			return (zfs_error(hdl, EZFS_BADPROP, errbuf));
-
 		case ENOTSUP:
 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
 			    "pool must be upgraded to set this "
@@ -3525,7 +3534,7 @@ zfs_clone(zfs_handle_t *zhp, const char *target, nvlist_t *props)
 			type = ZFS_TYPE_FILESYSTEM;
 		}
 		if ((props = zfs_valid_proplist(hdl, type, props, zoned,
-		    zhp, errbuf)) == NULL)
+		    zhp, zhp->zpool_hdl, errbuf)) == NULL)
 			return (-1);
 	}
 
@@ -3669,11 +3678,23 @@ zfs_snapshot_nvl(libzfs_handle_t *hdl, nvlist_t *snaps, nvlist_t *props)
 		}
 	}
 
+	/*
+	 * get pool handle for prop validation. assumes all snaps are in the
+	 * same pool, as does lzc_snapshot (below).
+	 */
+	char pool[MAXNAMELEN];
+	elem = nvlist_next_nvpair(snaps, NULL);
+	(void) strlcpy(pool, nvpair_name(elem), sizeof (pool));
+	pool[strcspn(pool, "/@")] = '\0';
+	zpool_handle_t *zpool_hdl = zpool_open(hdl, pool);
+
 	if (props != NULL &&
 	    (props = zfs_valid_proplist(hdl, ZFS_TYPE_SNAPSHOT,
-	    props, B_FALSE, NULL, errbuf)) == NULL) {
+	    props, B_FALSE, NULL, zpool_hdl, errbuf)) == NULL) {
+		zpool_close(zpool_hdl);
 		return (-1);
 	}
+	zpool_close(zpool_hdl);
 
 	ret = lzc_snapshot(snaps, props, &errors);
 
diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_pool.c b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_pool.c
index 5c78e813b690..f15a4570d4be 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_pool.c
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_pool.c
@@ -1164,8 +1164,8 @@ zpool_create(libzfs_handle_t *hdl, const char *pool, nvlist_t *nvroot,
 		    zfs_prop_to_name(ZFS_PROP_ZONED), &zonestr) == 0) &&
 		    strcmp(zonestr, "on") == 0);
 
-		if ((zc_fsprops = zfs_valid_proplist(hdl,
-		    ZFS_TYPE_FILESYSTEM, fsprops, zoned, NULL, msg)) == NULL) {
+		if ((zc_fsprops = zfs_valid_proplist(hdl, ZFS_TYPE_FILESYSTEM,
+		    fsprops, zoned, NULL, NULL, msg)) == NULL) {
 			goto create_failed;
 		}
 		if (!zc_props &&
@@ -1201,6 +1201,21 @@ zpool_create(libzfs_handle_t *hdl, const char *pool, nvlist_t *nvroot,
 			    "one or more vdevs refer to the same device"));
 			return (zfs_error(hdl, EZFS_BADDEV, msg));
 
+		case ERANGE:
+			/*
+			 * This happens if the record size is smaller or larger
+			 * than the allowed size range, or not a power of 2.
+			 *
+			 * NOTE: although zfs_valid_proplist is called earlier,
+			 * this case may have slipped through since the
+			 * pool does not exist yet and it is therefore
+			 * impossible to read properties e.g. max blocksize
+			 * from the pool.
+			 */
+			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+			    "record size invalid"));
+			return (zfs_error(hdl, EZFS_BADPROP, msg));
+
 		case EOVERFLOW:
 			/*
 			 * This occurs when one of the devices is below

From b26720e1df1ea61d1807e2509b26c7563dc423f6 Mon Sep 17 00:00:00 2001
From: melifaro <melifaro@FreeBSD.org>
Date: Sun, 18 Oct 2015 12:26:25 +0000
Subject: [PATCH 337/380] Fix deletion of ifaddr lle entries when deleting
 prefix from interface in   down state.

Regression appeared in r287789, where the "prefix has no corresponding
  installed route" case was forgotten. Additionally, lltable_delete_addr()
  was called with incorrect byte order (default is network for lltable code).
While here, improve comments on given cases and byte order.

PR:		203573
Submitted by:	phk
---
 sys/netinet/in.c | 27 +++++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/sys/netinet/in.c b/sys/netinet/in.c
index 1441936c3808..2607a1750ffd 100644
--- a/sys/netinet/in.c
+++ b/sys/netinet/in.c
@@ -734,14 +734,10 @@ in_scrubprefixlle(struct in_ifaddr *ia, int all, u_int flags)
 	struct sockaddr *saddr, *smask;
 	struct ifnet *ifp;
 
-	/*
-	 * remove all L2 entries on the given prefix
-	 */
 	saddr = (struct sockaddr *)&addr;
 	bzero(&addr, sizeof(addr));
 	addr.sin_len = sizeof(addr);
 	addr.sin_family = AF_INET;
-	addr.sin_addr.s_addr = ntohl(ia->ia_addr.sin_addr.s_addr);
 	smask = (struct sockaddr *)&mask;
 	bzero(&mask, sizeof(mask));
 	mask.sin_len = sizeof(mask);
@@ -749,10 +745,21 @@ in_scrubprefixlle(struct in_ifaddr *ia, int all, u_int flags)
 	mask.sin_addr.s_addr = ia->ia_subnetmask;
 	ifp = ia->ia_ifp;
 
-	if (all)
+	if (all) {
+
+		/*
+		 * Remove all L2 entries matching given prefix.
+		 * Convert address to host representation to avoid
+		 * doing this on every callback. ia_subnetmask is already
+		 * stored in host representation.
+		 */
+		addr.sin_addr.s_addr = ntohl(ia->ia_addr.sin_addr.s_addr);
 		lltable_prefix_free(AF_INET, saddr, smask, flags);
-	else
+	} else {
+		/* Remove interface address only */
+		addr.sin_addr.s_addr = ia->ia_addr.sin_addr.s_addr;
 		lltable_delete_addr(LLTABLE(ifp), LLE_IFADDR, saddr);
+	}
 }
 
 /*
@@ -808,6 +815,14 @@ in_scrubprefix(struct in_ifaddr *target, u_int flags)
 		fibnum = V_rt_add_addr_allfibs ? RT_ALL_FIBS :
 			target->ia_ifp->if_fib;
 		rt_addrmsg(RTM_DELETE, &target->ia_ifa, fibnum);
+	
+		/*
+		 * Removing address from !IFF_UP interface or
+		 * prefix which exists on other interface (along with route).
+		 * No entries should exist here except target addr.
+		 * Given that, delete this entry only.
+		 */
+		in_scrubprefixlle(target, 0, flags);
 		return (0);
 	}
 

From 2e36231acf2e90bdb35ec59d85cba8a0e7d2a047 Mon Sep 17 00:00:00 2001
From: andrew <andrew@FreeBSD.org>
Date: Sun, 18 Oct 2015 13:23:21 +0000
Subject: [PATCH 338/380] Correctly align the stack. The early csu assumed we
 passed the aux vector in through the stack pointer, however this may have
 been misaligned causing some userland applications to crash. A workaround was
 committed in r284707 where userland would check if the aux vector was passed
 using the old or new ABI and adjust the stack if needed. As 4 months have
 passed it is time to move to the new ABI, with the expectation the compat
 code in csu and the runtime linker to be removed in the future.

Sponsored by:	ABT Systems Ltd
---
 sys/arm64/arm64/machdep.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/sys/arm64/arm64/machdep.c b/sys/arm64/arm64/machdep.c
index 4bb3f674b9f4..e92a99ff3057 100644
--- a/sys/arm64/arm64/machdep.c
+++ b/sys/arm64/arm64/machdep.c
@@ -251,7 +251,13 @@ exec_setregs(struct thread *td, struct image_params *imgp, u_long stack)
 
 	memset(tf, 0, sizeof(struct trapframe));
 
-	tf->tf_sp = stack;
+	/*
+	 * We need to set x0 for init as it doesn't call
+	 * cpu_set_syscall_retval to copy the value. We also
+	 * need to set td_retval for the cases where we do.
+	 */
+	tf->tf_x[0] = td->td_retval[0] = stack;
+	tf->tf_sp = STACKALIGN(stack);
 	tf->tf_lr = imgp->entry_addr;
 	tf->tf_elr = imgp->entry_addr;
 }

From 5723997412f1009e9962e6b1790496e537a4065f Mon Sep 17 00:00:00 2001
From: ian <ian@FreeBSD.org>
Date: Sun, 18 Oct 2015 16:54:34 +0000
Subject: [PATCH 339/380] Rename arm_init_secondary_ic() ->
 arm_pic_init_secondary().  The latter is the name the function will have when
 the new ARM_INTRNG code is integrated, and doing this rename first will make
 it easier to toggle the new interrupt handling code on/off with a config
 option for debugging.

---
 sys/arm/allwinner/a20/a20_mp.c               | 2 +-
 sys/arm/altera/socfpga/socfpga_mp.c          | 2 +-
 sys/arm/amlogic/aml8726/aml8726_mp.c         | 2 +-
 sys/arm/annapurna/alpine/alpine_machdep_mp.c | 2 +-
 sys/arm/arm/gic.c                            | 2 +-
 sys/arm/freescale/imx/imx6_mp.c              | 2 +-
 sys/arm/include/intr.h                       | 2 +-
 sys/arm/qemu/virt_mp.c                       | 2 +-
 sys/arm/rockchip/rk30xx_mp.c                 | 2 +-
 sys/arm/samsung/exynos/exynos5_mp.c          | 2 +-
 sys/arm/ti/omap4/omap4_mp.c                  | 2 +-
 sys/arm/xilinx/zy7_mp.c                      | 2 +-
 12 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/sys/arm/allwinner/a20/a20_mp.c b/sys/arm/allwinner/a20/a20_mp.c
index 9429aaf2b6d5..4beac42481f1 100644
--- a/sys/arm/allwinner/a20/a20_mp.c
+++ b/sys/arm/allwinner/a20/a20_mp.c
@@ -61,7 +61,7 @@ void
 platform_mp_init_secondary(void)
 {
 
-	arm_init_secondary_ic();
+	arm_pic_init_secondary();
 }
 
 void
diff --git a/sys/arm/altera/socfpga/socfpga_mp.c b/sys/arm/altera/socfpga/socfpga_mp.c
index 24784e988814..461d3f4958bd 100644
--- a/sys/arm/altera/socfpga/socfpga_mp.c
+++ b/sys/arm/altera/socfpga/socfpga_mp.c
@@ -87,7 +87,7 @@ void
 platform_mp_init_secondary(void)
 {
 
-	arm_init_secondary_ic();
+	arm_pic_init_secondary();
 }
 
 void
diff --git a/sys/arm/amlogic/aml8726/aml8726_mp.c b/sys/arm/amlogic/aml8726/aml8726_mp.c
index 4f0bce067667..1d7ef032d5d8 100644
--- a/sys/arm/amlogic/aml8726/aml8726_mp.c
+++ b/sys/arm/amlogic/aml8726/aml8726_mp.c
@@ -351,7 +351,7 @@ platform_mp_init_secondary(void)
 	 * each AP.
 	 */
 
-	 arm_init_secondary_ic();
+	 arm_pic_init_secondary();
 }
 
 
diff --git a/sys/arm/annapurna/alpine/alpine_machdep_mp.c b/sys/arm/annapurna/alpine/alpine_machdep_mp.c
index 2c909175cb05..a4c45b8d07fb 100644
--- a/sys/arm/annapurna/alpine/alpine_machdep_mp.c
+++ b/sys/arm/annapurna/alpine/alpine_machdep_mp.c
@@ -122,7 +122,7 @@ void
 platform_mp_init_secondary(void)
 {
 
-	arm_init_secondary_ic();
+	arm_pic_init_secondary();
 }
 
 void
diff --git a/sys/arm/arm/gic.c b/sys/arm/arm/gic.c
index 6fcb7e082215..a9f171e8b837 100644
--- a/sys/arm/arm/gic.c
+++ b/sys/arm/arm/gic.c
@@ -492,7 +492,7 @@ arm_get_next_irq(int last_irq)
 }
 
 void
-arm_init_secondary_ic(void)
+arm_pic_init_secondary(void)
 {
 
 	arm_gic_init_secondary(arm_gic_sc->gic_dev);
diff --git a/sys/arm/freescale/imx/imx6_mp.c b/sys/arm/freescale/imx/imx6_mp.c
index b02588cfa22f..8957c547ad5e 100644
--- a/sys/arm/freescale/imx/imx6_mp.c
+++ b/sys/arm/freescale/imx/imx6_mp.c
@@ -69,7 +69,7 @@ void
 platform_mp_init_secondary(void)
 {
 
-	arm_init_secondary_ic();
+	arm_pic_init_secondary();
 }
 
 void
diff --git a/sys/arm/include/intr.h b/sys/arm/include/intr.h
index 7ab69d9405a0..6a46f00766a0 100644
--- a/sys/arm/include/intr.h
+++ b/sys/arm/include/intr.h
@@ -85,7 +85,7 @@ extern int (*arm_config_irq)(int irq, enum intr_trigger trig,
 
 void arm_irq_memory_barrier(uintptr_t);
 
-void arm_init_secondary_ic(void);
+void arm_pic_init_secondary(void);
 int  gic_decode_fdt(uint32_t iparentnode, uint32_t *intrcells, int *interrupt,
     int *trig, int *pol);
 
diff --git a/sys/arm/qemu/virt_mp.c b/sys/arm/qemu/virt_mp.c
index 4942745e109f..2fe1592af0fd 100644
--- a/sys/arm/qemu/virt_mp.c
+++ b/sys/arm/qemu/virt_mp.c
@@ -108,7 +108,7 @@ void
 platform_mp_init_secondary(void)
 {
 
-	arm_init_secondary_ic();
+	arm_pic_init_secondary();
 }
 
 void
diff --git a/sys/arm/rockchip/rk30xx_mp.c b/sys/arm/rockchip/rk30xx_mp.c
index 74c8b554c85c..f07605cfe18a 100644
--- a/sys/arm/rockchip/rk30xx_mp.c
+++ b/sys/arm/rockchip/rk30xx_mp.c
@@ -82,7 +82,7 @@ void
 platform_mp_init_secondary(void)
 {
 
-	arm_init_secondary_ic();
+	arm_pic_init_secondary();
 }
 
 void
diff --git a/sys/arm/samsung/exynos/exynos5_mp.c b/sys/arm/samsung/exynos/exynos5_mp.c
index 1b3afbbf0d6d..0f4a792b79d9 100644
--- a/sys/arm/samsung/exynos/exynos5_mp.c
+++ b/sys/arm/samsung/exynos/exynos5_mp.c
@@ -73,7 +73,7 @@ void
 platform_mp_init_secondary(void)
 {
 
-	arm_init_secondary_ic();
+	arm_pic_init_secondary();
 }
 
 void
diff --git a/sys/arm/ti/omap4/omap4_mp.c b/sys/arm/ti/omap4/omap4_mp.c
index 6423e568f24a..f53748e90a9a 100644
--- a/sys/arm/ti/omap4/omap4_mp.c
+++ b/sys/arm/ti/omap4/omap4_mp.c
@@ -44,7 +44,7 @@ __FBSDID("$FreeBSD$");
 void
 platform_mp_init_secondary(void)
 {
-	arm_init_secondary_ic();
+	arm_pic_init_secondary();
 }
 
 void
diff --git a/sys/arm/xilinx/zy7_mp.c b/sys/arm/xilinx/zy7_mp.c
index 305290ed2034..24e8595a5af3 100644
--- a/sys/arm/xilinx/zy7_mp.c
+++ b/sys/arm/xilinx/zy7_mp.c
@@ -49,7 +49,7 @@ void
 platform_mp_init_secondary(void)
 {
 
-	arm_init_secondary_ic();
+	arm_pic_init_secondary();
 }
 
 void

From 904ee0481b4afa4229326523ea6b0aad262cde1e Mon Sep 17 00:00:00 2001
From: dim <dim@FreeBSD.org>
Date: Sun, 18 Oct 2015 17:13:41 +0000
Subject: [PATCH 340/380] =?UTF-8?q?=EF=BB=BFPull=20in=20r248379=20from=20u?=
 =?UTF-8?q?pstream=20clang=20trunk=20(by=20J=C3=B6rg=20Sonnenberger):?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

  Refactor library decision for -fopenmp support from Darwin into a
  function for sharing with other platforms.

Pull in r248424 from upstream clang trunk (by Jörg Sonnenberger):

  Push OpenMP linker flags after linker input on Darwin. Don't add any
  libraries if -nostdlib is specified. Test.

Pull in r248426 from upstream clang trunk (by Jörg Sonnenberger):

  Support linking against OpenMP runtime on NetBSD.

Pull in r250657 from upstream clang trunk (by Dimitry Andric):

  Support linking against OpenMP runtime on FreeBSD.
---
 contrib/llvm/tools/clang/lib/Driver/Tools.cpp | 46 +++++++++++--------
 1 file changed, 28 insertions(+), 18 deletions(-)

diff --git a/contrib/llvm/tools/clang/lib/Driver/Tools.cpp b/contrib/llvm/tools/clang/lib/Driver/Tools.cpp
index 3c26fc5c86d3..7352537cf84f 100644
--- a/contrib/llvm/tools/clang/lib/Driver/Tools.cpp
+++ b/contrib/llvm/tools/clang/lib/Driver/Tools.cpp
@@ -2460,6 +2460,28 @@ static OpenMPRuntimeKind getOpenMPRuntime(const ToolChain &TC,
   return RT;
 }
 
+static void addOpenMPRuntime(ArgStringList &CmdArgs, const ToolChain &TC,
+                              const ArgList &Args) {
+  if (!Args.hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ,
+                    options::OPT_fno_openmp, false))
+    return;
+
+  switch (getOpenMPRuntime(TC, Args)) {
+  case OMPRT_OMP:
+    CmdArgs.push_back("-lomp");
+    break;
+  case OMPRT_GOMP:
+    CmdArgs.push_back("-lgomp");
+    break;
+  case OMPRT_IOMP5:
+    CmdArgs.push_back("-liomp5");
+    break;
+  case OMPRT_Unknown:
+    // Already diagnosed.
+    break;
+  }
+}
+
 static void addSanitizerRuntime(const ToolChain &TC, const ArgList &Args,
                                 ArgStringList &CmdArgs, StringRef Sanitizer,
                                 bool IsShared) {
@@ -6527,24 +6549,6 @@ void darwin::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   Args.AddAllArgs(CmdArgs, options::OPT_L);
 
-  if (Args.hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ,
-                   options::OPT_fno_openmp, false)) {
-    switch (getOpenMPRuntime(getToolChain(), Args)) {
-    case OMPRT_OMP:
-      CmdArgs.push_back("-lomp");
-      break;
-    case OMPRT_GOMP:
-      CmdArgs.push_back("-lgomp");
-      break;
-    case OMPRT_IOMP5:
-      CmdArgs.push_back("-liomp5");
-      break;
-    case OMPRT_Unknown:
-      // Already diagnosed.
-      break;
-    }
-  }
-
   AddLinkerInputs(getToolChain(), Inputs, Args, CmdArgs);
   // Build the input file for -filelist (list of linker input files) in case we
   // need it later
@@ -6563,6 +6567,10 @@ void darwin::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     InputFileList.push_back(II.getFilename());
   }
 
+  if (!Args.hasArg(options::OPT_nostdlib) &&
+      !Args.hasArg(options::OPT_nodefaultlibs))
+    addOpenMPRuntime(CmdArgs, getToolChain(), Args);
+
   if (isObjCRuntimeLinked(Args) && !Args.hasArg(options::OPT_nostdlib) &&
       !Args.hasArg(options::OPT_nodefaultlibs)) {
     // We use arclite library for both ARC and subscripting support.
@@ -7358,6 +7366,7 @@ void freebsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (!Args.hasArg(options::OPT_nostdlib) &&
       !Args.hasArg(options::OPT_nodefaultlibs)) {
+    addOpenMPRuntime(CmdArgs, ToolChain, Args);
     if (D.CCCIsCXX()) {
       ToolChain.AddCXXStdlibLibArgs(Args, CmdArgs);
       if (Args.hasArg(options::OPT_pg))
@@ -7673,6 +7682,7 @@ void netbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (!Args.hasArg(options::OPT_nostdlib) &&
       !Args.hasArg(options::OPT_nodefaultlibs)) {
+    addOpenMPRuntime(CmdArgs, getToolChain(), Args);
     if (D.CCCIsCXX()) {
       getToolChain().AddCXXStdlibLibArgs(Args, CmdArgs);
       CmdArgs.push_back("-lm");

From eed4f299ff7571714729dd1dfd1a38a19d24ae50 Mon Sep 17 00:00:00 2001
From: dim <dim@FreeBSD.org>
Date: Sun, 18 Oct 2015 17:14:45 +0000
Subject: [PATCH 341/380] Add clang patch corresponding to r289523.

---
 .../patch-09-clang-r250657-openmp.diff        | 182 ++++++++++++++++++
 1 file changed, 182 insertions(+)
 create mode 100644 contrib/llvm/patches/patch-09-clang-r250657-openmp.diff

diff --git a/contrib/llvm/patches/patch-09-clang-r250657-openmp.diff b/contrib/llvm/patches/patch-09-clang-r250657-openmp.diff
new file mode 100644
index 000000000000..e784673d0265
--- /dev/null
+++ b/contrib/llvm/patches/patch-09-clang-r250657-openmp.diff
@@ -0,0 +1,182 @@
+Pull in r248379 from upstream clang trunk (by Jörg Sonnenberger):
+
+  Refactor library decision for -fopenmp support from Darwin into a
+  function for sharing with other platforms.
+
+Pull in r248424 from upstream clang trunk (by Jörg Sonnenberger):
+
+  Push OpenMP linker flags after linker input on Darwin. Don't add any
+  libraries if -nostdlib is specified. Test.
+
+Pull in r248426 from upstream clang trunk (by Jörg Sonnenberger):
+
+  Support linking against OpenMP runtime on NetBSD.
+
+Pull in r250657 from upstream clang trunk (by Dimitry Andric):
+
+  Support linking against OpenMP runtime on FreeBSD.
+
+Introduced here: http://svnweb.freebsd.org/changeset/base/289523
+
+Index: tools/clang/lib/Driver/Tools.cpp
+===================================================================
+--- tools/clang/lib/Driver/Tools.cpp
++++ tools/clang/lib/Driver/Tools.cpp
+@@ -2460,6 +2460,28 @@ static OpenMPRuntimeKind getOpenMPRuntime(const To
+   return RT;
+ }
+ 
++static void addOpenMPRuntime(ArgStringList &CmdArgs, const ToolChain &TC,
++                              const ArgList &Args) {
++  if (!Args.hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ,
++                    options::OPT_fno_openmp, false))
++    return;
++
++  switch (getOpenMPRuntime(TC, Args)) {
++  case OMPRT_OMP:
++    CmdArgs.push_back("-lomp");
++    break;
++  case OMPRT_GOMP:
++    CmdArgs.push_back("-lgomp");
++    break;
++  case OMPRT_IOMP5:
++    CmdArgs.push_back("-liomp5");
++    break;
++  case OMPRT_Unknown:
++    // Already diagnosed.
++    break;
++  }
++}
++
+ static void addSanitizerRuntime(const ToolChain &TC, const ArgList &Args,
+                                 ArgStringList &CmdArgs, StringRef Sanitizer,
+                                 bool IsShared) {
+@@ -6527,24 +6549,6 @@ void darwin::Linker::ConstructJob(Compilation &C,
+ 
+   Args.AddAllArgs(CmdArgs, options::OPT_L);
+ 
+-  if (Args.hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ,
+-                   options::OPT_fno_openmp, false)) {
+-    switch (getOpenMPRuntime(getToolChain(), Args)) {
+-    case OMPRT_OMP:
+-      CmdArgs.push_back("-lomp");
+-      break;
+-    case OMPRT_GOMP:
+-      CmdArgs.push_back("-lgomp");
+-      break;
+-    case OMPRT_IOMP5:
+-      CmdArgs.push_back("-liomp5");
+-      break;
+-    case OMPRT_Unknown:
+-      // Already diagnosed.
+-      break;
+-    }
+-  }
+-
+   AddLinkerInputs(getToolChain(), Inputs, Args, CmdArgs);
+   // Build the input file for -filelist (list of linker input files) in case we
+   // need it later
+@@ -6563,6 +6567,10 @@ void darwin::Linker::ConstructJob(Compilation &C,
+     InputFileList.push_back(II.getFilename());
+   }
+ 
++  if (!Args.hasArg(options::OPT_nostdlib) &&
++      !Args.hasArg(options::OPT_nodefaultlibs))
++    addOpenMPRuntime(CmdArgs, getToolChain(), Args);
++
+   if (isObjCRuntimeLinked(Args) && !Args.hasArg(options::OPT_nostdlib) &&
+       !Args.hasArg(options::OPT_nodefaultlibs)) {
+     // We use arclite library for both ARC and subscripting support.
+@@ -7358,6 +7366,7 @@ void freebsd::Linker::ConstructJob(Compilation &C,
+ 
+   if (!Args.hasArg(options::OPT_nostdlib) &&
+       !Args.hasArg(options::OPT_nodefaultlibs)) {
++    addOpenMPRuntime(CmdArgs, ToolChain, Args);
+     if (D.CCCIsCXX()) {
+       ToolChain.AddCXXStdlibLibArgs(Args, CmdArgs);
+       if (Args.hasArg(options::OPT_pg))
+@@ -7673,6 +7682,7 @@ void netbsd::Linker::ConstructJob(Compilation &C,
+ 
+   if (!Args.hasArg(options::OPT_nostdlib) &&
+       !Args.hasArg(options::OPT_nodefaultlibs)) {
++    addOpenMPRuntime(CmdArgs, getToolChain(), Args);
+     if (D.CCCIsCXX()) {
+       getToolChain().AddCXXStdlibLibArgs(Args, CmdArgs);
+       CmdArgs.push_back("-lm");
+Index: tools/clang/test/Driver/fopenmp.c
+===================================================================
+--- tools/clang/test/Driver/fopenmp.c
++++ tools/clang/test/Driver/fopenmp.c
+@@ -1,6 +1,15 @@
+ // RUN: %clang -target x86_64-linux-gnu -fopenmp=libomp -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CC1-OPENMP
+ // RUN: %clang -target x86_64-linux-gnu -fopenmp=libgomp -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CC1-NO-OPENMP
+ // RUN: %clang -target x86_64-linux-gnu -fopenmp=libiomp5 -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CC1-OPENMP
++// RUN: %clang -target x86_64-apple-darwin -fopenmp=libomp -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CC1-OPENMP
++// RUN: %clang -target x86_64-apple-darwin -fopenmp=libgomp -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CC1-NO-OPENMP
++// RUN: %clang -target x86_64-apple-darwin -fopenmp=libiomp5 -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CC1-OPENMP
++// RUN: %clang -target x86_64-freebsd -fopenmp=libomp -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CC1-OPENMP
++// RUN: %clang -target x86_64-freebsd -fopenmp=libgomp -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CC1-NO-OPENMP
++// RUN: %clang -target x86_64-freebsd -fopenmp=libiomp5 -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CC1-OPENMP
++// RUN: %clang -target x86_64-netbsd -fopenmp=libomp -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CC1-OPENMP
++// RUN: %clang -target x86_64-netbsd -fopenmp=libgomp -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CC1-NO-OPENMP
++// RUN: %clang -target x86_64-netbsd -fopenmp=libiomp5 -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CC1-OPENMP
+ //
+ // CHECK-CC1-OPENMP: "-cc1"
+ // CHECK-CC1-OPENMP: "-fopenmp"
+@@ -12,6 +21,30 @@
+ // RUN: %clang -target x86_64-linux-gnu -fopenmp=libgomp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-GOMP
+ // RUN: %clang -target x86_64-linux-gnu -fopenmp=libiomp5 %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-IOMP5
+ //
++// RUN: %clang -nostdlib -target x86_64-linux-gnu -fopenmp=libomp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-NO-OMP
++// RUN: %clang -nostdlib -target x86_64-linux-gnu -fopenmp=libgomp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-NO-GOMP
++// RUN: %clang -nostdlib -target x86_64-linux-gnu -fopenmp=libiomp5 %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-NO-IOMP5
++//
++// RUN: %clang -target x86_64-darwin -fopenmp=libomp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-OMP
++// RUN: %clang -target x86_64-darwin -fopenmp=libgomp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-GOMP
++// RUN: %clang -target x86_64-darwin -fopenmp=libiomp5 %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-IOMP5
++//
++// RUN: %clang -nostdlib -target x86_64-darwin -fopenmp=libomp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-NO-OMP
++// RUN: %clang -nostdlib -target x86_64-darwin -fopenmp=libgomp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-NO-GOMP
++// RUN: %clang -nostdlib -target x86_64-darwin -fopenmp=libiomp5 %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-NO-IOMP5
++//
++// RUN: %clang -target x86_64-netbsd -fopenmp=libomp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-OMP
++// RUN: %clang -target x86_64-netbsd -fopenmp=libgomp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-GOMP
++// RUN: %clang -target x86_64-netbsd -fopenmp=libiomp5 %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-IOMP5
++//
++// RUN: %clang -nostdlib -target x86_64-freebsd -fopenmp=libomp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-NO-OMP
++// RUN: %clang -nostdlib -target x86_64-freebsd -fopenmp=libgomp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-NO-GOMP
++// RUN: %clang -nostdlib -target x86_64-freebsd -fopenmp=libiomp5 %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-NO-IOMP5
++//
++// RUN: %clang -nostdlib -target x86_64-netbsd -fopenmp=libomp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-NO-OMP
++// RUN: %clang -nostdlib -target x86_64-netbsd -fopenmp=libgomp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-NO-GOMP
++// RUN: %clang -nostdlib -target x86_64-netbsd -fopenmp=libiomp5 %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-NO-IOMP5
++//
+ // CHECK-LD-OMP: "{{.*}}ld{{(.exe)?}}"
+ // CHECK-LD-OMP: "-lomp"
+ //
+@@ -21,6 +54,15 @@
+ // CHECK-LD-IOMP5: "{{.*}}ld{{(.exe)?}}"
+ // CHECK-LD-IOMP5: "-liomp5"
+ //
++// CHECK-NO-OMP: "{{.*}}ld{{(.exe)?}}"
++// CHECK-NO-OMP-NOT: "-lomp"
++//
++// CHECK-NO-GOMP: "{{.*}}ld{{(.exe)?}}"
++// CHECK-NO-GOMP-NOT: "-lgomp"
++//
++// CHECK-NO-IOMP5: "{{.*}}ld{{(.exe)?}}"
++// CHECK-NO-IOMP5-NOT: "-liomp5"
++//
+ // We'd like to check that the default is sane, but until we have the ability
+ // to *always* semantically analyze OpenMP without always generating runtime
+ // calls (in the event of an unsupported runtime), we don't have a good way to
+@@ -28,6 +70,9 @@
+ // OpenMP runtime.
+ //
+ // RUN: %clang -target x86_64-linux-gnu -fopenmp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-ANY
++// RUN: %clang -target x86_64-darwin -fopenmp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-ANY
++// RUN: %clang -target x86_64-freebsd -fopenmp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-ANY
++// RUN: %clang -target x86_64-netbsd -fopenmp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-ANY
+ //
+ // CHECK-LD-ANY: "{{.*}}ld{{(.exe)?}}"
+ // CHECK-LD-ANY: "-l{{(omp|gomp|iomp5)}}"

From 5d3b6e9a38f0df1f6cb29b74d60e7a1d4f5dcf20 Mon Sep 17 00:00:00 2001
From: dim <dim@FreeBSD.org>
Date: Sun, 18 Oct 2015 17:18:19 +0000
Subject: [PATCH 342/380] Switch the default OpenMP runtime for clang to libomp
 (from the LLVM project), as libgomp is not supported anyway.  You can use the
 devel/llvm-devel port to install a recent copy of the OpenMP runtime.

---
 lib/clang/include/clang/Config/config.h | 2 +-
 lib/clang/include/llvm/Config/config.h  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/clang/include/clang/Config/config.h b/lib/clang/include/clang/Config/config.h
index 7ab95a0a1901..fecc361e1199 100644
--- a/lib/clang/include/clang/Config/config.h
+++ b/lib/clang/include/clang/Config/config.h
@@ -11,7 +11,7 @@
 #define BUG_REPORT_URL "https://bugs.freebsd.org/submit/"
 
 /* Default OpenMP runtime used by -fopenmp. */
-#define CLANG_DEFAULT_OPENMP_RUNTIME "libgomp"
+#define CLANG_DEFAULT_OPENMP_RUNTIME "libomp"
 
 /* Multilib suffix for libdir. */
 #define CLANG_LIBDIR_SUFFIX ""
diff --git a/lib/clang/include/llvm/Config/config.h b/lib/clang/include/llvm/Config/config.h
index bae066c96904..ef7295cb380a 100644
--- a/lib/clang/include/llvm/Config/config.h
+++ b/lib/clang/include/llvm/Config/config.h
@@ -15,7 +15,7 @@
 #define BUG_REPORT_URL "https://bugs.freebsd.org/submit/"
 
 /* Default OpenMP runtime used by -fopenmp. */
-#define CLANG_DEFAULT_OPENMP_RUNTIME "libgomp"
+#define CLANG_DEFAULT_OPENMP_RUNTIME "libomp"
 
 /* Define if we have libxml2 */
 /* #undef CLANG_HAVE_LIBXML */

From 1c4271d11bbe87be208c21d7f42a533582b2a2e4 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Sun, 18 Oct 2015 18:08:33 +0000
Subject: [PATCH 343/380] MFV r289526: 5561 support root pools on EFI/GPT
 partitioned disks 5125 update zpool/libzfs to manage bootable whole disk
 pools (EFI/GPT labeled disks)

Reviewed by: Jean McCormack <jean.mccormack@nexenta.com>
Reviewed by: Josef 'Jeff' Sipek <josef.sipek@nexenta.com>
Approved by: Dan McDonald <danmcd@omniti.com>
Author: Hans Rosenfeld <hans.rosenfeld@nexenta.com>

illumos/illumos-gate@1a902ef8628b0dffd6df5442354ab59bb8530962

This is NOP changes for FreeBSD.
---
 .../lib/libzfs/common/libzfs_pool.c           | 83 +------------------
 .../opensolaris/uts/common/fs/zfs/vdev.c      |  2 -
 2 files changed, 2 insertions(+), 83 deletions(-)

diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_pool.c b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_pool.c
index f15a4570d4be..18541604f421 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_pool.c
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_pool.c
@@ -402,29 +402,6 @@ bootfs_name_valid(const char *pool, char *bootfs)
 	return (B_FALSE);
 }
 
-/*
- * Inspect the configuration to determine if any of the devices contain
- * an EFI label.
- */
-static boolean_t
-pool_uses_efi(nvlist_t *config)
-{
-#ifdef illumos
-	nvlist_t **child;
-	uint_t c, children;
-
-	if (nvlist_lookup_nvlist_array(config, ZPOOL_CONFIG_CHILDREN,
-	    &child, &children) != 0)
-		return (read_efi_label(config, NULL) >= 0);
-
-	for (c = 0; c < children; c++) {
-		if (pool_uses_efi(child[c]))
-			return (B_TRUE);
-	}
-#endif	/* illumos */
-	return (B_FALSE);
-}
-
 boolean_t
 zpool_is_bootable(zpool_handle_t *zhp)
 {
@@ -453,7 +430,6 @@ zpool_valid_proplist(libzfs_handle_t *hdl, const char *poolname,
 	char *slash, *check;
 	struct stat64 statbuf;
 	zpool_handle_t *zhp;
-	nvlist_t *nvroot;
 
 	if (nvlist_alloc(&retprops, NV_UNIQUE_NAME, 0) != 0) {
 		(void) no_memory(hdl);
@@ -572,23 +548,6 @@ zpool_valid_proplist(libzfs_handle_t *hdl, const char *poolname,
 				(void) zfs_error(hdl, EZFS_OPENFAILED, errbuf);
 				goto error;
 			}
-			verify(nvlist_lookup_nvlist(zpool_get_config(zhp, NULL),
-			    ZPOOL_CONFIG_VDEV_TREE, &nvroot) == 0);
-
-#ifdef illumos
-			/*
-			 * bootfs property cannot be set on a disk which has
-			 * been EFI labeled.
-			 */
-			if (pool_uses_efi(nvroot)) {
-				zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
-				    "property '%s' not supported on "
-				    "EFI labeled devices"), propname);
-				(void) zfs_error(hdl, EZFS_POOL_NOTSUP, errbuf);
-				zpool_close(zhp);
-				goto error;
-			}
-#endif	/* illumos */
 			zpool_close(zhp);
 			break;
 
@@ -1326,25 +1285,6 @@ zpool_add(zpool_handle_t *zhp, nvlist_t *nvroot)
 		return (zfs_error(hdl, EZFS_BADVERSION, msg));
 	}
 
-	if (zpool_is_bootable(zhp) && nvlist_lookup_nvlist_array(nvroot,
-	    ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0) {
-		uint64_t s;
-
-		for (s = 0; s < nspares; s++) {
-			char *path;
-
-			if (nvlist_lookup_string(spares[s], ZPOOL_CONFIG_PATH,
-			    &path) == 0 && pool_uses_efi(spares[s])) {
-				zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
-				    "device '%s' contains an EFI label and "
-				    "cannot be used on root pools."),
-				    zpool_vdev_name(hdl, NULL, spares[s],
-				    B_FALSE));
-				return (zfs_error(hdl, EZFS_POOL_NOTSUP, msg));
-			}
-		}
-	}
-
 	if (zpool_get_prop_int(zhp, ZPOOL_PROP_VERSION, NULL) <
 	    SPA_VERSION_L2CACHE &&
 	    nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
@@ -2365,11 +2305,9 @@ zpool_get_config_physpath(nvlist_t *config, char *physpath, size_t phypath_size)
 		return (EZFS_INVALCONFIG);
 
 	/*
-	 * root pool can not have EFI labeled disks and can only have
-	 * a single top-level vdev.
+	 * root pool can only have a single top-level vdev.
 	 */
-	if (strcmp(type, VDEV_TYPE_ROOT) != 0 || count != 1 ||
-	    pool_uses_efi(vdev_root))
+	if (strcmp(type, VDEV_TYPE_ROOT) != 0 || count != 1)
 		return (EZFS_POOL_INVALARG);
 
 	(void) vdev_get_physpaths(child[0], physpath, phypath_size, &rsz,
@@ -2673,16 +2611,6 @@ zpool_vdev_attach(zpool_handle_t *zhp,
 		(void) snprintf(msg, sizeof (msg), dgettext(TEXT_DOMAIN,
 		    "cannot attach %s to %s"), new_disk, old_disk);
 
-	/*
-	 * If this is a root pool, make sure that we're not attaching an
-	 * EFI labeled device.
-	 */
-	if (rootpool && pool_uses_efi(nvroot)) {
-		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
-		    "EFI labeled devices are not supported on root pools."));
-		return (zfs_error(hdl, EZFS_POOL_NOTSUP, msg));
-	}
-
 	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
 	if ((tgt = zpool_find_vdev(zhp, old_disk, &avail_spare, &l2cache,
 	    &islog)) == 0)
@@ -3996,13 +3924,6 @@ zpool_label_disk(libzfs_handle_t *hdl, zpool_handle_t *zhp, const char *name)
 	if (zhp) {
 		nvlist_t *nvroot;
 
-		if (zpool_is_bootable(zhp)) {
-			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
-			    "EFI labeled devices are not supported on root "
-			    "pools."));
-			return (zfs_error(hdl, EZFS_POOL_NOTSUP, errbuf));
-		}
-
 		verify(nvlist_lookup_nvlist(zhp->zpool_config,
 		    ZPOOL_CONFIG_VDEV_TREE, &nvroot) == 0);
 
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c
index a8f1c5f5ca7f..429d31e1024c 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c
@@ -3412,8 +3412,6 @@ vdev_is_bootable(vdev_t *vd)
 		    strcmp(vdev_type, VDEV_TYPE_MISSING) == 0) {
 			return (B_FALSE);
 		}
-	} else if (vd->vdev_wholedisk == 1) {
-		return (B_FALSE);
 	}
 
 	for (int c = 0; c < vd->vdev_children; c++) {

From 93afe7bdc606349a681a0e5518afa7640d4ee4d6 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Sun, 18 Oct 2015 18:25:00 +0000
Subject: [PATCH 344/380] Reduce diff from upstream.

Should be no functional change.
---
 .../lib/libzfs/common/libzfs_pool.c           | 26 +++++++++----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_pool.c b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_pool.c
index 18541604f421..bbab81b6cd78 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_pool.c
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_pool.c
@@ -3729,17 +3729,18 @@ zpool_history_unpack(char *buf, uint64_t bytes_read, uint64_t *leftover,
 int
 zpool_get_history(zpool_handle_t *zhp, nvlist_t **nvhisp)
 {
-	char *buf = NULL;
-	uint64_t bufsize = HIS_BUF_LEN_DEF;
+	char *buf;
+	uint64_t buflen = HIS_BUF_LEN_DEF;
 	uint64_t off = 0;
 	nvlist_t **records = NULL;
 	uint_t numrecords = 0;
 	int err, i;
 
-	if ((buf = malloc(bufsize)) == NULL)
+	buf = malloc(buflen);
+	if (buf == NULL)
 		return (ENOMEM);
 	do {
-		uint64_t bytes_read = bufsize;
+		uint64_t bytes_read = buflen;
 		uint64_t leftover;
 
 		if ((err = get_history(zhp, buf, &off, &bytes_read)) != 0)
@@ -3753,18 +3754,16 @@ zpool_get_history(zpool_handle_t *zhp, nvlist_t **nvhisp)
 		    &leftover, &records, &numrecords)) != 0)
 			break;
 		off -= leftover;
-
-		/*
-		 * If the history block is too big, double the buffer
-		 * size and try again.
-		 */
 		if (leftover == bytes_read) {
+			/*
+			 * no progress made, because buffer is not big enough
+			 * to hold this record; resize and retry.
+			 */
+			buflen *= 2;
 			free(buf);
 			buf = NULL;
-
-			bufsize <<= 1;
-			if ((bufsize >= HIS_BUF_LEN_MAX) ||
-			    ((buf = malloc(bufsize)) == NULL)) {
+			if ((buflen >= HIS_BUF_LEN_MAX) ||
+			    ((buf = malloc(buflen)) == NULL)) {
 				err = ENOMEM;
 				break;
 			}
@@ -3772,6 +3771,7 @@ zpool_get_history(zpool_handle_t *zhp, nvlist_t **nvhisp)
 
 		/* CONSTCOND */
 	} while (1);
+
 	free(buf);
 
 	if (!err) {

From 8a5f64069af62a6675dc9ba61325d2a112ad869c Mon Sep 17 00:00:00 2001
From: ian <ian@FreeBSD.org>
Date: Sun, 18 Oct 2015 18:26:19 +0000
Subject: [PATCH 345/380] Import ARM_INTRNG, the "next generation" interrupt
 architecture for arm and armv6 architecures.  The primary enhancement over
 the old design is support for hierarchical interrupt controllers (such as a
 gpio driver which can receive interrupts from a root PIC and act as a PIC
 itself for clients interested in handling a change of gpio pin state as an
 interrupt).  The new code also provides an infrastructure for mapping
 interrupts described in metadata in the form of a "controller reference plus
 interrupt number" tuple into the simple "0-n" flat numeric space understood
 by rman and the bus resource mechanisms.

Use of the new code is enabled by setting the ARM_INTRNG option, and by
making a few simple changes to the platform's support code.  In addition
each existing PIC driver needs changes to be ready for INTRNG; this commit
contains the changes for the arm/gic driver, which most armv6 SoCs use, but
it does not enable the new code yet on any platform.

This project has been many years in the making, starting as a GSoC project
by Jakub Klama (jceel@) in 2012.  That didn't get committed right away and
the source base evolved out from under it to some degree.  In 2014 I rebased
the diffs to then -current and did some enhancements in the area of mapping
interrupt numbers and storing associated fdt data, then the project went
cold again for a while.  Eventually Svata Kraus took that work in progress
and did another big round of work on it, removing most of the remaining
rough edges.  Finally I took that and made one more pass through it, mostly
disabling the "INTR_SOLO" feature for now, pending further design
discussions on how to most efficiently dispatch a pending interrupt through
more than one layer of PIC.  The current code with the INTR_SOLO feature
disabled uses approximate 100 extra cpu cycles for each cascaded PIC the
interrupt has to be passed to, so what's left to do is about efficiency, not
correct operation.

Differential Revision:	https://reviews.freebsd.org/D2047
---
 sys/arm/arm/gic.c        |  630 +++++++++++++++-
 sys/arm/arm/intrng.c     | 1458 ++++++++++++++++++++++++++++++++++++++
 sys/arm/arm/mp_machdep.c |  125 +++-
 sys/arm/arm/nexus.c      |   47 +-
 sys/arm/arm/pic_if.m     |  124 ++++
 sys/arm/include/fdt.h    |    4 +
 sys/arm/include/intr.h   |  103 ++-
 sys/arm/include/smp.h    |   16 +
 sys/conf/files.arm       |    4 +-
 sys/conf/options.arm     |    1 +
 10 files changed, 2480 insertions(+), 32 deletions(-)
 create mode 100644 sys/arm/arm/intrng.c
 create mode 100644 sys/arm/arm/pic_if.m

diff --git a/sys/arm/arm/gic.c b/sys/arm/arm/gic.c
index a9f171e8b837..9ceac4865ea3 100644
--- a/sys/arm/arm/gic.c
+++ b/sys/arm/arm/gic.c
@@ -34,18 +34,25 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_platform.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bus.h>
 #include <sys/kernel.h>
 #include <sys/ktr.h>
 #include <sys/module.h>
+#include <sys/malloc.h>
 #include <sys/rman.h>
 #include <sys/pcpu.h>
 #include <sys/proc.h>
 #include <sys/cpuset.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
+#include <sys/smp.h>
+#ifdef ARM_INTRNG
+#include <sys/sched.h>
+#endif
 #include <machine/bus.h>
 #include <machine/intr.h>
 #include <machine/smp.h>
@@ -55,6 +62,10 @@ __FBSDID("$FreeBSD$");
 #include <dev/ofw/ofw_bus.h>
 #include <dev/ofw/ofw_bus_subr.h>
 
+#ifdef ARM_INTRNG
+#include "pic_if.h"
+#endif
+
 /* We are using GICv2 register naming */
 
 /* Distributor Registers */
@@ -83,8 +94,8 @@ __FBSDID("$FreeBSD$");
 #define GICC_ABPR		0x001C			/* v1 ICCABPR */
 #define GICC_IIDR		0x00FC			/* v1 ICCIIDR*/
 
-#define	GIC_FIRST_IPI		 0	/* Irqs 0-15 are SGIs/IPIs. */
-#define	GIC_LAST_IPI		15
+#define	GIC_FIRST_SGI		 0	/* Irqs 0-15 are SGIs/IPIs. */
+#define	GIC_LAST_SGI		15
 #define	GIC_FIRST_PPI		16	/* Irqs 16-31 are private (per */
 #define	GIC_LAST_PPI		31	/* core) peripheral interrupts. */
 #define	GIC_FIRST_SPI		32	/* Irqs 32+ are shared peripherals. */
@@ -102,8 +113,18 @@ __FBSDID("$FreeBSD$");
 #define	GIC_DEFAULT_ICFGR_INIT	0x00000000
 #endif
 
+#ifdef ARM_INTRNG
+static u_int gic_irq_cpu;
+static int arm_gic_intr(void *);
+static int arm_gic_bind(device_t dev, struct arm_irqsrc *isrc);
+#endif
+
 struct arm_gic_softc {
 	device_t		gic_dev;
+#ifdef ARM_INTRNG
+	void *			gic_intrhand;
+	struct arm_irqsrc **	gic_irqs;
+#endif
 	struct resource *	gic_res[3];
 	bus_space_tag_t		gic_c_bst;
 	bus_space_tag_t		gic_d_bst;
@@ -117,10 +138,13 @@ struct arm_gic_softc {
 static struct resource_spec arm_gic_spec[] = {
 	{ SYS_RES_MEMORY,	0,	RF_ACTIVE },	/* Distributor registers */
 	{ SYS_RES_MEMORY,	1,	RF_ACTIVE },	/* CPU Interrupt Intf. registers */
+#ifdef ARM_INTRNG
+	{ SYS_RES_IRQ,	  0, RF_ACTIVE | RF_OPTIONAL }, /* Parent interrupt */
+#endif
 	{ -1, 0 }
 };
 
-static struct arm_gic_softc *arm_gic_sc = NULL;
+static struct arm_gic_softc *gic_sc = NULL;
 
 #define	gic_c_read_4(_sc, _reg)		\
     bus_space_read_4((_sc)->gic_c_bst, (_sc)->gic_c_bsh, (_reg))
@@ -128,12 +152,16 @@ static struct arm_gic_softc *arm_gic_sc = NULL;
     bus_space_write_4((_sc)->gic_c_bst, (_sc)->gic_c_bsh, (_reg), (_val))
 #define	gic_d_read_4(_sc, _reg)		\
     bus_space_read_4((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg))
+#define	gic_d_write_1(_sc, _reg, _val)		\
+    bus_space_write_1((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg), (_val))
 #define	gic_d_write_4(_sc, _reg, _val)		\
     bus_space_write_4((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg), (_val))
 
+#ifndef ARM_INTRNG
 static int gic_config_irq(int irq, enum intr_trigger trig,
     enum intr_polarity pol);
 static void gic_post_filter(void *);
+#endif
 
 static struct ofw_compat_data compat_data[] = {
 	{"arm,gic",		true},	/* Non-standard, used in FreeBSD dts. */
@@ -159,6 +187,72 @@ arm_gic_probe(device_t dev)
 	return (BUS_PROBE_DEFAULT);
 }
 
+#ifdef ARM_INTRNG
+static inline void
+gic_irq_unmask(struct arm_gic_softc *sc, u_int irq)
+{
+
+	gic_d_write_4(sc, GICD_ISENABLER(irq >> 5), (1UL << (irq & 0x1F)));
+}
+
+static inline void
+gic_irq_mask(struct arm_gic_softc *sc, u_int irq)
+{
+
+	gic_d_write_4(sc, GICD_ICENABLER(irq >> 5), (1UL << (irq & 0x1F)));
+}
+#endif
+
+#ifdef SMP
+#ifdef ARM_INTRNG
+static void
+arm_gic_init_secondary(device_t dev)
+{
+	struct arm_gic_softc *sc = device_get_softc(dev);
+	struct arm_irqsrc *isrc;
+	u_int irq;
+
+	for (irq = 0; irq < sc->nirqs; irq += 4)
+		gic_d_write_4(sc, GICD_IPRIORITYR(irq >> 2), 0);
+
+	/* Set all the interrupts to be in Group 0 (secure) */
+	for (irq = 0; irq < sc->nirqs; irq += 32) {
+		gic_d_write_4(sc, GICD_IGROUPR(irq >> 5), 0);
+	}
+
+	/* Enable CPU interface */
+	gic_c_write_4(sc, GICC_CTLR, 1);
+
+	/* Set priority mask register. */
+	gic_c_write_4(sc, GICC_PMR, 0xff);
+
+	/* Enable interrupt distribution */
+	gic_d_write_4(sc, GICD_CTLR, 0x01);
+
+	/* Unmask attached SGI interrupts. */
+	for (irq = GIC_FIRST_SGI; irq <= GIC_LAST_SGI; irq++) {
+		isrc = sc->gic_irqs[irq];
+		if (isrc != NULL && isrc->isrc_handlers != 0) {
+			CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu);
+			gic_irq_unmask(sc, irq);
+		}
+	}
+
+	/* Unmask attached PPI interrupts. */
+	for (irq = GIC_FIRST_PPI; irq <= GIC_LAST_PPI; irq++) {
+		isrc = sc->gic_irqs[irq];
+		if (isrc == NULL || isrc->isrc_handlers == 0)
+			continue;
+		if (isrc->isrc_flags & ARM_ISRCF_BOUND) {
+			if (CPU_ISSET(PCPU_GET(cpuid), &isrc->isrc_cpu))
+				gic_irq_unmask(sc, irq);
+		} else {
+			CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu);
+			gic_irq_unmask(sc, irq);
+		}
+	}
+}
+#else
 static void
 arm_gic_init_secondary(device_t dev)
 {
@@ -189,7 +283,10 @@ arm_gic_init_secondary(device_t dev)
 	gic_d_write_4(sc, GICD_ISENABLER(29 >> 5), (1UL << (29 & 0x1F)));
 	gic_d_write_4(sc, GICD_ISENABLER(30 >> 5), (1UL << (30 & 0x1F)));
 }
-
+#endif /* ARM_INTRNG */
+#endif /* SMP */
+ 
+#ifndef ARM_INTRNG
 int
 gic_decode_fdt(uint32_t iparent, uint32_t *intr, int *interrupt,
     int *trig, int *pol)
@@ -234,6 +331,19 @@ gic_decode_fdt(uint32_t iparent, uint32_t *intr, int *interrupt,
 	}
 	return (0);
 }
+#endif
+
+#ifdef ARM_INTRNG
+static inline intptr_t
+gic_xref(device_t dev)
+{
+#ifdef FDT
+	return (OF_xref_from_node(ofw_bus_get_node(dev)));
+#else
+	return (0);
+#endif
+}
+#endif
 
 static int
 arm_gic_attach(device_t dev)
@@ -241,8 +351,11 @@ arm_gic_attach(device_t dev)
 	struct		arm_gic_softc *sc;
 	int		i;
 	uint32_t	icciidr;
+#ifdef ARM_INTRNG
+	intptr_t	xref = gic_xref(dev);
+#endif
 
-	if (arm_gic_sc)
+	if (gic_sc)
 		return (ENXIO);
 
 	sc = device_get_softc(dev);
@@ -253,7 +366,7 @@ arm_gic_attach(device_t dev)
 	}
 
 	sc->gic_dev = dev;
-	arm_gic_sc = sc;
+	gic_sc = sc;
 
 	/* Initialize mutex */
 	mtx_init(&sc->mutex, "GIC lock", "", MTX_SPIN);
@@ -273,9 +386,14 @@ arm_gic_attach(device_t dev)
 	sc->nirqs = gic_d_read_4(sc, GICD_TYPER);
 	sc->nirqs = 32 * ((sc->nirqs & 0x1f) + 1);
 
+#ifdef ARM_INTRNG
+	sc->gic_irqs = malloc(sc->nirqs * sizeof (*sc->gic_irqs), M_DEVBUF,
+	    M_WAITOK | M_ZERO);
+#else
 	/* Set up function pointers */
 	arm_post_filter = gic_post_filter;
 	arm_config_irq = gic_config_irq;
+#endif
 
 	icciidr = gic_c_read_4(sc, GICC_IIDR);
 	device_printf(dev,"pn 0x%x, arch 0x%x, rev 0x%x, implementer 0x%x irqs %u\n",
@@ -311,10 +429,455 @@ arm_gic_attach(device_t dev)
 
 	/* Enable interrupt distribution */
 	gic_d_write_4(sc, GICD_CTLR, 0x01);
+#ifndef ARM_INTRNG
+	return (0);
+#else
+	/*
+	 * Now, when everything is initialized, it's right time to
+	 * register interrupt controller to interrupt framefork.
+	 */
+	if (arm_pic_register(dev, xref) != 0) {
+		device_printf(dev, "could not register PIC\n");
+		goto cleanup;
+	}
 
+	if (sc->gic_res[2] == NULL) {
+		if (arm_pic_claim_root(dev, xref, arm_gic_intr, sc,
+		    GIC_LAST_SGI - GIC_FIRST_SGI + 1) != 0) {
+			device_printf(dev, "could not set PIC as a root\n");
+			arm_pic_unregister(dev, xref);
+			goto cleanup;
+		}
+	} else {
+		if (bus_setup_intr(dev, sc->gic_res[2], INTR_TYPE_CLK,
+		    arm_gic_intr, NULL, sc, &sc->gic_intrhand)) {
+			device_printf(dev, "could not setup irq handler\n");
+			arm_pic_unregister(dev, xref);
+			goto cleanup;
+		}
+	}
+
+	return (0);
+
+cleanup:
+	/*
+	 * XXX - not implemented arm_gic_detach() should be called !
+	 */
+	if (sc->gic_irqs != NULL)
+		free(sc->gic_irqs, M_DEVBUF);
+	bus_release_resources(dev, arm_gic_spec, sc->gic_res);
+	return(ENXIO);
+#endif
+}
+
+#ifdef ARM_INTRNG
+static int
+arm_gic_intr(void *arg)
+{
+	struct arm_gic_softc *sc = arg;
+	struct arm_irqsrc *isrc;
+	uint32_t irq_active_reg, irq;
+	struct trapframe *tf;
+
+	irq_active_reg = gic_c_read_4(sc, GICC_IAR);
+	irq = irq_active_reg & 0x3FF;
+
+	/*
+	 * 1. We do EOI here because recent read value from active interrupt
+	 *    register must be used for it. Another approach is to save this
+	 *    value into associated interrupt source.
+	 * 2. EOI must be done on same CPU where interrupt has fired. Thus
+	 *    we must ensure that interrupted thread does not migrate to
+	 *    another CPU.
+	 * 3. EOI cannot be delayed by any preemption which could happen on
+	 *    critical_exit() used in MI intr code, when interrupt thread is
+	 *    scheduled. See next point.
+	 * 4. IPI_RENDEZVOUS assumes that no preemption is permitted during
+	 *    an action and any use of critical_exit() could break this
+	 *    assumption. See comments within smp_rendezvous_action().
+	 * 5. We always return FILTER_HANDLED as this is an interrupt
+	 *    controller dispatch function. Otherwise, in cascaded interrupt
+	 *    case, the whole interrupt subtree would be masked.
+	 */
+
+	if (irq >= sc->nirqs) {
+		device_printf(sc->gic_dev, "Spurious interrupt detected\n");
+		gic_c_write_4(sc, GICC_EOIR, irq_active_reg);
+		return (FILTER_HANDLED);
+	}
+
+	tf = curthread->td_intr_frame;
+dispatch_irq:
+	isrc = sc->gic_irqs[irq];
+	if (isrc == NULL) {
+		device_printf(sc->gic_dev, "Stray interrupt %u detected\n", irq);
+		gic_irq_mask(sc, irq);
+		gic_c_write_4(sc, GICC_EOIR, irq_active_reg);
+		goto next_irq;
+	}
+
+	/*
+	 * Note that GIC_FIRST_SGI is zero and is not used in 'if' statement
+	 * as compiler complains that comparing u_int >= 0 is always true.
+	 */
+	if (irq <= GIC_LAST_SGI) {
+#ifdef SMP
+		/* Call EOI for all IPI before dispatch. */
+		gic_c_write_4(sc, GICC_EOIR, irq_active_reg);
+		arm_ipi_dispatch(isrc, tf);
+		goto next_irq;
+#else
+		printf("SGI %u on UP system detected\n", irq - GIC_FIRST_SGI);
+		gic_c_write_4(sc, GICC_EOIR, irq_active_reg);
+		goto next_irq;
+#endif
+	}
+
+	if (isrc->isrc_trig == INTR_TRIGGER_EDGE)
+		gic_c_write_4(sc, GICC_EOIR, irq_active_reg);
+
+	arm_irq_dispatch(isrc, tf);
+
+next_irq:
+//      arm_irq_memory_barrier(irq); /* XXX */
+//      irq_active_reg = gic_c_read_4(sc, GICC_IAR);
+//      irq = irq_active_reg & 0x3FF;
+	if (0 && irq < sc->nirqs)
+		goto dispatch_irq;
+
+	return (FILTER_HANDLED);
+}
+
+static int
+gic_attach_isrc(struct arm_gic_softc *sc, struct arm_irqsrc *isrc, u_int irq)
+{
+	const char *name;
+
+	/*
+	 * 1. The link between ISRC and controller must be set atomically.
+	 * 2. Just do things only once in rare case when consumers
+	 *    of shared interrupt came here at the same moment.
+	 */
+	mtx_lock_spin(&sc->mutex);
+	if (sc->gic_irqs[irq] != NULL) {
+		mtx_unlock_spin(&sc->mutex);
+		return (sc->gic_irqs[irq] == isrc ? 0 : EEXIST);
+	}
+	sc->gic_irqs[irq] = isrc;
+	isrc->isrc_data = irq;
+	mtx_unlock_spin(&sc->mutex);
+
+	name = device_get_nameunit(sc->gic_dev);
+	if (irq <= GIC_LAST_SGI)
+		arm_irq_set_name(isrc, "%s,i%u", name, irq - GIC_FIRST_SGI);
+	else if (irq <= GIC_LAST_PPI)
+		arm_irq_set_name(isrc, "%s,p%u", name, irq - GIC_FIRST_PPI);
+	else
+		arm_irq_set_name(isrc, "%s,s%u", name, irq - GIC_FIRST_SPI);
 	return (0);
 }
 
+static int
+gic_detach_isrc(struct arm_gic_softc *sc, struct arm_irqsrc *isrc, u_int irq)
+{
+
+	mtx_lock_spin(&sc->mutex);
+	if (sc->gic_irqs[irq] != isrc) {
+		mtx_unlock_spin(&sc->mutex);
+		return (sc->gic_irqs[irq] == NULL ? 0 : EINVAL);
+	}
+	sc->gic_irqs[irq] = NULL;
+	isrc->isrc_data = 0;
+	mtx_unlock_spin(&sc->mutex);
+
+	arm_irq_set_name(isrc, "");
+	return (0);
+}
+
+static void
+gic_config(struct arm_gic_softc *sc, u_int irq, enum intr_trigger trig,
+    enum intr_polarity pol)
+{
+	uint32_t reg;
+	uint32_t mask;
+
+	if (irq < GIC_FIRST_SPI)
+		return;
+
+	mtx_lock_spin(&sc->mutex);
+
+	reg = gic_d_read_4(sc, GICD_ICFGR(irq >> 4));
+	mask = (reg >> 2*(irq % 16)) & 0x3;
+
+	if (pol == INTR_POLARITY_LOW) {
+		mask &= ~GICD_ICFGR_POL_MASK;
+		mask |= GICD_ICFGR_POL_LOW;
+	} else if (pol == INTR_POLARITY_HIGH) {
+		mask &= ~GICD_ICFGR_POL_MASK;
+		mask |= GICD_ICFGR_POL_HIGH;
+	}
+
+	if (trig == INTR_TRIGGER_LEVEL) {
+		mask &= ~GICD_ICFGR_TRIG_MASK;
+		mask |= GICD_ICFGR_TRIG_LVL;
+	} else if (trig == INTR_TRIGGER_EDGE) {
+		mask &= ~GICD_ICFGR_TRIG_MASK;
+		mask |= GICD_ICFGR_TRIG_EDGE;
+	}
+
+	/* Set mask */
+	reg = reg & ~(0x3 << 2*(irq % 16));
+	reg = reg | (mask << 2*(irq % 16));
+	gic_d_write_4(sc, GICD_ICFGR(irq >> 4), reg);
+
+	mtx_unlock_spin(&sc->mutex);
+}
+
+static int
+gic_bind(struct arm_gic_softc *sc, u_int irq, cpuset_t *cpus)
+{
+	uint32_t cpu, end, mask;
+
+	end = min(mp_ncpus, 8);
+	for (cpu = end; cpu < MAXCPU; cpu++)
+		if (CPU_ISSET(cpu, cpus))
+			return (EINVAL);
+
+	for (mask = 0, cpu = 0; cpu < end; cpu++)
+		if (CPU_ISSET(cpu, cpus))
+			mask |= 1 << cpu;
+
+	gic_d_write_1(sc, GICD_ITARGETSR(0) + irq, mask);
+	return (0);
+}
+
+static int
+gic_irq_from_nspc(struct arm_gic_softc *sc, u_int type, u_int num, u_int *irqp)
+{
+
+	switch (type) {
+	case ARM_IRQ_NSPC_PLAIN:
+		*irqp = num;
+		return (*irqp < sc->nirqs ? 0 : EINVAL);
+
+	case ARM_IRQ_NSPC_IRQ:
+		*irqp = num + GIC_FIRST_PPI;
+		return (*irqp < sc->nirqs ? 0 : EINVAL);
+
+	case ARM_IRQ_NSPC_IPI:
+		*irqp = num + GIC_FIRST_SGI;
+		return (*irqp < GIC_LAST_SGI ? 0 : EINVAL);
+
+	default:
+		return (EINVAL);
+	}
+}
+
+static int
+gic_map_nspc(struct arm_gic_softc *sc, struct arm_irqsrc *isrc, u_int *irqp)
+{
+	int error;
+
+	error = gic_irq_from_nspc(sc, isrc->isrc_nspc_type, isrc->isrc_nspc_num,
+	    irqp);
+	if (error != 0)
+		return (error);
+	return (gic_attach_isrc(sc, isrc, *irqp));
+}
+
+#ifdef FDT
+static int
+gic_map_fdt(struct arm_gic_softc *sc, struct arm_irqsrc *isrc, u_int *irqp)
+{
+	u_int irq, tripol;
+	enum intr_trigger trig;
+	enum intr_polarity pol;
+	int error;
+
+	if (isrc->isrc_ncells == 1) {
+		irq = isrc->isrc_cells[0];
+		pol = INTR_POLARITY_CONFORM;
+		trig = INTR_TRIGGER_CONFORM;
+	} else if (isrc->isrc_ncells == 3) {
+		if (isrc->isrc_cells[0] == 0)
+			irq = isrc->isrc_cells[1] + GIC_FIRST_SPI;
+		else
+			irq = isrc->isrc_cells[1] + GIC_FIRST_PPI;
+
+		/*
+		 * In intr[2], bits[3:0] are trigger type and level flags.
+		 *   1 = low-to-high edge triggered
+		 *   2 = high-to-low edge triggered
+		 *   4 = active high level-sensitive
+		 *   8 = active low level-sensitive
+		 * The hardware only supports active-high-level or rising-edge.
+		 */
+		tripol = isrc->isrc_cells[2];
+		if (tripol & 0x0a) {
+			printf("unsupported trigger/polarity configuration "
+			    "0x%2x\n", tripol & 0x0f);
+			return (ENOTSUP);
+		}
+		pol = INTR_POLARITY_CONFORM;
+		if (tripol & 0x01)
+			trig = INTR_TRIGGER_EDGE;
+		else
+			trig = INTR_TRIGGER_LEVEL;
+	} else
+		return (EINVAL);
+
+	if (irq >= sc->nirqs)
+		return (EINVAL);
+
+	error = gic_attach_isrc(sc, isrc, irq);
+	if (error != 0)
+		return (error);
+
+	isrc->isrc_nspc_type = ARM_IRQ_NSPC_PLAIN;
+	isrc->isrc_nspc_num = irq;
+	isrc->isrc_trig = trig;
+	isrc->isrc_pol = pol;
+
+	*irqp = irq;
+	return (0);
+}
+#endif
+
+static int
+arm_gic_register(device_t dev, struct arm_irqsrc *isrc, boolean_t *is_percpu)
+{
+	struct arm_gic_softc *sc = device_get_softc(dev);
+	u_int irq;
+	int error;
+
+	if (isrc->isrc_type == ARM_ISRCT_NAMESPACE)
+		error = gic_map_nspc(sc, isrc, &irq);
+#ifdef FDT
+	else if (isrc->isrc_type == ARM_ISRCT_FDT)
+		error = gic_map_fdt(sc, isrc, &irq);
+#endif
+	else
+		return (EINVAL);
+
+	if (error == 0)
+		*is_percpu = irq < GIC_FIRST_SPI ? TRUE : FALSE;
+	return (error);
+}
+
+static void
+arm_gic_enable_intr(device_t dev, struct arm_irqsrc *isrc)
+{
+	struct arm_gic_softc *sc = device_get_softc(dev);
+	u_int irq = isrc->isrc_data;
+
+	if (isrc->isrc_trig == INTR_TRIGGER_CONFORM)
+		isrc->isrc_trig = INTR_TRIGGER_LEVEL;
+
+	/*
+	 * XXX - In case that per CPU interrupt is going to be enabled in time
+	 *       when SMP is already started, we need some IPI call which
+	 *       enables it on others CPUs. Further, it's more complicated as
+	 *       pic_enable_source() and pic_disable_source() should act on
+	 *       per CPU basis only. Thus, it should be solved here somehow.
+	 */
+	if (isrc->isrc_flags & ARM_ISRCF_PERCPU)
+		CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu);
+
+	gic_config(sc, irq, isrc->isrc_trig, isrc->isrc_pol);
+	arm_gic_bind(dev, isrc);
+}
+
+static void
+arm_gic_enable_source(device_t dev, struct arm_irqsrc *isrc)
+{
+	struct arm_gic_softc *sc = device_get_softc(dev);
+	u_int irq = isrc->isrc_data;
+
+	arm_irq_memory_barrier(irq);
+	gic_irq_unmask(sc, irq);
+}
+
+static void
+arm_gic_disable_source(device_t dev, struct arm_irqsrc *isrc)
+{
+	struct arm_gic_softc *sc = device_get_softc(dev);
+	u_int irq = isrc->isrc_data;
+
+	gic_irq_mask(sc, irq);
+}
+
+static int
+arm_gic_unregister(device_t dev, struct arm_irqsrc *isrc)
+{
+	struct arm_gic_softc *sc = device_get_softc(dev);
+	u_int irq = isrc->isrc_data;
+
+	return (gic_detach_isrc(sc, isrc, irq));
+}
+
+static void
+arm_gic_pre_ithread(device_t dev, struct arm_irqsrc *isrc)
+{
+	struct arm_gic_softc *sc = device_get_softc(dev);
+
+	arm_gic_disable_source(dev, isrc);
+	gic_c_write_4(sc, GICC_EOIR, isrc->isrc_data);
+}
+
+static void
+arm_gic_post_ithread(device_t dev, struct arm_irqsrc *isrc)
+{
+
+	arm_irq_memory_barrier(0);
+	arm_gic_enable_source(dev, isrc);
+}
+
+static void
+arm_gic_post_filter(device_t dev, struct arm_irqsrc *isrc)
+{
+	struct arm_gic_softc *sc = device_get_softc(dev);
+
+        /* EOI for edge-triggered done earlier. */
+	if (isrc->isrc_trig == INTR_TRIGGER_EDGE)
+		return;
+
+	arm_irq_memory_barrier(0);
+	gic_c_write_4(sc, GICC_EOIR, isrc->isrc_data);
+}
+
+#ifdef SMP
+static int
+arm_gic_bind(device_t dev, struct arm_irqsrc *isrc)
+{
+	struct arm_gic_softc *sc = device_get_softc(dev);
+	uint32_t irq = isrc->isrc_data;
+
+	if (irq < GIC_FIRST_SPI)
+		return (EINVAL);
+
+	if (CPU_EMPTY(&isrc->isrc_cpu)) {
+		gic_irq_cpu = arm_irq_next_cpu(gic_irq_cpu, &all_cpus);
+		CPU_SETOF(gic_irq_cpu, &isrc->isrc_cpu);
+	}
+	return (gic_bind(sc, irq, &isrc->isrc_cpu));
+}
+
+static void
+arm_gic_ipi_send(device_t dev, struct arm_irqsrc *isrc, cpuset_t cpus)
+{
+	struct arm_gic_softc *sc = device_get_softc(dev);
+	uint32_t irq, val = 0, i;
+
+	irq = isrc->isrc_data;
+
+	for (i = 0; i < MAXCPU; i++)
+		if (CPU_ISSET(i, &cpus))
+			val |= 1 << (16 + i);
+
+	gic_d_write_4(sc, GICD_SGIR(0), val | irq);
+}
+#endif
+#else
 static int
 arm_gic_next_irq(struct arm_gic_softc *sc, int last_irq)
 {
@@ -327,7 +890,7 @@ arm_gic_next_irq(struct arm_gic_softc *sc, int last_irq)
 	 * bits (ie CPU number), not just the IRQ number, and we do not
 	 * have this information later.
 	 */
-	if ((active_irq & 0x3ff) <= GIC_LAST_IPI)
+	if ((active_irq & 0x3ff) <= GIC_LAST_SGI)
 		gic_c_write_4(sc, GICC_EOIR, active_irq);
 	active_irq &= 0x3FF;
 
@@ -400,7 +963,7 @@ arm_gic_mask(device_t dev, int irq)
 	struct arm_gic_softc *sc = device_get_softc(dev);
 
 	gic_d_write_4(sc, GICD_ICENABLER(irq >> 5), (1UL << (irq & 0x1F)));
-	gic_c_write_4(sc, GICC_EOIR, irq);
+	gic_c_write_4(sc, GICC_EOIR, irq); /* XXX - not allowed */
 }
 
 static void
@@ -408,7 +971,7 @@ arm_gic_unmask(device_t dev, int irq)
 {
 	struct arm_gic_softc *sc = device_get_softc(dev);
 
-	if (irq > GIC_LAST_IPI)
+	if (irq > GIC_LAST_SGI)
 		arm_irq_memory_barrier(irq);
 
 	gic_d_write_4(sc, GICD_ISENABLER(irq >> 5), (1UL << (irq & 0x1F)));
@@ -455,10 +1018,10 @@ arm_gic_ipi_clear(device_t dev, int ipi)
 static void
 gic_post_filter(void *arg)
 {
-	struct arm_gic_softc *sc = arm_gic_sc;
+	struct arm_gic_softc *sc = gic_sc;
 	uintptr_t irq = (uintptr_t) arg;
 
-	if (irq > GIC_LAST_IPI)
+	if (irq > GIC_LAST_SGI)
 		arm_irq_memory_barrier(irq);
 	gic_c_write_4(sc, GICC_EOIR, irq);
 }
@@ -467,64 +1030,81 @@ static int
 gic_config_irq(int irq, enum intr_trigger trig, enum intr_polarity pol)
 {
 
-	return (arm_gic_config(arm_gic_sc->gic_dev, irq, trig, pol));
+	return (arm_gic_config(gic_sc->gic_dev, irq, trig, pol));
 }
 
 void
 arm_mask_irq(uintptr_t nb)
 {
 
-	arm_gic_mask(arm_gic_sc->gic_dev, nb);
+	arm_gic_mask(gic_sc->gic_dev, nb);
 }
 
 void
 arm_unmask_irq(uintptr_t nb)
 {
 
-	arm_gic_unmask(arm_gic_sc->gic_dev, nb);
+	arm_gic_unmask(gic_sc->gic_dev, nb);
 }
 
 int
 arm_get_next_irq(int last_irq)
 {
 
-	return (arm_gic_next_irq(arm_gic_sc, last_irq));
-}
-
-void
-arm_pic_init_secondary(void)
-{
-
-	arm_gic_init_secondary(arm_gic_sc->gic_dev);
+	return (arm_gic_next_irq(gic_sc, last_irq));
 }
 
 #ifdef SMP
+void
+arm_pic_init_secondary(void)
+{
+
+	arm_gic_init_secondary(gic_sc->gic_dev);
+}
+
 void
 pic_ipi_send(cpuset_t cpus, u_int ipi)
 {
 
-	arm_gic_ipi_send(arm_gic_sc->gic_dev, cpus, ipi);
+	arm_gic_ipi_send(gic_sc->gic_dev, cpus, ipi);
 }
 
 int
 pic_ipi_read(int i)
 {
 
-	return (arm_gic_ipi_read(arm_gic_sc->gic_dev, i));
+	return (arm_gic_ipi_read(gic_sc->gic_dev, i));
 }
 
 void
 pic_ipi_clear(int ipi)
 {
 
-	arm_gic_ipi_clear(arm_gic_sc->gic_dev, ipi);
+	arm_gic_ipi_clear(gic_sc->gic_dev, ipi);
 }
 #endif
+#endif /* ARM_INTRNG */
 
 static device_method_t arm_gic_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		arm_gic_probe),
 	DEVMETHOD(device_attach,	arm_gic_attach),
+#ifdef ARM_INTRNG
+	/* Interrupt controller interface */
+	DEVMETHOD(pic_disable_source,	arm_gic_disable_source),
+	DEVMETHOD(pic_enable_intr,	arm_gic_enable_intr),
+	DEVMETHOD(pic_enable_source,	arm_gic_enable_source),
+	DEVMETHOD(pic_post_filter,	arm_gic_post_filter),
+	DEVMETHOD(pic_post_ithread,	arm_gic_post_ithread),
+	DEVMETHOD(pic_pre_ithread,	arm_gic_pre_ithread),
+	DEVMETHOD(pic_register,		arm_gic_register),
+	DEVMETHOD(pic_unregister,	arm_gic_unregister),
+#ifdef SMP
+	DEVMETHOD(pic_bind,		arm_gic_bind),
+	DEVMETHOD(pic_init_secondary,	arm_gic_init_secondary),
+	DEVMETHOD(pic_ipi_send,		arm_gic_ipi_send),
+#endif
+#endif
 	{ 0, 0 }
 };
 
diff --git a/sys/arm/arm/intrng.c b/sys/arm/arm/intrng.c
new file mode 100644
index 000000000000..c30ec684bb34
--- /dev/null
+++ b/sys/arm/arm/intrng.c
@@ -0,0 +1,1458 @@
+/*-
+ * Copyright (c) 2012-2014 Jakub Wojciech Klama <jceel@FreeBSD.org>.
+ * Copyright (c) 2015 Svatopluk Kraus
+ * Copyright (c) 2015 Michal Meloun
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/*
+ *	ARM Interrupt Framework
+ *
+ *  TODO: - to support IPI (PPI) enabling on other CPUs if already started
+ *        - to complete things for removable PICs
+ */
+
+#include "opt_ddb.h"
+#include "opt_platform.h"
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/syslog.h>
+#include <sys/malloc.h>
+#include <sys/proc.h>
+#include <sys/queue.h>
+#include <sys/bus.h>
+#include <sys/interrupt.h>
+#include <sys/conf.h>
+#include <sys/cpuset.h>
+#include <sys/sched.h>
+#include <sys/smp.h>
+#include <machine/atomic.h>
+#include <machine/intr.h>
+#include <machine/cpu.h>
+#include <machine/smp.h>
+#include <machine/stdarg.h>
+
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include <dev/fdt/fdt_common.h>
+
+#ifdef DDB
+#include <ddb/ddb.h>
+#endif
+
+#include "pic_if.h"
+
+#define	INTRNAME_LEN	(2*MAXCOMLEN + 1)
+
+#ifdef DEBUG
+#define debugf(fmt, args...) do { printf("%s(): ", __func__);	\
+    printf(fmt,##args); } while (0)
+#else
+#define debugf(fmt, args...)
+#endif
+
+MALLOC_DECLARE(M_INTRNG);
+MALLOC_DEFINE(M_INTRNG, "intrng", "ARM interrupt handling");
+
+/* Main ARM interrupt handler called from assembler -> 'hidden' for C code. */
+void arm_irq_handler(struct trapframe *tf);
+
+/* Root interrupt controller stuff. */
+static struct arm_irqsrc *irq_root_isrc;
+static device_t irq_root_dev;
+static arm_irq_filter_t *irq_root_filter;
+static void *irq_root_arg;
+static u_int irq_root_ipicount;
+
+/* Interrupt controller definition. */
+struct arm_pic {
+	SLIST_ENTRY(arm_pic)	pic_next;
+	intptr_t		pic_xref;	/* hardware identification */
+	device_t		pic_dev;
+};
+
+static struct mtx pic_list_lock;
+static SLIST_HEAD(, arm_pic) pic_list;
+
+static struct arm_pic *pic_lookup(device_t dev, intptr_t xref);
+
+/* Interrupt source definition. */
+static struct mtx isrc_table_lock;
+static struct arm_irqsrc *irq_sources[NIRQ];
+u_int irq_next_free;
+
+#define IRQ_INVALID	nitems(irq_sources)
+
+#ifdef SMP
+static boolean_t irq_assign_cpu = FALSE;
+
+static struct arm_irqsrc ipi_sources[ARM_IPI_COUNT];
+static u_int ipi_next_num;
+#endif
+
+/*
+ * - 2 counters for each I/O interrupt.
+ * - MAXCPU counters for each IPI counters for SMP.
+ */
+#ifdef SMP
+#define INTRCNT_COUNT   (NIRQ * 2 + ARM_IPI_COUNT * MAXCPU)
+#else
+#define INTRCNT_COUNT   (NIRQ * 2)
+#endif
+
+/* Data for MI statistics reporting. */
+u_long intrcnt[INTRCNT_COUNT];
+char intrnames[INTRCNT_COUNT * INTRNAME_LEN];
+size_t sintrcnt = sizeof(intrcnt);
+size_t sintrnames = sizeof(intrnames);
+static u_int intrcnt_index;
+
+/*
+ *  Interrupt framework initialization routine.
+ */
+static void
+arm_irq_init(void *dummy __unused)
+{
+
+	SLIST_INIT(&pic_list);
+	mtx_init(&pic_list_lock, "arm pic list", NULL, MTX_DEF);
+	mtx_init(&isrc_table_lock, "arm isrc table", NULL, MTX_DEF);
+}
+SYSINIT(arm_irq_init, SI_SUB_INTR, SI_ORDER_FIRST, arm_irq_init, NULL);
+
+static void
+intrcnt_setname(const char *name, int index)
+{
+
+	snprintf(intrnames + INTRNAME_LEN * index, INTRNAME_LEN, "%-*s",
+	    INTRNAME_LEN - 1, name);
+}
+
+/*
+ *  Update name for interrupt source with interrupt event.
+ */
+static void
+intrcnt_updatename(struct arm_irqsrc *isrc)
+{
+
+	/* QQQ: What about stray counter name? */
+	mtx_assert(&isrc_table_lock, MA_OWNED);
+	intrcnt_setname(isrc->isrc_event->ie_fullname, isrc->isrc_index);
+}
+
+/*
+ *  Virtualization for interrupt source interrupt counter increment.
+ */
+static inline void
+isrc_increment_count(struct arm_irqsrc *isrc)
+{
+
+	/*
+	 * XXX - It should be atomic for PPI interrupts. It was proven that
+	 *       the lost is measurable easily for timer PPI interrupts.
+	 */
+	isrc->isrc_count[0]++;
+	/*atomic_add_long(&isrc->isrc_count[0], 1);*/
+}
+
+/*
+ *  Virtualization for interrupt source interrupt stray counter increment.
+ */
+static inline void
+isrc_increment_straycount(struct arm_irqsrc *isrc)
+{
+
+	isrc->isrc_count[1]++;
+}
+
+/*
+ *  Virtualization for interrupt source interrupt name update.
+ */
+static void
+isrc_update_name(struct arm_irqsrc *isrc, const char *name)
+{
+	char str[INTRNAME_LEN];
+
+	mtx_assert(&isrc_table_lock, MA_OWNED);
+
+	if (name != NULL) {
+		snprintf(str, INTRNAME_LEN, "%s: %s", isrc->isrc_name, name);
+		intrcnt_setname(str, isrc->isrc_index);
+		snprintf(str, INTRNAME_LEN, "stray %s: %s", isrc->isrc_name,
+		    name);
+		intrcnt_setname(str, isrc->isrc_index + 1);
+	} else {
+		snprintf(str, INTRNAME_LEN, "%s:", isrc->isrc_name);
+		intrcnt_setname(str, isrc->isrc_index);
+		snprintf(str, INTRNAME_LEN, "stray %s:", isrc->isrc_name);
+		intrcnt_setname(str, isrc->isrc_index + 1);
+	}
+}
+
+/*
+ *  Virtualization for interrupt source interrupt counters setup.
+ */
+static void
+isrc_setup_counters(struct arm_irqsrc *isrc)
+{
+	u_int index;
+
+	/*
+	 *  XXX - it does not work well with removable controllers and
+	 *        interrupt sources !!!
+	 */
+	index = atomic_fetchadd_int(&intrcnt_index, 2);
+	isrc->isrc_index = index;
+	isrc->isrc_count = &intrcnt[index];
+	isrc_update_name(isrc, NULL);
+}
+
+#ifdef SMP
+/*
+ *  Virtualization for interrupt source IPI counter increment.
+ */
+static inline void
+isrc_increment_ipi_count(struct arm_irqsrc *isrc, u_int cpu)
+{
+
+	isrc->isrc_count[cpu]++;
+}
+
+/*
+ *  Virtualization for interrupt source IPI counters setup.
+ */
+static void
+isrc_setup_ipi_counters(struct arm_irqsrc *isrc, const char *name)
+{
+	u_int index, i;
+	char str[INTRNAME_LEN];
+
+	index = atomic_fetchadd_int(&intrcnt_index, MAXCPU);
+	isrc->isrc_index = index;
+	isrc->isrc_count = &intrcnt[index];
+
+	for (i = 0; i < MAXCPU; i++) {
+		/*
+		 * We do not expect any race in IPI case here,
+		 * so locking is not needed.
+		 */
+		snprintf(str, INTRNAME_LEN, "cpu%d:%s", i, name);
+		intrcnt_setname(str, index + i);
+	}
+}
+#endif
+
+/*
+ *  Main ARM interrupt dispatch handler. It's called straight
+ *  from the assembler, where CPU interrupt is served.
+ */
+void
+arm_irq_handler(struct trapframe *tf)
+{
+	struct trapframe * oldframe;
+	struct thread * td;
+
+	KASSERT(irq_root_filter != NULL, ("%s: no filter", __func__));
+
+	PCPU_INC(cnt.v_intr);
+	critical_enter();
+	td = curthread;
+	oldframe = td->td_intr_frame;
+	td->td_intr_frame = tf;
+	irq_root_filter(irq_root_arg);
+	td->td_intr_frame = oldframe;
+	critical_exit();
+}
+
+/*
+ *  ARM interrupt controller dispatch function for interrupts. It should
+ *  be called straight from the interrupt controller, when associated interrupt
+ *  source is learned.
+ */
+void
+arm_irq_dispatch(struct arm_irqsrc *isrc, struct trapframe *tf)
+{
+
+	KASSERT(isrc != NULL, ("%s: no source", __func__));
+
+	isrc_increment_count(isrc);
+
+#ifdef INTR_SOLO
+	if (isrc->isrc_filter != NULL) {
+		int error;
+		error = isrc->isrc_filter(isrc->isrc_arg, tf);
+		PIC_POST_FILTER(isrc->isrc_dev, isrc);
+		if (error == FILTER_HANDLED)
+			return;
+	} else 
+#endif
+	if (isrc->isrc_event != NULL) {
+		if (intr_event_handle(isrc->isrc_event, tf) == 0)
+			return;
+	}
+
+	isrc_increment_straycount(isrc);
+	PIC_DISABLE_SOURCE(isrc->isrc_dev, isrc);
+
+	device_printf(isrc->isrc_dev, "stray irq <%s> disabled",
+	    isrc->isrc_name);
+}
+
+/*
+ *  Allocate interrupt source.
+ */
+static struct arm_irqsrc *
+isrc_alloc(u_int type, u_int extsize)
+{
+	struct arm_irqsrc *isrc;
+
+	isrc = malloc(sizeof(*isrc) + extsize, M_INTRNG, M_WAITOK | M_ZERO);
+	isrc->isrc_irq = IRQ_INVALID;	/* just to be safe */
+	isrc->isrc_type = type;
+	isrc->isrc_nspc_type = ARM_IRQ_NSPC_NONE;
+	isrc->isrc_trig = INTR_TRIGGER_CONFORM;
+	isrc->isrc_pol = INTR_POLARITY_CONFORM;
+	CPU_ZERO(&isrc->isrc_cpu);
+	return (isrc);
+}
+
+/*
+ *  Free interrupt source.
+ */
+static void
+isrc_free(struct arm_irqsrc *isrc)
+{
+
+	free(isrc, M_INTRNG);
+}
+
+void
+arm_irq_set_name(struct arm_irqsrc *isrc, const char *fmt, ...)
+{
+	va_list ap;
+
+	va_start(ap, fmt);
+	vsnprintf(isrc->isrc_name, ARM_ISRC_NAMELEN, fmt, ap);
+	va_end(ap);
+}
+
+/*
+ *  Alloc unique interrupt number (resource handle) for interrupt source.
+ *
+ *  There could be various strategies how to allocate free interrupt number
+ *  (resource handle) for new interrupt source.
+ *
+ *  1. Handles are always allocated forward, so handles are not recycled
+ *     immediately. However, if only one free handle left which is reused
+ *     constantly...
+ */
+static int
+isrc_alloc_irq_locked(struct arm_irqsrc *isrc)
+{
+	u_int maxirqs, irq;
+
+	mtx_assert(&isrc_table_lock, MA_OWNED);
+
+	maxirqs = nitems(irq_sources);
+	if (irq_next_free >= maxirqs)
+		return (ENOSPC);
+
+	for (irq = irq_next_free; irq < maxirqs; irq++) {
+		if (irq_sources[irq] == NULL)
+			goto found;
+	}
+	for (irq = 0; irq < irq_next_free; irq++) {
+		if (irq_sources[irq] == NULL)
+			goto found;
+	}
+
+	irq_next_free = maxirqs;
+	return (ENOSPC);
+
+found:
+	isrc->isrc_irq = irq;
+	irq_sources[irq] = isrc;
+
+	arm_irq_set_name(isrc, "irq%u", irq);
+	isrc_setup_counters(isrc);
+
+	irq_next_free = irq + 1;
+	if (irq_next_free >= maxirqs)
+		irq_next_free = 0;
+	return (0);
+}
+#ifdef notyet
+/*
+ *  Free unique interrupt number (resource handle) from interrupt source.
+ */
+static int
+isrc_free_irq(struct arm_irqsrc *isrc)
+{
+	u_int maxirqs;
+
+	mtx_assert(&isrc_table_lock, MA_NOTOWNED);
+
+	maxirqs = nitems(irq_sources);
+	if (isrc->isrc_irq >= maxirqs)
+		return (EINVAL);
+
+	mtx_lock(&isrc_table_lock);
+	if (irq_sources[isrc->isrc_irq] != isrc) {
+		mtx_unlock(&isrc_table_lock);
+		return (EINVAL);
+	}
+
+	irq_sources[isrc->isrc_irq] = NULL;
+	isrc->isrc_irq = IRQ_INVALID;	/* just to be safe */
+	mtx_unlock(&isrc_table_lock);
+
+	return (0);
+}
+#endif
+/*
+ *  Lookup interrupt source by interrupt number (resource handle).
+ */
+static struct arm_irqsrc *
+isrc_lookup(u_int irq)
+{
+
+	if (irq < nitems(irq_sources))
+		return (irq_sources[irq]);
+	return (NULL);
+}
+
+/*
+ *  Lookup interrupt source by namespace description.
+ */
+static struct arm_irqsrc *
+isrc_namespace_lookup(device_t dev, uint16_t type, uint16_t num)
+{
+	u_int irq;
+	struct arm_irqsrc *isrc;
+
+	mtx_assert(&isrc_table_lock, MA_OWNED);
+
+	for (irq = 0; irq < nitems(irq_sources); irq++) {
+		isrc = irq_sources[irq];
+		if (isrc != NULL && isrc->isrc_dev == dev &&
+		    isrc->isrc_nspc_type == type && isrc->isrc_nspc_num == num)
+			return (isrc);
+	}
+	return (NULL);
+}
+
+/*
+ *  Map interrupt source according to namespace into framework. If such mapping
+ *  does not exist, create it. Return unique interrupt number (resource handle)
+ *  associated with mapped interrupt source.
+ */
+u_int
+arm_namespace_map_irq(device_t dev, uint16_t type, uint16_t num)
+{
+	struct arm_irqsrc *isrc, *new_isrc;
+	int error;
+
+	new_isrc = isrc_alloc(ARM_ISRCT_NAMESPACE, 0);
+
+	mtx_lock(&isrc_table_lock);
+	isrc = isrc_namespace_lookup(dev, type, num);
+	if (isrc != NULL) {
+		mtx_unlock(&isrc_table_lock);
+		isrc_free(new_isrc);
+		return (isrc->isrc_irq);	/* already mapped */
+	}
+
+	error = isrc_alloc_irq_locked(new_isrc);
+	if (error != 0) {
+		mtx_unlock(&isrc_table_lock);
+		isrc_free(new_isrc);
+		return (IRQ_INVALID);		/* no space left */
+	}
+
+	new_isrc->isrc_dev = dev;
+	new_isrc->isrc_nspc_type = type;
+	new_isrc->isrc_nspc_num = num;
+	mtx_unlock(&isrc_table_lock);
+
+	return (new_isrc->isrc_irq);
+}
+
+#ifdef FDT
+/*
+ *  Lookup interrupt source by FDT description.
+ */
+static struct arm_irqsrc *
+isrc_fdt_lookup(intptr_t xref, pcell_t *cells, u_int ncells)
+{
+	u_int irq, cellsize;
+	struct arm_irqsrc *isrc;
+
+	mtx_assert(&isrc_table_lock, MA_OWNED);
+
+	cellsize = ncells * sizeof(*cells);
+	for (irq = 0; irq < nitems(irq_sources); irq++) {
+		isrc = irq_sources[irq];
+		if (isrc != NULL && isrc->isrc_type == ARM_ISRCT_FDT &&
+		    isrc->isrc_xref == xref && isrc->isrc_ncells == ncells &&
+		    memcmp(isrc->isrc_cells, cells, cellsize) == 0)
+			return (isrc);
+	}
+	return (NULL);
+}
+
+/*
+ *  Map interrupt source according to FDT data into framework. If such mapping
+ *  does not exist, create it. Return unique interrupt number (resource handle)
+ *  associated with mapped interrupt source.
+ */
+u_int
+arm_fdt_map_irq(phandle_t node, pcell_t *cells, u_int ncells)
+{
+	struct arm_irqsrc *isrc, *new_isrc;
+	u_int cellsize;
+	intptr_t xref;
+	int error;
+
+	xref = (intptr_t)node;	/* It's so simple for now. */
+
+	cellsize = ncells * sizeof(*cells);
+	new_isrc = isrc_alloc(ARM_ISRCT_FDT, cellsize);
+
+	mtx_lock(&isrc_table_lock);
+	isrc = isrc_fdt_lookup(xref, cells, ncells);
+	if (isrc != NULL) {
+		mtx_unlock(&isrc_table_lock);
+		isrc_free(new_isrc);
+		return (isrc->isrc_irq);	/* already mapped */
+	}
+
+	error = isrc_alloc_irq_locked(new_isrc);
+	if (error != 0) {
+		mtx_unlock(&isrc_table_lock);
+		isrc_free(new_isrc);
+		return (IRQ_INVALID);		/* no space left */
+	}
+
+	new_isrc->isrc_xref = xref;
+	new_isrc->isrc_ncells = ncells;
+	memcpy(new_isrc->isrc_cells, cells, cellsize);
+	mtx_unlock(&isrc_table_lock);
+
+	return (new_isrc->isrc_irq);
+}
+#endif
+
+/*
+ *  Register interrupt source into interrupt controller.
+ */
+static int
+isrc_register(struct arm_irqsrc *isrc)
+{
+	struct arm_pic *pic;
+	boolean_t is_percpu;
+	int error;
+
+	if (isrc->isrc_flags & ARM_ISRCF_REGISTERED)
+		return (0);
+
+	if (isrc->isrc_dev == NULL) {
+		pic = pic_lookup(NULL, isrc->isrc_xref);
+		if (pic == NULL || pic->pic_dev == NULL)
+			return (ESRCH);
+		isrc->isrc_dev = pic->pic_dev;
+	}
+
+	error = PIC_REGISTER(isrc->isrc_dev, isrc, &is_percpu);
+	if (error != 0)
+		return (error);
+
+	mtx_lock(&isrc_table_lock);
+	isrc->isrc_flags |= ARM_ISRCF_REGISTERED;
+	if (is_percpu)
+		isrc->isrc_flags |= ARM_ISRCF_PERCPU;
+	isrc_update_name(isrc, NULL);
+	mtx_unlock(&isrc_table_lock);
+	return (0);
+}
+
+#ifdef INTR_SOLO
+/*
+ *  Setup filter into interrupt source.
+ */
+static int
+iscr_setup_filter(struct arm_irqsrc *isrc, const char *name,
+    arm_irq_filter_t *filter, void *arg, void **cookiep)
+{
+
+	if (filter == NULL)
+		return (EINVAL);
+
+	mtx_lock(&isrc_table_lock);
+	/*
+	 * Make sure that we do not mix the two ways
+	 * how we handle interrupt sources.
+	 */
+	if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) {
+		mtx_unlock(&isrc_table_lock);
+		return (EBUSY);
+	}
+	isrc->isrc_filter = filter;
+	isrc->isrc_arg = arg;
+	isrc_update_name(isrc, name);
+	mtx_unlock(&isrc_table_lock);
+
+	*cookiep = isrc;
+	return (0);
+}
+#endif
+
+/*
+ *  Interrupt source pre_ithread method for MI interrupt framework.
+ */
+static void
+arm_isrc_pre_ithread(void *arg)
+{
+	struct arm_irqsrc *isrc = arg;
+
+	PIC_PRE_ITHREAD(isrc->isrc_dev, isrc);
+}
+
+/*
+ *  Interrupt source post_ithread method for MI interrupt framework.
+ */
+static void
+arm_isrc_post_ithread(void *arg)
+{
+	struct arm_irqsrc *isrc = arg;
+
+	PIC_POST_ITHREAD(isrc->isrc_dev, isrc);
+}
+
+/*
+ *  Interrupt source post_filter method for MI interrupt framework.
+ */
+static void
+arm_isrc_post_filter(void *arg)
+{
+	struct arm_irqsrc *isrc = arg;
+
+	PIC_POST_FILTER(isrc->isrc_dev, isrc);
+}
+
+/*
+ *  Interrupt source assign_cpu method for MI interrupt framework.
+ */
+static int
+arm_isrc_assign_cpu(void *arg, int cpu)
+{
+#ifdef SMP
+	struct arm_irqsrc *isrc = arg;
+	int error;
+
+	if (isrc->isrc_dev != irq_root_dev)
+		return (EINVAL);
+
+	mtx_lock(&isrc_table_lock);
+	if (cpu == NOCPU) {
+		CPU_ZERO(&isrc->isrc_cpu);
+		isrc->isrc_flags &= ~ARM_ISRCF_BOUND;
+	} else {
+		CPU_SETOF(cpu, &isrc->isrc_cpu);
+		isrc->isrc_flags |= ARM_ISRCF_BOUND;
+	}
+
+	/*
+	 * In NOCPU case, it's up to PIC to either leave ISRC on same CPU or
+	 * re-balance it to another CPU or enable it on more CPUs. However,
+	 * PIC is expected to change isrc_cpu appropriately to keep us well
+	 * informed if the call is successfull.
+	 */
+	if (irq_assign_cpu) {
+		error = PIC_BIND(isrc->isrc_dev, isrc);
+		if (error) {
+			CPU_ZERO(&isrc->isrc_cpu);
+			mtx_unlock(&isrc_table_lock);
+			return (error);
+		}
+	}
+	mtx_unlock(&isrc_table_lock);
+	return (0);
+#else
+	return (EOPNOTSUPP);
+#endif
+}
+
+/*
+ *  Create interrupt event for interrupt source.
+ */
+static int
+isrc_event_create(struct arm_irqsrc *isrc)
+{
+	struct intr_event *ie;
+	int error;
+
+	error = intr_event_create(&ie, isrc, 0, isrc->isrc_irq,
+	    arm_isrc_pre_ithread, arm_isrc_post_ithread, arm_isrc_post_filter,
+	    arm_isrc_assign_cpu, "%s:", isrc->isrc_name);
+	if (error)
+		return (error);
+
+	mtx_lock(&isrc_table_lock);
+	/*
+	 * Make sure that we do not mix the two ways
+	 * how we handle interrupt sources. Let contested event wins.
+	 */
+	if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) {
+		mtx_unlock(&isrc_table_lock);
+		intr_event_destroy(ie);
+		return (isrc->isrc_event != NULL ? EBUSY : 0);
+	}
+	isrc->isrc_event = ie;
+	mtx_unlock(&isrc_table_lock);
+
+	return (0);
+}
+#ifdef notyet
+/*
+ *  Destroy interrupt event for interrupt source.
+ */
+static void
+isrc_event_destroy(struct arm_irqsrc *isrc)
+{
+	struct intr_event *ie;
+
+	mtx_lock(&isrc_table_lock);
+	ie = isrc->isrc_event;
+	isrc->isrc_event = NULL;
+	mtx_unlock(&isrc_table_lock);
+
+	if (ie != NULL)
+		intr_event_destroy(ie);
+}
+#endif
+/*
+ *  Add handler to interrupt source.
+ */
+static int
+isrc_add_handler(struct arm_irqsrc *isrc, const char *name,
+    driver_filter_t filter, driver_intr_t handler, void *arg,
+    enum intr_type flags, void **cookiep)
+{
+	int error;
+
+	if (isrc->isrc_event == NULL) {
+		error = isrc_event_create(isrc);
+		if (error)
+			return (error);
+	}
+
+	error = intr_event_add_handler(isrc->isrc_event, name, filter, handler,
+	    arg, intr_priority(flags), flags, cookiep);
+	if (error == 0) {
+		mtx_lock(&isrc_table_lock);
+		intrcnt_updatename(isrc);
+		mtx_unlock(&isrc_table_lock);
+	}
+
+	return (error);
+}
+
+/*
+ *  Lookup interrupt controller locked.
+ */
+static struct arm_pic *
+pic_lookup_locked(device_t dev, intptr_t xref)
+{
+	struct arm_pic *pic;
+
+	mtx_assert(&pic_list_lock, MA_OWNED);
+
+	SLIST_FOREACH(pic, &pic_list, pic_next) {
+		if (pic->pic_xref != xref)
+			continue;
+		if (pic->pic_xref != 0 || pic->pic_dev == dev)
+			return (pic);
+	}
+	return (NULL);
+}
+
+/*
+ *  Lookup interrupt controller.
+ */
+static struct arm_pic *
+pic_lookup(device_t dev, intptr_t xref)
+{
+	struct arm_pic *pic;
+
+	mtx_lock(&pic_list_lock);
+	pic = pic_lookup_locked(dev, xref);
+	mtx_unlock(&pic_list_lock);
+
+	return (pic);
+}
+
+/*
+ *  Create interrupt controller.
+ */
+static struct arm_pic *
+pic_create(device_t dev, intptr_t xref)
+{
+	struct arm_pic *pic;
+
+	mtx_lock(&pic_list_lock);
+	pic = pic_lookup_locked(dev, xref);
+	if (pic != NULL) {
+		mtx_unlock(&pic_list_lock);
+		return (pic);
+	}
+	pic = malloc(sizeof(*pic), M_INTRNG, M_NOWAIT | M_ZERO);
+	pic->pic_xref = xref;
+	pic->pic_dev = dev;
+	SLIST_INSERT_HEAD(&pic_list, pic, pic_next);
+	mtx_unlock(&pic_list_lock);
+
+	return (pic);
+}
+#ifdef notyet
+/*
+ *  Destroy interrupt controller.
+ */
+static void
+pic_destroy(device_t dev, intptr_t xref)
+{
+	struct arm_pic *pic;
+
+	mtx_lock(&pic_list_lock);
+	pic = pic_lookup_locked(dev, xref);
+	if (pic == NULL) {
+		mtx_unlock(&pic_list_lock);
+		return;
+	}
+	SLIST_REMOVE(&pic_list, pic, arm_pic, pic_next);
+	mtx_unlock(&pic_list_lock);
+
+	free(pic, M_INTRNG);
+}
+#endif
+/*
+ *  Register interrupt controller.
+ */
+int
+arm_pic_register(device_t dev, intptr_t xref)
+{
+	struct arm_pic *pic;
+
+	pic = pic_create(dev, xref);
+	if (pic == NULL)
+		return (ENOMEM);
+	if (pic->pic_dev != dev)
+		return (EINVAL);	/* XXX it could be many things. */
+
+	debugf("PIC %p registered for %s <xref %x>\n", pic,
+	    device_get_nameunit(dev), xref);
+	return (0);
+}
+
+/*
+ *  Unregister interrupt controller.
+ */
+int
+arm_pic_unregister(device_t dev, intptr_t xref)
+{
+
+	panic("%s: not implemented", __func__);
+}
+
+/*
+ *  Mark interrupt controller (itself) as a root one.
+ *
+ *  Note that only an interrupt controller can really know its position
+ *  in interrupt controller's tree. So root PIC must claim itself as a root.
+ *
+ *  In FDT case, according to ePAPR approved version 1.1 from 08 April 2011,
+ *  page 30:
+ *    "The root of the interrupt tree is determined when traversal
+ *     of the interrupt tree reaches an interrupt controller node without
+ *     an interrupts property and thus no explicit interrupt parent."
+ */
+int
+arm_pic_claim_root(device_t dev, intptr_t xref, arm_irq_filter_t *filter,
+    void *arg, u_int ipicount)
+{
+	int error;
+	u_int rootirq;
+
+	if (pic_lookup(dev, xref) == NULL) {
+		device_printf(dev, "not registered\n");
+		return (EINVAL);
+	}
+	if (filter == NULL) {
+		device_printf(dev, "filter missing\n");
+		return (EINVAL);
+	}
+
+	/*
+	 * Only one interrupt controllers could be on the root for now.
+	 * Note that we further suppose that there is not threaded interrupt
+	 * routine (handler) on the root. See arm_irq_handler().
+	 */
+	if (irq_root_dev != NULL) {
+		device_printf(dev, "another root already set\n");
+		return (EBUSY);
+	}
+
+	rootirq = arm_namespace_map_irq(device_get_parent(dev), 0, 0);
+	if (rootirq == IRQ_INVALID) {
+		device_printf(dev, "failed to map an irq for the root pic\n");
+		return (ENOMEM);
+	}
+
+        /* Create the isrc. */
+	irq_root_isrc = isrc_lookup(rootirq);
+
+        /* XXX "register" with the PIC.  We are the "pic" here, so fake it. */
+	irq_root_isrc->isrc_flags |= ARM_ISRCF_REGISTERED;
+
+	error = arm_irq_add_handler(device_get_parent(dev), 
+		(void*)filter, NULL, arg, rootirq, INTR_TYPE_CLK, NULL);
+	if (error != 0) {
+		device_printf(dev, "failed to install root pic handler\n");
+		return (error);
+	}
+	irq_root_dev = dev;
+	irq_root_filter = filter;
+	irq_root_arg = arg;
+	irq_root_ipicount = ipicount;
+
+	debugf("irq root set to %s\n", device_get_nameunit(dev));
+	return (0);
+}
+
+int
+arm_irq_add_handler(device_t dev, driver_filter_t filt, driver_intr_t hand,
+    void *arg, u_int irq, int flags, void **cookiep)
+{
+	const char *name;
+	struct arm_irqsrc *isrc;
+	int error;
+
+	name = device_get_nameunit(dev);
+
+#ifdef INTR_SOLO
+	/*
+	 * Standard handling is done thru MI interrupt framework. However,
+	 * some interrupts could request solely own special handling. This
+	 * non standard handling can be used for interrupt controllers without
+	 * handler (filter only), so in case that interrupt controllers are
+	 * chained, MI interrupt framework is called only in leaf controller.
+	 *
+	 * Note that root interrupt controller routine is served as well,
+	 * however in arm_irq_handler(), i.e. main system dispatch routine.
+	 */
+	if (flags & INTR_SOLO && hand != NULL) {
+		debugf("irq %u cannot solo on %s\n", irq, name);
+		return (EINVAL);
+	}
+#endif
+
+	isrc = isrc_lookup(irq);
+	if (isrc == NULL) {
+		debugf("irq %u without source on %s\n", irq, name);
+		return (EINVAL);
+	}
+
+	error = isrc_register(isrc);
+	if (error != 0) {
+		debugf("irq %u map error %d on %s\n", irq, error, name);
+		return (error);
+	}
+
+#ifdef INTR_SOLO
+	if (flags & INTR_SOLO) {
+		error = iscr_setup_filter(isrc, name, (arm_irq_filter_t *)filt,
+		    arg, cookiep);
+		debugf("irq %u setup filter error %d on %s\n", irq, error,
+		    name);
+	} else
+#endif
+		{
+		error = isrc_add_handler(isrc, name, filt, hand, arg, flags,
+		    cookiep);
+		debugf("irq %u add handler error %d on %s\n", irq, error, name);
+	}
+	if (error != 0)
+		return (error);
+
+	mtx_lock(&isrc_table_lock);
+	isrc->isrc_handlers++;
+	if (isrc->isrc_handlers == 1) {
+		PIC_ENABLE_INTR(isrc->isrc_dev, isrc);
+		PIC_ENABLE_SOURCE(isrc->isrc_dev, isrc);
+	}
+	mtx_unlock(&isrc_table_lock);
+	return (0);
+}
+
+int
+arm_irq_remove_handler(device_t dev, u_int irq, void *cookie)
+{
+	struct arm_irqsrc *isrc;
+	int error;
+
+	isrc = isrc_lookup(irq);
+	if (isrc == NULL || isrc->isrc_handlers == 0)
+		return (EINVAL);
+
+	if (isrc->isrc_filter != NULL) {
+		if (isrc != cookie)
+			return (EINVAL);
+
+		mtx_lock(&isrc_table_lock);
+		isrc->isrc_filter = NULL;
+		isrc->isrc_arg = NULL;
+		isrc->isrc_handlers = 0;
+		PIC_DISABLE_SOURCE(isrc->isrc_dev, isrc);
+		PIC_DISABLE_INTR(isrc->isrc_dev, isrc);
+		isrc_update_name(isrc, NULL);
+		mtx_unlock(&isrc_table_lock);
+		return (0);
+	}
+
+	if (isrc != intr_handler_source(cookie))
+		return (EINVAL);
+
+	error = intr_event_remove_handler(cookie);
+	if (error == 0) {
+		mtx_lock(&isrc_table_lock);
+		isrc->isrc_handlers--;
+		if (isrc->isrc_handlers == 0) {
+			PIC_DISABLE_SOURCE(isrc->isrc_dev, isrc);
+			PIC_DISABLE_INTR(isrc->isrc_dev, isrc);
+		}
+		intrcnt_updatename(isrc);
+		mtx_unlock(&isrc_table_lock);
+	}
+	return (error);
+}
+
+int
+arm_irq_config(u_int irq, enum intr_trigger trig, enum intr_polarity pol)
+{
+	struct arm_irqsrc *isrc;
+
+	isrc = isrc_lookup(irq);
+	if (isrc == NULL)
+		return (EINVAL);
+
+	if (isrc->isrc_handlers != 0)
+		return (EBUSY);	/* interrrupt is enabled (active) */
+
+	/*
+	 * Once an interrupt is enabled, we do not change its configuration.
+	 * A controller PIC_ENABLE_INTR() method is called when an interrupt
+	 * is going to be enabled. In this method, a controller should setup
+	 * the interrupt according to saved configuration parameters.
+	 */
+	isrc->isrc_trig = trig;
+	isrc->isrc_pol = pol;
+
+	return (0);
+}
+
+int
+arm_irq_describe(u_int irq, void *cookie, const char *descr)
+{
+	struct arm_irqsrc *isrc;
+	int error;
+
+	isrc = isrc_lookup(irq);
+	if (isrc == NULL || isrc->isrc_handlers == 0)
+		return (EINVAL);
+
+	if (isrc->isrc_filter != NULL) {
+		if (isrc != cookie)
+			return (EINVAL);
+
+		mtx_lock(&isrc_table_lock);
+		isrc_update_name(isrc, descr);
+		mtx_unlock(&isrc_table_lock);
+		return (0);
+	}
+
+	error = intr_event_describe_handler(isrc->isrc_event, cookie, descr);
+	if (error == 0) {
+		mtx_lock(&isrc_table_lock);
+		intrcnt_updatename(isrc);
+		mtx_unlock(&isrc_table_lock);
+	}
+	return (error);
+}
+
+#ifdef SMP
+int
+arm_irq_bind(u_int irq, int cpu)
+{
+	struct arm_irqsrc *isrc;
+
+	isrc = isrc_lookup(irq);
+	if (isrc == NULL || isrc->isrc_handlers == 0)
+		return (EINVAL);
+
+	if (isrc->isrc_filter != NULL)
+		return (arm_isrc_assign_cpu(isrc, cpu));
+
+	return (intr_event_bind(isrc->isrc_event, cpu));
+}
+
+/*
+ * Return the CPU that the next interrupt source should use.
+ * For now just returns the next CPU according to round-robin.
+ */
+u_int
+arm_irq_next_cpu(u_int last_cpu, cpuset_t *cpumask)
+{
+
+	if (!irq_assign_cpu || mp_ncpus == 1)
+		return (PCPU_GET(cpuid));
+
+	do {
+		last_cpu++;
+		if (last_cpu > mp_maxid)
+			last_cpu = 0;
+	} while (!CPU_ISSET(last_cpu, cpumask));
+	return (last_cpu);
+}
+
+/*
+ *  Distribute all the interrupt sources among the available
+ *  CPUs once the AP's have been launched.
+ */
+static void
+arm_irq_shuffle(void *arg __unused)
+{
+	struct arm_irqsrc *isrc;
+	u_int i;
+
+	if (mp_ncpus == 1)
+		return;
+
+	mtx_lock(&isrc_table_lock);
+	irq_assign_cpu = TRUE;
+	for (i = 0; i < NIRQ; i++) {
+		isrc = irq_sources[i];
+		if (isrc == NULL || isrc->isrc_handlers == 0 ||
+		    isrc->isrc_flags & ARM_ISRCF_PERCPU)
+			continue;
+
+		if (isrc->isrc_event != NULL &&
+		    isrc->isrc_flags & ARM_ISRCF_BOUND &&
+		    isrc->isrc_event->ie_cpu != CPU_FFS(&isrc->isrc_cpu) - 1)
+			panic("%s: CPU inconsistency", __func__);
+
+		if ((isrc->isrc_flags & ARM_ISRCF_BOUND) == 0)
+			CPU_ZERO(&isrc->isrc_cpu); /* start again */
+
+		/*
+		 * We are in wicked position here if the following call fails
+		 * for bound ISRC. The best thing we can do is to clear
+		 * isrc_cpu so inconsistency with ie_cpu will be detectable.
+		 */
+		if (PIC_BIND(isrc->isrc_dev, isrc) != 0)
+			CPU_ZERO(&isrc->isrc_cpu);
+	}
+	mtx_unlock(&isrc_table_lock);
+}
+SYSINIT(arm_irq_shuffle, SI_SUB_SMP, SI_ORDER_SECOND, arm_irq_shuffle, NULL);
+
+#else
+u_int
+arm_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask)
+{
+
+	return (PCPU_GET(cpuid));
+}
+#endif
+
+void dosoftints(void);
+void
+dosoftints(void)
+{
+}
+
+/*
+ * arm_irq_memory_barrier()
+ *
+ * Ensure all writes to device memory have reached devices before proceeding.
+ *
+ * This is intended to be called from the post-filter and post-thread routines
+ * of an interrupt controller implementation.  A peripheral device driver should
+ * use bus_space_barrier() if it needs to ensure a write has reached the
+ * hardware for some reason other than clearing interrupt conditions.
+ *
+ * The need for this function arises from the ARM weak memory ordering model.
+ * Writes to locations mapped with the Device attribute bypass any caches, but
+ * are buffered.  Multiple writes to the same device will be observed by that
+ * device in the order issued by the cpu.  Writes to different devices may
+ * appear at those devices in a different order than issued by the cpu.  That
+ * is, if the cpu writes to device A then device B, the write to device B could
+ * complete before the write to device A.
+ *
+ * Consider a typical device interrupt handler which services the interrupt and
+ * writes to a device status-acknowledge register to clear the interrupt before
+ * returning.  That write is posted to the L2 controller which "immediately"
+ * places it in a store buffer and automatically drains that buffer.  This can
+ * be less immediate than you'd think... There may be no free slots in the store
+ * buffers, so an existing buffer has to be drained first to make room.  The
+ * target bus may be busy with other traffic (such as DMA for various devices),
+ * delaying the drain of the store buffer for some indeterminate time.  While
+ * all this delay is happening, execution proceeds on the CPU, unwinding its way
+ * out of the interrupt call stack to the point where the interrupt driver code
+ * is ready to EOI and unmask the interrupt.  The interrupt controller may be
+ * accessed via a faster bus than the hardware whose handler just ran; the write
+ * to unmask and EOI the interrupt may complete quickly while the device write
+ * to ack and clear the interrupt source is still lingering in a store buffer
+ * waiting for access to a slower bus.  With the interrupt unmasked at the
+ * interrupt controller but still active at the device, as soon as interrupts
+ * are enabled on the core the device re-interrupts immediately: now you've got
+ * a spurious interrupt on your hands.
+ *
+ * The right way to fix this problem is for every device driver to use the
+ * proper bus_space_barrier() calls in its interrupt handler.  For ARM a single
+ * barrier call at the end of the handler would work.  This would have to be
+ * done to every driver in the system, not just arm-specific drivers.
+ *
+ * Another potential fix is to map all device memory as Strongly-Ordered rather
+ * than Device memory, which takes the store buffers out of the picture.  This
+ * has a pretty big impact on overall system performance, because each strongly
+ * ordered memory access causes all L2 store buffers to be drained.
+ *
+ * A compromise solution is to have the interrupt controller implementation call
+ * this function to establish a barrier between writes to the interrupt-source
+ * device and writes to the interrupt controller device.
+ *
+ * This takes the interrupt number as an argument, and currently doesn't use it.
+ * The plan is that maybe some day there is a way to flag certain interrupts as
+ * "memory barrier safe" and we can avoid this overhead with them.
+ */
+void
+arm_irq_memory_barrier(uintptr_t irq)
+{
+
+	dsb();
+	cpu_l2cache_drain_writebuf();
+}
+
+#ifdef SMP
+/*
+ *  Lookup IPI source.
+ */
+static struct arm_irqsrc *
+arm_ipi_lookup(u_int ipi)
+{
+
+	if (ipi >= ARM_IPI_COUNT)
+		panic("%s: no such IPI %u", __func__, ipi);
+
+	return (&ipi_sources[ipi]);
+}
+
+/*
+ *  ARM interrupt controller dispatch function for IPIs. It should
+ *  be called straight from the interrupt controller, when associated
+ *  interrupt source is learned. Or from anybody who has an interrupt
+ *  source mapped.
+ */
+void
+arm_ipi_dispatch(struct arm_irqsrc *isrc, struct trapframe *tf)
+{
+	void *arg;
+
+	KASSERT(isrc != NULL, ("%s: no source", __func__));
+
+	isrc_increment_ipi_count(isrc, PCPU_GET(cpuid));
+
+	/*
+	 * Supply ipi filter with trapframe argument
+	 * if none is registered.
+	 */
+	arg = isrc->isrc_arg != NULL ? isrc->isrc_arg : tf;
+	isrc->isrc_ipifilter(arg);
+}
+
+/*
+ *  Map IPI into interrupt controller.
+ *
+ *  Not SMP coherent.
+ */
+static int
+ipi_map(struct arm_irqsrc *isrc, u_int ipi)
+{
+	boolean_t is_percpu;
+	int error;
+
+	if (ipi >= ARM_IPI_COUNT)
+		panic("%s: no such IPI %u", __func__, ipi);
+
+	KASSERT(irq_root_dev != NULL, ("%s: no root attached", __func__));
+
+	isrc->isrc_type = ARM_ISRCT_NAMESPACE;
+	isrc->isrc_nspc_type = ARM_IRQ_NSPC_IPI;
+	isrc->isrc_nspc_num = ipi_next_num;
+
+	error = PIC_REGISTER(irq_root_dev, isrc, &is_percpu);
+
+	debugf("ipi %u mapped to %u on %s - error %d\n", ipi, ipi_next_num,
+	    device_get_nameunit(irq_root_dev), error);
+
+	if (error == 0) {
+		isrc->isrc_dev = irq_root_dev;
+		ipi_next_num++;
+	}
+	return (error);
+}
+
+/*
+ *  Setup IPI handler to interrupt source.
+ *
+ *  Note that there could be more ways how to send and receive IPIs
+ *  on a platform like fast interrupts for example. In that case,
+ *  one can call this function with ASIF_NOALLOC flag set and then
+ *  call arm_ipi_dispatch() when appropriate.
+ *
+ *  Not SMP coherent.
+ */
+int
+arm_ipi_set_handler(u_int ipi, const char *name, arm_ipi_filter_t *filter,
+    void *arg, u_int flags)
+{
+	struct arm_irqsrc *isrc;
+	int error;
+
+	if (filter == NULL)
+		return(EINVAL);
+
+	isrc = arm_ipi_lookup(ipi);
+	if (isrc->isrc_ipifilter != NULL)
+		return (EEXIST);
+
+	if ((flags & AISHF_NOALLOC) == 0) {
+		error = ipi_map(isrc, ipi);
+		if (error != 0)
+			return (error);
+	}
+
+	isrc->isrc_ipifilter = filter;
+	isrc->isrc_arg = arg;
+	isrc->isrc_handlers = 1;
+	isrc_setup_ipi_counters(isrc, name);
+
+	if (isrc->isrc_dev != NULL) {
+		mtx_lock(&isrc_table_lock);
+		PIC_ENABLE_INTR(isrc->isrc_dev, isrc);
+		PIC_ENABLE_SOURCE(isrc->isrc_dev, isrc);
+		mtx_unlock(&isrc_table_lock);
+	}
+	return (0);
+}
+
+/*
+ *  Send IPI thru interrupt controller.
+ */
+void
+pic_ipi_send(cpuset_t cpus, u_int ipi)
+{
+	struct arm_irqsrc *isrc;
+
+	isrc = arm_ipi_lookup(ipi);
+
+	KASSERT(irq_root_dev != NULL, ("%s: no root attached", __func__));
+	PIC_IPI_SEND(irq_root_dev, isrc, cpus);
+}
+
+/*
+ *  Init interrupt controller on another CPU.
+ */
+void
+arm_pic_init_secondary(void)
+{
+
+	/*
+	 * QQQ: Only root PIC is aware of other CPUs ???
+	 */
+	KASSERT(irq_root_dev != NULL, ("%s: no root attached", __func__));
+
+	//mtx_lock(&isrc_table_lock);
+	PIC_INIT_SECONDARY(irq_root_dev);
+	//mtx_unlock(&isrc_table_lock);
+}
+#endif
+
+#ifdef DDB
+DB_SHOW_COMMAND(irqs, db_show_irqs)
+{
+	u_int i, irqsum;
+	struct arm_irqsrc *isrc;
+
+#ifdef SMP
+	for (i = 0; i <= mp_maxid; i++) {
+		struct pcpu *pc;
+		u_int ipi, ipisum;
+
+		pc = pcpu_find(i);
+		if (pc != NULL) {
+			for (ipisum = 0, ipi = 0; ipi < ARM_IPI_COUNT; ipi++) {
+				isrc = arm_ipi_lookup(ipi);
+				if (isrc->isrc_count != NULL)
+					ipisum += isrc->isrc_count[i];
+			}
+			printf ("cpu%u: total %u ipis %u\n", i,
+			    pc->pc_cnt.v_intr, ipisum);
+		}
+	}
+	db_printf("\n");
+#endif
+
+	for (irqsum = 0, i = 0; i < NIRQ; i++) {
+		isrc = irq_sources[i];
+		if (isrc == NULL)
+			continue;
+
+		db_printf("irq%-3u <%s>: cpu %02lx%s cnt %lu\n", i,
+		    isrc->isrc_name, isrc->isrc_cpu.__bits[0],
+		    isrc->isrc_flags & ARM_ISRCF_BOUND ? " (bound)" : "",
+		    isrc->isrc_count[0]);
+		irqsum += isrc->isrc_count[0];
+	}
+	db_printf("irq total %u\n", irqsum);
+}
+#endif
diff --git a/sys/arm/arm/mp_machdep.c b/sys/arm/arm/mp_machdep.c
index 083e62efb8f4..5d1043fde6eb 100644
--- a/sys/arm/arm/mp_machdep.c
+++ b/sys/arm/arm/mp_machdep.c
@@ -74,7 +74,9 @@ volatile int mp_naps;
 /* Set to 1 once we're ready to let the APs out of the pen. */
 volatile int aps_ready = 0;
 
+#ifndef ARM_INTRNG
 static int ipi_handler(void *arg);
+#endif
 void set_stackptrs(int cpu);
 
 /* Temporary variables for init_secondary()  */
@@ -134,7 +136,6 @@ cpu_mp_start(void)
 	else
 		for (i = 1; i < mp_ncpus; i++)
 			CPU_SET(i, &all_cpus);
-
 }
 
 /* Introduce rest of cores to the world */
@@ -150,7 +151,9 @@ init_secondary(int cpu)
 {
 	struct pcpu *pc;
 	uint32_t loop_counter;
+#ifndef ARM_INTRNG
 	int start = 0, end = 0;
+#endif
 
 #ifdef ARM_NEW_PMAP
 	pmap_set_tex();
@@ -211,11 +214,12 @@ init_secondary(int cpu)
 
 	mtx_unlock_spin(&ap_boot_mtx);
 
+#ifndef ARM_INTRNG
 	/* Enable ipi */
 #ifdef IPI_IRQ_START
 	start = IPI_IRQ_START;
 #ifdef IPI_IRQ_END
-  	end = IPI_IRQ_END;
+	end = IPI_IRQ_END;
 #else
 	end = IPI_IRQ_START;
 #endif
@@ -223,6 +227,7 @@ init_secondary(int cpu)
 
 	for (int i = start; i <= end; i++)
 		arm_unmask_irq(i);
+#endif /* INTRNG */
 	enable_interrupts(PSR_I);
 
 	loop_counter = 0;
@@ -245,6 +250,108 @@ init_secondary(int cpu)
 	/* NOTREACHED */
 }
 
+#ifdef ARM_INTRNG
+static void
+ipi_rendezvous(void *dummy __unused)
+{
+
+	CTR0(KTR_SMP, "IPI_RENDEZVOUS");
+	smp_rendezvous_action();
+}
+
+static void
+ipi_ast(void *dummy __unused)
+{
+
+	CTR0(KTR_SMP, "IPI_AST");
+}
+
+static void
+ipi_stop(void *dummy __unused)
+{
+	u_int cpu;
+
+	/*
+	 * IPI_STOP_HARD is mapped to IPI_STOP.
+	 */
+	CTR0(KTR_SMP, "IPI_STOP or IPI_STOP_HARD");
+
+	cpu = PCPU_GET(cpuid);
+	savectx(&stoppcbs[cpu]);
+
+	/*
+	 * CPUs are stopped when entering the debugger and at
+	 * system shutdown, both events which can precede a
+	 * panic dump.  For the dump to be correct, all caches
+	 * must be flushed and invalidated, but on ARM there's
+	 * no way to broadcast a wbinv_all to other cores.
+	 * Instead, we have each core do the local wbinv_all as
+	 * part of stopping the core.  The core requesting the
+	 * stop will do the l2 cache flush after all other cores
+	 * have done their l1 flushes and stopped.
+	 */
+	cpu_idcache_wbinv_all();
+
+	/* Indicate we are stopped */
+	CPU_SET_ATOMIC(cpu, &stopped_cpus);
+
+	/* Wait for restart */
+	while (!CPU_ISSET(cpu, &started_cpus))
+		cpu_spinwait();
+
+	CPU_CLR_ATOMIC(cpu, &started_cpus);
+	CPU_CLR_ATOMIC(cpu, &stopped_cpus);
+	CTR0(KTR_SMP, "IPI_STOP (restart)");
+}
+
+static void
+ipi_preempt(void *arg)
+{
+	struct trapframe *oldframe;
+	struct thread *td;
+
+	critical_enter();
+	td = curthread;
+	td->td_intr_nesting_level++;
+	oldframe = td->td_intr_frame;
+	td->td_intr_frame = (struct trapframe *)arg;
+
+	CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__);
+	sched_preempt(td);
+
+	td->td_intr_frame = oldframe;
+	td->td_intr_nesting_level--;
+	critical_exit();
+}
+
+static void
+ipi_hardclock(void *arg)
+{
+	struct trapframe *oldframe;
+	struct thread *td;
+
+	critical_enter();
+	td = curthread;
+	td->td_intr_nesting_level++;
+	oldframe = td->td_intr_frame;
+	td->td_intr_frame = (struct trapframe *)arg;
+
+	CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__);
+	hardclockintr();
+
+	td->td_intr_frame = oldframe;
+	td->td_intr_nesting_level--;
+	critical_exit();
+}
+
+static void
+ipi_tlb(void *dummy __unused)
+{
+
+	CTR1(KTR_SMP, "%s: IPI_TLB", __func__);
+	cpufuncs.cf_tlb_flushID();
+}
+#else
 static int
 ipi_handler(void *arg)
 {
@@ -320,15 +427,28 @@ ipi_handler(void *arg)
 
 	return (FILTER_HANDLED);
 }
+#endif
 
 static void
 release_aps(void *dummy __unused)
 {
 	uint32_t loop_counter;
+#ifndef ARM_INTRNG
 	int start = 0, end = 0;
+#endif
 
 	if (mp_ncpus == 1)
 		return;
+
+#ifdef ARM_INTRNG
+	arm_ipi_set_handler(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL, 0);
+	arm_ipi_set_handler(IPI_AST, "ast", ipi_ast, NULL, 0);
+	arm_ipi_set_handler(IPI_STOP, "stop", ipi_stop, NULL, 0);
+	arm_ipi_set_handler(IPI_PREEMPT, "preempt", ipi_preempt, NULL, 0);
+	arm_ipi_set_handler(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL, 0);
+	arm_ipi_set_handler(IPI_TLB, "tlb", ipi_tlb, NULL, 0);
+
+#else
 #ifdef IPI_IRQ_START
 	start = IPI_IRQ_START;
 #ifdef IPI_IRQ_END
@@ -353,6 +473,7 @@ release_aps(void *dummy __unused)
 		/* Enable ipi */
 		arm_unmask_irq(i);
 	}
+#endif
 	atomic_store_rel_int(&aps_ready, 1);
 	/* Wake the other threads up */
 #if __ARM_ARCH >= 7
diff --git a/sys/arm/arm/nexus.c b/sys/arm/arm/nexus.c
index def6c1ad3cc4..e2deeb0e29fd 100644
--- a/sys/arm/arm/nexus.c
+++ b/sys/arm/arm/nexus.c
@@ -85,8 +85,17 @@ static	struct resource *nexus_alloc_resource(device_t, device_t, int, int *,
     u_long, u_long, u_long, u_int);
 static	int nexus_activate_resource(device_t, device_t, int, int,
     struct resource *);
+#ifdef ARM_INTRNG
+#ifdef SMP
+static	int nexus_bind_intr(device_t, device_t, struct resource *, int);
+#endif
+#endif
 static int nexus_config_intr(device_t dev, int irq, enum intr_trigger trig,
     enum intr_polarity pol);
+#ifdef ARM_INTRNG
+static	int nexus_describe_intr(device_t dev, device_t child,
+    struct resource *irq, void *cookie, const char *descr);
+#endif
 static	int nexus_deactivate_resource(device_t, device_t, int, int,
     struct resource *);
 static int nexus_release_resource(device_t, device_t, int, int,
@@ -115,6 +124,12 @@ static device_method_t nexus_methods[] = {
 	DEVMETHOD(bus_release_resource,	nexus_release_resource),
 	DEVMETHOD(bus_setup_intr,	nexus_setup_intr),
 	DEVMETHOD(bus_teardown_intr,	nexus_teardown_intr),
+#ifdef ARM_INTRNG
+	DEVMETHOD(bus_describe_intr,	nexus_describe_intr),
+#ifdef SMP
+	DEVMETHOD(bus_bind_intr,	nexus_bind_intr),
+#endif
+#endif
 #ifdef FDT
 	DEVMETHOD(ofw_bus_map_intr,	nexus_ofw_map_intr),
 #endif
@@ -251,9 +266,12 @@ nexus_config_intr(device_t dev, int irq, enum intr_trigger trig,
 {
 	int ret = ENODEV;
 
+#ifdef ARM_INTRNG
+	ret = arm_irq_config(irq, trig, pol);
+#else
 	if (arm_config_irq)
 		ret = (*arm_config_irq)(irq, trig, pol);
-
+#endif
 	return (ret);
 }
 
@@ -267,9 +285,14 @@ nexus_setup_intr(device_t dev, device_t child, struct resource *res, int flags,
 		flags |= INTR_EXCL;
 
 	for (irq = rman_get_start(res); irq <= rman_get_end(res); irq++) {
+#ifdef ARM_INTRNG
+		arm_irq_add_handler(child, filt, intr, arg, irq, flags,
+		    cookiep);
+#else
 		arm_setup_irqhandler(device_get_nameunit(child),
 		    filt, intr, arg, irq, flags, cookiep);
 		arm_unmask_irq(irq);
+#endif
 	}
 	return (0);
 }
@@ -278,9 +301,31 @@ static int
 nexus_teardown_intr(device_t dev, device_t child, struct resource *r, void *ih)
 {
 
+#ifdef ARM_INTRNG
+	return (arm_irq_remove_handler(child, rman_get_start(r), ih));
+#else
 	return (arm_remove_irqhandler(rman_get_start(r), ih));
+#endif
 }
 
+#ifdef ARM_INTRNG
+static int
+nexus_describe_intr(device_t dev, device_t child, struct resource *irq,
+    void *cookie, const char *descr)
+{
+
+	return (arm_irq_describe(rman_get_start(irq), cookie, descr));
+}
+
+#ifdef SMP
+static int
+nexus_bind_intr(device_t dev, device_t child, struct resource *irq, int cpu)
+{
+
+	return (arm_irq_bind(rman_get_start(irq), cpu));
+}
+#endif
+#endif
 
 static int
 nexus_activate_resource(device_t bus, device_t child, int type, int rid,
diff --git a/sys/arm/arm/pic_if.m b/sys/arm/arm/pic_if.m
new file mode 100644
index 000000000000..d3002ea49561
--- /dev/null
+++ b/sys/arm/arm/pic_if.m
@@ -0,0 +1,124 @@
+#-
+# Copyright (c) 2012 Jakub Wojciech Klama <jceel@FreeBSD.org>
+# Copyright (c) 2015 Svatopluk Kraus
+# Copyright (c) 2015 Michal Meloun
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+#
+
+#include <sys/bus.h>
+#include <sys/cpuset.h>
+#include <machine/frame.h>
+#include <machine/intr.h>
+
+INTERFACE pic;
+
+CODE {
+	static int null_pic_bind(device_t dev, struct arm_irqsrc *isrc)
+	{
+		return (EOPNOTSUPP);
+	}
+
+	static void null_pic_disable_intr(device_t dev, struct arm_irqsrc *isrc)
+	{
+		return;
+	}
+
+	static void null_pic_enable_intr(device_t dev, struct arm_irqsrc *isrc)
+	{
+		return;
+	}
+
+	static void null_pic_init_secondary(device_t dev)
+	{
+		return;
+	}
+
+	static void null_pic_ipi_send(device_t dev, cpuset_t cpus, u_int ipi)
+	{
+		return;
+	}
+};
+
+METHOD int register {
+	device_t		dev;
+	struct arm_irqsrc	*isrc;
+	boolean_t		*is_percpu;
+};
+
+METHOD int unregister {
+	device_t		dev;
+	struct arm_irqsrc	*isrc;
+};
+
+METHOD void disable_intr {
+	device_t		dev;
+	struct arm_irqsrc	*isrc;
+} DEFAULT null_pic_disable_intr;
+
+METHOD void disable_source {
+	device_t		dev;
+	struct arm_irqsrc	*isrc;
+};
+
+METHOD void enable_source {
+	device_t		dev;
+	struct arm_irqsrc	*isrc;
+};
+
+METHOD void enable_intr {
+	device_t		dev;
+	struct arm_irqsrc	*isrc;
+} DEFAULT null_pic_enable_intr;
+
+METHOD void pre_ithread {
+	device_t		dev;
+	struct arm_irqsrc	*isrc;
+};
+
+METHOD void post_ithread {
+	device_t		dev;
+	struct arm_irqsrc	*isrc;
+};
+
+METHOD void post_filter {
+	device_t		dev;
+	struct arm_irqsrc	*isrc;
+};
+
+METHOD int bind {
+	device_t		dev;
+	struct arm_irqsrc	*isrc;
+} DEFAULT null_pic_bind;
+
+METHOD void init_secondary {
+	device_t	dev;
+} DEFAULT null_pic_init_secondary;
+
+METHOD void ipi_send {
+	device_t		dev;
+	struct arm_irqsrc	*isrc;
+	cpuset_t		cpus;
+} DEFAULT null_pic_ipi_send;
diff --git a/sys/arm/include/fdt.h b/sys/arm/include/fdt.h
index c1b785e8c1f8..e8302d6a7b3b 100644
--- a/sys/arm/include/fdt.h
+++ b/sys/arm/include/fdt.h
@@ -34,12 +34,16 @@
 
 #include <machine/bus.h>
 
+#ifndef INTRNG
+
 /* Max interrupt number */
 #define FDT_INTR_MAX	NIRQ
 
 /* Map phandle/intpin pair to global IRQ number */
 #define	FDT_MAP_IRQ(node, pin)	(pin)
 
+#endif
+
 /*
  * Bus space tag. XXX endianess info needs to be derived from the blob.
  */
diff --git a/sys/arm/include/intr.h b/sys/arm/include/intr.h
index 6a46f00766a0..ed075dfd39c1 100644
--- a/sys/arm/include/intr.h
+++ b/sys/arm/include/intr.h
@@ -43,6 +43,102 @@
 #include <dev/ofw/openfirm.h>
 #endif
 
+#ifdef ARM_INTRNG
+
+#ifndef NIRQ
+#define	NIRQ		1024	/* XXX - It should be an option. */
+#endif
+
+#ifdef notyet
+#define	INTR_SOLO	INTR_MD1
+typedef int arm_irq_filter_t(void *arg, struct trapframe *tf);
+#else
+typedef int arm_irq_filter_t(void *arg);
+#endif
+
+#define ARM_ISRC_NAMELEN	(MAXCOMLEN + 1)
+
+typedef void arm_ipi_filter_t(void *arg);
+
+enum arm_isrc_type {
+	ARM_ISRCT_NAMESPACE,
+	ARM_ISRCT_FDT
+};
+
+#define ARM_ISRCF_REGISTERED	0x01	/* registered in a controller */
+#define ARM_ISRCF_PERCPU	0x02	/* per CPU interrupt */
+#define ARM_ISRCF_BOUND		0x04	/* bound to a CPU */
+
+/* Interrupt source definition. */
+struct arm_irqsrc {
+	device_t		isrc_dev;	/* where isrc is mapped */
+	intptr_t		isrc_xref;	/* device reference key */
+	uintptr_t		isrc_data;	/* device data for isrc */
+	u_int			isrc_irq;	/* unique identificator */
+	enum arm_isrc_type	isrc_type;	/* how is isrc decribed */
+	u_int			isrc_flags;
+	char			isrc_name[ARM_ISRC_NAMELEN];
+	uint16_t		isrc_nspc_type;
+	uint16_t		isrc_nspc_num;
+	enum intr_trigger	isrc_trig;
+	enum intr_polarity	isrc_pol;
+	cpuset_t		isrc_cpu;	/* on which CPUs is enabled */
+	u_int			isrc_index;
+	u_long *		isrc_count;
+	u_int			isrc_handlers;
+	struct intr_event *	isrc_event;
+	arm_irq_filter_t *	isrc_filter;
+	arm_ipi_filter_t *	isrc_ipifilter;
+	void *			isrc_arg;
+#ifdef FDT
+	u_int			isrc_ncells;
+	pcell_t			isrc_cells[];	/* leave it last */
+#endif
+};
+
+void arm_irq_set_name(struct arm_irqsrc *isrc, const char *fmt, ...)
+    __printflike(2, 3);
+
+void arm_irq_dispatch(struct arm_irqsrc *isrc, struct trapframe *tf);
+
+#define ARM_IRQ_NSPC_NONE	0
+#define ARM_IRQ_NSPC_PLAIN	1
+#define ARM_IRQ_NSPC_IRQ	2
+#define ARM_IRQ_NSPC_IPI	3
+
+u_int arm_namespace_map_irq(device_t dev, uint16_t type, uint16_t num);
+#ifdef FDT
+u_int arm_fdt_map_irq(phandle_t, pcell_t *, u_int);
+#endif
+
+int arm_pic_register(device_t dev, intptr_t xref);
+int arm_pic_unregister(device_t dev, intptr_t xref);
+int arm_pic_claim_root(device_t dev, intptr_t xref, arm_irq_filter_t *filter,
+    void *arg, u_int ipicount);
+
+int arm_irq_add_handler(device_t dev, driver_filter_t, driver_intr_t, void *,
+    u_int, int, void **);
+int arm_irq_remove_handler(device_t dev, u_int, void *);
+int arm_irq_config(u_int, enum intr_trigger, enum intr_polarity);
+int arm_irq_describe(u_int, void *, const char *);
+
+u_int arm_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask);
+
+#ifdef SMP
+int arm_irq_bind(u_int, int);
+
+void arm_ipi_dispatch(struct arm_irqsrc *isrc, struct trapframe *tf);
+
+#define AISHF_NOALLOC	0x0001
+
+int arm_ipi_set_handler(u_int ipi, const char *name, arm_ipi_filter_t *filter,
+    void *arg, u_int flags);
+
+void arm_pic_init_secondary(void);
+#endif
+
+#else /* ARM_INTRNG */
+
 /* XXX move to std.* files? */
 #ifdef CPU_XSCALE_81342
 #define NIRQ		128
@@ -71,7 +167,6 @@
 #define NIRQ		32
 #endif
 
-
 int arm_get_next_irq(int);
 void arm_mask_irq(uintptr_t);
 void arm_unmask_irq(uintptr_t);
@@ -83,8 +178,6 @@ extern void (*arm_post_filter)(void *);
 extern int (*arm_config_irq)(int irq, enum intr_trigger trig,
     enum intr_polarity pol);
 
-void arm_irq_memory_barrier(uintptr_t);
-
 void arm_pic_init_secondary(void);
 int  gic_decode_fdt(uint32_t iparentnode, uint32_t *intrcells, int *interrupt,
     int *trig, int *pol);
@@ -93,4 +186,8 @@ int  gic_decode_fdt(uint32_t iparentnode, uint32_t *intrcells, int *interrupt,
 int arm_fdt_map_irq(phandle_t, pcell_t *, int);
 #endif
 
+#endif /* ARM_INTRNG */
+
+void arm_irq_memory_barrier(uintptr_t);
+
 #endif	/* _MACHINE_INTR_H */
diff --git a/sys/arm/include/smp.h b/sys/arm/include/smp.h
index bce8b4f342c0..1c90431a853f 100644
--- a/sys/arm/include/smp.h
+++ b/sys/arm/include/smp.h
@@ -6,6 +6,19 @@
 #include <sys/_cpuset.h>
 #include <machine/pcb.h>
 
+#ifdef ARM_INTRNG
+enum {
+	IPI_AST,
+	IPI_PREEMPT,
+	IPI_RENDEZVOUS,
+	IPI_STOP,
+	IPI_STOP_HARD = IPI_STOP, /* These are synonyms on arm. */
+	IPI_HARDCLOCK,
+	IPI_TLB,
+	IPI_CACHE,
+	ARM_IPI_COUNT
+};
+#else
 #define IPI_AST		0
 #define IPI_PREEMPT	2
 #define IPI_RENDEZVOUS	3
@@ -14,6 +27,7 @@
 #define IPI_HARDCLOCK	6
 #define IPI_TLB		7
 #define IPI_CACHE	8
+#endif /* INTRNG */
 
 void	init_secondary(int cpu);
 void	mpentry(void);
@@ -24,8 +38,10 @@ void	ipi_selected(cpuset_t cpus, u_int ipi);
 
 /* PIC interface */
 void	pic_ipi_send(cpuset_t cpus, u_int ipi);
+#ifndef ARM_INTRNG
 void	pic_ipi_clear(int ipi);
 int	pic_ipi_read(int arg);
+#endif
 
 /* Platform interface */
 void	platform_mp_setmaxid(void);
diff --git a/sys/conf/files.arm b/sys/conf/files.arm
index a2abcc243c54..33838ae146ab 100644
--- a/sys/conf/files.arm
+++ b/sys/conf/files.arm
@@ -41,7 +41,8 @@ arm/arm/gic.c			optional	gic
 arm/arm/identcpu.c		standard
 arm/arm/in_cksum.c		optional	inet | inet6
 arm/arm/in_cksum_arm.S		optional	inet | inet6
-arm/arm/intr.c			standard
+arm/arm/intr.c			optional	!arm_intrng
+arm/arm/intrng.c		optional	arm_intrng
 arm/arm/locore.S		standard	no-obj
 arm/arm/machdep.c		standard
 arm/arm/mem.c			optional	mem
@@ -49,6 +50,7 @@ arm/arm/minidump_machdep.c	optional	mem
 arm/arm/mp_machdep.c		optional	smp
 arm/arm/nexus.c			standard
 arm/arm/physmem.c		standard
+arm/arm/pic_if.m		optional	arm_intrng
 arm/arm/pl190.c			optional	pl190
 arm/arm/pl310.c			optional	pl310
 arm/arm/platform.c		optional	platform
diff --git a/sys/conf/options.arm b/sys/conf/options.arm
index 57cef6a18dc6..aee546a03863 100644
--- a/sys/conf/options.arm
+++ b/sys/conf/options.arm
@@ -1,6 +1,7 @@
 #$FreeBSD$
 ARMV6			opt_global.h
 ARM_CACHE_LOCK_ENABLE	opt_global.h
+ARM_INTRNG		opt_global.h
 ARM_KERN_DIRECTMAP	opt_vm.h
 ARM_L2_PIPT		opt_global.h
 ARM_MANY_BOARD		opt_global.h

From 14f613391579f707498faa0327abbbdbe979a1bb Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Sun, 18 Oct 2015 18:32:22 +0000
Subject: [PATCH 346/380] MFV r289530: 5847 libzfs_diff should check
 zfs_prop_get() return

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Albert Lee <trisk@omniti.com>
Approved by: Dan McDonald <danmcd@omniti.com>
Author: Alexander Eremin <a.eremin@nexenta.com>

illumos/illumos-gate@8430278980a48338e04c7dd52b495b7f1551367a
---
 .../opensolaris/lib/libzfs/common/libzfs_diff.c        | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_diff.c b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_diff.c
index dde329a8226f..bb49eba0f835 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_diff.c
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_diff.c
@@ -21,6 +21,7 @@
 
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
  */
 
 /*
@@ -624,9 +625,12 @@ get_snapshot_names(differ_info_t *di, const char *fromsnap,
 
 		zhp = zfs_open(hdl, di->ds, ZFS_TYPE_FILESYSTEM);
 		while (zhp != NULL) {
-			(void) zfs_prop_get(zhp, ZFS_PROP_ORIGIN,
-			    origin, sizeof (origin), &src, NULL, 0, B_FALSE);
-
+			if (zfs_prop_get(zhp, ZFS_PROP_ORIGIN, origin,
+			    sizeof (origin), &src, NULL, 0, B_FALSE) != 0) {
+				(void) zfs_close(zhp);
+				zhp = NULL;
+				break;
+			}
 			if (strncmp(origin, fromsnap, fsnlen) == 0)
 				break;
 

From fbeda1e06d2489d7521e1a679200cd7ae0f59a8e Mon Sep 17 00:00:00 2001
From: ian <ian@FreeBSD.org>
Date: Sun, 18 Oct 2015 18:39:16 +0000
Subject: [PATCH 347/380] Enable ARM_INTRNG on the pandaboard platform.

Differential Revision:	https://reviews.freebsd.org/D2048
---
 sys/arm/conf/PANDABOARD    | 2 ++
 sys/arm/ti/omap4/std.omap4 | 4 ----
 sys/arm/ti/ti_common.c     | 2 +-
 3 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/sys/arm/conf/PANDABOARD b/sys/arm/conf/PANDABOARD
index d04b81a637fa..fe62e8993655 100644
--- a/sys/arm/conf/PANDABOARD
+++ b/sys/arm/conf/PANDABOARD
@@ -30,6 +30,8 @@ hints		"PANDABOARD.hints"
 include 	"std.armv6"
 include 	"../ti/omap4/pandaboard/std.pandaboard"
 
+options		ARM_INTRNG		# new interrupt framework
+
 options 	HZ=100
 options 	SCHED_ULE		# ULE scheduler
 options 	PLATFORM
diff --git a/sys/arm/ti/omap4/std.omap4 b/sys/arm/ti/omap4/std.omap4
index 27ec5ff4bbe9..02fc8a2d28a5 100644
--- a/sys/arm/ti/omap4/std.omap4
+++ b/sys/arm/ti/omap4/std.omap4
@@ -17,7 +17,3 @@ makeoptions	KERNVIRTADDR=0xc0200000
 options		SOC_OMAP4
 
 options		ARM_L2_PIPT
-
-options		IPI_IRQ_START=0
-options		IPI_IRQ_END=15
-
diff --git a/sys/arm/ti/ti_common.c b/sys/arm/ti/ti_common.c
index 50072f2acfa3..31a6a32eed7a 100644
--- a/sys/arm/ti/ti_common.c
+++ b/sys/arm/ti/ti_common.c
@@ -70,7 +70,7 @@ fdt_aintc_decode_ic(phandle_t node, pcell_t *intr, int *interrupt, int *trig,
 #endif
 
 fdt_pic_decode_t fdt_pic_table[] = {
-#ifdef SOC_OMAP4
+#if defined(SOC_OMAP4) && !defined(ARM_INTRNG)
 	&gic_decode_fdt,
 #endif
 #ifdef SOC_TI_AM335X

From d057275ef4a67049243a83d6dccbd63a22ef9bf5 Mon Sep 17 00:00:00 2001
From: sbruno <sbruno@FreeBSD.org>
Date: Sun, 18 Oct 2015 18:40:11 +0000
Subject: [PATCH 348/380] Add VM_KMEM_SIZE_SCALE=1 as these systems are going
 to have super small amount of RAM, e.g. 16M or 32M

Reviewed by:	adrian
---
 sys/mips/conf/AR933X_BASE | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/sys/mips/conf/AR933X_BASE b/sys/mips/conf/AR933X_BASE
index 6669de668ef4..38f847db12eb 100644
--- a/sys/mips/conf/AR933X_BASE
+++ b/sys/mips/conf/AR933X_BASE
@@ -40,6 +40,9 @@ options 	SCSI_NO_OP_STRINGS
 # .. And no sysctl strings
 options 	NO_SYSCTL_DESCR
 
+# For small memory footprints
+options		VM_KMEM_SIZE_SCALE=1
+
 # Limit IO size
 options 	NBUF=128
 

From 467d89cb6f2ea2fc3628ccea44e2c47f9546151c Mon Sep 17 00:00:00 2001
From: sbruno <sbruno@FreeBSD.org>
Date: Sun, 18 Oct 2015 18:41:30 +0000
Subject: [PATCH 349/380] Remove geom_uncompress from TP-MR3020 config.  Its
 now using root on USB and there's no need for it now.

---
 sys/mips/conf/TP-MR3020 | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/sys/mips/conf/TP-MR3020 b/sys/mips/conf/TP-MR3020
index a19f41bcb070..b388e9bbebb9 100644
--- a/sys/mips/conf/TP-MR3020
+++ b/sys/mips/conf/TP-MR3020
@@ -44,9 +44,6 @@ device	arswitch
 # redboot stuff.
 options	AR71XX_ENV_UBOOT
 
-# uzip - to boot natively from flash
-device	geom_uncompress
-
 # Used for the static uboot partition map
 device	geom_map
 

From 82b76061b617fdfb2755d8ab5bdaed2f707ea818 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Sun, 18 Oct 2015 19:05:56 +0000
Subject: [PATCH 350/380] MFV r289535: 5767 fix several problems with zfs test
 suite

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Christopher Siden <christopher.siden@delphix.com>
Approved by: Gordon Ross <gwr@nexenta.com>
Author: John Wren Kennedy <john.kennedy@delphix.com>

illumos/illumos-gate@52244c0958bdf281ca42932b449f644b4decfdc2

In fact, only unrelated part of that commit is applicable:
8. zpool list -v doesn't print spares
It also doesn't correctly identify log devices.
---
 .../opensolaris/cmd/zpool/zpool_main.c        | 50 ++++++++++++++-----
 1 file changed, 38 insertions(+), 12 deletions(-)

diff --git a/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c b/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c
index a3eabd1e0eb4..52ad81d05988 100644
--- a/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c
+++ b/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c
@@ -2940,6 +2940,9 @@ print_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
 	uint_t c, children;
 	char *vname;
 	boolean_t scripted = cb->cb_scripted;
+	uint64_t islog = B_FALSE;
+	boolean_t haslog = B_FALSE;
+	char *dashes = "%-*s      -      -      -         -      -      -\n";
 
 	verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
 	    (uint64_t **)&vs, &c) == 0);
@@ -2990,24 +2993,47 @@ print_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
 		    ZPOOL_CONFIG_IS_HOLE, &ishole) == 0 && ishole)
 			continue;
 
+		if (nvlist_lookup_uint64(child[c],
+		    ZPOOL_CONFIG_IS_LOG, &islog) == 0 && islog) {
+			haslog = B_TRUE;
+			continue;
+		}
+
 		vname = zpool_vdev_name(g_zfs, zhp, child[c], B_FALSE);
 		print_list_stats(zhp, vname, child[c], cb, depth + 2);
 		free(vname);
 	}
 
-	/*
-	 * Include level 2 ARC devices in iostat output
-	 */
-	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
-	    &child, &children) != 0)
-		return;
-
-	if (children > 0) {
-		(void) printf("%-*s      -      -      -      -      -      "
-		    "-\n", cb->cb_namewidth, "cache");
+	if (haslog == B_TRUE) {
+		/* LINTED E_SEC_PRINTF_VAR_FMT */
+		(void) printf(dashes, cb->cb_namewidth, "log");
 		for (c = 0; c < children; c++) {
-			vname = zpool_vdev_name(g_zfs, zhp, child[c],
-			    B_FALSE);
+			if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
+			    &islog) != 0 || !islog)
+				continue;
+			vname = zpool_vdev_name(g_zfs, zhp, child[c], B_FALSE);
+			print_list_stats(zhp, vname, child[c], cb, depth + 2);
+			free(vname);
+		}
+	}
+
+	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
+	    &child, &children) == 0 && children > 0) {
+		/* LINTED E_SEC_PRINTF_VAR_FMT */
+		(void) printf(dashes, cb->cb_namewidth, "cache");
+		for (c = 0; c < children; c++) {
+			vname = zpool_vdev_name(g_zfs, zhp, child[c], B_FALSE);
+			print_list_stats(zhp, vname, child[c], cb, depth + 2);
+			free(vname);
+		}
+	}
+
+	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES, &child,
+	    &children) == 0 && children > 0) {
+		/* LINTED E_SEC_PRINTF_VAR_FMT */
+		(void) printf(dashes, cb->cb_namewidth, "spare");
+		for (c = 0; c < children; c++) {
+			vname = zpool_vdev_name(g_zfs, zhp, child[c], B_FALSE);
 			print_list_stats(zhp, vname, child[c], cb, depth + 2);
 			free(vname);
 		}

From be375ae276cd9f6a505a43abbf557266b981f0ed Mon Sep 17 00:00:00 2001
From: ian <ian@FreeBSD.org>
Date: Sun, 18 Oct 2015 19:54:11 +0000
Subject: [PATCH 351/380] Enable ARM_INTRNG on IMX6 platforms, and make the
 imx_gpio driver an interrupt controller.

The latter is required for INTRNG, because of the hardware erratum
workaround installed by the linux folks into the imx6 FDT data, which remaps
an ethernet interrupt to the gpio device.  In the non-INTRNG world we
intercept the call to map the interrupt and map it back to the ethernet
hardware (because we don't need linux's workaround), but in the INTRNG world
we lose the hookpoint where that remapping was happening, but we gain the
ability to work the way linux does by having the gpio driver dispatch the
interrupt.
---
 sys/arm/conf/IMX6                    |   2 +
 sys/arm/freescale/imx/imx6_machdep.c |   2 +
 sys/arm/freescale/imx/imx_gpio.c     | 401 +++++++++++++++++++++------
 3 files changed, 319 insertions(+), 86 deletions(-)

diff --git a/sys/arm/conf/IMX6 b/sys/arm/conf/IMX6
index fb293a9d6d6b..cf2f34cdf376 100644
--- a/sys/arm/conf/IMX6
+++ b/sys/arm/conf/IMX6
@@ -22,6 +22,8 @@ ident		IMX6
 include 	"std.armv6"
 include 	"../freescale/imx/std.imx6"
 
+options 	ARM_INTRNG
+
 options 	SOC_IMX6
 
 options 	HZ=500			# Scheduling quantum is 2 milliseconds.
diff --git a/sys/arm/freescale/imx/imx6_machdep.c b/sys/arm/freescale/imx/imx6_machdep.c
index 2322cffab0d5..ce936b69579c 100644
--- a/sys/arm/freescale/imx/imx6_machdep.c
+++ b/sys/arm/freescale/imx/imx6_machdep.c
@@ -58,6 +58,7 @@ struct fdt_fixup_entry fdt_fixup_table[] = {
 
 static uint32_t gpio1_node;
 
+#ifndef ARM_INTRNG
 /*
  * Work around the linux workaround for imx6 erratum 006687, in which some
  * ethernet interrupts don't go to the GPC and thus won't wake the system from
@@ -91,6 +92,7 @@ fdt_pic_decode_t fdt_pic_table[] = {
 	&imx6_decode_fdt,
 	NULL
 };
+#endif
 
 static vm_offset_t
 imx6_lastaddr(platform_t plat)
diff --git a/sys/arm/freescale/imx/imx_gpio.c b/sys/arm/freescale/imx/imx_gpio.c
index e70b710a6c1e..3c10efa1c89b 100644
--- a/sys/arm/freescale/imx/imx_gpio.c
+++ b/sys/arm/freescale/imx/imx_gpio.c
@@ -34,6 +34,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_platform.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bus.h>
@@ -44,8 +46,10 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/gpio.h>
+#include <sys/proc.h>
 
 #include <machine/bus.h>
+#include <machine/intr.h>
 #include <machine/resource.h>
 
 #include <dev/fdt/fdt_common.h>
@@ -56,13 +60,9 @@ __FBSDID("$FreeBSD$");
 
 #include "gpio_if.h"
 
-#define GPIO_LOCK(_sc)		mtx_lock(&(_sc)->sc_mtx)
-#define	GPIO_UNLOCK(_sc)	mtx_unlock(&(_sc)->sc_mtx)
-#define GPIO_LOCK_INIT(_sc)	mtx_init(&_sc->sc_mtx, 			\
-	    device_get_nameunit(_sc->sc_dev), "imx_gpio", MTX_DEF)
-#define GPIO_LOCK_DESTROY(_sc)	mtx_destroy(&_sc->sc_mtx);
-#define GPIO_ASSERT_LOCKED(_sc)	mtx_assert(&_sc->sc_mtx, MA_OWNED);
-#define GPIO_ASSERT_UNLOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_NOTOWNED);
+#ifdef ARM_INTRNG
+#include "pic_if.h"
+#endif
 
 #define	WRITE4(_sc, _r, _v)						\
 	    bus_space_write_4((_sc)->sc_iot, (_sc)->sc_ioh, (_r), (_v))
@@ -95,13 +95,13 @@ struct imx51_gpio_softc {
 	device_t		dev;
 	device_t		sc_busdev;
 	struct mtx		sc_mtx;
-	struct resource		*sc_res[11]; /* 1 x mem, 2 x IRQ, 8 x IRQ */
-	void			*gpio_ih[11]; /* 1 ptr is not a big waste */
-	int			sc_l_irq; /* Last irq resource */
+	struct resource		*sc_res[3]; /* 1 x mem, 2 x IRQ */
+	void			*gpio_ih[2];
 	bus_space_tag_t		sc_iot;
 	bus_space_handle_t	sc_ioh;
 	int			gpio_npins;
 	struct gpio_pin		gpio_pins[NGPIO];
+	struct arm_irqsrc 	*gpio_pic_irqsrc[NGPIO];
 };
 
 static struct ofw_compat_data compat_data[] = {
@@ -118,18 +118,6 @@ static struct resource_spec imx_gpio_spec[] = {
 	{ -1, 0 }
 };
 
-static struct resource_spec imx_gpio0irq_spec[] = {
-	{ SYS_RES_IRQ,		2,	RF_ACTIVE },
-	{ SYS_RES_IRQ,		3,	RF_ACTIVE },
-	{ SYS_RES_IRQ,		4,	RF_ACTIVE },
-	{ SYS_RES_IRQ,		5,	RF_ACTIVE },
-	{ SYS_RES_IRQ,		6,	RF_ACTIVE },
-	{ SYS_RES_IRQ,		7,	RF_ACTIVE },
-	{ SYS_RES_IRQ,		8,	RF_ACTIVE },
-	{ SYS_RES_IRQ,		9,	RF_ACTIVE },
-	{ -1, 0 }
-};
-
 /*
  * Helpers
  */
@@ -142,7 +130,6 @@ static void imx51_gpio_pin_configure(struct imx51_gpio_softc *,
 static int imx51_gpio_probe(device_t);
 static int imx51_gpio_attach(device_t);
 static int imx51_gpio_detach(device_t);
-static int imx51_gpio_intr(void *);
 
 /*
  * GPIO interface
@@ -157,12 +144,258 @@ static int imx51_gpio_pin_set(device_t, uint32_t, unsigned int);
 static int imx51_gpio_pin_get(device_t, uint32_t, unsigned int *);
 static int imx51_gpio_pin_toggle(device_t, uint32_t pin);
 
+#ifdef ARM_INTRNG
+/*
+ * this is teardown_intr
+ */
+static void
+gpio_pic_disable_intr(device_t dev, struct arm_irqsrc *isrc)
+{
+	struct imx51_gpio_softc *sc;
+	u_int irq;
+
+	sc = device_get_softc(dev);
+	irq = isrc->isrc_data;
+
+	// XXX Not sure this is necessary
+	mtx_lock_spin(&sc->sc_mtx);
+	CLEAR4(sc, IMX_GPIO_IMR_REG, (1U << irq));
+	WRITE4(sc, IMX_GPIO_ISR_REG, (1U << irq));
+	mtx_unlock_spin(&sc->sc_mtx);
+}
+
+/*
+ * this is mask_intr
+ */
+static void
+gpio_pic_disable_source(device_t dev, struct arm_irqsrc *isrc)
+{
+	struct imx51_gpio_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	mtx_lock_spin(&sc->sc_mtx);
+	CLEAR4(sc, IMX_GPIO_IMR_REG, (1U << isrc->isrc_data));
+	mtx_unlock_spin(&sc->sc_mtx);
+}
+
+/*
+ * this is setup_intr
+ */
+static void
+gpio_pic_enable_intr(device_t dev, struct arm_irqsrc *isrc)
+{
+	struct imx51_gpio_softc *sc;
+	int icfg;
+	u_int irq, reg, shift, wrk;
+
+	sc = device_get_softc(dev);
+
+	if (isrc->isrc_trig == INTR_TRIGGER_LEVEL) {
+		if (isrc->isrc_pol == INTR_POLARITY_LOW)
+			icfg = GPIO_ICR_COND_LOW;
+		else
+			icfg = GPIO_ICR_COND_HIGH;
+	} else {
+		if (isrc->isrc_pol == INTR_POLARITY_HIGH)
+			icfg = GPIO_ICR_COND_FALL;
+		else
+			icfg = GPIO_ICR_COND_RISE;
+	}
+
+	irq = isrc->isrc_data;
+	if (irq < 16) {
+		reg = IMX_GPIO_ICR1_REG;
+		shift = 2 * irq;
+	} else {
+		reg = IMX_GPIO_ICR2_REG;
+		shift = 2 * (irq - 16);
+	}
+
+	mtx_lock_spin(&sc->sc_mtx);
+	CLEAR4(sc, IMX_GPIO_IMR_REG, (1U << irq));
+	WRITE4(sc, IMX_GPIO_ISR_REG, (1U << irq));
+	wrk = READ4(sc, reg);
+	wrk &= ~(0x03 << shift);
+	wrk |= icfg << shift;
+	WRITE4(sc, reg, wrk);
+	mtx_unlock_spin(&sc->sc_mtx);
+}
+
+/*
+ * this is unmask_intr
+ */
+static void
+gpio_pic_enable_source(device_t dev, struct arm_irqsrc *isrc)
+{
+	struct imx51_gpio_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	mtx_lock_spin(&sc->sc_mtx);
+	SET4(sc, IMX_GPIO_IMR_REG, (1U << isrc->isrc_data));
+	mtx_unlock_spin(&sc->sc_mtx);
+}
+
+static void
+gpio_pic_post_filter(device_t dev, struct arm_irqsrc *isrc)
+{
+	struct imx51_gpio_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	arm_irq_memory_barrier(0);
+        /* EOI.  W1C reg so no r-m-w, no locking needed. */
+	WRITE4(sc, IMX_GPIO_ISR_REG, (1U << isrc->isrc_data));
+}
+
+static void
+gpio_pic_post_ithread(device_t dev, struct arm_irqsrc *isrc)
+{
+
+	arm_irq_memory_barrier(0);
+	gpio_pic_enable_source(dev, isrc);
+}
+
+static void
+gpio_pic_pre_ithread(device_t dev, struct arm_irqsrc *isrc)
+{
+
+	gpio_pic_disable_source(dev, isrc);
+}
+
+/*
+ * intrng calls this to make a new isrc known to us.
+ */
+static int
+gpio_pic_register(device_t dev, struct arm_irqsrc *isrc, boolean_t *is_percpu)
+{
+	struct imx51_gpio_softc *sc;
+	u_int irq, tripol;
+
+	sc = device_get_softc(dev);
+
+	/*
+	 * From devicetree/bindings/gpio/fsl-imx-gpio.txt:
+	 *  #interrupt-cells:  2. The first cell is the GPIO number. The second
+	 *  cell bits[3:0] is used to specify trigger type and level flags:
+	 *    1 = low-to-high edge triggered.
+	 *    2 = high-to-low edge triggered.
+	 *    4 = active high level-sensitive.
+	 *    8 = active low level-sensitive.
+         * We can do any single one of these modes, but nothing in combo.
+	 */
+
+	if (isrc->isrc_ncells != 2) {
+		device_printf(sc->dev, "Invalid #interrupt-cells");
+		return (EINVAL);
+	}
+
+	irq = isrc->isrc_cells[0];
+	tripol = isrc->isrc_cells[1];
+	if (irq >= sc->gpio_npins) {
+		device_printf(sc->dev, "Invalid interrupt number %d", irq);
+		return (EINVAL);
+	}
+	switch (tripol)
+	{
+	case 1:
+		isrc->isrc_trig = INTR_TRIGGER_EDGE; 
+		isrc->isrc_pol  = INTR_POLARITY_HIGH;
+		break;
+	case 2:
+		isrc->isrc_trig = INTR_TRIGGER_EDGE; 
+		isrc->isrc_pol  = INTR_POLARITY_LOW;
+		break;
+	case 4:
+		isrc->isrc_trig = INTR_TRIGGER_LEVEL; 
+		isrc->isrc_pol  = INTR_POLARITY_HIGH;
+		break;
+	case 8:
+		isrc->isrc_trig = INTR_TRIGGER_LEVEL; 
+		isrc->isrc_pol  = INTR_POLARITY_LOW;
+		break;
+	default:
+		device_printf(sc->dev, "unsupported trigger/polarity 0x%2x\n",
+		    tripol);
+		return (ENOTSUP);
+	}
+	isrc->isrc_nspc_type = ARM_IRQ_NSPC_PLAIN;
+	isrc->isrc_nspc_num = irq;
+
+	/*
+	 * 1. The link between ISRC and controller must be set atomically.
+	 * 2. Just do things only once in rare case when consumers
+	 *    of shared interrupt came here at the same moment.
+	 */
+	mtx_lock_spin(&sc->sc_mtx);
+	if (sc->gpio_pic_irqsrc[irq] != NULL) {
+		mtx_unlock_spin(&sc->sc_mtx);
+		return (sc->gpio_pic_irqsrc[irq] == isrc ? 0 : EEXIST);
+	}
+	sc->gpio_pic_irqsrc[irq] = isrc;
+	isrc->isrc_data = irq;
+	mtx_unlock_spin(&sc->sc_mtx);
+
+	arm_irq_set_name(isrc, "%s,%u", device_get_nameunit(sc->dev), irq);
+	return (0);
+}
+
+static int
+gpio_pic_unregister(device_t dev, struct arm_irqsrc *isrc)
+{
+	struct imx51_gpio_softc *sc;
+	u_int irq;
+
+	sc = device_get_softc(dev);
+
+	mtx_lock_spin(&sc->sc_mtx);
+	irq = isrc->isrc_data;
+	if (sc->gpio_pic_irqsrc[irq] != isrc) {
+		mtx_unlock_spin(&sc->sc_mtx);
+		return (sc->gpio_pic_irqsrc[irq] == NULL ? 0 : EINVAL);
+	}
+	sc->gpio_pic_irqsrc[irq] = NULL;
+	isrc->isrc_data = 0;
+	mtx_unlock_spin(&sc->sc_mtx);
+
+	arm_irq_set_name(isrc, "");
+	return (0);
+}
+
+static int
+gpio_pic_filter(void *arg)
+{
+	struct imx51_gpio_softc *sc;
+	uint32_t i, interrupts;
+
+	sc = arg;
+	mtx_lock_spin(&sc->sc_mtx);
+	interrupts = READ4(sc, IMX_GPIO_ISR_REG) & READ4(sc, IMX_GPIO_IMR_REG);
+	mtx_unlock_spin(&sc->sc_mtx);
+
+	for (i = 0; interrupts != 0; i++, interrupts >>= 1) {
+		if ((interrupts & 0x1) == 0)
+			continue;
+		if (sc->gpio_pic_irqsrc[i])
+			arm_irq_dispatch(sc->gpio_pic_irqsrc[i], curthread->td_intr_frame);
+		else
+			device_printf(sc->dev, "spurious interrupt %d\n", i);
+	}
+
+	return (FILTER_HANDLED);
+}
+#endif
+
+/*
+ *
+ */
 static void
 imx51_gpio_pin_configure(struct imx51_gpio_softc *sc, struct gpio_pin *pin,
     unsigned int flags)
 {
 
-	GPIO_LOCK(sc);
+	mtx_lock_spin(&sc->sc_mtx);
 
 	/*
 	 * Manage input/output
@@ -171,15 +404,15 @@ imx51_gpio_pin_configure(struct imx51_gpio_softc *sc, struct gpio_pin *pin,
 		pin->gp_flags &= ~(GPIO_PIN_INPUT|GPIO_PIN_OUTPUT);
 		if (flags & GPIO_PIN_OUTPUT) {
 			pin->gp_flags |= GPIO_PIN_OUTPUT;
-			SET4(sc, IMX_GPIO_OE_REG, (1 << pin->gp_pin));
+			SET4(sc, IMX_GPIO_OE_REG, (1U << pin->gp_pin));
 		}
 		else {
 			pin->gp_flags |= GPIO_PIN_INPUT;
-			CLEAR4(sc, IMX_GPIO_OE_REG, (1 << pin->gp_pin));
+			CLEAR4(sc, IMX_GPIO_OE_REG, (1U << pin->gp_pin));
 		}
 	}
 
-	GPIO_UNLOCK(sc);
+	mtx_unlock_spin(&sc->sc_mtx);
 }
 
 static device_t
@@ -195,8 +428,11 @@ imx51_gpio_get_bus(device_t dev)
 static int
 imx51_gpio_pin_max(device_t dev, int *maxpin)
 {
+	struct imx51_gpio_softc *sc;
+
+	sc = device_get_softc(dev);
+	*maxpin = sc->gpio_npins - 1;
 
-	*maxpin = NGPIO - 1;
 	return (0);
 }
 
@@ -215,9 +451,9 @@ imx51_gpio_pin_getcaps(device_t dev, uint32_t pin, uint32_t *caps)
 	if (i >= sc->gpio_npins)
 		return (EINVAL);
 
-	GPIO_LOCK(sc);
+	mtx_lock_spin(&sc->sc_mtx);
 	*caps = sc->gpio_pins[i].gp_caps;
-	GPIO_UNLOCK(sc);
+	mtx_unlock_spin(&sc->sc_mtx);
 
 	return (0);
 }
@@ -237,9 +473,9 @@ imx51_gpio_pin_getflags(device_t dev, uint32_t pin, uint32_t *flags)
 	if (i >= sc->gpio_npins)
 		return (EINVAL);
 
-	GPIO_LOCK(sc);
+	mtx_lock_spin(&sc->sc_mtx);
 	*flags = sc->gpio_pins[i].gp_flags;
-	GPIO_UNLOCK(sc);
+	mtx_unlock_spin(&sc->sc_mtx);
 
 	return (0);
 }
@@ -259,9 +495,9 @@ imx51_gpio_pin_getname(device_t dev, uint32_t pin, char *name)
 	if (i >= sc->gpio_npins)
 		return (EINVAL);
 
-	GPIO_LOCK(sc);
+	mtx_lock_spin(&sc->sc_mtx);
 	memcpy(name, sc->gpio_pins[i].gp_name, GPIOMAXNAME);
-	GPIO_UNLOCK(sc);
+	mtx_unlock_spin(&sc->sc_mtx);
 
 	return (0);
 }
@@ -301,12 +537,12 @@ imx51_gpio_pin_set(device_t dev, uint32_t pin, unsigned int value)
 	if (i >= sc->gpio_npins)
 		return (EINVAL);
 
-	GPIO_LOCK(sc);
+	mtx_lock_spin(&sc->sc_mtx);
 	if (value)
-		SET4(sc, IMX_GPIO_DR_REG, (1 << i));
+		SET4(sc, IMX_GPIO_DR_REG, (1U << i));
 	else
-		CLEAR4(sc, IMX_GPIO_DR_REG, (1 << i));
-	GPIO_UNLOCK(sc);
+		CLEAR4(sc, IMX_GPIO_DR_REG, (1U << i));
+	mtx_unlock_spin(&sc->sc_mtx);
 
 	return (0);
 }
@@ -326,9 +562,9 @@ imx51_gpio_pin_get(device_t dev, uint32_t pin, unsigned int *val)
 	if (i >= sc->gpio_npins)
 		return (EINVAL);
 
-	GPIO_LOCK(sc);
+	mtx_lock_spin(&sc->sc_mtx);
 	*val = (READ4(sc, IMX_GPIO_DR_REG) >> i) & 1;
-	GPIO_UNLOCK(sc);
+	mtx_unlock_spin(&sc->sc_mtx);
 
 	return (0);
 }
@@ -348,34 +584,14 @@ imx51_gpio_pin_toggle(device_t dev, uint32_t pin)
 	if (i >= sc->gpio_npins)
 		return (EINVAL);
 
-	GPIO_LOCK(sc);
+	mtx_lock_spin(&sc->sc_mtx);
 	WRITE4(sc, IMX_GPIO_DR_REG,
-	    (READ4(sc, IMX_GPIO_DR_REG) ^ (1 << i)));
-	GPIO_UNLOCK(sc);
+	    (READ4(sc, IMX_GPIO_DR_REG) ^ (1U << i)));
+	mtx_unlock_spin(&sc->sc_mtx);
 
 	return (0);
 }
 
-static int
-imx51_gpio_intr(void *arg)
-{
-	struct imx51_gpio_softc *sc;
-	uint32_t input, value;
-
-	sc = arg;
-	input = READ4(sc, IMX_GPIO_ISR_REG);
-	value = input & READ4(sc, IMX_GPIO_IMR_REG);
-	WRITE4(sc, IMX_GPIO_ISR_REG, input);
-
-	if (!value)
-		goto intr_done;
-
-	/* TODO: interrupt handling */
-
-intr_done:
-	return (FILTER_HANDLED);
-}
-
 static int
 imx51_gpio_probe(device_t dev)
 {
@@ -395,10 +611,13 @@ static int
 imx51_gpio_attach(device_t dev)
 {
 	struct imx51_gpio_softc *sc;
-	int i, irq;
+	int i, irq, unit;
 
 	sc = device_get_softc(dev);
-	mtx_init(&sc->sc_mtx, device_get_nameunit(dev), NULL, MTX_DEF);
+	sc->dev = dev;
+	sc->gpio_npins = NGPIO;
+
+	mtx_init(&sc->sc_mtx, device_get_nameunit(sc->dev), NULL, MTX_SPIN);
 
 	if (bus_alloc_resources(dev, imx_gpio_spec, sc->sc_res)) {
 		device_printf(dev, "could not allocate resources\n");
@@ -407,40 +626,40 @@ imx51_gpio_attach(device_t dev)
 		return (ENXIO);
 	}
 
-	sc->dev = dev;
-	sc->gpio_npins = NGPIO;
-	sc->sc_l_irq = 2;
 	sc->sc_iot = rman_get_bustag(sc->sc_res[0]);
 	sc->sc_ioh = rman_get_bushandle(sc->sc_res[0]);
-
-	if (bus_alloc_resources(dev, imx_gpio0irq_spec, &sc->sc_res[3]) == 0) {
-		/*
-		 * First GPIO unit able to serve +8 interrupts for 8 first
-		 * pins.
-		 */
-		sc->sc_l_irq = 10;
-	}
-
-	for (irq = 1; irq <= sc->sc_l_irq; irq ++) {
-		if ((bus_setup_intr(dev, sc->sc_res[irq], INTR_TYPE_MISC,
-		    imx51_gpio_intr, NULL, sc, &sc->gpio_ih[irq]))) {
+	/*
+	 * Mask off all interrupts in hardware, then set up interrupt handling.
+	 */
+	WRITE4(sc, IMX_GPIO_IMR_REG, 0);
+	for (irq = 0; irq < 2; irq++) {
+#ifdef ARM_INTRNG
+		if ((bus_setup_intr(dev, sc->sc_res[1 + irq], INTR_TYPE_CLK,
+		    gpio_pic_filter, NULL, sc, &sc->gpio_ih[irq]))) {
 			device_printf(dev,
 			    "WARNING: unable to register interrupt handler\n");
 			imx51_gpio_detach(dev);
 			return (ENXIO);
 		}
+#endif		
 	}
 
+	unit = device_get_unit(dev);
 	for (i = 0; i < sc->gpio_npins; i++) {
  		sc->gpio_pins[i].gp_pin = i;
  		sc->gpio_pins[i].gp_caps = DEFAULT_CAPS;
  		sc->gpio_pins[i].gp_flags =
- 		    (READ4(sc, IMX_GPIO_OE_REG) & (1 << i)) ? GPIO_PIN_OUTPUT:
+ 		    (READ4(sc, IMX_GPIO_OE_REG) & (1U << i)) ? GPIO_PIN_OUTPUT :
  		    GPIO_PIN_INPUT;
  		snprintf(sc->gpio_pins[i].gp_name, GPIOMAXNAME,
- 		    "imx_gpio%d.%d", device_get_unit(dev), i);
+ 		    "imx_gpio%d.%d", unit, i);
 	}
+
+#ifdef ARM_INTRNG
+	arm_pic_register(dev, OF_xref_from_node(ofw_bus_get_node(dev)));
+#endif
 	sc->sc_busdev = gpiobus_attach_bus(dev);
+	
 	if (sc->sc_busdev == NULL) {
 		imx51_gpio_detach(dev);
 		return (ENXIO);
@@ -457,14 +676,11 @@ imx51_gpio_detach(device_t dev)
 
 	sc = device_get_softc(dev);
 
-	KASSERT(mtx_initialized(&sc->sc_mtx), ("gpio mutex not initialized"));
-
 	gpiobus_detach_bus(dev);
-	for (irq = 1; irq <= sc->sc_l_irq; irq ++) {
+	for (irq = 1; irq <= 2; irq++) {
 		if (sc->gpio_ih[irq])
 			bus_teardown_intr(dev, sc->sc_res[irq], sc->gpio_ih[irq]);
 	}
-	bus_release_resources(dev, imx_gpio0irq_spec, &sc->sc_res[3]);
 	bus_release_resources(dev, imx_gpio_spec, sc->sc_res);
 	mtx_destroy(&sc->sc_mtx);
 
@@ -476,6 +692,19 @@ static device_method_t imx51_gpio_methods[] = {
 	DEVMETHOD(device_attach,	imx51_gpio_attach),
 	DEVMETHOD(device_detach,	imx51_gpio_detach),
 
+#ifdef ARM_INTRNG
+	/* Interrupt controller interface */
+	DEVMETHOD(pic_disable_intr,	gpio_pic_disable_intr),
+	DEVMETHOD(pic_disable_source,	gpio_pic_disable_source),
+	DEVMETHOD(pic_enable_intr,	gpio_pic_enable_intr),
+	DEVMETHOD(pic_enable_source,	gpio_pic_enable_source),
+	DEVMETHOD(pic_post_filter,	gpio_pic_post_filter),
+	DEVMETHOD(pic_post_ithread,	gpio_pic_post_ithread),
+	DEVMETHOD(pic_pre_ithread,	gpio_pic_pre_ithread),
+	DEVMETHOD(pic_register,		gpio_pic_register),
+	DEVMETHOD(pic_unregister,	gpio_pic_unregister),
+#endif
+
 	/* GPIO protocol */
 	DEVMETHOD(gpio_get_bus,		imx51_gpio_get_bus),
 	DEVMETHOD(gpio_pin_max,		imx51_gpio_pin_max),

From 8bc8b643610f938b6e14ed5edfd313cc5fb7bd6d Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Sun, 18 Oct 2015 20:19:44 +0000
Subject: [PATCH 352/380] NTB: Rename some constants to match Linux

No functional change.

Obtained from:	Linux (part of e26a5843) (Dual BSD/GPL driver)
Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/ntb_hw/ntb_hw.c   | 177 ++++++++++++++++++----------------
 sys/dev/ntb/ntb_hw/ntb_hw.h   |   3 +-
 sys/dev/ntb/ntb_hw/ntb_regs.h |  38 ++++----
 3 files changed, 113 insertions(+), 105 deletions(-)

diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c
index 7a6d0abf47d7..9491da40ad4f 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.c
@@ -66,7 +66,7 @@ __FBSDID("$FreeBSD$");
 #define NTB_MAX_BARS	4
 #define NTB_MW_TO_BAR(mw) ((mw) + 1)
 
-#define MAX_MSIX_INTERRUPTS MAX(XEON_MAX_DB_BITS, SOC_MAX_DB_BITS)
+#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, SOC_DB_COUNT)
 
 #define NTB_HB_TIMEOUT	1 /* second */
 #define SOC_LINK_RECOVERY_TIME	500
@@ -246,18 +246,18 @@ static struct ntb_hw_info pci_ids[] = {
 
 	/* XXX: PS/SS IDs left out until they are supported. */
 	{ 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B",
-		NTB_XEON, NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 },
+		NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 },
 	{ 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B",
-		NTB_XEON, NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 },
+		NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 },
 	{ 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON,
-		NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP
-		    | NTB_BAR_SIZE_4K },
+		NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
+		    NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K },
 	{ 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON,
-		NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP
-	},
+		NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
+		    NTB_SB01BASE_LOCKUP },
 	{ 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON,
-		NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP
-	},
+		NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
+		    NTB_SB01BASE_LOCKUP },
 
 	{ 0x00000000, NULL, NTB_SOC, 0 }
 };
@@ -390,7 +390,7 @@ ntb_map_pci_bars(struct ntb_softc *ntb)
 		return (rc);
 
 	ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4);
-	if (HAS_FEATURE(NTB_REGS_THRU_MW) && !HAS_FEATURE(NTB_SPLIT_BAR))
+	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP) && !HAS_FEATURE(NTB_SPLIT_BAR))
 		rc = map_pci_bar(ntb, map_mmr_bar,
 		    &ntb->bar_info[NTB_B2B_BAR_2]);
 	else
@@ -400,7 +400,7 @@ ntb_map_pci_bars(struct ntb_softc *ntb)
 		return (rc);
 
 	ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5);
-	if (HAS_FEATURE(NTB_REGS_THRU_MW))
+	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
 		rc = map_pci_bar(ntb, map_mmr_bar,
 		    &ntb->bar_info[NTB_B2B_BAR_3]);
 	else
@@ -650,7 +650,7 @@ ntb_setup_interrupts(struct ntb_softc *ntb)
 	 */
 	mask = 0;
 	if (ntb->type == NTB_XEON)
-		mask = (1 << XEON_LINK_DB);
+		mask = (1 << XEON_DB_LINK);
 	db_iowrite(ntb, ntb->reg_ofs.ldb_mask, ~mask);
 
 	num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device),
@@ -852,7 +852,7 @@ handle_xeon_event_irq(void *arg)
 		device_printf(ntb->device, "Error determining link status\n");
 
 	/* bit 15 is always the link bit */
-	db_iowrite(ntb, ntb->reg_ofs.ldb, 1 << XEON_LINK_DB);
+	db_iowrite(ntb, ntb->reg_ofs.ldb, 1 << XEON_DB_LINK);
 }
 
 static void
@@ -864,9 +864,9 @@ ntb_handle_legacy_interrupt(void *arg)
 
 	ldb = db_ioread(ntb, ntb->reg_ofs.ldb);
 
-	if (ntb->type == NTB_XEON && (ldb & XEON_DB_HW_LINK) != 0) {
+	if (ntb->type == NTB_XEON && (ldb & XEON_DB_LINK_BIT) != 0) {
 		handle_xeon_event_irq(ntb);
-		ldb &= ~XEON_DB_HW_LINK;
+		ldb &= ~XEON_DB_LINK_BIT;
 	}
 
 	while (ldb != 0) {
@@ -1014,21 +1014,21 @@ ntb_setup_xeon(struct ntb_softc *ntb)
 	case NTB_CONN_B2B:
 		/*
 		 * reg_ofs.rdb and reg_ofs.spad_remote are effectively ignored
-		 * with the NTB_REGS_THRU_MW errata mode enabled.  (See
+		 * with the NTB_SDOORBELL_LOCKUP errata mode enabled.  (See
 		 * ntb_ring_doorbell() and ntb_read/write_remote_spad().)
 		 */
 		ntb->reg_ofs.rdb	 = XEON_B2B_DOORBELL_OFFSET;
 		ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET;
 
-		ntb->limits.max_spads	 = XEON_MAX_SPADS;
+		ntb->limits.max_spads	 = XEON_SPAD_COUNT;
 		break;
 
 	case NTB_CONN_RP:
 		/*
-		 * Every Xeon today needs NTB_REGS_THRU_MW, so punt on RP for
+		 * Every Xeon today needs NTB_SDOORBELL_LOCKUP, so punt on RP for
 		 * now.
 		 */
-		KASSERT(HAS_FEATURE(NTB_REGS_THRU_MW),
+		KASSERT(HAS_FEATURE(NTB_SDOORBELL_LOCKUP),
 		    ("Xeon without MW errata unimplemented"));
 		device_printf(ntb->device,
 		    "NTB-RP disabled to due hardware errata.\n");
@@ -1053,7 +1053,7 @@ ntb_setup_xeon(struct ntb_softc *ntb)
 	 * This should already be the case based on the driver defaults, but
 	 * write the limit registers first just in case.
 	 */
-	if (HAS_FEATURE(NTB_REGS_THRU_MW)) {
+	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) {
 		/*
 		 * Set the Limit register to 4k, the minimum size, to prevent
 		 * an illegal access.
@@ -1076,9 +1076,9 @@ ntb_setup_xeon(struct ntb_softc *ntb)
 	ntb->reg_ofs.lnk_stat	 = XEON_LINK_STATUS_OFFSET;
 	ntb->reg_ofs.spci_cmd	 = XEON_PCICMD_OFFSET;
 
-	ntb->limits.max_db_bits	 = XEON_MAX_DB_BITS;
-	ntb->limits.msix_cnt	 = XEON_MSIX_CNT;
-	ntb->bits_per_vector	 = XEON_DB_BITS_PER_VEC;
+	ntb->limits.max_db_bits	 = XEON_DB_COUNT;
+	ntb->limits.msix_cnt	 = XEON_DB_MSIX_VECTOR_COUNT;
+	ntb->bits_per_vector	 = XEON_DB_MSIX_VECTOR_SHIFT;
 
 	/*
 	 * HW Errata on bit 14 of b2bdoorbell register.  Writes will not be
@@ -1089,9 +1089,9 @@ ntb_setup_xeon(struct ntb_softc *ntb)
 	 * anyway.  Nor for non-B2B connection types.
 	 */
 	if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14) &&
-	    !HAS_FEATURE(NTB_REGS_THRU_MW) &&
+	    !HAS_FEATURE(NTB_SDOORBELL_LOCKUP) &&
 	    ntb->conn_type == NTB_CONN_B2B)
-		ntb->limits.max_db_bits = XEON_MAX_DB_BITS - 1;
+		ntb->limits.max_db_bits = XEON_DB_COUNT - 1;
 
 	configure_xeon_secondary_side_bars(ntb);
 
@@ -1128,10 +1128,10 @@ ntb_setup_soc(struct ntb_softc *ntb)
 	ntb->reg_ofs.spad_remote = SOC_B2B_SPAD_OFFSET;
 	ntb->reg_ofs.spci_cmd	 = SOC_PCICMD_OFFSET;
 
-	ntb->limits.max_spads	 = SOC_MAX_SPADS;
-	ntb->limits.max_db_bits	 = SOC_MAX_DB_BITS;
-	ntb->limits.msix_cnt	 = SOC_MSIX_CNT;
-	ntb->bits_per_vector	 = SOC_DB_BITS_PER_VEC;
+	ntb->limits.max_spads	 = SOC_SPAD_COUNT;
+	ntb->limits.max_db_bits	 = SOC_DB_COUNT;
+	ntb->limits.msix_cnt	 = SOC_DB_MSIX_VECTOR_COUNT;
+	ntb->bits_per_vector	 = SOC_DB_MSIX_VECTOR_SHIFT;
 
 	/*
 	 * FIXME - MSI-X bug on early SOC HW, remove once internal issue is
@@ -1155,15 +1155,17 @@ configure_soc_secondary_side_bars(struct ntb_softc *ntb)
 {
 
 	if (ntb->dev_type == NTB_DEV_USD) {
-		ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, MBAR23_DSD_ADDR);
-		ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, MBAR4_DSD_ADDR);
-		ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_USD_ADDR);
-		ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR4_USD_ADDR);
+		ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET,
+		    XEON_B2B_BAR2_DSD_ADDR);
+		ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, XEON_B2B_BAR4_DSD_ADDR);
+		ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_USD_ADDR);
+		ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_USD_ADDR);
 	} else {
-		ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, MBAR23_USD_ADDR);
-		ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, MBAR4_USD_ADDR);
-		ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_DSD_ADDR);
-		ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR4_DSD_ADDR);
+		ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET,
+		    XEON_B2B_BAR2_USD_ADDR);
+		ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, XEON_B2B_BAR4_USD_ADDR);
+		ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_DSD_ADDR);
+		ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_DSD_ADDR);
 	}
 }
 
@@ -1172,68 +1174,77 @@ configure_xeon_secondary_side_bars(struct ntb_softc *ntb)
 {
 
 	if (ntb->dev_type == NTB_DEV_USD) {
-		ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, MBAR23_DSD_ADDR);
-		if (HAS_FEATURE(NTB_REGS_THRU_MW))
+		ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET,
+		    XEON_B2B_BAR2_DSD_ADDR);
+		if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
 			ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
-			    MBAR01_DSD_ADDR);
+			    XEON_B2B_BAR0_DSD_ADDR);
 		else {
 			if (HAS_FEATURE(NTB_SPLIT_BAR)) {
 				ntb_reg_write(4, XEON_PBAR4XLAT_OFFSET,
-				    MBAR4_DSD_ADDR);
+				    XEON_B2B_BAR4_DSD_ADDR);
 				ntb_reg_write(4, XEON_PBAR5XLAT_OFFSET,
-				    MBAR5_DSD_ADDR);
+				    XEON_B2B_BAR5_DSD_ADDR);
 			} else
 				ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
-				    MBAR4_DSD_ADDR);
+				    XEON_B2B_BAR4_DSD_ADDR);
 			/*
 			 * B2B_XLAT_OFFSET is a 64-bit register but can only be
 			 * written 32 bits at a time.
 			 */
 			ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL,
-			    MBAR01_DSD_ADDR & 0xffffffff);
+			    XEON_B2B_BAR0_DSD_ADDR & 0xffffffff);
 			ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU,
-			    MBAR01_DSD_ADDR >> 32);
+			    XEON_B2B_BAR0_DSD_ADDR >> 32);
 		}
-		ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_USD_ADDR);
-		ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_USD_ADDR);
-		if (HAS_FEATURE(NTB_SPLIT_BAR)) {
-			ntb_reg_write(4, XEON_SBAR4BASE_OFFSET, MBAR4_USD_ADDR);
-			ntb_reg_write(4, XEON_SBAR5BASE_OFFSET, MBAR5_USD_ADDR);
-		} else
-			ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR4_USD_ADDR);
-	} else {
-		ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, MBAR23_USD_ADDR);
-		if (HAS_FEATURE(NTB_REGS_THRU_MW))
-			ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
-			    MBAR01_USD_ADDR);
-		else {
-			if (HAS_FEATURE(NTB_SPLIT_BAR)) {
-				ntb_reg_write(4, XEON_PBAR4XLAT_OFFSET,
-				    MBAR4_USD_ADDR);
-				ntb_reg_write(4, XEON_PBAR5XLAT_OFFSET,
-				    MBAR5_USD_ADDR);
-			} else
-				ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
-				    MBAR4_USD_ADDR);
-			/*
-			 * B2B_XLAT_OFFSET is a 64-bit register but can only be
-			 * written 32 bits at a time.
-			 */
-			ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL,
-			    MBAR01_USD_ADDR & 0xffffffff);
-			ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU,
-			    MBAR01_USD_ADDR >> 32);
-		}
-		ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_DSD_ADDR);
-		ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_DSD_ADDR);
+		ntb_reg_write(8, XEON_SBAR0BASE_OFFSET,
+		    XEON_B2B_BAR0_USD_ADDR);
+		ntb_reg_write(8, XEON_SBAR2BASE_OFFSET,
+		    XEON_B2B_BAR2_USD_ADDR);
 		if (HAS_FEATURE(NTB_SPLIT_BAR)) {
 			ntb_reg_write(4, XEON_SBAR4BASE_OFFSET,
-			    MBAR4_DSD_ADDR);
+			    XEON_B2B_BAR4_USD_ADDR);
 			ntb_reg_write(4, XEON_SBAR5BASE_OFFSET,
-			    MBAR5_DSD_ADDR);
+			    XEON_B2B_BAR5_USD_ADDR);
 		} else
 			ntb_reg_write(8, XEON_SBAR4BASE_OFFSET,
-			    MBAR4_DSD_ADDR);
+			    XEON_B2B_BAR4_USD_ADDR);
+	} else {
+		ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET,
+		    XEON_B2B_BAR2_USD_ADDR);
+		if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
+			ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
+			    XEON_B2B_BAR0_USD_ADDR);
+		else {
+			if (HAS_FEATURE(NTB_SPLIT_BAR)) {
+				ntb_reg_write(4, XEON_PBAR4XLAT_OFFSET,
+				    XEON_B2B_BAR4_USD_ADDR);
+				ntb_reg_write(4, XEON_PBAR5XLAT_OFFSET,
+				    XEON_B2B_BAR5_USD_ADDR);
+			} else
+				ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
+				    XEON_B2B_BAR4_USD_ADDR);
+			/*
+			 * B2B_XLAT_OFFSET is a 64-bit register but can only be
+			 * written 32 bits at a time.
+			 */
+			ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL,
+			    XEON_B2B_BAR0_USD_ADDR & 0xffffffff);
+			ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU,
+			    XEON_B2B_BAR0_USD_ADDR >> 32);
+		}
+		ntb_reg_write(8, XEON_SBAR0BASE_OFFSET,
+		    XEON_B2B_BAR0_DSD_ADDR);
+		ntb_reg_write(8, XEON_SBAR2BASE_OFFSET,
+		    XEON_B2B_BAR2_DSD_ADDR);
+		if (HAS_FEATURE(NTB_SPLIT_BAR)) {
+			ntb_reg_write(4, XEON_SBAR4BASE_OFFSET,
+			    XEON_B2B_BAR4_DSD_ADDR);
+			ntb_reg_write(4, XEON_SBAR5BASE_OFFSET,
+			    XEON_B2B_BAR5_DSD_ADDR);
+		} else
+			ntb_reg_write(8, XEON_SBAR4BASE_OFFSET,
+			    XEON_B2B_BAR4_DSD_ADDR);
 	}
 }
 
@@ -1718,7 +1729,7 @@ ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
 	if (idx >= ntb->limits.max_spads)
 		return (EINVAL);
 
-	if (HAS_FEATURE(NTB_REGS_THRU_MW))
+	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
 		ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val);
 	else
 		ntb_reg_write(4, ntb->reg_ofs.spad_remote + idx * 4, val);
@@ -1744,7 +1755,7 @@ ntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
 	if (idx >= ntb->limits.max_spads)
 		return (EINVAL);
 
-	if (HAS_FEATURE(NTB_REGS_THRU_MW))
+	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
 		*val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4);
 	else
 		*val = ntb_reg_read(4, ntb->reg_ofs.spad_remote + idx * 4);
@@ -1853,7 +1864,7 @@ ntb_ring_doorbell(struct ntb_softc *ntb, unsigned int db)
 		bit = ((1 << ntb->bits_per_vector) - 1) <<
 		    (db * ntb->bits_per_vector);
 
-	if (HAS_FEATURE(NTB_REGS_THRU_MW)) {
+	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) {
 		ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, bit);
 		return;
 	}
diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.h b/sys/dev/ntb/ntb_hw/ntb_hw.h
index e6d90add8b71..a990c5437b70 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.h
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.h
@@ -80,8 +80,7 @@ device_t ntb_get_device(struct ntb_softc *ntb);
 
 /* Hardware owns the low 32 bits of features. */
 #define NTB_BAR_SIZE_4K		(1 << 0)
-/* REGS_THRU_MW is the equivalent of Linux's NTB_HWERR_SDOORBELL_LOCKUP */
-#define NTB_REGS_THRU_MW	(1 << 1)
+#define NTB_SDOORBELL_LOCKUP	(1 << 1)
 #define NTB_SB01BASE_LOCKUP	(1 << 2)
 #define NTB_B2BDOORBELL_BIT14	(1 << 3)
 /* Software/configuration owns the top 32 bits. */
diff --git a/sys/dev/ntb/ntb_hw/ntb_regs.h b/sys/dev/ntb/ntb_hw/ntb_regs.h
index 6cb8098ee3cb..957353f0a7a5 100644
--- a/sys/dev/ntb/ntb_hw/ntb_regs.h
+++ b/sys/dev/ntb/ntb_hw/ntb_regs.h
@@ -33,17 +33,15 @@
 #define NTB_LINK_SPEED_MASK	0x000f
 #define NTB_LINK_WIDTH_MASK	0x03f0
 
-#define XEON_MSIX_CNT		4
 #define XEON_SNB_MAX_MW		2
 #define XEON_HSXSPLIT_MAX_MW	3
-#define XEON_MAX_SPADS		16
-#define XEON_MAX_COMPAT_SPADS	16
 /* Reserve the uppermost bit for link interrupt */
-#define XEON_MAX_DB_BITS	15
-#define XEON_LINK_DB		15
-#define XEON_DB_BITS_PER_VEC	5
-
-#define XEON_DB_HW_LINK		0x8000
+#define XEON_DB_COUNT		15
+#define XEON_DB_LINK		15
+#define XEON_DB_MSIX_VECTOR_COUNT	4
+#define XEON_DB_MSIX_VECTOR_SHIFT	5
+#define XEON_DB_LINK_BIT	(1 << XEON_DB_LINK)
+#define XEON_SPAD_COUNT		16
 
 #define XEON_PCICMD_OFFSET	0x0504
 #define XEON_DEVCTRL_OFFSET	0x0598
@@ -81,11 +79,11 @@
 #define XEON_B2B_XLAT_OFFSETL	0x0144
 #define XEON_B2B_XLAT_OFFSETU	0x0148
 
-#define SOC_MSIX_CNT		34
 #define SOC_MAX_MW		2
-#define SOC_MAX_SPADS		16
-#define SOC_MAX_DB_BITS		34
-#define SOC_DB_BITS_PER_VEC	1
+#define SOC_DB_COUNT		34
+#define SOC_DB_MSIX_VECTOR_COUNT	34
+#define SOC_DB_MSIX_VECTOR_SHIFT	1
+#define SOC_SPAD_COUNT		16
 
 #define SOC_PCICMD_OFFSET	0xb004
 #define SOC_MBAR23_OFFSET	0xb018
@@ -152,14 +150,14 @@
 #define NTB_DEV_USD	0
 
 /* All addresses are in low 32-bit space so 32-bit BARs can function */
-#define MBAR01_USD_ADDR		0x2100000cull
-#define MBAR23_USD_ADDR		0x4100000cull
-#define MBAR4_USD_ADDR		0x8100000cull
-#define MBAR5_USD_ADDR		0xa100000cull
-#define MBAR01_DSD_ADDR		0x2000000cull
-#define MBAR23_DSD_ADDR		0x4000000cull
-#define MBAR4_DSD_ADDR		0x8000000cull
-#define MBAR5_DSD_ADDR		0xa000000cull
+#define XEON_B2B_BAR0_USD_ADDR		0x2100000cull
+#define XEON_B2B_BAR2_USD_ADDR		0x4100000cull
+#define XEON_B2B_BAR4_USD_ADDR		0x8100000cull
+#define XEON_B2B_BAR5_USD_ADDR		0xa100000cull
+#define XEON_B2B_BAR0_DSD_ADDR		0x2000000cull
+#define XEON_B2B_BAR2_DSD_ADDR		0x4000000cull
+#define XEON_B2B_BAR4_DSD_ADDR		0x8000000cull
+#define XEON_B2B_BAR5_DSD_ADDR		0xa000000cull
 
 /* XEON Shadowed MMIO Space */
 #define XEON_SHADOW_PDOORBELL_OFFSET	0x60

From 09ef809907bd613f2f59b78c721b4e80380443e0 Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Sun, 18 Oct 2015 20:19:53 +0000
Subject: [PATCH 353/380] NTB: Rename some variables/functions to match Linux

No functional change.

Still part of the huge e26a5843 rewrite.  I'm trying to make it less of
a complete rewrite in the FreeBSD version of the driver.  Still, it
helps if our names match Linux.

Obtained from:	Linux (e26a5843) (Dual BSD/GPL driver)
Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/if_ntb/if_ntb.c   |  10 +-
 sys/dev/ntb/ntb_hw/ntb_hw.c   | 206 +++++++++++++++++++---------------
 sys/dev/ntb/ntb_hw/ntb_hw.h   |   2 +-
 sys/dev/ntb/ntb_hw/ntb_regs.h |   6 +-
 4 files changed, 122 insertions(+), 102 deletions(-)

diff --git a/sys/dev/ntb/if_ntb/if_ntb.c b/sys/dev/ntb/if_ntb/if_ntb.c
index 8b1fa7a5bd63..e7d7a91c8a26 100644
--- a/sys/dev/ntb/if_ntb/if_ntb.c
+++ b/sys/dev/ntb/if_ntb/if_ntb.c
@@ -220,7 +220,7 @@ enum {
 	IF_NTB_MAX_SPAD,
 };
 
-#define QP_TO_MW(ntb, qp)	((qp) % ntb_get_max_mw(ntb))
+#define QP_TO_MW(ntb, qp)	((qp) % ntb_mw_count(ntb))
 #define NTB_QP_DEF_NUM_ENTRIES	100
 #define NTB_LINK_DOWN_TIMEOUT	10
 
@@ -493,7 +493,7 @@ ntb_transport_init(struct ntb_softc *ntb)
 	int rc, i;
 
 	if (max_num_clients == 0)
-		nt->max_qps = MIN(ntb_get_max_cbs(ntb), ntb_get_max_mw(ntb));
+		nt->max_qps = MIN(ntb_get_max_cbs(ntb), ntb_mw_count(ntb));
 	else
 		nt->max_qps = MIN(ntb_get_max_cbs(ntb), max_num_clients);
 
@@ -562,7 +562,7 @@ ntb_transport_init_queue(struct ntb_netdev *nt, unsigned int qp_num)
 	unsigned int num_qps_mw, tx_size;
 	uint8_t mw_num, mw_max;
 
-	mw_max = ntb_get_max_mw(nt->ntb);
+	mw_max = ntb_mw_count(nt->ntb);
 	mw_num = QP_TO_MW(nt->ntb, qp_num);
 
 	qp = &nt->qps[qp_num];
@@ -1055,7 +1055,7 @@ ntb_transport_link_work(void *arg)
 	uint32_t val, i, num_mw;
 	int rc;
 
-	num_mw = ntb_get_max_mw(ntb);
+	num_mw = ntb_mw_count(ntb);
 
 	/* send the local info, in the opposite order of the way we read it */
 	for (i = 0; i < num_mw; i++) {
@@ -1214,7 +1214,7 @@ ntb_transport_setup_qp_mw(struct ntb_netdev *nt, unsigned int qp_num)
 	uint8_t mw_num, mw_max;
 	unsigned int i;
 
-	mw_max = ntb_get_max_mw(nt->ntb);
+	mw_max = ntb_mw_count(nt->ntb);
 	mw_num = QP_TO_MW(nt->ntb, qp_num);
 
 	if (nt->max_qps % mw_max && mw_num + 1 < nt->max_qps / mw_max)
diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c
index 9491da40ad4f..39fbb1d82123 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.c
@@ -59,17 +59,10 @@ __FBSDID("$FreeBSD$");
  * be picked up and redistributed in Linux with a dual GPL/BSD license.
  */
 
-#define NTB_CONFIG_BAR	0
-#define NTB_B2B_BAR_1	1
-#define NTB_B2B_BAR_2	2
-#define NTB_B2B_BAR_3	3
-#define NTB_MAX_BARS	4
-#define NTB_MW_TO_BAR(mw) ((mw) + 1)
-
 #define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, SOC_DB_COUNT)
 
-#define NTB_HB_TIMEOUT	1 /* second */
-#define SOC_LINK_RECOVERY_TIME	500
+#define NTB_HB_TIMEOUT		1 /* second */
+#define SOC_LINK_RECOVERY_TIME	500 /* ms */
 
 #define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev))
 
@@ -78,6 +71,14 @@ enum ntb_device_type {
 	NTB_SOC
 };
 
+enum ntb_bar {
+	NTB_CONFIG_BAR = 0,
+	NTB_B2B_BAR_1,
+	NTB_B2B_BAR_2,
+	NTB_B2B_BAR_3,
+	NTB_MAX_BARS
+};
+
 /* Device features and workarounds */
 #define HAS_FEATURE(feature)	\
 	((ntb->features & (feature)) != 0)
@@ -131,12 +132,6 @@ struct ntb_softc {
 	struct ntb_db_cb	*db_cb;
 	uint8_t			max_cbs;
 
-	struct {
-		uint8_t max_mw;
-		uint8_t max_spads;
-		uint8_t max_db_bits;
-		uint8_t msix_cnt;
-	} limits;
 	struct {
 		uint32_t ldb;
 		uint32_t ldb_mask;
@@ -153,10 +148,15 @@ struct ntb_softc {
 	uint32_t ppd;
 	uint8_t conn_type;
 	uint8_t dev_type;
-	uint8_t bits_per_vector;
 	uint8_t link_status;
 	uint8_t link_width;
 	uint8_t link_speed;
+
+	uint8_t				mw_count;
+	uint8_t				spad_count;
+	uint8_t				db_count;
+	uint8_t				db_vec_count;
+	uint8_t				db_vec_shift;
 };
 
 #ifdef __i386__
@@ -189,9 +189,9 @@ bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle,
 #define ntb_reg_write(SIZE, offset, val) \
 	    ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val)
 #define ntb_mw_read(SIZE, offset) \
-	    ntb_bar_read(SIZE, NTB_MW_TO_BAR(ntb->limits.max_mw), offset)
+	    ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->mw_count), offset)
 #define ntb_mw_write(SIZE, offset, val) \
-	    ntb_bar_write(SIZE, NTB_MW_TO_BAR(ntb->limits.max_mw), \
+	    ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->mw_count), \
 		offset, val)
 
 typedef int (*bar_map_strategy)(struct ntb_softc *ntb,
@@ -200,6 +200,7 @@ typedef int (*bar_map_strategy)(struct ntb_softc *ntb,
 static int ntb_probe(device_t device);
 static int ntb_attach(device_t device);
 static int ntb_detach(device_t device);
+static inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw);
 static int ntb_map_pci_bars(struct ntb_softc *ntb);
 static int map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy,
     struct ntb_pci_bar_info *bar);
@@ -218,10 +219,10 @@ static void handle_xeon_irq(void *arg);
 static void handle_xeon_event_irq(void *arg);
 static void ntb_handle_legacy_interrupt(void *arg);
 static void ntb_irq_work(void *arg);
-static uint64_t db_ioread(struct ntb_softc *, uint32_t regoff);
-static void db_iowrite(struct ntb_softc *, uint32_t regoff, uint64_t val);
-static void mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx);
-static void unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx);
+static inline uint64_t ntb_db_read(struct ntb_softc *, uint64_t regoff);
+static inline void ntb_db_write(struct ntb_softc *, uint64_t regoff, uint64_t val);
+static inline void mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx);
+static inline void unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx);
 static int ntb_create_callbacks(struct ntb_softc *ntb, uint32_t num_vectors);
 static void ntb_free_callbacks(struct ntb_softc *ntb);
 static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id);
@@ -373,6 +374,16 @@ ntb_detach(device_t device)
 	return (0);
 }
 
+static inline enum ntb_bar
+ntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw)
+{
+
+	KASSERT(mw < ntb->mw_count, ("%s: mw:%u > count:%u", __func__, mw,
+	    (unsigned)ntb->mw_count));
+
+	return (NTB_B2B_BAR_1 + mw);
+}
+
 static int
 ntb_map_pci_bars(struct ntb_softc *ntb)
 {
@@ -651,10 +662,10 @@ ntb_setup_interrupts(struct ntb_softc *ntb)
 	mask = 0;
 	if (ntb->type == NTB_XEON)
 		mask = (1 << XEON_DB_LINK);
-	db_iowrite(ntb, ntb->reg_ofs.ldb_mask, ~mask);
+	ntb_db_write(ntb, ntb->reg_ofs.ldb_mask, ~mask);
 
 	num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device),
-	    ntb->limits.max_db_bits);
+	    ntb->db_count);
 	if (desired_vectors >= 1) {
 		rc = pci_alloc_msix(ntb->device, &num_vectors);
 
@@ -675,34 +686,36 @@ ntb_setup_interrupts(struct ntb_softc *ntb)
 	} else
 		num_vectors = 1;
 
-	/*
-	 * If allocating MSI-X interrupts succeeds, limit callbacks to the
-	 * number of MSI-X slots available.
-	 */
-	ntb_create_callbacks(ntb, num_vectors);
-
-	if (ntb->type == NTB_XEON)
-		rc = ntb_setup_xeon_msix(ntb, num_vectors);
-	else
-		rc = ntb_setup_soc_msix(ntb, num_vectors);
-	if (rc != 0) {
-		device_printf(ntb->device,
-		    "Error allocating MSI-X interrupts: %d\n", rc);
-
+	if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) {
 		/*
 		 * If allocating MSI-X interrupts failed and we're forced to
 		 * use legacy INTx anyway, the only limit on individual
 		 * callbacks is the number of doorbell bits.
-		 *
-		 * CEM: This seems odd to me but matches the behavior of the
-		 * Linux driver ca. September 2013
 		 */
-		ntb_free_callbacks(ntb);
-		ntb_create_callbacks(ntb, ntb->limits.max_db_bits);
-	}
-
-	if (ntb->type == NTB_XEON && rc == ENOSPC)
+		ntb->db_vec_count = 1;
+		ntb->db_vec_shift = ntb->db_count;
+		ntb_create_callbacks(ntb, ntb->db_count);
 		rc = ntb_setup_legacy_interrupt(ntb);
+	} else {
+		ntb_create_callbacks(ntb, num_vectors);
+		if (ntb->type == NTB_XEON)
+			rc = ntb_setup_xeon_msix(ntb, num_vectors);
+		else
+			rc = ntb_setup_soc_msix(ntb, num_vectors);
+		if (rc == 0 && ntb->type == NTB_XEON) {
+			/*
+			 * Prevent consumers from registering callbacks on the link event irq
+			 * slot, from which they will never be called back.
+			 */
+			ntb->db_cb[num_vectors - 1].reserved = true;
+			ntb->max_cbs--;
+		}
+	}
+	if (rc != 0) {
+		device_printf(ntb->device,
+		    "Error allocating interrupts: %d\n", rc);
+		ntb_free_callbacks(ntb);
+	}
 
 	return (rc);
 }
@@ -759,8 +772,8 @@ ntb_teardown_interrupts(struct ntb_softc *ntb)
  * Doorbell register and mask are 64-bit on SoC, 16-bit on Xeon.  Abstract it
  * out to make code clearer.
  */
-static uint64_t
-db_ioread(struct ntb_softc *ntb, uint32_t regoff)
+static inline uint64_t
+ntb_db_read(struct ntb_softc *ntb, uint64_t regoff)
 {
 
 	if (ntb->type == NTB_SOC)
@@ -771,8 +784,8 @@ db_ioread(struct ntb_softc *ntb, uint32_t regoff)
 	return (ntb_reg_read(2, regoff));
 }
 
-static void
-db_iowrite(struct ntb_softc *ntb, uint32_t regoff, uint64_t val)
+static inline void
+ntb_db_write(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)
 {
 
 	if (ntb->type == NTB_SOC) {
@@ -789,9 +802,9 @@ mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx)
 {
 	uint64_t mask;
 
-	mask = db_ioread(ntb, ntb->reg_ofs.ldb_mask);
-	mask |= 1 << (idx * ntb->bits_per_vector);
-	db_iowrite(ntb, ntb->reg_ofs.ldb_mask, mask);
+	mask = ntb_db_read(ntb, ntb->reg_ofs.ldb_mask);
+	mask |= 1 << (idx * ntb->db_vec_shift);
+	ntb_db_write(ntb, ntb->reg_ofs.ldb_mask, mask);
 }
 
 static void
@@ -799,9 +812,9 @@ unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx)
 {
 	uint64_t mask;
 
-	mask = db_ioread(ntb, ntb->reg_ofs.ldb_mask);
-	mask &= ~(1 << (idx * ntb->bits_per_vector));
-	db_iowrite(ntb, ntb->reg_ofs.ldb_mask, mask);
+	mask = ntb_db_read(ntb, ntb->reg_ofs.ldb_mask);
+	mask &= ~(1 << (idx * ntb->db_vec_shift));
+	ntb_db_write(ntb, ntb->reg_ofs.ldb_mask, mask);
 }
 
 static void
@@ -810,7 +823,7 @@ handle_soc_irq(void *arg)
 	struct ntb_db_cb *db_cb = arg;
 	struct ntb_softc *ntb = db_cb->ntb;
 
-	db_iowrite(ntb, ntb->reg_ofs.ldb, (uint64_t) 1 << db_cb->db_num);
+	ntb_db_write(ntb, ntb->reg_ofs.ldb, (uint64_t) 1 << db_cb->db_num);
 
 	if (db_cb->callback != NULL) {
 		mask_ldb_interrupt(ntb, db_cb->db_num);
@@ -830,9 +843,9 @@ handle_xeon_irq(void *arg)
 	 * vectors, with the 4th having a single bit for link
 	 * interrupts.
 	 */
-	db_iowrite(ntb, ntb->reg_ofs.ldb,
-	    ((1 << ntb->bits_per_vector) - 1) <<
-	    (db_cb->db_num * ntb->bits_per_vector));
+	ntb_db_write(ntb, ntb->reg_ofs.ldb,
+	    ((1 << ntb->db_vec_shift) - 1) <<
+	    (db_cb->db_num * ntb->db_vec_shift));
 
 	if (db_cb->callback != NULL) {
 		mask_ldb_interrupt(ntb, db_cb->db_num);
@@ -852,7 +865,7 @@ handle_xeon_event_irq(void *arg)
 		device_printf(ntb->device, "Error determining link status\n");
 
 	/* bit 15 is always the link bit */
-	db_iowrite(ntb, ntb->reg_ofs.ldb, 1 << XEON_DB_LINK);
+	ntb_db_write(ntb, ntb->reg_ofs.ldb, 1 << XEON_DB_LINK);
 }
 
 static void
@@ -862,7 +875,7 @@ ntb_handle_legacy_interrupt(void *arg)
 	unsigned int i;
 	uint64_t ldb;
 
-	ldb = db_ioread(ntb, ntb->reg_ofs.ldb);
+	ldb = ntb_db_read(ntb, ntb->reg_ofs.ldb);
 
 	if (ntb->type == NTB_XEON && (ldb & XEON_DB_LINK_BIT) != 0) {
 		handle_xeon_event_irq(ntb);
@@ -900,10 +913,14 @@ ntb_free_callbacks(struct ntb_softc *ntb)
 {
 	uint8_t i;
 
+	if (ntb->db_cb == NULL)
+		return;
+
 	for (i = 0; i < ntb->max_cbs; i++)
 		ntb_unregister_db_callback(ntb, i);
 
 	free(ntb->db_cb, M_NTB);
+	ntb->db_cb = NULL;
 	ntb->max_cbs = 0;
 }
 
@@ -932,14 +949,14 @@ ntb_detect_max_mw(struct ntb_softc *ntb)
 {
 
 	if (ntb->type == NTB_SOC) {
-		ntb->limits.max_mw = SOC_MAX_MW;
+		ntb->mw_count = SOC_MW_COUNT;
 		return;
 	}
 
 	if (HAS_FEATURE(NTB_SPLIT_BAR))
-		ntb->limits.max_mw = XEON_HSXSPLIT_MAX_MW;
+		ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT;
 	else
-		ntb->limits.max_mw = XEON_SNB_MAX_MW;
+		ntb->mw_count = XEON_SNB_MW_COUNT;
 }
 
 static int
@@ -1020,7 +1037,7 @@ ntb_setup_xeon(struct ntb_softc *ntb)
 		ntb->reg_ofs.rdb	 = XEON_B2B_DOORBELL_OFFSET;
 		ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET;
 
-		ntb->limits.max_spads	 = XEON_SPAD_COUNT;
+		ntb->spad_count	 = XEON_SPAD_COUNT;
 		break;
 
 	case NTB_CONN_RP:
@@ -1063,7 +1080,7 @@ ntb_setup_xeon(struct ntb_softc *ntb)
 		ntb_reg_write(8, XEON_PBAR4LMT_OFFSET,
 		    ntb_get_mw_size(ntb, 1) + 0x1000);
 		/* Reserve the last MW for mapping remote spad */
-		ntb->limits.max_mw--;
+		ntb->mw_count--;
 	} else
 		/*
 		 * Disable the limit register, just in case it is set to
@@ -1076,9 +1093,9 @@ ntb_setup_xeon(struct ntb_softc *ntb)
 	ntb->reg_ofs.lnk_stat	 = XEON_LINK_STATUS_OFFSET;
 	ntb->reg_ofs.spci_cmd	 = XEON_PCICMD_OFFSET;
 
-	ntb->limits.max_db_bits	 = XEON_DB_COUNT;
-	ntb->limits.msix_cnt	 = XEON_DB_MSIX_VECTOR_COUNT;
-	ntb->bits_per_vector	 = XEON_DB_MSIX_VECTOR_SHIFT;
+	ntb->db_count		 = XEON_DB_COUNT;
+	ntb->db_vec_count	 = XEON_DB_MSIX_VECTOR_COUNT;
+	ntb->db_vec_shift	 = XEON_DB_MSIX_VECTOR_SHIFT;
 
 	/*
 	 * HW Errata on bit 14 of b2bdoorbell register.  Writes will not be
@@ -1091,7 +1108,7 @@ ntb_setup_xeon(struct ntb_softc *ntb)
 	if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14) &&
 	    !HAS_FEATURE(NTB_SDOORBELL_LOCKUP) &&
 	    ntb->conn_type == NTB_CONN_B2B)
-		ntb->limits.max_db_bits = XEON_DB_COUNT - 1;
+		ntb->db_count = XEON_DB_COUNT - 1;
 
 	configure_xeon_secondary_side_bars(ntb);
 
@@ -1128,10 +1145,10 @@ ntb_setup_soc(struct ntb_softc *ntb)
 	ntb->reg_ofs.spad_remote = SOC_B2B_SPAD_OFFSET;
 	ntb->reg_ofs.spci_cmd	 = SOC_PCICMD_OFFSET;
 
-	ntb->limits.max_spads	 = SOC_SPAD_COUNT;
-	ntb->limits.max_db_bits	 = SOC_DB_COUNT;
-	ntb->limits.msix_cnt	 = SOC_DB_MSIX_VECTOR_COUNT;
-	ntb->bits_per_vector	 = SOC_DB_MSIX_VECTOR_SHIFT;
+	ntb->spad_count	 = SOC_SPAD_COUNT;
+	ntb->db_count		 = SOC_DB_COUNT;
+	ntb->db_vec_count	 = SOC_DB_MSIX_VECTOR_COUNT;
+	ntb->db_vec_shift	 = SOC_DB_MSIX_VECTOR_SHIFT;
 
 	/*
 	 * FIXME - MSI-X bug on early SOC HW, remove once internal issue is
@@ -1648,7 +1665,7 @@ uint8_t
 ntb_get_max_spads(struct ntb_softc *ntb)
 {
 
-	return (ntb->limits.max_spads);
+	return (ntb->spad_count);
 }
 
 uint8_t
@@ -1659,10 +1676,10 @@ ntb_get_max_cbs(struct ntb_softc *ntb)
 }
 
 uint8_t
-ntb_get_max_mw(struct ntb_softc *ntb)
+ntb_mw_count(struct ntb_softc *ntb)
 {
 
-	return (ntb->limits.max_mw);
+	return (ntb->mw_count);
 }
 
 /**
@@ -1680,7 +1697,7 @@ int
 ntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
 {
 
-	if (idx >= ntb->limits.max_spads)
+	if (idx >= ntb->spad_count)
 		return (EINVAL);
 
 	ntb_reg_write(4, ntb->reg_ofs.spad_local + idx * 4, val);
@@ -1703,7 +1720,7 @@ int
 ntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
 {
 
-	if (idx >= ntb->limits.max_spads)
+	if (idx >= ntb->spad_count)
 		return (EINVAL);
 
 	*val = ntb_reg_read(4, ntb->reg_ofs.spad_local + idx * 4);
@@ -1726,7 +1743,7 @@ int
 ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
 {
 
-	if (idx >= ntb->limits.max_spads)
+	if (idx >= ntb->spad_count)
 		return (EINVAL);
 
 	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
@@ -1752,7 +1769,7 @@ int
 ntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
 {
 
-	if (idx >= ntb->limits.max_spads)
+	if (idx >= ntb->spad_count)
 		return (EINVAL);
 
 	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
@@ -1777,20 +1794,20 @@ void *
 ntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw)
 {
 
-	if (mw >= ntb_get_max_mw(ntb))
+	if (mw >= ntb_mw_count(ntb))
 		return (NULL);
 
-	return (ntb->bar_info[NTB_MW_TO_BAR(mw)].vbase);
+	return (ntb->bar_info[ntb_mw_to_bar(ntb, mw)].vbase);
 }
 
 vm_paddr_t
 ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw)
 {
 
-	if (mw >= ntb_get_max_mw(ntb))
+	if (mw >= ntb_mw_count(ntb))
 		return (0);
 
-	return (ntb->bar_info[NTB_MW_TO_BAR(mw)].pbase);
+	return (ntb->bar_info[ntb_mw_to_bar(ntb, mw)].pbase);
 }
 
 /**
@@ -1806,10 +1823,10 @@ u_long
 ntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw)
 {
 
-	if (mw >= ntb_get_max_mw(ntb))
+	if (mw >= ntb_mw_count(ntb))
 		return (0);
 
-	return (ntb->bar_info[NTB_MW_TO_BAR(mw)].size);
+	return (ntb->bar_info[ntb_mw_to_bar(ntb, mw)].size);
 }
 
 /**
@@ -1826,10 +1843,10 @@ void
 ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr)
 {
 
-	if (mw >= ntb_get_max_mw(ntb))
+	if (mw >= ntb_mw_count(ntb))
 		return;
 
-	switch (NTB_MW_TO_BAR(mw)) {
+	switch (ntb_mw_to_bar(ntb, mw)) {
 	case NTB_B2B_BAR_1:
 		ntb_reg_write(8, ntb->reg_ofs.bar2_xlat, addr);
 		break;
@@ -1842,6 +1859,9 @@ ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr)
 	case NTB_B2B_BAR_3:
 		ntb_reg_write(4, ntb->reg_ofs.bar5_xlat, addr);
 		break;
+	default:
+		KASSERT(false, ("invalid BAR"));
+		break;
 	}
 }
 
@@ -1861,15 +1881,15 @@ ntb_ring_doorbell(struct ntb_softc *ntb, unsigned int db)
 	if (ntb->type == NTB_SOC)
 		bit = 1 << db;
 	else
-		bit = ((1 << ntb->bits_per_vector) - 1) <<
-		    (db * ntb->bits_per_vector);
+		bit = ((1 << ntb->db_vec_shift) - 1) <<
+		    (db * ntb->db_vec_shift);
 
 	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) {
 		ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, bit);
 		return;
 	}
 
-	db_iowrite(ntb, ntb->reg_ofs.rdb, bit);
+	ntb_db_write(ntb, ntb->reg_ofs.rdb, bit);
 }
 
 /**
diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.h b/sys/dev/ntb/ntb_hw/ntb_hw.h
index a990c5437b70..475dfe402693 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.h
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.h
@@ -62,7 +62,7 @@ struct ntb_softc *ntb_register_transport(struct ntb_softc *ntb,
     void *transport);
 void ntb_unregister_transport(struct ntb_softc *ntb);
 uint8_t ntb_get_max_cbs(struct ntb_softc *ntb);
-uint8_t ntb_get_max_mw(struct ntb_softc *ntb);
+uint8_t ntb_mw_count(struct ntb_softc *ntb);
 uint8_t ntb_get_max_spads(struct ntb_softc *ntb);
 int ntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val);
 int ntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val);
diff --git a/sys/dev/ntb/ntb_hw/ntb_regs.h b/sys/dev/ntb/ntb_hw/ntb_regs.h
index 957353f0a7a5..7dd78407bcd9 100644
--- a/sys/dev/ntb/ntb_hw/ntb_regs.h
+++ b/sys/dev/ntb/ntb_hw/ntb_regs.h
@@ -33,8 +33,8 @@
 #define NTB_LINK_SPEED_MASK	0x000f
 #define NTB_LINK_WIDTH_MASK	0x03f0
 
-#define XEON_SNB_MAX_MW		2
-#define XEON_HSXSPLIT_MAX_MW	3
+#define XEON_SNB_MW_COUNT	2
+#define XEON_HSX_SPLIT_MW_COUNT	3
 /* Reserve the uppermost bit for link interrupt */
 #define XEON_DB_COUNT		15
 #define XEON_DB_LINK		15
@@ -79,7 +79,7 @@
 #define XEON_B2B_XLAT_OFFSETL	0x0144
 #define XEON_B2B_XLAT_OFFSETU	0x0148
 
-#define SOC_MAX_MW		2
+#define SOC_MW_COUNT		2
 #define SOC_DB_COUNT		34
 #define SOC_DB_MSIX_VECTOR_COUNT	34
 #define SOC_DB_MSIX_VECTOR_SHIFT	1

From ea4ede852685bbecd8c44b72333fed2d792435be Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Sun, 18 Oct 2015 20:20:02 +0000
Subject: [PATCH 354/380] NTB: Simplify interrupt handling by merging SoC/Xeon

Some interrupt-related function names changed to match Linux.

No functional change.  Still part of the huge e26a5843 rewrite in Linux.

Obtained from:	Linux (e26a5843) (Dual BSD/GPL driver)
Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/ntb_hw/ntb_hw.c | 147 +++++++++---------------------------
 1 file changed, 36 insertions(+), 111 deletions(-)

diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c
index 39fbb1d82123..a2b95b625279 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.c
@@ -209,14 +209,12 @@ static int map_memory_window_bar(struct ntb_softc *ntb,
     struct ntb_pci_bar_info *bar);
 static void ntb_unmap_pci_bar(struct ntb_softc *ntb);
 static int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail);
-static int ntb_setup_interrupts(struct ntb_softc *ntb);
+static int ntb_init_isr(struct ntb_softc *ntb);
 static int ntb_setup_legacy_interrupt(struct ntb_softc *ntb);
-static int ntb_setup_xeon_msix(struct ntb_softc *ntb, uint32_t num_vectors);
-static int ntb_setup_soc_msix(struct ntb_softc *ntb, uint32_t num_vectors);
+static int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors);
 static void ntb_teardown_interrupts(struct ntb_softc *ntb);
-static void handle_soc_irq(void *arg);
-static void handle_xeon_irq(void *arg);
-static void handle_xeon_event_irq(void *arg);
+static inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector);
+static void handle_irq(void *arg);
 static void ntb_handle_legacy_interrupt(void *arg);
 static void ntb_irq_work(void *arg);
 static inline uint64_t ntb_db_read(struct ntb_softc *, uint64_t regoff);
@@ -340,7 +338,7 @@ ntb_attach(device_t device)
 		error = ntb_setup_xeon(ntb);
 	if (error)
 		goto out;
-	error = ntb_setup_interrupts(ntb);
+	error = ntb_init_isr(ntb);
 	if (error)
 		goto out;
 
@@ -526,55 +524,7 @@ ntb_unmap_pci_bar(struct ntb_softc *ntb)
 }
 
 static int
-ntb_setup_xeon_msix(struct ntb_softc *ntb, uint32_t num_vectors)
-{
-	void (*interrupt_handler)(void *);
-	void *int_arg;
-	uint32_t i;
-	int rc;
-
-	if (num_vectors < 4)
-		return (ENOSPC);
-
-	for (i = 0; i < num_vectors; i++) {
-		ntb->int_info[i].rid = i + 1;
-		ntb->int_info[i].res = bus_alloc_resource_any(ntb->device,
-		    SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE);
-		if (ntb->int_info[i].res == NULL) {
-			device_printf(ntb->device,
-			    "bus_alloc_resource failed\n");
-			return (ENOMEM);
-		}
-		ntb->int_info[i].tag = NULL;
-		ntb->allocated_interrupts++;
-		if (i == num_vectors - 1) {
-			interrupt_handler = handle_xeon_event_irq;
-			int_arg = ntb;
-		} else {
-			interrupt_handler = handle_xeon_irq;
-			int_arg = &ntb->db_cb[i];
-		}
-		rc = bus_setup_intr(ntb->device, ntb->int_info[i].res,
-		    INTR_MPSAFE | INTR_TYPE_MISC, NULL, interrupt_handler,
-		    int_arg, &ntb->int_info[i].tag);
-		if (rc != 0) {
-			device_printf(ntb->device,
-			    "bus_setup_intr failed\n");
-			return (ENXIO);
-		}
-	}
-
-	/*
-	 * Prevent consumers from registering callbacks on the link event irq
-	 * slot, from which they will never be called back.
-	 */
-	ntb->db_cb[num_vectors - 1].reserved = true;
-	ntb->max_cbs--;
-	return (0);
-}
-
-static int
-ntb_setup_soc_msix(struct ntb_softc *ntb, uint32_t num_vectors)
+ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors)
 {
 	uint32_t i;
 	int rc;
@@ -591,7 +541,7 @@ ntb_setup_soc_msix(struct ntb_softc *ntb, uint32_t num_vectors)
 		ntb->int_info[i].tag = NULL;
 		ntb->allocated_interrupts++;
 		rc = bus_setup_intr(ntb->device, ntb->int_info[i].res,
-		    INTR_MPSAFE | INTR_TYPE_MISC, NULL, handle_soc_irq,
+		    INTR_MPSAFE | INTR_TYPE_MISC, NULL, handle_irq,
 		    &ntb->db_cb[i], &ntb->int_info[i].tag);
 		if (rc != 0) {
 			device_printf(ntb->device, "bus_setup_intr failed\n");
@@ -647,7 +597,7 @@ ntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail)
 }
 
 static int
-ntb_setup_interrupts(struct ntb_softc *ntb)
+ntb_init_isr(struct ntb_softc *ntb)
 {
 	uint32_t desired_vectors, num_vectors;
 	uint64_t mask;
@@ -698,10 +648,7 @@ ntb_setup_interrupts(struct ntb_softc *ntb)
 		rc = ntb_setup_legacy_interrupt(ntb);
 	} else {
 		ntb_create_callbacks(ntb, num_vectors);
-		if (ntb->type == NTB_XEON)
-			rc = ntb_setup_xeon_msix(ntb, num_vectors);
-		else
-			rc = ntb_setup_soc_msix(ntb, num_vectors);
+		rc = ntb_setup_msix(ntb, num_vectors);
 		if (rc == 0 && ntb->type == NTB_XEON) {
 			/*
 			 * Prevent consumers from registering callbacks on the link event irq
@@ -807,7 +754,7 @@ mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx)
 	ntb_db_write(ntb, ntb->reg_ofs.ldb_mask, mask);
 }
 
-static void
+static inline void
 unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx)
 {
 	uint64_t mask;
@@ -817,55 +764,42 @@ unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx)
 	ntb_db_write(ntb, ntb->reg_ofs.ldb_mask, mask);
 }
 
-static void
-handle_soc_irq(void *arg)
+static inline uint64_t
+ntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector)
 {
-	struct ntb_db_cb *db_cb = arg;
-	struct ntb_softc *ntb = db_cb->ntb;
+	uint64_t shift, mask;
 
-	ntb_db_write(ntb, ntb->reg_ofs.ldb, (uint64_t) 1 << db_cb->db_num);
-
-	if (db_cb->callback != NULL) {
-		mask_ldb_interrupt(ntb, db_cb->db_num);
-		callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb);
-	}
+	shift = ntb->db_vec_shift;
+	mask = (1ull << shift) - 1;
+	return (mask << (shift * db_vector));
 }
 
 static void
-handle_xeon_irq(void *arg)
+handle_irq(void *arg)
 {
 	struct ntb_db_cb *db_cb = arg;
 	struct ntb_softc *ntb = db_cb->ntb;
-
-	/*
-	 * On Xeon, there are 16 bits in the interrupt register
-	 * but only 4 vectors.  So, 5 bits are assigned to the first 3
-	 * vectors, with the 4th having a single bit for link
-	 * interrupts.
-	 */
-	ntb_db_write(ntb, ntb->reg_ofs.ldb,
-	    ((1 << ntb->db_vec_shift) - 1) <<
-	    (db_cb->db_num * ntb->db_vec_shift));
-
-	if (db_cb->callback != NULL) {
-		mask_ldb_interrupt(ntb, db_cb->db_num);
-		callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb);
-	}
-}
-
-/* Since we do not have a HW doorbell in SOC, this is only used in JF/JT */
-static void
-handle_xeon_event_irq(void *arg)
-{
-	struct ntb_softc *ntb = arg;
+	uint64_t vec_mask;
 	int rc;
 
-	rc = ntb_check_link_status(ntb);
-	if (rc != 0)
-		device_printf(ntb->device, "Error determining link status\n");
+	vec_mask = ntb_vec_mask(ntb, db_cb->db_num);
 
-	/* bit 15 is always the link bit */
-	ntb_db_write(ntb, ntb->reg_ofs.ldb, 1 << XEON_DB_LINK);
+	if (ntb->type == NTB_XEON && (vec_mask & XEON_DB_LINK_BIT) != 0) {
+		rc = ntb_check_link_status(ntb);
+		if (rc != 0)
+			device_printf(ntb->device,
+			    "Error determining link status\n");
+	}
+
+	if (db_cb->callback != NULL) {
+		KASSERT(!db_cb->reserved, ("user callback on link event cb"));
+		mask_ldb_interrupt(ntb, db_cb->db_num);
+	}
+
+	ntb_db_write(ntb, ntb->reg_ofs.ldb, vec_mask);
+
+	if (db_cb->callback != NULL)
+		callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb);
 }
 
 static void
@@ -876,19 +810,10 @@ ntb_handle_legacy_interrupt(void *arg)
 	uint64_t ldb;
 
 	ldb = ntb_db_read(ntb, ntb->reg_ofs.ldb);
-
-	if (ntb->type == NTB_XEON && (ldb & XEON_DB_LINK_BIT) != 0) {
-		handle_xeon_event_irq(ntb);
-		ldb &= ~XEON_DB_LINK_BIT;
-	}
-
 	while (ldb != 0) {
 		i = ffs(ldb);
 		ldb &= ldb - 1;
-		if (ntb->type == NTB_SOC)
-			handle_soc_irq(&ntb->db_cb[i]);
-		else
-			handle_xeon_irq(&ntb->db_cb[i]);
+		handle_irq(&ntb->db_cb[i]);
 	}
 }
 

From accb1ad2a254837cd5e33b42cb84098259c2a754 Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Sun, 18 Oct 2015 20:20:11 +0000
Subject: [PATCH 355/380] NTB: Simplify ntb_map_pci_bars

Skip using a function pointer for shared error logging.

Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/ntb_hw/ntb_hw.c | 53 +++++++++++++++----------------------
 1 file changed, 21 insertions(+), 32 deletions(-)

diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c
index a2b95b625279..46b3823a6661 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.c
@@ -194,16 +194,12 @@ bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle,
 	    ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->mw_count), \
 		offset, val)
 
-typedef int (*bar_map_strategy)(struct ntb_softc *ntb,
-    struct ntb_pci_bar_info *bar);
-
 static int ntb_probe(device_t device);
 static int ntb_attach(device_t device);
 static int ntb_detach(device_t device);
 static inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw);
 static int ntb_map_pci_bars(struct ntb_softc *ntb);
-static int map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy,
-    struct ntb_pci_bar_info *bar);
+static void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *);
 static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar);
 static int map_memory_window_bar(struct ntb_softc *ntb,
     struct ntb_pci_bar_info *bar);
@@ -388,53 +384,44 @@ ntb_map_pci_bars(struct ntb_softc *ntb)
 	int rc;
 
 	ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0);
-	rc = map_pci_bar(ntb, map_mmr_bar, &ntb->bar_info[NTB_CONFIG_BAR]);
+	rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]);
 	if (rc != 0)
-		return (rc);
+		goto out;
 
 	ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2);
-	rc = map_pci_bar(ntb, map_memory_window_bar,
-	    &ntb->bar_info[NTB_B2B_BAR_1]);
+	rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]);
 	if (rc != 0)
-		return (rc);
+		goto out;
 
 	ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4);
 	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP) && !HAS_FEATURE(NTB_SPLIT_BAR))
-		rc = map_pci_bar(ntb, map_mmr_bar,
-		    &ntb->bar_info[NTB_B2B_BAR_2]);
+		rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]);
 	else
-		rc = map_pci_bar(ntb, map_memory_window_bar,
-		    &ntb->bar_info[NTB_B2B_BAR_2]);
+		rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]);
 	if (!HAS_FEATURE(NTB_SPLIT_BAR))
-		return (rc);
+		goto out;
 
 	ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5);
 	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
-		rc = map_pci_bar(ntb, map_mmr_bar,
-		    &ntb->bar_info[NTB_B2B_BAR_3]);
+		rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]);
 	else
-		rc = map_pci_bar(ntb, map_memory_window_bar,
-		    &ntb->bar_info[NTB_B2B_BAR_3]);
-	return (rc);
-}
+		rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]);
 
-static int
-map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy,
-    struct ntb_pci_bar_info *bar)
-{
-	int rc;
-
-	rc = strategy(ntb, bar);
+out:
 	if (rc != 0)
 		device_printf(ntb->device,
 		    "unable to allocate pci resource\n");
-	else
-		device_printf(ntb->device,
-		    "Bar size = %lx, v %p, p %p\n",
-		    bar->size, bar->vbase, (void *)(bar->pbase));
 	return (rc);
 }
 
+static void
+print_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar)
+{
+
+	device_printf(ntb->device, "Bar size = %lx, v %p, p %p\n",
+	    bar->size, bar->vbase, (void *)(bar->pbase));
+}
+
 static int
 map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar)
 {
@@ -445,6 +432,7 @@ map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar)
 		return (ENXIO);
 
 	save_bar_parameters(bar);
+	print_map_success(ntb, bar);
 	return (0);
 }
 
@@ -505,6 +493,7 @@ map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar)
 		    "unable to mark bar as WRITE_COMBINING\n");
 		return (rc);
 	}
+	print_map_success(ntb, bar);
 	return (0);
 }
 

From 759a205fcee452d0c32e52108466c3c12e4ac511 Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Sun, 18 Oct 2015 20:20:20 +0000
Subject: [PATCH 356/380] NTB: "Split ntb_hw_intel and ntb_transport drivers"

This Linux commit was more or less a rewrite.  Unfortunately, the commit
log does not give a lot of context for the rewrite.  I have tried to
faithfully follow the changes made upstream, including matching function
names where possible, while churning the FreeBSD driver as little as
possible.

This is the bulk of the rewrite.  There are two groups of changes to
follow in separate commits: fleshing out the rest of the changes to
xeon_setup_b2b_mw(), and some changes to if_ntb.

Yes, this is a big patch (3 files changed, 416 insertions(+), 237
deletions(-)), but the Linux patch was 13 files changed, 2,589
additions(+) and 2,195 deletions(-).

Original Linux commit log:
Change ntb_hw_intel to use the new NTB hardware abstraction layer.

Split ntb_transport into its own driver.  Change it to use the new NTB
hardware abstraction layer.

Authored by:	Allen Hubbe
Obtained from:	Linux (e26a5843) (Dual BSD/GPL driver)
Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/ntb_hw/ntb_hw.c   | 628 ++++++++++++++++++++++------------
 sys/dev/ntb/ntb_hw/ntb_hw.h   |   4 +-
 sys/dev/ntb/ntb_hw/ntb_regs.h |  22 +-
 3 files changed, 417 insertions(+), 237 deletions(-)

diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c
index 46b3823a6661..96d7d296ed89 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.c
@@ -1,5 +1,6 @@
 /*-
  * Copyright (C) 2013 Intel Corporation
+ * Copyright (C) 2015 EMC Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -115,6 +116,33 @@ struct ntb_db_cb {
 	bool			reserved;
 };
 
+struct ntb_reg {
+	uint32_t	ntb_ctl;
+	uint32_t	lnk_sta;
+	uint8_t		db_size;
+	unsigned	mw_bar[NTB_MAX_BARS];
+};
+
+struct ntb_alt_reg {
+	uint32_t	db_bell;
+	uint32_t	db_mask;
+	uint32_t	spad;
+};
+
+struct ntb_xlat_reg {
+	uint64_t	bar0_base;
+	uint64_t	bar2_xlat;
+	uint64_t	bar2_limit;
+};
+
+struct ntb_b2b_addr {
+	uint64_t	bar0_addr;
+	uint64_t	bar2_addr64;
+	uint64_t	bar4_addr64;
+	uint64_t	bar4_addr32;
+	uint64_t	bar5_addr32;
+};
+
 struct ntb_softc {
 	device_t		device;
 	enum ntb_device_type	type;
@@ -135,14 +163,9 @@ struct ntb_softc {
 	struct {
 		uint32_t ldb;
 		uint32_t ldb_mask;
-		uint32_t rdb;
-		uint32_t bar2_xlat;
 		uint32_t bar4_xlat;
 		uint32_t bar5_xlat;
-		uint32_t spad_remote;
 		uint32_t spad_local;
-		uint32_t lnk_cntl;
-		uint32_t lnk_stat;
 		uint32_t spci_cmd;
 	} reg_ofs;
 	uint32_t ppd;
@@ -152,11 +175,36 @@ struct ntb_softc {
 	uint8_t link_width;
 	uint8_t link_speed;
 
+	/* Offset of peer bar0 in B2B BAR */
+	uint64_t			b2b_off;
+	/* Memory window used to access peer bar0 */
+	uint8_t				b2b_mw_idx;
+
 	uint8_t				mw_count;
 	uint8_t				spad_count;
 	uint8_t				db_count;
 	uint8_t				db_vec_count;
 	uint8_t				db_vec_shift;
+
+	/* Protects local DB mask and (h). */
+#define HW_LOCK(sc)	mtx_lock_spin(&(sc)->db_mask_lock)
+#define HW_UNLOCK(sc)	mtx_unlock_spin(&(sc)->db_mask_lock)
+#define HW_ASSERT(sc,f)	mtx_assert(&(sc)->db_mask_lock, (f))
+	struct mtx			db_mask_lock;
+
+	uint32_t			ntb_ctl;	/* (h) - SOC only */
+	uint32_t			lnk_sta;	/* (h) - SOC only */
+
+	uint64_t			db_valid_mask;
+	uint64_t			db_link_mask;
+	uint64_t			db_mask;	/* (h) */
+
+	int				last_ts;	/* ticks @ last irq */
+
+	const struct ntb_reg		*reg;
+	const struct ntb_alt_reg	*self_reg;
+	const struct ntb_alt_reg	*peer_reg;
+	const struct ntb_xlat_reg	*xlat_reg;
 };
 
 #ifdef __i386__
@@ -189,9 +237,9 @@ bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle,
 #define ntb_reg_write(SIZE, offset, val) \
 	    ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val)
 #define ntb_mw_read(SIZE, offset) \
-	    ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->mw_count), offset)
+	    ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset)
 #define ntb_mw_write(SIZE, offset, val) \
-	    ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->mw_count), \
+	    ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \
 		offset, val)
 
 static int ntb_probe(device_t device);
@@ -217,23 +265,27 @@ static inline uint64_t ntb_db_read(struct ntb_softc *, uint64_t regoff);
 static inline void ntb_db_write(struct ntb_softc *, uint64_t regoff, uint64_t val);
 static inline void mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx);
 static inline void unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx);
+static inline void ntb_db_set_mask(struct ntb_softc *, uint64_t bits);
+static inline void ntb_db_clear_mask(struct ntb_softc *, uint64_t bits);
 static int ntb_create_callbacks(struct ntb_softc *ntb, uint32_t num_vectors);
 static void ntb_free_callbacks(struct ntb_softc *ntb);
 static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id);
 static void ntb_detect_max_mw(struct ntb_softc *ntb);
 static int ntb_detect_xeon(struct ntb_softc *ntb);
 static int ntb_detect_soc(struct ntb_softc *ntb);
-static int ntb_setup_xeon(struct ntb_softc *ntb);
-static int ntb_setup_soc(struct ntb_softc *ntb);
+static int ntb_xeon_init_dev(struct ntb_softc *ntb);
+static int ntb_soc_init_dev(struct ntb_softc *ntb);
 static void ntb_teardown_xeon(struct ntb_softc *ntb);
 static void configure_soc_secondary_side_bars(struct ntb_softc *ntb);
 static void configure_xeon_secondary_side_bars(struct ntb_softc *ntb);
-static void ntb_handle_heartbeat(void *arg);
+static int xeon_setup_b2b_mw(struct ntb_softc *,
+    const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr);
+static void soc_link_hb(void *arg);
 static void ntb_handle_link_event(struct ntb_softc *ntb, int link_state);
-static void ntb_hw_link_down(struct ntb_softc *ntb);
-static void ntb_hw_link_up(struct ntb_softc *ntb);
+static void ntb_link_disable(struct ntb_softc *ntb);
+static void ntb_link_enable(struct ntb_softc *ntb);
 static void recover_soc_link(void *arg);
-static int ntb_check_link_status(struct ntb_softc *ntb);
+static int ntb_poll_link(struct ntb_softc *ntb);
 static void save_bar_parameters(struct ntb_pci_bar_info *bar);
 
 static struct ntb_hw_info pci_ids[] = {
@@ -257,6 +309,61 @@ static struct ntb_hw_info pci_ids[] = {
 	{ 0x00000000, NULL, NTB_SOC, 0 }
 };
 
+static const struct ntb_reg soc_reg = {
+	.ntb_ctl = SOC_NTBCNTL_OFFSET,
+	.lnk_sta = SOC_LINK_STATUS_OFFSET,
+	.db_size = sizeof(uint64_t),
+	.mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 },
+};
+
+static const struct ntb_alt_reg soc_b2b_reg = {
+	.db_bell = SOC_B2B_DOORBELL_OFFSET,
+	.spad = SOC_B2B_SPAD_OFFSET,
+};
+
+static const struct ntb_xlat_reg soc_sec_xlat = {
+#if 0
+	/* "FIXME" says the Linux driver. */
+	.bar0_base = SOC_SBAR0BASE_OFFSET,
+	.bar2_limit = SOC_SBAR2LMT_OFFSET,
+#endif
+	.bar2_xlat = SOC_SBAR2XLAT_OFFSET,
+};
+
+static const struct ntb_reg xeon_reg = {
+	.ntb_ctl = XEON_NTBCNTL_OFFSET,
+	.lnk_sta = XEON_LINK_STATUS_OFFSET,
+	.db_size = sizeof(uint16_t),
+	.mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 },
+};
+
+static const struct ntb_alt_reg xeon_b2b_reg = {
+	.db_bell = XEON_B2B_DOORBELL_OFFSET,
+	.spad = XEON_B2B_SPAD_OFFSET,
+};
+
+static const struct ntb_xlat_reg xeon_sec_xlat = {
+	.bar0_base = XEON_SBAR0BASE_OFFSET,
+	.bar2_limit = XEON_SBAR2LMT_OFFSET,
+	.bar2_xlat = XEON_SBAR2XLAT_OFFSET,
+};
+
+static const struct ntb_b2b_addr xeon_b2b_usd_addr = {
+	.bar0_addr = XEON_B2B_BAR0_USD_ADDR,
+	.bar2_addr64 = XEON_B2B_BAR2_USD_ADDR64,
+	.bar4_addr64 = XEON_B2B_BAR4_USD_ADDR64,
+	.bar4_addr32 = XEON_B2B_BAR4_USD_ADDR32,
+	.bar5_addr32 = XEON_B2B_BAR5_USD_ADDR32,
+};
+
+static const struct ntb_b2b_addr xeon_b2b_dsd_addr = {
+	.bar0_addr = XEON_B2B_BAR0_DSD_ADDR,
+	.bar2_addr64 = XEON_B2B_BAR2_DSD_ADDR64,
+	.bar4_addr64 = XEON_B2B_BAR4_DSD_ADDR64,
+	.bar4_addr32 = XEON_B2B_BAR4_DSD_ADDR32,
+	.bar5_addr32 = XEON_B2B_BAR5_DSD_ADDR32,
+};
+
 /*
  * OS <-> Driver interface structures
  */
@@ -311,10 +418,12 @@ ntb_attach(device_t device)
 	ntb->device = device;
 	ntb->type = p->type;
 	ntb->features = p->features;
+	ntb->b2b_mw_idx = UINT8_MAX;
 
 	/* Heartbeat timer for NTB_SOC since there is no link interrupt */
 	callout_init(&ntb->heartbeat_timer, 1);
 	callout_init(&ntb->lr_timer, 1);
+	mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN);
 
 	if (ntb->type == NTB_SOC)
 		error = ntb_detect_soc(ntb);
@@ -329,9 +438,9 @@ ntb_attach(device_t device)
 	if (error)
 		goto out;
 	if (ntb->type == NTB_SOC)
-		error = ntb_setup_soc(ntb);
+		error = ntb_soc_init_dev(ntb);
 	else
-		error = ntb_setup_xeon(ntb);
+		error = ntb_xeon_init_dev(ntb);
 	if (error)
 		goto out;
 	error = ntb_init_isr(ntb);
@@ -352,12 +461,16 @@ ntb_detach(device_t device)
 	struct ntb_softc *ntb;
 
 	ntb = DEVICE2SOFTC(device);
+
+	ntb_db_set_mask(ntb, ntb->db_valid_mask);
 	callout_drain(&ntb->heartbeat_timer);
 	callout_drain(&ntb->lr_timer);
 	if (ntb->type == NTB_XEON)
 		ntb_teardown_xeon(ntb);
 	ntb_teardown_interrupts(ntb);
 
+	mtx_destroy(&ntb->db_mask_lock);
+
 	/*
 	 * Redetect total MWs so we unmap properly -- in case we lowered the
 	 * maximum to work around Xeon errata.
@@ -368,14 +481,17 @@ ntb_detach(device_t device)
 	return (0);
 }
 
+/*
+ * Driver internal routines
+ */
 static inline enum ntb_bar
 ntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw)
 {
 
-	KASSERT(mw < ntb->mw_count, ("%s: mw:%u > count:%u", __func__, mw,
-	    (unsigned)ntb->mw_count));
+	KASSERT(mw < ntb->mw_count || (mw != UINT8_MAX && mw == ntb->b2b_mw_idx),
+	    ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count));
 
-	return (NTB_B2B_BAR_1 + mw);
+	return (ntb->reg->mw_bar[mw]);
 }
 
 static int
@@ -593,15 +709,16 @@ ntb_init_isr(struct ntb_softc *ntb)
 	int rc;
 
 	ntb->allocated_interrupts = 0;
+	ntb->last_ts = ticks;
 
 	/*
 	 * On SOC, disable all interrupts.  On XEON, disable all but Link
 	 * Interrupt.  The rest will be unmasked as callbacks are registered.
 	 */
-	mask = 0;
+	mask = ntb->db_valid_mask;
 	if (ntb->type == NTB_XEON)
-		mask = (1 << XEON_DB_LINK);
-	ntb_db_write(ntb, ntb->reg_ofs.ldb_mask, ~mask);
+		mask &= ~ntb->db_link_mask;
+	ntb_db_set_mask(ntb, mask);
 
 	num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device),
 	    ntb->db_count);
@@ -724,6 +841,14 @@ static inline void
 ntb_db_write(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)
 {
 
+	KASSERT((val & ~ntb->db_valid_mask) == 0,
+	    ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__,
+	     (uintmax_t)(val & ~ntb->db_valid_mask),
+	     (uintmax_t)ntb->db_valid_mask));
+
+	if (regoff == ntb->reg_ofs.ldb_mask)
+		HW_ASSERT(ntb, MA_OWNED);
+
 	if (ntb->type == NTB_SOC) {
 		ntb_reg_write(8, regoff, val);
 		return;
@@ -733,14 +858,38 @@ ntb_db_write(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)
 	ntb_reg_write(2, regoff, (uint16_t)val);
 }
 
-static void
+static inline void
+ntb_db_set_mask(struct ntb_softc *ntb, uint64_t bits)
+{
+
+	HW_LOCK(ntb);
+	ntb->db_mask |= bits;
+	ntb_db_write(ntb, ntb->reg_ofs.ldb_mask, ntb->db_mask);
+	HW_UNLOCK(ntb);
+}
+
+static inline void
+ntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits)
+{
+
+	KASSERT((bits & ~ntb->db_valid_mask) == 0,
+	    ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__,
+	     (uintmax_t)(bits & ~ntb->db_valid_mask),
+	     (uintmax_t)ntb->db_valid_mask));
+
+	HW_LOCK(ntb);
+	ntb->db_mask &= ~bits;
+	ntb_db_write(ntb, ntb->reg_ofs.ldb_mask, ntb->db_mask);
+	HW_UNLOCK(ntb);
+}
+
+static inline void
 mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx)
 {
 	uint64_t mask;
 
-	mask = ntb_db_read(ntb, ntb->reg_ofs.ldb_mask);
-	mask |= 1 << (idx * ntb->db_vec_shift);
-	ntb_db_write(ntb, ntb->reg_ofs.ldb_mask, mask);
+	mask = 1ull << (idx * ntb->db_vec_shift);
+	ntb_db_set_mask(ntb, mask);
 }
 
 static inline void
@@ -748,9 +897,8 @@ unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx)
 {
 	uint64_t mask;
 
-	mask = ntb_db_read(ntb, ntb->reg_ofs.ldb_mask);
-	mask &= ~(1 << (idx * ntb->db_vec_shift));
-	ntb_db_write(ntb, ntb->reg_ofs.ldb_mask, mask);
+	mask = 1ull << (idx * ntb->db_vec_shift);
+	ntb_db_clear_mask(ntb, mask);
 }
 
 static inline uint64_t
@@ -771,10 +919,11 @@ handle_irq(void *arg)
 	uint64_t vec_mask;
 	int rc;
 
+	ntb->last_ts = ticks;
 	vec_mask = ntb_vec_mask(ntb, db_cb->db_num);
 
-	if (ntb->type == NTB_XEON && (vec_mask & XEON_DB_LINK_BIT) != 0) {
-		rc = ntb_check_link_status(ntb);
+	if ((vec_mask & ntb->db_link_mask) != 0) {
+		rc = ntb_poll_link(ntb);
 		if (rc != 0)
 			device_printf(ntb->device,
 			    "Error determining link status\n");
@@ -855,7 +1004,7 @@ static void
 ntb_teardown_xeon(struct ntb_softc *ntb)
 {
 
-	ntb_hw_link_down(ntb);
+	ntb_link_disable(ntb);
 }
 
 static void
@@ -889,6 +1038,10 @@ ntb_detect_xeon(struct ntb_softc *ntb)
 	if ((ppd & XEON_PPD_SPLIT_BAR) != 0)
 		ntb->features |= NTB_SPLIT_BAR;
 
+	/* SB01BASE_LOCKUP errata is a superset of SDOORBELL errata */
+	if (HAS_FEATURE(NTB_SB01BASE_LOCKUP))
+		ntb->features |= NTB_SDOORBELL_LOCKUP;
+
 	conn_type = ppd & XEON_PPD_CONN_TYPE;
 	switch (conn_type) {
 	case NTB_CONN_B2B:
@@ -930,48 +1083,34 @@ ntb_detect_soc(struct ntb_softc *ntb)
 }
 
 static int
-ntb_setup_xeon(struct ntb_softc *ntb)
+ntb_xeon_init_dev(struct ntb_softc *ntb)
 {
+	int rc;
 
 	ntb->reg_ofs.ldb	= XEON_PDOORBELL_OFFSET;
 	ntb->reg_ofs.ldb_mask	= XEON_PDBMSK_OFFSET;
 	ntb->reg_ofs.spad_local	= XEON_SPAD_OFFSET;
-	ntb->reg_ofs.bar2_xlat	= XEON_SBAR2XLAT_OFFSET;
 	ntb->reg_ofs.bar4_xlat	= XEON_SBAR4XLAT_OFFSET;
 	if (HAS_FEATURE(NTB_SPLIT_BAR))
 		ntb->reg_ofs.bar5_xlat = XEON_SBAR5XLAT_OFFSET;
+	ntb->reg_ofs.spci_cmd	= XEON_PCICMD_OFFSET;
 
-	switch (ntb->conn_type) {
-	case NTB_CONN_B2B:
-		/*
-		 * reg_ofs.rdb and reg_ofs.spad_remote are effectively ignored
-		 * with the NTB_SDOORBELL_LOCKUP errata mode enabled.  (See
-		 * ntb_ring_doorbell() and ntb_read/write_remote_spad().)
-		 */
-		ntb->reg_ofs.rdb	 = XEON_B2B_DOORBELL_OFFSET;
-		ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET;
+	ntb->spad_count		= XEON_SPAD_COUNT;
+	ntb->db_count		= XEON_DB_COUNT;
+	ntb->db_link_mask	= XEON_DB_LINK_BIT;
+	ntb->db_vec_count	= XEON_DB_MSIX_VECTOR_COUNT;
+	ntb->db_vec_shift	= XEON_DB_MSIX_VECTOR_SHIFT;
 
-		ntb->spad_count	 = XEON_SPAD_COUNT;
-		break;
-
-	case NTB_CONN_RP:
-		/*
-		 * Every Xeon today needs NTB_SDOORBELL_LOCKUP, so punt on RP for
-		 * now.
-		 */
-		KASSERT(HAS_FEATURE(NTB_SDOORBELL_LOCKUP),
-		    ("Xeon without MW errata unimplemented"));
-		device_printf(ntb->device,
-		    "NTB-RP disabled to due hardware errata.\n");
-		return (ENXIO);
-
-	case NTB_CONN_TRANSPARENT:
-	default:
+	if (ntb->conn_type != NTB_CONN_B2B) {
 		device_printf(ntb->device, "Connection type %d not supported\n",
 		    ntb->conn_type);
 		return (ENXIO);
 	}
 
+	ntb->reg = &xeon_reg;
+	ntb->peer_reg = &xeon_b2b_reg;
+	ntb->xlat_reg = &xeon_sec_xlat;
+
 	/*
 	 * There is a Xeon hardware errata related to writes to SDOORBELL or
 	 * B2BDOORBELL in conjunction with inbound access to NTB MMIO space,
@@ -985,17 +1124,21 @@ ntb_setup_xeon(struct ntb_softc *ntb)
 	 * write the limit registers first just in case.
 	 */
 	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) {
+		/* Reserve the last MW for mapping remote spad */
+		ntb->b2b_mw_idx = ntb->mw_count - 1;
+		ntb->mw_count--;
 		/*
 		 * Set the Limit register to 4k, the minimum size, to prevent
 		 * an illegal access.
-		 *
-		 * XXX: Should this be PBAR5LMT / get_mw_size(, max_mw - 1)?
 		 */
-		ntb_reg_write(8, XEON_PBAR4LMT_OFFSET,
-		    ntb_get_mw_size(ntb, 1) + 0x1000);
-		/* Reserve the last MW for mapping remote spad */
-		ntb->mw_count--;
-	} else
+		if (HAS_FEATURE(NTB_SPLIT_BAR)) {
+			ntb_reg_write(4, XEON_PBAR4LMT_OFFSET, 0);
+			ntb_reg_write(4, XEON_PBAR5LMT_OFFSET,
+			    ntb_get_mw_size(ntb, ntb->b2b_mw_idx) + 0x1000);
+		} else
+			ntb_reg_write(8, XEON_PBAR4LMT_OFFSET,
+			    ntb_get_mw_size(ntb, ntb->b2b_mw_idx) + 0x1000);
+	} else {
 		/*
 		 * Disable the limit register, just in case it is set to
 		 * something silly.  A 64-bit write will also clear PBAR5LMT in
@@ -1003,66 +1146,61 @@ ntb_setup_xeon(struct ntb_softc *ntb)
 		 */
 		ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0);
 
-	ntb->reg_ofs.lnk_cntl	 = XEON_NTBCNTL_OFFSET;
-	ntb->reg_ofs.lnk_stat	 = XEON_LINK_STATUS_OFFSET;
-	ntb->reg_ofs.spci_cmd	 = XEON_PCICMD_OFFSET;
+		/*
+		 * HW Errata on bit 14 of b2bdoorbell register.  Writes will not be
+		 * mirrored to the remote system.  Shrink the number of bits by one,
+		 * since bit 14 is the last bit.
+		 *
+		 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register
+		 * anyway.  Nor for non-B2B connection types.
+		 */
+		if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14))
+			ntb->db_count = XEON_DB_COUNT - 1;
+	}
 
-	ntb->db_count		 = XEON_DB_COUNT;
-	ntb->db_vec_count	 = XEON_DB_MSIX_VECTOR_COUNT;
-	ntb->db_vec_shift	 = XEON_DB_MSIX_VECTOR_SHIFT;
+	ntb->db_valid_mask = (1ull << ntb->db_count) - 1;
 
-	/*
-	 * HW Errata on bit 14 of b2bdoorbell register.  Writes will not be
-	 * mirrored to the remote system.  Shrink the number of bits by one,
-	 * since bit 14 is the last bit.
-	 *
-	 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register
-	 * anyway.  Nor for non-B2B connection types.
-	 */
-	if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14) &&
-	    !HAS_FEATURE(NTB_SDOORBELL_LOCKUP) &&
-	    ntb->conn_type == NTB_CONN_B2B)
-		ntb->db_count = XEON_DB_COUNT - 1;
-
-	configure_xeon_secondary_side_bars(ntb);
+	if (ntb->dev_type == NTB_DEV_USD)
+		rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr,
+		    &xeon_b2b_usd_addr);
+	else
+		rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr,
+		    &xeon_b2b_dsd_addr);
+	if (rc != 0)
+		return (rc);
 
 	/* Enable Bus Master and Memory Space on the secondary side */
-	if (ntb->conn_type == NTB_CONN_B2B)
-		ntb_reg_write(2, ntb->reg_ofs.spci_cmd,
-		    PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
+	ntb_reg_write(2, ntb->reg_ofs.spci_cmd,
+	    PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
 
 	/* Enable link training */
-	ntb_hw_link_up(ntb);
+	ntb_link_enable(ntb);
 
 	return (0);
 }
 
 static int
-ntb_setup_soc(struct ntb_softc *ntb)
+ntb_soc_init_dev(struct ntb_softc *ntb)
 {
 
 	KASSERT(ntb->conn_type == NTB_CONN_B2B,
 	    ("Unsupported NTB configuration (%d)\n", ntb->conn_type));
 
-	/* Initiate PCI-E link training */
-	pci_write_config(ntb->device, NTB_PPD_OFFSET,
-	    ntb->ppd | SOC_PPD_INIT_LINK, 4);
-
 	ntb->reg_ofs.ldb	 = SOC_PDOORBELL_OFFSET;
 	ntb->reg_ofs.ldb_mask	 = SOC_PDBMSK_OFFSET;
-	ntb->reg_ofs.rdb	 = SOC_B2B_DOORBELL_OFFSET;
-	ntb->reg_ofs.bar2_xlat	 = SOC_SBAR2XLAT_OFFSET;
 	ntb->reg_ofs.bar4_xlat	 = SOC_SBAR4XLAT_OFFSET;
-	ntb->reg_ofs.lnk_cntl	 = SOC_NTBCNTL_OFFSET;
-	ntb->reg_ofs.lnk_stat	 = SOC_LINK_STATUS_OFFSET;
 	ntb->reg_ofs.spad_local	 = SOC_SPAD_OFFSET;
-	ntb->reg_ofs.spad_remote = SOC_B2B_SPAD_OFFSET;
 	ntb->reg_ofs.spci_cmd	 = SOC_PCICMD_OFFSET;
 
-	ntb->spad_count	 = SOC_SPAD_COUNT;
+	ntb->spad_count		 = SOC_SPAD_COUNT;
 	ntb->db_count		 = SOC_DB_COUNT;
 	ntb->db_vec_count	 = SOC_DB_MSIX_VECTOR_COUNT;
 	ntb->db_vec_shift	 = SOC_DB_MSIX_VECTOR_SHIFT;
+	ntb->db_valid_mask	 = (1ull << ntb->db_count) - 1;
+
+	ntb->reg = &soc_reg;
+	ntb->peer_reg = &soc_b2b_reg;
+	ntb->xlat_reg = &soc_sec_xlat;
 
 	/*
 	 * FIXME - MSI-X bug on early SOC HW, remove once internal issue is
@@ -1076,118 +1214,95 @@ ntb_setup_soc(struct ntb_softc *ntb)
 	ntb_reg_write(2, ntb->reg_ofs.spci_cmd,
 	    PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
 
-	callout_reset(&ntb->heartbeat_timer, 0, ntb_handle_heartbeat, ntb);
+	/* Initiate PCI-E link training */
+	ntb_link_enable(ntb);
+
+	callout_reset(&ntb->heartbeat_timer, 0, soc_link_hb, ntb);
 
 	return (0);
 }
 
+/* XXX: Linux driver doesn't seem to do any of this for SoC. */
 static void
 configure_soc_secondary_side_bars(struct ntb_softc *ntb)
 {
 
 	if (ntb->dev_type == NTB_DEV_USD) {
 		ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET,
-		    XEON_B2B_BAR2_DSD_ADDR);
-		ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, XEON_B2B_BAR4_DSD_ADDR);
-		ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_USD_ADDR);
-		ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_USD_ADDR);
+		    XEON_B2B_BAR2_DSD_ADDR64);
+		ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET,
+		    XEON_B2B_BAR4_DSD_ADDR64);
+		ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_USD_ADDR64);
+		ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_USD_ADDR64);
 	} else {
 		ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET,
-		    XEON_B2B_BAR2_USD_ADDR);
-		ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, XEON_B2B_BAR4_USD_ADDR);
-		ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_DSD_ADDR);
-		ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_DSD_ADDR);
+		    XEON_B2B_BAR2_USD_ADDR64);
+		ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET,
+		    XEON_B2B_BAR4_USD_ADDR64);
+		ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_DSD_ADDR64);
+		ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_DSD_ADDR64);
 	}
 }
 
-static void
-configure_xeon_secondary_side_bars(struct ntb_softc *ntb)
+static int
+xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr,
+    const struct ntb_b2b_addr *peer_addr)
 {
 
-	if (ntb->dev_type == NTB_DEV_USD) {
-		ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET,
-		    XEON_B2B_BAR2_DSD_ADDR);
-		if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
-			ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
-			    XEON_B2B_BAR0_DSD_ADDR);
-		else {
-			if (HAS_FEATURE(NTB_SPLIT_BAR)) {
-				ntb_reg_write(4, XEON_PBAR4XLAT_OFFSET,
-				    XEON_B2B_BAR4_DSD_ADDR);
-				ntb_reg_write(4, XEON_PBAR5XLAT_OFFSET,
-				    XEON_B2B_BAR5_DSD_ADDR);
-			} else
-				ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
-				    XEON_B2B_BAR4_DSD_ADDR);
-			/*
-			 * B2B_XLAT_OFFSET is a 64-bit register but can only be
-			 * written 32 bits at a time.
-			 */
-			ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL,
-			    XEON_B2B_BAR0_DSD_ADDR & 0xffffffff);
-			ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU,
-			    XEON_B2B_BAR0_DSD_ADDR >> 32);
-		}
-		ntb_reg_write(8, XEON_SBAR0BASE_OFFSET,
-		    XEON_B2B_BAR0_USD_ADDR);
-		ntb_reg_write(8, XEON_SBAR2BASE_OFFSET,
-		    XEON_B2B_BAR2_USD_ADDR);
+	/* Local addresses */
+	ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, addr->bar2_addr64);
+	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
+		ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, addr->bar0_addr);
+	else {
 		if (HAS_FEATURE(NTB_SPLIT_BAR)) {
-			ntb_reg_write(4, XEON_SBAR4BASE_OFFSET,
-			    XEON_B2B_BAR4_USD_ADDR);
-			ntb_reg_write(4, XEON_SBAR5BASE_OFFSET,
-			    XEON_B2B_BAR5_USD_ADDR);
+			ntb_reg_write(4, XEON_PBAR4XLAT_OFFSET,
+			    addr->bar4_addr32);
+			ntb_reg_write(4, XEON_PBAR5XLAT_OFFSET,
+			    addr->bar5_addr32);
 		} else
-			ntb_reg_write(8, XEON_SBAR4BASE_OFFSET,
-			    XEON_B2B_BAR4_USD_ADDR);
-	} else {
-		ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET,
-		    XEON_B2B_BAR2_USD_ADDR);
-		if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
 			ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
-			    XEON_B2B_BAR0_USD_ADDR);
-		else {
-			if (HAS_FEATURE(NTB_SPLIT_BAR)) {
-				ntb_reg_write(4, XEON_PBAR4XLAT_OFFSET,
-				    XEON_B2B_BAR4_USD_ADDR);
-				ntb_reg_write(4, XEON_PBAR5XLAT_OFFSET,
-				    XEON_B2B_BAR5_USD_ADDR);
-			} else
-				ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
-				    XEON_B2B_BAR4_USD_ADDR);
-			/*
-			 * B2B_XLAT_OFFSET is a 64-bit register but can only be
-			 * written 32 bits at a time.
-			 */
-			ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL,
-			    XEON_B2B_BAR0_USD_ADDR & 0xffffffff);
-			ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU,
-			    XEON_B2B_BAR0_USD_ADDR >> 32);
-		}
-		ntb_reg_write(8, XEON_SBAR0BASE_OFFSET,
-		    XEON_B2B_BAR0_DSD_ADDR);
-		ntb_reg_write(8, XEON_SBAR2BASE_OFFSET,
-		    XEON_B2B_BAR2_DSD_ADDR);
-		if (HAS_FEATURE(NTB_SPLIT_BAR)) {
-			ntb_reg_write(4, XEON_SBAR4BASE_OFFSET,
-			    XEON_B2B_BAR4_DSD_ADDR);
-			ntb_reg_write(4, XEON_SBAR5BASE_OFFSET,
-			    XEON_B2B_BAR5_DSD_ADDR);
-		} else
-			ntb_reg_write(8, XEON_SBAR4BASE_OFFSET,
-			    XEON_B2B_BAR4_DSD_ADDR);
+			    addr->bar4_addr64);
+		/*
+		 * B2B_XLAT_OFFSET is a 64-bit register but can only be
+		 * written 32 bits at a time.
+		 */
+		ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL,
+		    addr->bar0_addr & 0xffffffff);
+		ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, addr->bar0_addr >> 32);
 	}
+
+	/* Peer addresses */
+	ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, peer_addr->bar0_addr);
+	ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, peer_addr->bar2_addr64);
+	if (HAS_FEATURE(NTB_SPLIT_BAR)) {
+		ntb_reg_write(4, XEON_SBAR4BASE_OFFSET,
+		    peer_addr->bar4_addr32);
+		ntb_reg_write(4, XEON_SBAR5BASE_OFFSET,
+		    peer_addr->bar5_addr32);
+	} else
+		ntb_reg_write(8, XEON_SBAR4BASE_OFFSET,
+		    peer_addr->bar4_addr64);
+	return (0);
 }
 
 /* SOC does not have link status interrupt, poll on that platform */
 static void
-ntb_handle_heartbeat(void *arg)
+soc_link_hb(void *arg)
 {
 	struct ntb_softc *ntb = arg;
 	uint32_t status32;
 	int rc;
 
-	rc = ntb_check_link_status(ntb);
+	/*
+	 * Delay polling the link status if an interrupt was received, unless
+	 * the cached link status says the link is down.
+	 */
+	if ((long)ticks - ((long)ntb->last_ts + NTB_HB_TIMEOUT * hz) < 0 &&
+	    (ntb->ntb_ctl & SOC_CNTL_LINK_DOWN) == 0)
+		goto out;
+
+
+	rc = ntb_poll_link(ntb);
 	if (rc != 0)
 		device_printf(ntb->device,
 		    "Error determining link status\n");
@@ -1202,8 +1317,9 @@ ntb_handle_heartbeat(void *arg)
 		}
 	}
 
-	callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz,
-	    ntb_handle_heartbeat, ntb);
+out:
+	callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, soc_link_hb,
+	    ntb);
 }
 
 static void
@@ -1261,7 +1377,7 @@ ntb_handle_link_event(struct ntb_softc *ntb, int link_state)
 
 		if (ntb->type == NTB_SOC ||
 		    ntb->conn_type == NTB_CONN_TRANSPARENT)
-			status = ntb_reg_read(2, ntb->reg_ofs.lnk_stat);
+			status = ntb_reg_read(2, ntb->reg->lnk_sta);
 		else
 			status = pci_read_config(ntb->device,
 			    XEON_LINK_STATUS_OFFSET, 2);
@@ -1270,7 +1386,7 @@ ntb_handle_link_event(struct ntb_softc *ntb, int link_state)
 		device_printf(ntb->device, "Link Width %d, Link Speed %d\n",
 		    ntb->link_width, ntb->link_speed);
 		callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz,
-		    ntb_handle_heartbeat, ntb);
+		    soc_link_hb, ntb);
 	} else {
 		device_printf(ntb->device, "Link Down\n");
 		ntb->link_status = NTB_LINK_DOWN;
@@ -1284,26 +1400,32 @@ ntb_handle_link_event(struct ntb_softc *ntb, int link_state)
 }
 
 static void
-ntb_hw_link_up(struct ntb_softc *ntb)
+ntb_link_enable(struct ntb_softc *ntb)
 {
 	uint32_t cntl;
 
+	if (ntb->type == NTB_SOC) {
+		pci_write_config(ntb->device, NTB_PPD_OFFSET,
+		    ntb->ppd | SOC_PPD_INIT_LINK, 4);
+		return;
+	}
+
 	if (ntb->conn_type == NTB_CONN_TRANSPARENT) {
 		ntb_handle_link_event(ntb, NTB_LINK_UP);
 		return;
 	}
 
-	cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl);
+	cntl = ntb_reg_read(4, ntb->reg->ntb_ctl);
 	cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK);
 	cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP;
 	cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP;
 	if (HAS_FEATURE(NTB_SPLIT_BAR))
 		cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP;
-	ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, cntl);
+	ntb_reg_write(4, ntb->reg->ntb_ctl, cntl);
 }
 
 static void
-ntb_hw_link_down(struct ntb_softc *ntb)
+ntb_link_disable(struct ntb_softc *ntb)
 {
 	uint32_t cntl;
 
@@ -1312,13 +1434,13 @@ ntb_hw_link_down(struct ntb_softc *ntb)
 		return;
 	}
 
-	cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl);
+	cntl = ntb_reg_read(4, ntb->reg->ntb_ctl);
 	cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP);
 	cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP);
 	if (HAS_FEATURE(NTB_SPLIT_BAR))
 		cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP);
 	cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK;
-	ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, cntl);
+	ntb_reg_write(4, ntb->reg->ntb_ctl, cntl);
 }
 
 static void
@@ -1327,7 +1449,6 @@ recover_soc_link(void *arg)
 	struct ntb_softc *ntb = arg;
 	uint8_t speed, width;
 	uint32_t status32;
-	uint16_t status16;
 
 	soc_perform_link_restart(ntb);
 
@@ -1348,19 +1469,19 @@ recover_soc_link(void *arg)
 	if ((status32 & SOC_IBIST_ERR_OFLOW) != 0)
 		goto retry;
 
-	status32 = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl);
+	status32 = ntb_reg_read(4, ntb->reg->ntb_ctl);
 	if ((status32 & SOC_CNTL_LINK_DOWN) != 0)
 		goto out;
 
-	status16 = ntb_reg_read(2, ntb->reg_ofs.lnk_stat);
-	width = (status16 & NTB_LINK_WIDTH_MASK) >> 4;
-	speed = (status16 & NTB_LINK_SPEED_MASK);
+	status32 = ntb_reg_read(4, ntb->reg->lnk_sta);
+	width = (status32 & NTB_LINK_WIDTH_MASK) >> 4;
+	speed = (status32 & NTB_LINK_SPEED_MASK);
 	if (ntb->link_width != width || ntb->link_speed != speed)
 		goto retry;
 
 out:
-	callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz,
-	    ntb_handle_heartbeat, ntb);
+	callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, soc_link_hb,
+	    ntb);
 	return;
 
 retry:
@@ -1369,21 +1490,32 @@ retry:
 }
 
 static int
-ntb_check_link_status(struct ntb_softc *ntb)
+ntb_poll_link(struct ntb_softc *ntb)
 {
 	int link_state;
 	uint32_t ntb_cntl;
 	uint16_t status;
 
 	if (ntb->type == NTB_SOC) {
-		ntb_cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl);
+		HW_LOCK(ntb);
+		ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl);
+		if (ntb_cntl == ntb->ntb_ctl) {
+			HW_UNLOCK(ntb);
+			return (0);
+		}
+		ntb->ntb_ctl = ntb_cntl;
+		ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta);
+		HW_UNLOCK(ntb);
+
 		if ((ntb_cntl & SOC_CNTL_LINK_DOWN) != 0)
 			link_state = NTB_LINK_DOWN;
 		else
 			link_state = NTB_LINK_UP;
 	} else {
-		status = pci_read_config(ntb->device, XEON_LINK_STATUS_OFFSET,
-		    2);
+		status = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2);
+		if (status == ntb->lnk_sta)
+			return (0);
+		ntb->lnk_sta = status;
 
 		if ((status & NTB_LINK_STATUS_ACTIVE) != 0)
 			link_state = NTB_LINK_UP;
@@ -1392,10 +1524,32 @@ ntb_check_link_status(struct ntb_softc *ntb)
 	}
 
 	ntb_handle_link_event(ntb, link_state);
-
 	return (0);
 }
 
+static void
+ntb_irq_work(void *arg)
+{
+	struct ntb_db_cb *db_cb = arg;
+	struct ntb_softc *ntb;
+	int rc;
+
+	rc = db_cb->callback(db_cb->data, db_cb->db_num);
+	/* Poll if forward progress was made. */
+	if (rc != 0) {
+		callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb);
+		return;
+	}
+
+	/* Unmask interrupt if no progress was made. */
+	ntb = db_cb->ntb;
+	unmask_ldb_interrupt(ntb, db_cb->db_num);
+}
+
+/*
+ * Public API to the rest of the OS
+ */
+
 /**
  * ntb_register_event_callback() - register event callback
  * @ntb: pointer to ntb_softc instance
@@ -1431,25 +1585,6 @@ ntb_unregister_event_callback(struct ntb_softc *ntb)
 	ntb->event_cb = NULL;
 }
 
-static void
-ntb_irq_work(void *arg)
-{
-	struct ntb_db_cb *db_cb = arg;
-	struct ntb_softc *ntb;
-	int rc;
-
-	rc = db_cb->callback(db_cb->data, db_cb->db_num);
-	/* Poll if forward progress was made. */
-	if (rc != 0) {
-		callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb);
-		return;
-	}
-
-	/* Unmask interrupt if no progress was made. */
-	ntb = db_cb->ntb;
-	unmask_ldb_interrupt(ntb, db_cb->db_num);
-}
-
 /**
  * ntb_register_db_callback() - register a callback for doorbell interrupt
  * @ntb: pointer to ntb_softc instance
@@ -1663,7 +1798,7 @@ ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
 	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
 		ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val);
 	else
-		ntb_reg_write(4, ntb->reg_ofs.spad_remote + idx * 4, val);
+		ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val);
 
 	return (0);
 }
@@ -1689,7 +1824,7 @@ ntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
 	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
 		*val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4);
 	else
-		*val = ntb_reg_read(4, ntb->reg_ofs.spad_remote + idx * 4);
+		*val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4);
 
 	return (0);
 }
@@ -1714,7 +1849,7 @@ ntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw)
 	return (ntb->bar_info[ntb_mw_to_bar(ntb, mw)].vbase);
 }
 
-vm_paddr_t
+bus_addr_t
 ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw)
 {
 
@@ -1762,7 +1897,7 @@ ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr)
 
 	switch (ntb_mw_to_bar(ntb, mw)) {
 	case NTB_B2B_BAR_1:
-		ntb_reg_write(8, ntb->reg_ofs.bar2_xlat, addr);
+		ntb_reg_write(8, ntb->xlat_reg->bar2_xlat, addr);
 		break;
 	case NTB_B2B_BAR_2:
 		if (HAS_FEATURE(NTB_SPLIT_BAR))
@@ -1803,7 +1938,44 @@ ntb_ring_doorbell(struct ntb_softc *ntb, unsigned int db)
 		return;
 	}
 
-	ntb_db_write(ntb, ntb->reg_ofs.rdb, bit);
+	ntb_db_write(ntb, ntb->peer_reg->db_bell, bit);
+}
+
+/*
+ * ntb_get_peer_db_addr() - Return the address of the remote doorbell register,
+ * as well as the size of the register (via *sz_out).
+ *
+ * This function allows a caller using I/OAT DMA to chain the remote doorbell
+ * ring to its memory window write.
+ *
+ * Note that writing the peer doorbell via a memory window will *not* generate
+ * an interrupt on the remote host; that must be done seperately.
+ */
+bus_addr_t
+ntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out)
+{
+	struct ntb_pci_bar_info *bar;
+	uint64_t regoff;
+
+	KASSERT(sz_out != NULL, ("must be non-NULL"));
+
+	if (!HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) {
+		bar = &ntb->bar_info[NTB_CONFIG_BAR];
+		regoff = ntb->peer_reg->db_bell;
+	} else {
+		KASSERT((HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 2) ||
+		    (!HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 1),
+		    ("mw_count invalid after setup"));
+		KASSERT(ntb->b2b_mw_idx != UINT8_MAX, ("invalid b2b idx"));
+
+		bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)];
+		regoff = XEON_SHADOW_PDOORBELL_OFFSET;
+	}
+	KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh"));
+
+	*sz_out = ntb->reg->db_size;
+	/* HACK: Specific to current x86 bus implementation. */
+	return ((uint64_t)bar->pci_bus_handle + regoff);
 }
 
 /**
diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.h b/sys/dev/ntb/ntb_hw/ntb_hw.h
index 475dfe402693..37bfa6823c3f 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.h
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.h
@@ -1,5 +1,6 @@
 /*-
  * Copyright (C) 2013 Intel Corporation
+ * Copyright (C) 2015 EMC Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -71,10 +72,11 @@ int ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx,
 int ntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx,
     uint32_t *val);
 void *ntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw);
-vm_paddr_t ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw);
+bus_addr_t ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw);
 u_long ntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw);
 void ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr);
 void ntb_ring_doorbell(struct ntb_softc *ntb, unsigned int db);
+bus_addr_t ntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out);
 bool ntb_query_link_status(struct ntb_softc *ntb);
 device_t ntb_get_device(struct ntb_softc *ntb);
 
diff --git a/sys/dev/ntb/ntb_hw/ntb_regs.h b/sys/dev/ntb/ntb_hw/ntb_regs.h
index 7dd78407bcd9..d56bb95a685d 100644
--- a/sys/dev/ntb/ntb_hw/ntb_regs.h
+++ b/sys/dev/ntb/ntb_hw/ntb_regs.h
@@ -1,5 +1,6 @@
 /*-
  * Copyright (C) 2013 Intel Corporation
+ * Copyright (C) 2015 EMC Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -150,14 +151,19 @@
 #define NTB_DEV_USD	0
 
 /* All addresses are in low 32-bit space so 32-bit BARs can function */
-#define XEON_B2B_BAR0_USD_ADDR		0x2100000cull
-#define XEON_B2B_BAR2_USD_ADDR		0x4100000cull
-#define XEON_B2B_BAR4_USD_ADDR		0x8100000cull
-#define XEON_B2B_BAR5_USD_ADDR		0xa100000cull
-#define XEON_B2B_BAR0_DSD_ADDR		0x2000000cull
-#define XEON_B2B_BAR2_DSD_ADDR		0x4000000cull
-#define XEON_B2B_BAR4_DSD_ADDR		0x8000000cull
-#define XEON_B2B_BAR5_DSD_ADDR		0xa000000cull
+#define XEON_B2B_BAR0_USD_ADDR		0x1000000000000000ull
+#define XEON_B2B_BAR2_USD_ADDR64	0x2000000000000000ull
+#define XEON_B2B_BAR4_USD_ADDR64	0x4000000000000000ull
+#define XEON_B2B_BAR4_USD_ADDR32	0x20000000ull
+#define XEON_B2B_BAR5_USD_ADDR32	0x40000000ull
+#define XEON_B2B_BAR0_DSD_ADDR		0x9000000000000000ull
+#define XEON_B2B_BAR2_DSD_ADDR64	0xa000000000000000ull
+#define XEON_B2B_BAR4_DSD_ADDR64	0xc000000000000000ull
+#define XEON_B2B_BAR4_DSD_ADDR32	0xa0000000ull
+#define XEON_B2B_BAR5_DSD_ADDR32	0xc0000000ull
+
+/* The peer ntb secondary config space is 32KB fixed size */
+#define XEON_B2B_MIN_SIZE		0x8000
 
 /* XEON Shadowed MMIO Space */
 #define XEON_SHADOW_PDOORBELL_OFFSET	0x60

From f05887a9e3f79d31557cad47b616ae1fd2fdd282 Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Sun, 18 Oct 2015 20:20:29 +0000
Subject: [PATCH 357/380] NTB: Flesh out the rest of the xeon_setup_b2b_mw
 changes

Move all Xeon secondary register setup to the setup_b2b_mw routine.  We
use subroutines to make it a bit less wordy than the Linux version.

Adds a new tunable, 'hw.ntb.b2b_mw_share'.  By default, it is off
(zero).  If both sides enable it (any non-zero value), the NTB driver
attempts to use only half of a memory window for remote register MMIO
access.

This is still part of the large Linux rewrite (e26a5843).

Authored by:	Allen Hubbe
Obtained from:	Linux (e26a5843) (Dual BSD/GPL driver)
Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/ntb_hw/ntb_hw.c   | 286 ++++++++++++++++++++++++++--------
 sys/dev/ntb/ntb_hw/ntb_regs.h |   5 +
 2 files changed, 228 insertions(+), 63 deletions(-)

diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c
index 96d7d296ed89..72754dae2464 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.c
@@ -99,6 +99,13 @@ struct ntb_pci_bar_info {
 	vm_paddr_t		pbase;
 	void			*vbase;
 	u_long			size;
+
+	/* Configuration register offsets */
+	uint32_t		psz_off;
+	uint32_t		ssz_off;
+	uint32_t		sbarbase_off;
+	uint32_t		sbarlmt_off;
+	uint32_t		pbarxlat_off;
 };
 
 struct ntb_int_info {
@@ -178,6 +185,7 @@ struct ntb_softc {
 	/* Offset of peer bar0 in B2B BAR */
 	uint64_t			b2b_off;
 	/* Memory window used to access peer bar0 */
+#define B2B_MW_DISABLED			UINT8_MAX
 	uint8_t				b2b_mw_idx;
 
 	uint8_t				mw_count;
@@ -277,7 +285,12 @@ static int ntb_xeon_init_dev(struct ntb_softc *ntb);
 static int ntb_soc_init_dev(struct ntb_softc *ntb);
 static void ntb_teardown_xeon(struct ntb_softc *ntb);
 static void configure_soc_secondary_side_bars(struct ntb_softc *ntb);
-static void configure_xeon_secondary_side_bars(struct ntb_softc *ntb);
+static void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx,
+    enum ntb_bar regbar);
+static void xeon_set_sbar_base_and_limit(struct ntb_softc *,
+    uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar);
+static void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr,
+    enum ntb_bar idx);
 static int xeon_setup_b2b_mw(struct ntb_softc *,
     const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr);
 static void soc_link_hb(void *arg);
@@ -418,7 +431,7 @@ ntb_attach(device_t device)
 	ntb->device = device;
 	ntb->type = p->type;
 	ntb->features = p->features;
-	ntb->b2b_mw_idx = UINT8_MAX;
+	ntb->b2b_mw_idx = B2B_MW_DISABLED;
 
 	/* Heartbeat timer for NTB_SOC since there is no link interrupt */
 	callout_init(&ntb->heartbeat_timer, 1);
@@ -488,7 +501,8 @@ static inline enum ntb_bar
 ntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw)
 {
 
-	KASSERT(mw < ntb->mw_count || (mw != UINT8_MAX && mw == ntb->b2b_mw_idx),
+	KASSERT(mw < ntb->mw_count ||
+	    (mw != B2B_MW_DISABLED && mw == ntb->b2b_mw_idx),
 	    ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count));
 
 	return (ntb->reg->mw_bar[mw]);
@@ -508,12 +522,24 @@ ntb_map_pci_bars(struct ntb_softc *ntb)
 	rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]);
 	if (rc != 0)
 		goto out;
+	ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET;
+	ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET;
+	ntb->bar_info[NTB_B2B_BAR_1].sbarbase_off = XEON_SBAR2BASE_OFFSET;
+	ntb->bar_info[NTB_B2B_BAR_1].sbarlmt_off = XEON_SBAR2LMT_OFFSET;
+	ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET;
 
 	ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4);
+	/* XXX Are shared MW B2Bs write-combining? */
 	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP) && !HAS_FEATURE(NTB_SPLIT_BAR))
 		rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]);
 	else
 		rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]);
+	ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET;
+	ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET;
+	ntb->bar_info[NTB_B2B_BAR_2].sbarbase_off = XEON_SBAR4BASE_OFFSET;
+	ntb->bar_info[NTB_B2B_BAR_2].sbarlmt_off = XEON_SBAR4LMT_OFFSET;
+	ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET;
+
 	if (!HAS_FEATURE(NTB_SPLIT_BAR))
 		goto out;
 
@@ -522,6 +548,11 @@ ntb_map_pci_bars(struct ntb_softc *ntb)
 		rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]);
 	else
 		rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]);
+	ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET;
+	ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET;
+	ntb->bar_info[NTB_B2B_BAR_3].sbarbase_off = XEON_SBAR5BASE_OFFSET;
+	ntb->bar_info[NTB_B2B_BAR_3].sbarlmt_off = XEON_SBAR5LMT_OFFSET;
+	ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET;
 
 out:
 	if (rc != 0)
@@ -1117,35 +1148,11 @@ ntb_xeon_init_dev(struct ntb_softc *ntb)
 	 * which may hang the system.  To workaround this use the second memory
 	 * window to access the interrupt and scratch pad registers on the
 	 * remote system.
-	 *
-	 * There is another HW errata on the limit registers -- they can only
-	 * be written when the base register is (?)4GB aligned and < 32-bit.
-	 * This should already be the case based on the driver defaults, but
-	 * write the limit registers first just in case.
 	 */
-	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) {
-		/* Reserve the last MW for mapping remote spad */
+	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
+		/* Use the last MW for mapping remote spad */
 		ntb->b2b_mw_idx = ntb->mw_count - 1;
-		ntb->mw_count--;
-		/*
-		 * Set the Limit register to 4k, the minimum size, to prevent
-		 * an illegal access.
-		 */
-		if (HAS_FEATURE(NTB_SPLIT_BAR)) {
-			ntb_reg_write(4, XEON_PBAR4LMT_OFFSET, 0);
-			ntb_reg_write(4, XEON_PBAR5LMT_OFFSET,
-			    ntb_get_mw_size(ntb, ntb->b2b_mw_idx) + 0x1000);
-		} else
-			ntb_reg_write(8, XEON_PBAR4LMT_OFFSET,
-			    ntb_get_mw_size(ntb, ntb->b2b_mw_idx) + 0x1000);
-	} else {
-		/*
-		 * Disable the limit register, just in case it is set to
-		 * something silly.  A 64-bit write will also clear PBAR5LMT in
-		 * split-bar mode, and this is desired.
-		 */
-		ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0);
-
+	else if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14))
 		/*
 		 * HW Errata on bit 14 of b2bdoorbell register.  Writes will not be
 		 * mirrored to the remote system.  Shrink the number of bits by one,
@@ -1154,9 +1161,7 @@ ntb_xeon_init_dev(struct ntb_softc *ntb)
 		 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register
 		 * anyway.  Nor for non-B2B connection types.
 		 */
-		if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14))
-			ntb->db_count = XEON_DB_COUNT - 1;
-	}
+		ntb->db_count = XEON_DB_COUNT - 1;
 
 	ntb->db_valid_mask = (1ull << ntb->db_count) - 1;
 
@@ -1244,44 +1249,198 @@ configure_soc_secondary_side_bars(struct ntb_softc *ntb)
 	}
 }
 
+
+/*
+ * When working around Xeon SDOORBELL errata by remapping remote registers in a
+ * MW, limit the B2B MW to half a MW.  By sharing a MW, half the shared MW
+ * remains for use by a higher layer.
+ *
+ * Will only be used if working around SDOORBELL errata and the BIOS-configured
+ * MW size is sufficiently large.
+ */
+static unsigned int ntb_b2b_mw_share;
+SYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share,
+    0, "If enabled (non-zero), prefer to share half of the B2B peer register "
+    "MW with higher level consumers.  Both sides of the NTB MUST set the same "
+    "value here.");
+
+static void
+xeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx,
+    enum ntb_bar regbar)
+{
+	struct ntb_pci_bar_info *bar;
+	uint8_t bar_sz;
+
+	if (!HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3)
+		return;
+
+	bar = &ntb->bar_info[idx];
+	bar_sz = pci_read_config(ntb->device, bar->psz_off, 1);
+	if (idx == regbar) {
+		if (ntb->b2b_off != 0)
+			bar_sz--;
+		else
+			bar_sz = 0;
+	}
+	pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1);
+	bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1);
+	(void)bar_sz;
+}
+
+static void
+xeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t base_addr,
+    enum ntb_bar idx, enum ntb_bar regbar)
+{
+	struct ntb_pci_bar_info *bar;
+	vm_paddr_t bar_addr;
+
+	bar = &ntb->bar_info[idx];
+	bar_addr = base_addr + ((idx == regbar) ? ntb->b2b_off : 0);
+
+	if (HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) {
+		ntb_reg_write(4, bar->sbarbase_off, bar_addr);
+		ntb_reg_write(4, bar->sbarlmt_off, bar_addr);
+		bar_addr = ntb_reg_read(4, bar->sbarbase_off);
+		(void)bar_addr;
+		bar_addr = ntb_reg_read(4, bar->sbarlmt_off);
+	} else {
+		ntb_reg_write(8, bar->sbarbase_off, bar_addr);
+		ntb_reg_write(8, bar->sbarlmt_off, bar_addr);
+		bar_addr = ntb_reg_read(8, bar->sbarbase_off);
+		(void)bar_addr;
+		bar_addr = ntb_reg_read(8, bar->sbarlmt_off);
+	}
+	(void)bar_addr;
+}
+
+static void
+xeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx)
+{
+	struct ntb_pci_bar_info *bar;
+
+	bar = &ntb->bar_info[idx];
+	if (HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) {
+		ntb_reg_write(4, bar->pbarxlat_off, base_addr);
+		base_addr = ntb_reg_read(4, bar->pbarxlat_off);
+	} else {
+		ntb_reg_write(8, bar->pbarxlat_off, base_addr);
+		base_addr = ntb_reg_read(8, bar->pbarxlat_off);
+	}
+	(void)base_addr;
+}
+
 static int
 xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr,
     const struct ntb_b2b_addr *peer_addr)
 {
+	struct ntb_pci_bar_info *b2b_bar;
+	vm_size_t bar_size;
+	uint64_t bar_addr;
+	enum ntb_bar b2b_bar_num, i;
 
-	/* Local addresses */
-	ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, addr->bar2_addr64);
-	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
-		ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, addr->bar0_addr);
-	else {
-		if (HAS_FEATURE(NTB_SPLIT_BAR)) {
-			ntb_reg_write(4, XEON_PBAR4XLAT_OFFSET,
-			    addr->bar4_addr32);
-			ntb_reg_write(4, XEON_PBAR5XLAT_OFFSET,
-			    addr->bar5_addr32);
-		} else
-			ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
-			    addr->bar4_addr64);
-		/*
-		 * B2B_XLAT_OFFSET is a 64-bit register but can only be
-		 * written 32 bits at a time.
-		 */
-		ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL,
-		    addr->bar0_addr & 0xffffffff);
-		ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, addr->bar0_addr >> 32);
+	if (ntb->b2b_mw_idx == B2B_MW_DISABLED) {
+		b2b_bar = NULL;
+		b2b_bar_num = NTB_CONFIG_BAR;
+		ntb->b2b_off = 0;
+	} else {
+		b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx);
+		KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS,
+		    ("invalid b2b mw bar"));
+
+		b2b_bar = &ntb->bar_info[b2b_bar_num];
+		bar_size = b2b_bar->size;
+
+		if (ntb_b2b_mw_share != 0 &&
+		    (bar_size >> 1) >= XEON_B2B_MIN_SIZE)
+			ntb->b2b_off = bar_size >> 1;
+		else if (bar_size >= XEON_B2B_MIN_SIZE) {
+			ntb->b2b_off = 0;
+			ntb->mw_count--;
+		} else {
+			device_printf(ntb->device,
+			    "B2B bar size is too small!\n");
+			return (EIO);
+		}
 	}
 
-	/* Peer addresses */
-	ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, peer_addr->bar0_addr);
-	ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, peer_addr->bar2_addr64);
+	/*
+	 * Reset the secondary bar sizes to match the primary bar sizes.
+	 * (Except, disable or halve the size of the B2B secondary bar.)
+	 */
+	for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++)
+		xeon_reset_sbar_size(ntb, i, b2b_bar_num);
+
+	bar_addr = 0;
+	if (b2b_bar_num == NTB_CONFIG_BAR)
+		bar_addr = addr->bar0_addr;
+	else if (b2b_bar_num == NTB_B2B_BAR_1)
+		bar_addr = addr->bar2_addr64;
+	else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR))
+		bar_addr = addr->bar4_addr64;
+	else if (b2b_bar_num == NTB_B2B_BAR_2)
+		bar_addr = addr->bar4_addr32;
+	else if (b2b_bar_num == NTB_B2B_BAR_3)
+		bar_addr = addr->bar5_addr32;
+	else
+		KASSERT(false, ("invalid bar"));
+
+	ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr);
+
+	/*
+	 * Other SBARs are normally hit by the PBAR xlat, except for the b2b
+	 * register BAR.  The B2B BAR is either disabled above or configured
+	 * half-size.  It starts at PBAR xlat + offset.
+	 *
+	 * Also set up incoming BAR limits == base (zero length window).
+	 */
+	xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1,
+	    b2b_bar_num);
 	if (HAS_FEATURE(NTB_SPLIT_BAR)) {
-		ntb_reg_write(4, XEON_SBAR4BASE_OFFSET,
-		    peer_addr->bar4_addr32);
-		ntb_reg_write(4, XEON_SBAR5BASE_OFFSET,
-		    peer_addr->bar5_addr32);
+		xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32,
+		    NTB_B2B_BAR_2, b2b_bar_num);
+		xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32,
+		    NTB_B2B_BAR_3, b2b_bar_num);
 	} else
-		ntb_reg_write(8, XEON_SBAR4BASE_OFFSET,
-		    peer_addr->bar4_addr64);
+		xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64,
+		    NTB_B2B_BAR_2, b2b_bar_num);
+
+	/* Zero incoming translation addrs */
+	ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0);
+	ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0);
+
+	/* Zero outgoing translation limits (whole bar size windows) */
+	ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0);
+	ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0);
+
+	/* Set outgoing translation offsets */
+	xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1);
+	if (HAS_FEATURE(NTB_SPLIT_BAR)) {
+		xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2);
+		xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3);
+	} else
+		xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2);
+
+	/* Set the translation offset for B2B registers */
+	bar_addr = 0;
+	if (b2b_bar_num == NTB_CONFIG_BAR)
+		bar_addr = peer_addr->bar0_addr;
+	else if (b2b_bar_num == NTB_B2B_BAR_1)
+		bar_addr = peer_addr->bar2_addr64;
+	else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR))
+		bar_addr = peer_addr->bar4_addr64;
+	else if (b2b_bar_num == NTB_B2B_BAR_2)
+		bar_addr = peer_addr->bar4_addr32;
+	else if (b2b_bar_num == NTB_B2B_BAR_3)
+		bar_addr = peer_addr->bar5_addr32;
+	else
+		KASSERT(false, ("invalid bar"));
+
+	/*
+	 * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits
+	 * at a time.
+	 */
+	ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff);
+	ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32);
 	return (0);
 }
 
@@ -1966,7 +2125,8 @@ ntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out)
 		KASSERT((HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 2) ||
 		    (!HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 1),
 		    ("mw_count invalid after setup"));
-		KASSERT(ntb->b2b_mw_idx != UINT8_MAX, ("invalid b2b idx"));
+		KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED,
+		    ("invalid b2b idx"));
 
 		bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)];
 		regoff = XEON_SHADOW_PDOORBELL_OFFSET;
diff --git a/sys/dev/ntb/ntb_hw/ntb_regs.h b/sys/dev/ntb/ntb_hw/ntb_regs.h
index d56bb95a685d..2daf681fd918 100644
--- a/sys/dev/ntb/ntb_hw/ntb_regs.h
+++ b/sys/dev/ntb/ntb_hw/ntb_regs.h
@@ -135,6 +135,11 @@
 
 #define XEON_PBAR23SZ_OFFSET	0x00d0
 #define XEON_PBAR45SZ_OFFSET	0x00d1
+#define XEON_PBAR4SZ_OFFSET	0x00d1
+#define XEON_PBAR5SZ_OFFSET	0x00d5
+#define XEON_SBAR23SZ_OFFSET	0x00d2
+#define XEON_SBAR4SZ_OFFSET	0x00d3
+#define XEON_SBAR5SZ_OFFSET	0x00d6
 #define NTB_PPD_OFFSET		0x00d4
 #define XEON_PPD_CONN_TYPE	0x0003
 #define XEON_PPD_DEV_TYPE	0x0010

From 724f085cd4120f49f2fe657f6aa9f4265372227c Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Sun, 18 Oct 2015 20:20:38 +0000
Subject: [PATCH 358/380] if_ntb: Replace handmade bitset macros with
 sys/bitset.h

No functional change.

Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/if_ntb/if_ntb.c | 35 ++++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 13 deletions(-)

diff --git a/sys/dev/ntb/if_ntb/if_ntb.c b/sys/dev/ntb/if_ntb/if_ntb.c
index e7d7a91c8a26..8c87fb78e278 100644
--- a/sys/dev/ntb/if_ntb/if_ntb.c
+++ b/sys/dev/ntb/if_ntb/if_ntb.c
@@ -30,6 +30,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/kernel.h>
 #include <sys/systm.h>
+#include <sys/bitset.h>
 #include <sys/bus.h>
 #include <sys/ktr.h>
 #include <sys/limits.h>
@@ -42,14 +43,17 @@ __FBSDID("$FreeBSD$");
 #include <sys/sockio.h>
 #include <sys/sysctl.h>
 #include <sys/taskqueue.h>
+
 #include <net/if.h>
 #include <net/if_media.h>
 #include <net/if_types.h>
 #include <net/if_var.h>
 #include <net/bpf.h>
 #include <net/ethernet.h>
+
 #include <vm/vm.h>
 #include <vm/pmap.h>
+
 #include <machine/bus.h>
 #include <machine/cpufunc.h>
 #include <machine/pmap.h>
@@ -67,10 +71,12 @@ __FBSDID("$FreeBSD$");
  * be picked up and redistributed in Linux with a dual GPL/BSD license.
  */
 
-/* TODO: These functions should really be part of the kernel */
-#define test_bit(pos, bitmap_addr)  (*(bitmap_addr) & 1UL << (pos))
-#define set_bit(pos, bitmap_addr)   *(bitmap_addr) |= 1UL << (pos)
-#define clear_bit(pos, bitmap_addr) *(bitmap_addr) &= ~(1UL << (pos))
+#define QP_SETSIZE	64
+BITSET_DEFINE(_qpset, QP_SETSIZE);
+#define test_bit(pos, addr)	BIT_ISSET(QP_SETSIZE, (pos), (addr))
+#define set_bit(pos, addr)	BIT_SET(QP_SETSIZE, (pos), (addr))
+#define clear_bit(pos, addr)	BIT_CLR(QP_SETSIZE, (pos), (addr))
+#define ffs_bit(addr)		BIT_FFS(QP_SETSIZE, (addr))
 
 #define KTR_NTB KTR_SPARE3
 
@@ -176,9 +182,9 @@ struct ntb_netdev {
 	struct ifnet		*ifp;
 	struct ntb_transport_mw	mw[NTB_MAX_NUM_MW];
 	struct ntb_transport_qp	*qps;
-	uint64_t		max_qps;
-	uint64_t		qp_bitmap;
-	bool			transport_link;
+	struct _qpset		qp_bitmap;
+	uint8_t			max_qps;
+	enum ntb_link_event	transport_link;
 	struct callout		link_work;
 	struct ntb_transport_qp *qp;
 	uint64_t		bufsize;
@@ -490,7 +496,8 @@ static int
 ntb_transport_init(struct ntb_softc *ntb)
 {
 	struct ntb_netdev *nt = &net_softc;
-	int rc, i;
+	int rc;
+	uint8_t i;
 
 	if (max_num_clients == 0)
 		nt->max_qps = MIN(ntb_get_max_cbs(ntb), ntb_mw_count(ntb));
@@ -504,10 +511,12 @@ ntb_transport_init(struct ntb_softc *ntb)
 	nt->qps = malloc(nt->max_qps * sizeof(struct ntb_transport_qp),
 			  M_NTB_IF, M_WAITOK|M_ZERO);
 
-	nt->qp_bitmap = ((uint64_t) 1 << nt->max_qps) - 1;
+	KASSERT(nt->max_qps <= QP_SETSIZE, ("invalid max_qps"));
 
-	for (i = 0; i < nt->max_qps; i++)
+	for (i = 0; i < nt->max_qps; i++) {
+		set_bit(i, &nt->qp_bitmap);
 		ntb_transport_init_queue(nt, i);
+	}
 
 	callout_init(&nt->link_work, 0);
 
@@ -535,7 +544,7 @@ ntb_transport_free(void *transport)
 {
 	struct ntb_netdev *nt = transport;
 	struct ntb_softc *ntb = nt->ntb;
-	int i;
+	uint8_t i;
 
 	ntb_transport_link_cleanup(nt);
 
@@ -656,7 +665,7 @@ ntb_transport_create_queue(void *data, struct ntb_softc *pdev,
 	if (nt == NULL)
 		goto err;
 
-	free_queue = ffs(nt->qp_bitmap);
+	free_queue = ffs_bit(&nt->qp_bitmap);
 	if (free_queue == 0)
 		goto err;
 
@@ -1284,7 +1293,7 @@ ntb_qp_link_work(void *arg)
 static void
 ntb_transport_link_cleanup(struct ntb_netdev *nt)
 {
-	int i;
+	uint8_t i;
 
 	/* Pass along the info to any clients */
 	for (i = 0; i < nt->max_qps; i++)

From 5d075fd18fef51e7cd9ab30ff1f17d780502b861 Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Sun, 18 Oct 2015 20:20:48 +0000
Subject: [PATCH 359/380] if_ntb: Rename things to match Linux driver

No functional change.  Part of the huge rewrite (e26a5843).

Obtained from:	Linux (e26a5843) (Dual BSD/GPL driver)
Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/if_ntb/if_ntb.c | 232 ++++++++++++++++++------------------
 sys/dev/ntb/ntb_hw/ntb_hw.c |  33 ++---
 sys/dev/ntb/ntb_hw/ntb_hw.h |  12 +-
 3 files changed, 136 insertions(+), 141 deletions(-)

diff --git a/sys/dev/ntb/if_ntb/if_ntb.c b/sys/dev/ntb/if_ntb/if_ntb.c
index 8c87fb78e278..3e45a8968ff8 100644
--- a/sys/dev/ntb/if_ntb/if_ntb.c
+++ b/sys/dev/ntb/if_ntb/if_ntb.c
@@ -1,5 +1,6 @@
 /*-
  * Copyright (C) 2013 Intel Corporation
+ * Copyright (C) 2015 EMC Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -110,13 +111,13 @@ struct ntb_rx_info {
 };
 
 struct ntb_transport_qp {
-	struct ntb_netdev	*transport;
+	struct ntb_transport_ctx	*transport;
 	struct ntb_softc	*ntb;
 
 	void			*cb_data;
 
 	bool			client_ready;
-	bool			qp_link;
+	bool			link_is_up;
 	uint8_t			qp_num;	/* Only 64 QPs are allowed.  0-63 */
 
 	struct ntb_rx_info	*rx_info;
@@ -172,19 +173,20 @@ struct ntb_queue_handlers {
 
 
 struct ntb_transport_mw {
-	size_t		size;
+	size_t		xlat_size;
 	void		*virt_addr;
 	vm_paddr_t	dma_addr;
 };
 
-struct ntb_netdev {
+struct ntb_transport_ctx {
 	struct ntb_softc	*ntb;
 	struct ifnet		*ifp;
-	struct ntb_transport_mw	mw[NTB_MAX_NUM_MW];
-	struct ntb_transport_qp	*qps;
+	struct ntb_transport_mw	mw_vec[NTB_MAX_NUM_MW];
+	struct ntb_transport_qp	*qp_vec;
 	struct _qpset		qp_bitmap;
-	uint8_t			max_qps;
-	enum ntb_link_event	transport_link;
+	unsigned		mw_count;
+	unsigned		qp_count;
+	enum ntb_link_event	link_is_up;
 	struct callout		link_work;
 	struct ntb_transport_qp *qp;
 	uint64_t		bufsize;
@@ -193,7 +195,7 @@ struct ntb_netdev {
 	struct mtx		rx_lock;
 };
 
-static struct ntb_netdev net_softc;
+static struct ntb_transport_ctx net_softc;
 
 enum {
 	IF_NTB_DESC_DONE_FLAG = 1 << 0,
@@ -226,7 +228,7 @@ enum {
 	IF_NTB_MAX_SPAD,
 };
 
-#define QP_TO_MW(ntb, qp)	((qp) % ntb_mw_count(ntb))
+#define QP_TO_MW(nt, qp)	((qp) % nt->mw_count)
 #define NTB_QP_DEF_NUM_ENTRIES	100
 #define NTB_LINK_DOWN_TIMEOUT	10
 
@@ -243,7 +245,7 @@ static void ntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data,
 static void ntb_net_event_handler(void *data, enum ntb_link_event status);
 static int ntb_transport_init(struct ntb_softc *ntb);
 static void ntb_transport_free(void *transport);
-static void ntb_transport_init_queue(struct ntb_netdev *nt,
+static void ntb_transport_init_queue(struct ntb_transport_ctx *nt,
     unsigned int qp_num);
 static void ntb_transport_free_queue(struct ntb_transport_qp *qp);
 static struct ntb_transport_qp * ntb_transport_create_queue(void *data,
@@ -264,12 +266,12 @@ static void ntb_rx_copy_task(struct ntb_transport_qp *qp,
 static void ntb_rx_completion_task(void *arg, int pending);
 static void ntb_transport_event_callback(void *data, enum ntb_hw_event event);
 static void ntb_transport_link_work(void *arg);
-static int ntb_set_mw(struct ntb_netdev *nt, int num_mw, unsigned int size);
-static void ntb_free_mw(struct ntb_netdev *nt, int num_mw);
-static void ntb_transport_setup_qp_mw(struct ntb_netdev *nt,
+static int ntb_set_mw(struct ntb_transport_ctx *, int num_mw, unsigned size);
+static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw);
+static void ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,
     unsigned int qp_num);
 static void ntb_qp_link_work(void *arg);
-static void ntb_transport_link_cleanup(struct ntb_netdev *nt);
+static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt);
 static void ntb_qp_link_down(struct ntb_transport_qp *qp);
 static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp);
 static void ntb_transport_link_down(struct ntb_transport_qp *qp);
@@ -380,7 +382,7 @@ ntb_teardown_interface(void)
 static void
 ntb_net_init(void *arg)
 {
-	struct ntb_netdev *ntb_softc = arg;
+	struct ntb_transport_ctx *ntb_softc = arg;
 	struct ifnet *ifp = ntb_softc->ifp;
 
 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
@@ -392,7 +394,7 @@ ntb_net_init(void *arg)
 static int
 ntb_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 {
-	struct ntb_netdev *nt = ifp->if_softc;
+	struct ntb_transport_ctx *nt = ifp->if_softc;
 	struct ifreq *ifr = (struct ifreq *)data;
 	int error = 0;
 
@@ -421,7 +423,7 @@ static void
 ntb_start(struct ifnet *ifp)
 {
 	struct mbuf *m_head;
-	struct ntb_netdev *nt = ifp->if_softc;
+	struct ntb_transport_ctx *nt = ifp->if_softc;
 	int rc;
 
 	mtx_lock(&nt->tx_lock);
@@ -495,37 +497,37 @@ ntb_net_event_handler(void *data, enum ntb_link_event status)
 static int
 ntb_transport_init(struct ntb_softc *ntb)
 {
-	struct ntb_netdev *nt = &net_softc;
+	struct ntb_transport_ctx *nt = &net_softc;
 	int rc;
 	uint8_t i;
 
+	nt->mw_count = ntb_mw_count(ntb);
 	if (max_num_clients == 0)
-		nt->max_qps = MIN(ntb_get_max_cbs(ntb), ntb_mw_count(ntb));
+		nt->qp_count = MIN(ntb_get_max_cbs(ntb), ntb_mw_count(ntb));
 	else
-		nt->max_qps = MIN(ntb_get_max_cbs(ntb), max_num_clients);
+		nt->qp_count = MIN(ntb_get_max_cbs(ntb), max_num_clients);
 
 	ntb_register_transport(ntb, nt);
 	mtx_init(&nt->tx_lock, "ntb transport tx", NULL, MTX_DEF);
 	mtx_init(&nt->rx_lock, "ntb transport rx", NULL, MTX_DEF);
 
-	nt->qps = malloc(nt->max_qps * sizeof(struct ntb_transport_qp),
-			  M_NTB_IF, M_WAITOK|M_ZERO);
+	nt->qp_vec = malloc(nt->qp_count * sizeof(*nt->qp_vec), M_NTB_IF,
+	    M_WAITOK | M_ZERO);
 
-	KASSERT(nt->max_qps <= QP_SETSIZE, ("invalid max_qps"));
+	KASSERT(nt->qp_count <= QP_SETSIZE, ("invalid qp_count"));
 
-	for (i = 0; i < nt->max_qps; i++) {
+	for (i = 0; i < nt->qp_count; i++) {
 		set_bit(i, &nt->qp_bitmap);
 		ntb_transport_init_queue(nt, i);
 	}
 
 	callout_init(&nt->link_work, 0);
 
-	rc = ntb_register_event_callback(ntb,
-					 ntb_transport_event_callback);
+	rc = ntb_register_event_callback(ntb, ntb_transport_event_callback);
 	if (rc != 0)
 		goto err;
 
-	if (ntb_query_link_status(ntb)) {
+	if (ntb_link_is_up(ntb)) {
 		if (bootverbose)
 			device_printf(ntb_get_device(ntb), "link up\n");
 		callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt);
@@ -534,7 +536,7 @@ ntb_transport_init(struct ntb_softc *ntb)
 	return (0);
 
 err:
-	free(nt->qps, M_NTB_IF);
+	free(nt->qp_vec, M_NTB_IF);
 	ntb_unregister_transport(ntb);
 	return (rc);
 }
@@ -542,7 +544,7 @@ err:
 static void
 ntb_transport_free(void *transport)
 {
-	struct ntb_netdev *nt = transport;
+	struct ntb_transport_ctx *nt = transport;
 	struct ntb_softc *ntb = nt->ntb;
 	uint8_t i;
 
@@ -551,46 +553,46 @@ ntb_transport_free(void *transport)
 	callout_drain(&nt->link_work);
 
 	/* verify that all the qps are freed */
-	for (i = 0; i < nt->max_qps; i++)
+	for (i = 0; i < nt->qp_count; i++)
 		if (!test_bit(i, &nt->qp_bitmap))
-			ntb_transport_free_queue(&nt->qps[i]);
+			ntb_transport_free_queue(&nt->qp_vec[i]);
 
 	ntb_unregister_event_callback(ntb);
 
 	for (i = 0; i < NTB_MAX_NUM_MW; i++)
 		ntb_free_mw(nt, i);
 
-	free(nt->qps, M_NTB_IF);
+	free(nt->qp_vec, M_NTB_IF);
 	ntb_unregister_transport(ntb);
 }
 
 static void
-ntb_transport_init_queue(struct ntb_netdev *nt, unsigned int qp_num)
+ntb_transport_init_queue(struct ntb_transport_ctx *nt, unsigned int qp_num)
 {
 	struct ntb_transport_qp *qp;
 	unsigned int num_qps_mw, tx_size;
-	uint8_t mw_num, mw_max;
+	uint8_t mw_num, mw_count;
 
-	mw_max = ntb_mw_count(nt->ntb);
-	mw_num = QP_TO_MW(nt->ntb, qp_num);
+	mw_count = ntb_mw_count(nt->ntb);
+	mw_num = QP_TO_MW(nt, qp_num);
 
-	qp = &nt->qps[qp_num];
+	qp = &nt->qp_vec[qp_num];
 	qp->qp_num = qp_num;
 	qp->transport = nt;
 	qp->ntb = nt->ntb;
-	qp->qp_link = NTB_LINK_DOWN;
-	qp->client_ready = NTB_LINK_DOWN;
+	qp->link_is_up = false;
+	qp->client_ready = false;
 	qp->event_handler = NULL;
 
-	if (nt->max_qps % mw_max && mw_num + 1 < nt->max_qps / mw_max)
-		num_qps_mw = nt->max_qps / mw_max + 1;
+	if (nt->qp_count % mw_count && mw_num + 1 < nt->qp_count / mw_count)
+		num_qps_mw = nt->qp_count / mw_count + 1;
 	else
-		num_qps_mw = nt->max_qps / mw_max;
+		num_qps_mw = nt->qp_count / mw_count;
 
-	tx_size = (unsigned int) ntb_get_mw_size(qp->ntb, mw_num) / num_qps_mw;
+	tx_size = ntb_get_mw_size(qp->ntb, mw_num) / num_qps_mw;
 	qp->rx_info = (struct ntb_rx_info *)
 	    ((char *)ntb_get_mw_vbase(qp->ntb, mw_num) +
-	    (qp_num / mw_max * tx_size));
+	    (qp_num / mw_count * tx_size));
 	tx_size -= sizeof(struct ntb_rx_info);
 
 	qp->tx_mw = qp->rx_info + 1;
@@ -657,7 +659,7 @@ ntb_transport_create_queue(void *data, struct ntb_softc *pdev,
 {
 	struct ntb_queue_entry *entry;
 	struct ntb_transport_qp *qp;
-	struct ntb_netdev *nt;
+	struct ntb_transport_ctx *nt;
 	unsigned int free_queue;
 	int rc, i;
 
@@ -674,7 +676,7 @@ ntb_transport_create_queue(void *data, struct ntb_softc *pdev,
 
 	clear_bit(free_queue, &nt->qp_bitmap);
 
-	qp = &nt->qps[free_queue];
+	qp = &nt->qp_vec[free_queue];
 	qp->cb_data = data;
 	qp->rx_handler = handlers->rx_handler;
 	qp->tx_handler = handlers->tx_handler;
@@ -725,11 +727,11 @@ ntb_transport_link_up(struct ntb_transport_qp *qp)
 	if (qp == NULL)
 		return;
 
-	qp->client_ready = NTB_LINK_UP;
+	qp->client_ready = true;
 	if (bootverbose)
 		device_printf(ntb_get_device(qp->ntb), "qp client ready\n");
 
-	if (qp->transport->transport_link == NTB_LINK_UP)
+	if (qp->transport->link_is_up)
 		callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp);
 }
 
@@ -757,7 +759,7 @@ ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,
 	struct ntb_queue_entry *entry;
 	int rc;
 
-	if (qp == NULL || qp->qp_link != NTB_LINK_UP || len == 0) {
+	if (qp == NULL || !qp->link_is_up || len == 0) {
 		CTR0(KTR_NTB, "TX: link not up");
 		return (EINVAL);
 	}
@@ -839,7 +841,7 @@ ntb_tx_copy_task(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry,
 	/* TODO: replace with bus_space_write */
 	hdr->flags = entry->flags | IF_NTB_DESC_DONE_FLAG;
 
-	ntb_ring_doorbell(qp->ntb, qp->qp_num);
+	ntb_peer_db_set(qp->ntb, 1ull << qp->qp_num);
 
 	/*
 	 * The entry length can only be zero if the packet is intended to be a
@@ -1013,7 +1015,7 @@ ntb_rx_completion_task(void *arg, int pending)
 	while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) {
 		m = entry->buf;
 		CTR2(KTR_NTB, "RX: completing entry %p, mbuf %p", entry, m);
-		if (qp->rx_handler && qp->client_ready == NTB_LINK_UP)
+		if (qp->rx_handler && qp->client_ready)
 			qp->rx_handler(qp, qp->cb_data, m, entry->len);
 
 		entry->buf = NULL;
@@ -1035,7 +1037,7 @@ ntb_rx_completion_task(void *arg, int pending)
 static void
 ntb_transport_event_callback(void *data, enum ntb_hw_event event)
 {
-	struct ntb_netdev *nt = data;
+	struct ntb_transport_ctx *nt = data;
 
 	switch (event) {
 	case NTB_EVENT_HW_LINK_UP:
@@ -1057,7 +1059,7 @@ ntb_transport_event_callback(void *data, enum ntb_hw_event event)
 static void
 ntb_transport_link_work(void *arg)
 {
-	struct ntb_netdev *nt = arg;
+	struct ntb_transport_ctx *nt = arg;
 	struct ntb_softc *ntb = nt->ntb;
 	struct ntb_transport_qp *qp;
 	uint64_t val64;
@@ -1068,45 +1070,45 @@ ntb_transport_link_work(void *arg)
 
 	/* send the local info, in the opposite order of the way we read it */
 	for (i = 0; i < num_mw; i++) {
-		rc = ntb_write_remote_spad(ntb, IF_NTB_MW0_SZ_HIGH + (i * 2),
+		rc = ntb_peer_spad_write(ntb, IF_NTB_MW0_SZ_HIGH + (i * 2),
 		    (uint64_t)ntb_get_mw_size(ntb, i) >> 32);
 		if (rc != 0)
 			goto out;
 
-		rc = ntb_write_remote_spad(ntb, IF_NTB_MW0_SZ_LOW + (i * 2),
+		rc = ntb_peer_spad_write(ntb, IF_NTB_MW0_SZ_LOW + (i * 2),
 		    (uint32_t)ntb_get_mw_size(ntb, i));
 		if (rc != 0)
 			goto out;
 	}
 
-	rc = ntb_write_remote_spad(ntb, IF_NTB_NUM_MWS, num_mw);
+	rc = ntb_peer_spad_write(ntb, IF_NTB_NUM_MWS, num_mw);
 	if (rc != 0)
 		goto out;
 
-	rc = ntb_write_remote_spad(ntb, IF_NTB_NUM_QPS, nt->max_qps);
+	rc = ntb_peer_spad_write(ntb, IF_NTB_NUM_QPS, nt->qp_count);
 	if (rc != 0)
 		goto out;
 
-	rc = ntb_write_remote_spad(ntb, IF_NTB_VERSION, NTB_TRANSPORT_VERSION);
+	rc = ntb_peer_spad_write(ntb, IF_NTB_VERSION, NTB_TRANSPORT_VERSION);
 	if (rc != 0)
 		goto out;
 
 	/* Query the remote side for its info */
-	rc = ntb_read_local_spad(ntb, IF_NTB_VERSION, &val);
+	rc = ntb_spad_read(ntb, IF_NTB_VERSION, &val);
 	if (rc != 0)
 		goto out;
 
 	if (val != NTB_TRANSPORT_VERSION)
 		goto out;
 
-	rc = ntb_read_local_spad(ntb, IF_NTB_NUM_QPS, &val);
+	rc = ntb_spad_read(ntb, IF_NTB_NUM_QPS, &val);
 	if (rc != 0)
 		goto out;
 
-	if (val != nt->max_qps)
+	if (val != nt->qp_count)
 		goto out;
 
-	rc = ntb_read_local_spad(ntb, IF_NTB_NUM_MWS, &val);
+	rc = ntb_spad_read(ntb, IF_NTB_NUM_MWS, &val);
 	if (rc != 0)
 		goto out;
 
@@ -1114,14 +1116,14 @@ ntb_transport_link_work(void *arg)
 		goto out;
 
 	for (i = 0; i < num_mw; i++) {
-		rc = ntb_read_local_spad(ntb, IF_NTB_MW0_SZ_HIGH + (i * 2),
+		rc = ntb_spad_read(ntb, IF_NTB_MW0_SZ_HIGH + (i * 2),
 		    &val);
 		if (rc != 0)
 			goto free_mws;
 
 		val64 = (uint64_t)val << 32;
 
-		rc = ntb_read_local_spad(ntb, IF_NTB_MW0_SZ_LOW + (i * 2),
+		rc = ntb_spad_read(ntb, IF_NTB_MW0_SZ_LOW + (i * 2),
 		    &val);
 		if (rc != 0)
 			goto free_mws;
@@ -1133,16 +1135,16 @@ ntb_transport_link_work(void *arg)
 			goto free_mws;
 	}
 
-	nt->transport_link = NTB_LINK_UP;
+	nt->link_is_up = true;
 	if (bootverbose)
 		device_printf(ntb_get_device(ntb), "transport link up\n");
 
-	for (i = 0; i < nt->max_qps; i++) {
-		qp = &nt->qps[i];
+	for (i = 0; i < nt->qp_count; i++) {
+		qp = &nt->qp_vec[i];
 
 		ntb_transport_setup_qp_mw(nt, i);
 
-		if (qp->client_ready == NTB_LINK_UP)
+		if (qp->client_ready)
 			callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp);
 	}
 
@@ -1152,32 +1154,32 @@ free_mws:
 	for (i = 0; i < NTB_MAX_NUM_MW; i++)
 		ntb_free_mw(nt, i);
 out:
-	if (ntb_query_link_status(ntb))
+	if (ntb_link_is_up(ntb))
 		callout_reset(&nt->link_work,
 		    NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_transport_link_work, nt);
 }
 
 static int
-ntb_set_mw(struct ntb_netdev *nt, int num_mw, unsigned int size)
+ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw, unsigned int size)
 {
-	struct ntb_transport_mw *mw = &nt->mw[num_mw];
+	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
 
 	/* No need to re-setup */
-	if (mw->size == size)
+	if (mw->xlat_size == size)
 		return (0);
 
-	if (mw->size != 0)
+	if (mw->xlat_size != 0)
 		ntb_free_mw(nt, num_mw);
 
 	/* Alloc memory for receiving data.  Must be 4k aligned */
-	mw->size = size;
+	mw->xlat_size = size;
 
-	mw->virt_addr = contigmalloc(mw->size, M_NTB_IF, M_ZERO, 0,
-	    BUS_SPACE_MAXADDR, mw->size, 0);
+	mw->virt_addr = contigmalloc(mw->xlat_size, M_NTB_IF, M_ZERO, 0,
+	    BUS_SPACE_MAXADDR, mw->xlat_size, 0);
 	if (mw->virt_addr == NULL) {
-		mw->size = 0;
+		mw->xlat_size = 0;
 		printf("ntb: Unable to allocate MW buffer of size %zu\n",
-		    mw->size);
+		    mw->xlat_size);
 		return (ENOMEM);
 	}
 	/* TODO: replace with bus_space_* functions */
@@ -1203,37 +1205,37 @@ ntb_set_mw(struct ntb_netdev *nt, int num_mw, unsigned int size)
 }
 
 static void
-ntb_free_mw(struct ntb_netdev *nt, int num_mw)
+ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw)
 {
-	struct ntb_transport_mw *mw = &nt->mw[num_mw];
+	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
 
 	if (mw->virt_addr == NULL)
 		return;
 
-	contigfree(mw->virt_addr, mw->size, M_NTB_IF);
+	contigfree(mw->virt_addr, mw->xlat_size, M_NTB_IF);
 	mw->virt_addr = NULL;
 }
 
 static void
-ntb_transport_setup_qp_mw(struct ntb_netdev *nt, unsigned int qp_num)
+ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt, unsigned int qp_num)
 {
-	struct ntb_transport_qp *qp = &nt->qps[qp_num];
+	struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
 	void *offset;
 	unsigned int rx_size, num_qps_mw;
-	uint8_t mw_num, mw_max;
+	uint8_t mw_num, mw_count;
 	unsigned int i;
 
-	mw_max = ntb_mw_count(nt->ntb);
-	mw_num = QP_TO_MW(nt->ntb, qp_num);
+	mw_count = ntb_mw_count(nt->ntb);
+	mw_num = QP_TO_MW(nt, qp_num);
 
-	if (nt->max_qps % mw_max && mw_num + 1 < nt->max_qps / mw_max)
-		num_qps_mw = nt->max_qps / mw_max + 1;
+	if (nt->qp_count % mw_count && mw_num + 1 < nt->qp_count / mw_count)
+		num_qps_mw = nt->qp_count / mw_count + 1;
 	else
-		num_qps_mw = nt->max_qps / mw_max;
+		num_qps_mw = nt->qp_count / mw_count;
 
-	rx_size = (unsigned int) nt->mw[mw_num].size / num_qps_mw;
-	qp->remote_rx_info = (void *)((uint8_t *)nt->mw[mw_num].virt_addr +
-			     (qp_num / mw_max * rx_size));
+	rx_size = nt->mw_vec[mw_num].xlat_size / num_qps_mw;
+	qp->remote_rx_info = (void *)((uint8_t *)nt->mw_vec[mw_num].virt_addr +
+	    (qp_num / mw_count * rx_size));
 	rx_size -= sizeof(struct ntb_rx_info);
 
 	qp->rx_buff = qp->remote_rx_info + 1;
@@ -1263,27 +1265,27 @@ ntb_qp_link_work(void *arg)
 {
 	struct ntb_transport_qp *qp = arg;
 	struct ntb_softc *ntb = qp->ntb;
-	struct ntb_netdev *nt = qp->transport;
+	struct ntb_transport_ctx *nt = qp->transport;
 	int rc, val;
 
 
-	rc = ntb_read_remote_spad(ntb, IF_NTB_QP_LINKS, &val);
+	rc = ntb_peer_spad_read(ntb, IF_NTB_QP_LINKS, &val);
 	if (rc != 0)
 		return;
 
-	rc = ntb_write_remote_spad(ntb, IF_NTB_QP_LINKS, val | 1 << qp->qp_num);
+	rc = ntb_peer_spad_write(ntb, IF_NTB_QP_LINKS, val | 1 << qp->qp_num);
 
 	/* query remote spad for qp ready bits */
-	rc = ntb_read_local_spad(ntb, IF_NTB_QP_LINKS, &val);
+	rc = ntb_spad_read(ntb, IF_NTB_QP_LINKS, &val);
 
 	/* See if the remote side is up */
 	if ((1 << qp->qp_num & val) != 0) {
-		qp->qp_link = NTB_LINK_UP;
+		qp->link_is_up = true;
 		if (qp->event_handler != NULL)
 			qp->event_handler(qp->cb_data, NTB_LINK_UP);
 		if (bootverbose)
 			device_printf(ntb_get_device(ntb), "qp link up\n");
-	} else if (nt->transport_link == NTB_LINK_UP) {
+	} else if (nt->link_is_up) {
 		callout_reset(&qp->link_work,
 		    NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp);
 	}
@@ -1291,19 +1293,19 @@ ntb_qp_link_work(void *arg)
 
 /* Link down event*/
 static void
-ntb_transport_link_cleanup(struct ntb_netdev *nt)
+ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)
 {
 	uint8_t i;
 
 	/* Pass along the info to any clients */
-	for (i = 0; i < nt->max_qps; i++)
+	for (i = 0; i < nt->qp_count; i++)
 		if (!test_bit(i, &nt->qp_bitmap))
-			ntb_qp_link_down(&nt->qps[i]);
+			ntb_qp_link_down(&nt->qp_vec[i]);
 
-	if (nt->transport_link == NTB_LINK_DOWN)
+	if (!nt->link_is_up)
 		callout_drain(&nt->link_work);
 	else
-		nt->transport_link = NTB_LINK_DOWN;
+		nt->link_is_up = false;
 
 	/*
 	 * The scratchpad registers keep the values if the remote side
@@ -1311,7 +1313,7 @@ ntb_transport_link_cleanup(struct ntb_netdev *nt)
 	 * time they are accessed
 	 */
 	for (i = 0; i < IF_NTB_MAX_SPAD; i++)
-		ntb_write_local_spad(nt->ntb, i, 0);
+		ntb_spad_write(nt->ntb, i, 0);
 }
 
 
@@ -1325,9 +1327,9 @@ ntb_qp_link_down(struct ntb_transport_qp *qp)
 static void
 ntb_qp_link_cleanup(struct ntb_transport_qp *qp)
 {
-	struct ntb_netdev *nt = qp->transport;
+	struct ntb_transport_ctx *nt = qp->transport;
 
-	if (qp->qp_link == NTB_LINK_DOWN) {
+	if (!qp->link_is_up) {
 		callout_drain(&qp->link_work);
 		return;
 	}
@@ -1335,9 +1337,9 @@ ntb_qp_link_cleanup(struct ntb_transport_qp *qp)
 	if (qp->event_handler != NULL)
 		qp->event_handler(qp->cb_data, NTB_LINK_DOWN);
 
-	qp->qp_link = NTB_LINK_DOWN;
+	qp->link_is_up = false;
 
-	if (nt->transport_link == NTB_LINK_UP)
+	if (nt->link_is_up)
 		callout_reset(&qp->link_work,
 		    NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp);
 }
@@ -1359,16 +1361,16 @@ ntb_transport_link_down(struct ntb_transport_qp *qp)
 	if (qp == NULL)
 		return;
 
-	qp->client_ready = NTB_LINK_DOWN;
+	qp->client_ready = false;
 
-	rc = ntb_read_remote_spad(qp->ntb, IF_NTB_QP_LINKS, &val);
+	rc = ntb_peer_spad_read(qp->ntb, IF_NTB_QP_LINKS, &val);
 	if (rc != 0)
 		return;
 
-	rc = ntb_write_remote_spad(qp->ntb, IF_NTB_QP_LINKS,
+	rc = ntb_peer_spad_write(qp->ntb, IF_NTB_QP_LINKS,
 	   val & ~(1 << qp->qp_num));
 
-	if (qp->qp_link == NTB_LINK_UP)
+	if (qp->link_is_up)
 		ntb_send_link_down(qp);
 	else
 		callout_drain(&qp->link_work);
@@ -1381,10 +1383,10 @@ ntb_send_link_down(struct ntb_transport_qp *qp)
 	struct ntb_queue_entry *entry;
 	int i, rc;
 
-	if (qp->qp_link == NTB_LINK_DOWN)
+	if (!qp->link_is_up)
 		return;
 
-	qp->qp_link = NTB_LINK_DOWN;
+	qp->link_is_up = false;
 
 	for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) {
 		entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q);
diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c
index 72754dae2464..cee2424db8d4 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.c
@@ -1891,7 +1891,7 @@ ntb_mw_count(struct ntb_softc *ntb)
 }
 
 /**
- * ntb_write_local_spad() - write to the secondary scratchpad register
+ * ntb_spad_write() - write to the secondary scratchpad register
  * @ntb: pointer to ntb_softc instance
  * @idx: index to the scratchpad register, 0 based
  * @val: the data value to put into the register
@@ -1902,7 +1902,7 @@ ntb_mw_count(struct ntb_softc *ntb)
  * RETURNS: An appropriate ERRNO error value on error, or zero for success.
  */
 int
-ntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
+ntb_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
 {
 
 	if (idx >= ntb->spad_count)
@@ -1914,7 +1914,7 @@ ntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
 }
 
 /**
- * ntb_read_local_spad() - read from the primary scratchpad register
+ * ntb_spad_read() - read from the primary scratchpad register
  * @ntb: pointer to ntb_softc instance
  * @idx: index to scratchpad register, 0 based
  * @val: pointer to 32bit integer for storing the register value
@@ -1925,7 +1925,7 @@ ntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
  * RETURNS: An appropriate ERRNO error value on error, or zero for success.
  */
 int
-ntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
+ntb_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
 {
 
 	if (idx >= ntb->spad_count)
@@ -1937,7 +1937,7 @@ ntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
 }
 
 /**
- * ntb_write_remote_spad() - write to the secondary scratchpad register
+ * ntb_peer_spad_write() - write to the secondary scratchpad register
  * @ntb: pointer to ntb_softc instance
  * @idx: index to the scratchpad register, 0 based
  * @val: the data value to put into the register
@@ -1948,7 +1948,7 @@ ntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
  * RETURNS: An appropriate ERRNO error value on error, or zero for success.
  */
 int
-ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
+ntb_peer_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
 {
 
 	if (idx >= ntb->spad_count)
@@ -1963,7 +1963,7 @@ ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
 }
 
 /**
- * ntb_read_remote_spad() - read from the primary scratchpad register
+ * ntb_peer_spad_read() - read from the primary scratchpad register
  * @ntb: pointer to ntb_softc instance
  * @idx: index to scratchpad register, 0 based
  * @val: pointer to 32bit integer for storing the register value
@@ -1974,7 +1974,7 @@ ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
  * RETURNS: An appropriate ERRNO error value on error, or zero for success.
  */
 int
-ntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
+ntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
 {
 
 	if (idx >= ntb->spad_count)
@@ -2074,23 +2074,16 @@ ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr)
 }
 
 /**
- * ntb_ring_doorbell() - Set the doorbell on the secondary/external side
+ * ntb_peer_db_set() - Set the doorbell on the secondary/external side
  * @ntb: pointer to ntb_softc instance
- * @db: doorbell to ring
+ * @bit: doorbell bits to ring
  *
  * This function allows triggering of a doorbell on the secondary/external
  * side that will initiate an interrupt on the remote host
  */
 void
-ntb_ring_doorbell(struct ntb_softc *ntb, unsigned int db)
+ntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit)
 {
-	uint64_t bit;
-
-	if (ntb->type == NTB_SOC)
-		bit = 1 << db;
-	else
-		bit = ((1 << ntb->db_vec_shift) - 1) <<
-		    (db * ntb->db_vec_shift);
 
 	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) {
 		ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, bit);
@@ -2139,7 +2132,7 @@ ntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out)
 }
 
 /**
- * ntb_query_link_status() - return the hardware link status
+ * ntb_link_is_up() - return the hardware link status
  * @ndev: pointer to ntb_device instance
  *
  * Returns true if the hardware is connected to the remote system
@@ -2147,7 +2140,7 @@ ntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out)
  * RETURNS: true or false based on the hardware link state
  */
 bool
-ntb_query_link_status(struct ntb_softc *ntb)
+ntb_link_is_up(struct ntb_softc *ntb)
 {
 
 	return (ntb->link_status == NTB_LINK_UP);
diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.h b/sys/dev/ntb/ntb_hw/ntb_hw.h
index 37bfa6823c3f..05a5f90b5837 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.h
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.h
@@ -65,19 +65,19 @@ void ntb_unregister_transport(struct ntb_softc *ntb);
 uint8_t ntb_get_max_cbs(struct ntb_softc *ntb);
 uint8_t ntb_mw_count(struct ntb_softc *ntb);
 uint8_t ntb_get_max_spads(struct ntb_softc *ntb);
-int ntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val);
-int ntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val);
-int ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx,
+int ntb_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val);
+int ntb_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val);
+int ntb_peer_spad_write(struct ntb_softc *ntb, unsigned int idx,
     uint32_t val);
-int ntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx,
+int ntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx,
     uint32_t *val);
 void *ntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw);
 bus_addr_t ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw);
 u_long ntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw);
 void ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr);
-void ntb_ring_doorbell(struct ntb_softc *ntb, unsigned int db);
+void ntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit);
 bus_addr_t ntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out);
-bool ntb_query_link_status(struct ntb_softc *ntb);
+bool ntb_link_is_up(struct ntb_softc *ntb);
 device_t ntb_get_device(struct ntb_softc *ntb);
 
 /* Hardware owns the low 32 bits of features. */

From c8ef511f2c0bf6629830d96f2f318ce10b240df1 Mon Sep 17 00:00:00 2001
From: cem <cem@FreeBSD.org>
Date: Sun, 18 Oct 2015 20:20:57 +0000
Subject: [PATCH 360/380] if_ntb: MFV e26a5843: Move MW/DB management to if_ntb

This is the last e26a5843 patch.  The general thrust of the rewrite was
to move more responsibility for Memory Window and Doorbell interrupt
management from the ntb_hw driver to if_ntb.

A number of APIs have been added, removed, or replaced.  The old
DB callback mechanism has been excised.  Instead, callers (if_ntb) are
responsible for configuring MWs and handling their interrupts more
directly.

This adds a tunable, hw.ntb.max_mw_size, allowing users to limit the
size of memory windows used by if_ntb (identical to the Linux modparam
of the same name).

Despite attempts to keep mechanical name changes to separate commits,
some have snuck in here.  At least the driver should be much more
similar to the latest Linux one now -- making porting fixes easier.

Authored by:	Allen Hubbe
Obtained from:	Linux (Dual BSD/GPL driver)
Sponsored by:	EMC / Isilon Storage Division
---
 sys/dev/ntb/if_ntb/if_ntb.c   |  584 +++++++++++--------
 sys/dev/ntb/ntb_hw/ntb_hw.c   | 1029 +++++++++++++++++----------------
 sys/dev/ntb/ntb_hw/ntb_hw.h   |   81 ++-
 sys/dev/ntb/ntb_hw/ntb_regs.h |    1 +
 4 files changed, 940 insertions(+), 755 deletions(-)

diff --git a/sys/dev/ntb/if_ntb/if_ntb.c b/sys/dev/ntb/if_ntb/if_ntb.c
index 3e45a8968ff8..6a1fd92e5c88 100644
--- a/sys/dev/ntb/if_ntb/if_ntb.c
+++ b/sys/dev/ntb/if_ntb/if_ntb.c
@@ -81,12 +81,22 @@ BITSET_DEFINE(_qpset, QP_SETSIZE);
 
 #define KTR_NTB KTR_SPARE3
 
-#define NTB_TRANSPORT_VERSION	3
+#define NTB_TRANSPORT_VERSION	4
 #define NTB_RX_MAX_PKTS		64
 #define	NTB_RXQ_SIZE		300
 
+enum ntb_link_event {
+	NTB_LINK_DOWN = 0,
+	NTB_LINK_UP,
+};
+
 static unsigned int transport_mtu = 0x4000 + ETHER_HDR_LEN + ETHER_CRC_LEN;
 
+static uint64_t max_mw_size;
+SYSCTL_UQUAD(_hw_ntb, OID_AUTO, max_mw_size, CTLFLAG_RDTUN, &max_mw_size, 0,
+    "If enabled (non-zero), limit the size of large memory windows. "
+    "Both sides of the NTB MUST set the same value here.");
+
 static unsigned int max_num_clients;
 SYSCTL_UINT(_hw_ntb, OID_AUTO, max_num_clients, CTLFLAG_RDTUN,
     &max_num_clients, 0, "Maximum number of NTB transport clients.  "
@@ -99,11 +109,15 @@ struct ntb_queue_entry {
 	/* ntb_queue list reference */
 	STAILQ_ENTRY(ntb_queue_entry) entry;
 
-	/* info on data to be transfered */
+	/* info on data to be transferred */
 	void		*cb_data;
 	void		*buf;
-	uint64_t	len;
-	uint64_t	flags;
+	unsigned	len;
+	unsigned	flags;
+
+	struct ntb_transport_qp		*qp;
+	struct ntb_payload_header	*x_hdr;
+	unsigned	index;
 };
 
 struct ntb_rx_info {
@@ -128,6 +142,7 @@ struct ntb_transport_qp {
 	struct ntb_queue_list	tx_free_q;
 	struct mtx		ntb_tx_free_q_lock;
 	void			*tx_mw;
+	bus_addr_t		tx_mw_phys;
 	uint64_t		tx_index;
 	uint64_t		tx_max_entry;
 	uint64_t		tx_max_frame;
@@ -139,6 +154,7 @@ struct ntb_transport_qp {
 	struct mtx		ntb_rx_pend_q_lock;
 	struct mtx		ntb_rx_free_q_lock;
 	struct task		rx_completion_task;
+	struct task		rxc_db_work;
 	void			*rx_buff;
 	uint64_t		rx_index;
 	uint64_t		rx_max_entry;
@@ -171,11 +187,18 @@ struct ntb_queue_handlers {
 	void (*event_handler)(void *data, enum ntb_link_event status);
 };
 
-
 struct ntb_transport_mw {
+	vm_paddr_t	phys_addr;
+	size_t		phys_size;
+	size_t		xlat_align;
+	size_t		xlat_align_size;
+	/* Tx buff is off vbase / phys_addr */
+	void		*vbase;
 	size_t		xlat_size;
+	size_t		buff_size;
+	/* Rx buff is off virt_addr / dma_addr */
 	void		*virt_addr;
-	vm_paddr_t	dma_addr;
+	bus_addr_t	dma_addr;
 };
 
 struct ntb_transport_ctx {
@@ -184,15 +207,18 @@ struct ntb_transport_ctx {
 	struct ntb_transport_mw	mw_vec[NTB_MAX_NUM_MW];
 	struct ntb_transport_qp	*qp_vec;
 	struct _qpset		qp_bitmap;
+	struct _qpset		qp_bitmap_free;
 	unsigned		mw_count;
 	unsigned		qp_count;
 	enum ntb_link_event	link_is_up;
 	struct callout		link_work;
-	struct ntb_transport_qp *qp;
 	uint64_t		bufsize;
 	u_char			eaddr[ETHER_ADDR_LEN];
 	struct mtx		tx_lock;
 	struct mtx		rx_lock;
+
+	/* The hardcoded single queuepair in ntb_setup_interface() */
+	struct ntb_transport_qp *qp;
 };
 
 static struct ntb_transport_ctx net_softc;
@@ -244,31 +270,32 @@ static void ntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data,
     void *data, int len);
 static void ntb_net_event_handler(void *data, enum ntb_link_event status);
 static int ntb_transport_init(struct ntb_softc *ntb);
-static void ntb_transport_free(void *transport);
+static void ntb_transport_free(struct ntb_transport_ctx *);
 static void ntb_transport_init_queue(struct ntb_transport_ctx *nt,
     unsigned int qp_num);
 static void ntb_transport_free_queue(struct ntb_transport_qp *qp);
-static struct ntb_transport_qp * ntb_transport_create_queue(void *data,
+static struct ntb_transport_qp *ntb_transport_create_queue(void *data,
     struct ntb_softc *pdev, const struct ntb_queue_handlers *handlers);
 static void ntb_transport_link_up(struct ntb_transport_qp *qp);
 static int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb,
     void *data, unsigned int len);
 static int ntb_process_tx(struct ntb_transport_qp *qp,
     struct ntb_queue_entry *entry);
-static void ntb_tx_copy_task(struct ntb_transport_qp *qp,
+static void ntb_memcpy_tx(struct ntb_transport_qp *qp,
     struct ntb_queue_entry *entry, void *offset);
 static void ntb_qp_full(void *arg);
-static int ntb_transport_rxc_db(void *arg, int dummy);
+static void ntb_transport_rxc_db(void *arg, int pending);
 static void ntb_rx_pendq_full(void *arg);
 static int ntb_process_rxc(struct ntb_transport_qp *qp);
 static void ntb_rx_copy_task(struct ntb_transport_qp *qp,
     struct ntb_queue_entry *entry, void *offset);
-static void ntb_rx_completion_task(void *arg, int pending);
-static void ntb_transport_event_callback(void *data, enum ntb_hw_event event);
+static void ntb_complete_rxc(void *arg, int pending);
+static void ntb_transport_doorbell_callback(void *data, int vector);
+static void ntb_transport_event_callback(void *data);
 static void ntb_transport_link_work(void *arg);
 static int ntb_set_mw(struct ntb_transport_ctx *, int num_mw, unsigned size);
 static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw);
-static void ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,
+static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,
     unsigned int qp_num);
 static void ntb_qp_link_work(void *arg);
 static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt);
@@ -283,6 +310,11 @@ static struct ntb_queue_entry *ntb_list_rm(struct mtx *lock,
 static void create_random_local_eui48(u_char *eaddr);
 static unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp);
 
+static const struct ntb_ctx_ops ntb_transport_ops = {
+	.link_event = ntb_transport_event_callback,
+	.db_event = ntb_transport_doorbell_callback,
+};
+
 MALLOC_DEFINE(M_NTB_IF, "if_ntb", "ntb network driver");
 
 /* Module setup and teardown */
@@ -320,6 +352,7 @@ ntb_setup_interface(void)
 	struct ifnet *ifp;
 	struct ntb_queue_handlers handlers = { ntb_net_rx_handler,
 	    ntb_net_tx_handler, ntb_net_event_handler };
+	int rc;
 
 	net_softc.ntb = devclass_get_softc(devclass_find("ntb_hw"), 0);
 	if (net_softc.ntb == NULL) {
@@ -327,11 +360,16 @@ ntb_setup_interface(void)
 		return (ENXIO);
 	}
 
-	ntb_transport_init(net_softc.ntb);
+	rc = ntb_transport_init(net_softc.ntb);
+	if (rc != 0) {
+		printf("ntb: Cannot init transport: %d\n", rc);
+		return (rc);
+	}
 
 	ifp = net_softc.ifp = if_alloc(IFT_ETHER);
 	if (ifp == NULL) {
-		printf("ntb: cannot allocate ifnet structure\n");
+		ntb_transport_free(&net_softc);
+		printf("ntb: Cannot allocate ifnet structure\n");
 		return (ENOMEM);
 	}
 
@@ -498,83 +536,107 @@ static int
 ntb_transport_init(struct ntb_softc *ntb)
 {
 	struct ntb_transport_ctx *nt = &net_softc;
+	struct ntb_transport_mw *mw;
+	uint64_t qp_bitmap;
 	int rc;
-	uint8_t i;
+	unsigned i;
 
 	nt->mw_count = ntb_mw_count(ntb);
-	if (max_num_clients == 0)
-		nt->qp_count = MIN(ntb_get_max_cbs(ntb), ntb_mw_count(ntb));
-	else
-		nt->qp_count = MIN(ntb_get_max_cbs(ntb), max_num_clients);
+	for (i = 0; i < nt->mw_count; i++) {
+		mw = &nt->mw_vec[i];
+
+		rc = ntb_mw_get_range(ntb, i, &mw->phys_addr, &mw->vbase,
+		    &mw->phys_size, &mw->xlat_align, &mw->xlat_align_size);
+		if (rc != 0)
+			goto err;
+
+		mw->buff_size = 0;
+		mw->xlat_size = 0;
+		mw->virt_addr = 0;
+		mw->dma_addr = 0;
+	}
+
+	qp_bitmap = ntb_db_valid_mask(ntb);
+	nt->qp_count = flsll(qp_bitmap);
+	KASSERT(nt->qp_count != 0, ("bogus db bitmap"));
+	nt->qp_count -= 1;
+
+	if (max_num_clients != 0 && max_num_clients < nt->qp_count)
+		nt->qp_count = max_num_clients;
+	else if (nt->mw_count < nt->qp_count)
+		nt->qp_count = nt->mw_count;
+	KASSERT(nt->qp_count <= QP_SETSIZE, ("invalid qp_count"));
 
-	ntb_register_transport(ntb, nt);
 	mtx_init(&nt->tx_lock, "ntb transport tx", NULL, MTX_DEF);
 	mtx_init(&nt->rx_lock, "ntb transport rx", NULL, MTX_DEF);
 
 	nt->qp_vec = malloc(nt->qp_count * sizeof(*nt->qp_vec), M_NTB_IF,
 	    M_WAITOK | M_ZERO);
 
-	KASSERT(nt->qp_count <= QP_SETSIZE, ("invalid qp_count"));
-
 	for (i = 0; i < nt->qp_count; i++) {
 		set_bit(i, &nt->qp_bitmap);
+		set_bit(i, &nt->qp_bitmap_free);
 		ntb_transport_init_queue(nt, i);
 	}
 
 	callout_init(&nt->link_work, 0);
 
-	rc = ntb_register_event_callback(ntb, ntb_transport_event_callback);
+	rc = ntb_set_ctx(ntb, nt, &ntb_transport_ops);
 	if (rc != 0)
 		goto err;
 
-	if (ntb_link_is_up(ntb)) {
-		if (bootverbose)
-			device_printf(ntb_get_device(ntb), "link up\n");
-		callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt);
-	}
-
+	nt->link_is_up = false;
+	ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
+	ntb_link_event(ntb);
 	return (0);
 
 err:
 	free(nt->qp_vec, M_NTB_IF);
-	ntb_unregister_transport(ntb);
+	nt->qp_vec = NULL;
 	return (rc);
 }
 
 static void
-ntb_transport_free(void *transport)
+ntb_transport_free(struct ntb_transport_ctx *nt)
 {
-	struct ntb_transport_ctx *nt = transport;
 	struct ntb_softc *ntb = nt->ntb;
+	struct _qpset qp_bitmap_alloc;
 	uint8_t i;
 
 	ntb_transport_link_cleanup(nt);
 
 	callout_drain(&nt->link_work);
 
-	/* verify that all the qps are freed */
+	BIT_COPY(QP_SETSIZE, &nt->qp_bitmap, &qp_bitmap_alloc);
+	BIT_NAND(QP_SETSIZE, &qp_bitmap_alloc, &nt->qp_bitmap_free);
+
+	/* Verify that all the QPs are freed */
 	for (i = 0; i < nt->qp_count; i++)
-		if (!test_bit(i, &nt->qp_bitmap))
+		if (test_bit(i, &qp_bitmap_alloc))
 			ntb_transport_free_queue(&nt->qp_vec[i]);
 
-	ntb_unregister_event_callback(ntb);
+	ntb_link_disable(ntb);
+	ntb_clear_ctx(ntb);
 
-	for (i = 0; i < NTB_MAX_NUM_MW; i++)
+	for (i = 0; i < nt->mw_count; i++)
 		ntb_free_mw(nt, i);
 
 	free(nt->qp_vec, M_NTB_IF);
-	ntb_unregister_transport(ntb);
 }
 
 static void
 ntb_transport_init_queue(struct ntb_transport_ctx *nt, unsigned int qp_num)
 {
+	struct ntb_transport_mw *mw;
 	struct ntb_transport_qp *qp;
-	unsigned int num_qps_mw, tx_size;
-	uint8_t mw_num, mw_count;
+	vm_paddr_t mw_base;
+	uint64_t mw_size, qp_offset;
+	size_t tx_size;
+	unsigned num_qps_mw, mw_num, mw_count;
 
-	mw_count = ntb_mw_count(nt->ntb);
+	mw_count = nt->mw_count;
 	mw_num = QP_TO_MW(nt, qp_num);
+	mw = &nt->mw_vec[mw_num];
 
 	qp = &nt->qp_vec[qp_num];
 	qp->qp_num = qp_num;
@@ -589,13 +651,22 @@ ntb_transport_init_queue(struct ntb_transport_ctx *nt, unsigned int qp_num)
 	else
 		num_qps_mw = nt->qp_count / mw_count;
 
-	tx_size = ntb_get_mw_size(qp->ntb, mw_num) / num_qps_mw;
-	qp->rx_info = (struct ntb_rx_info *)
-	    ((char *)ntb_get_mw_vbase(qp->ntb, mw_num) +
-	    (qp_num / mw_count * tx_size));
-	tx_size -= sizeof(struct ntb_rx_info);
+	mw_base = mw->phys_addr;
+	mw_size = mw->phys_size;
+
+	tx_size = mw_size / num_qps_mw;
+	qp_offset = tx_size * qp_num / mw_count;
+
+	qp->tx_mw = (char *)mw->vbase + qp_offset;
+	KASSERT(qp->tx_mw != NULL, ("uh oh?"));
+
+	/* XXX Assumes that a vm_paddr_t is equivalent to bus_addr_t */
+	qp->tx_mw_phys = mw_base + qp_offset;
+	KASSERT(qp->tx_mw_phys != 0, ("uh oh?"));
+
+	tx_size -= sizeof(struct ntb_rx_info);
+	qp->rx_info = (void *)((char *)qp->tx_mw + tx_size);
 
-	qp->tx_mw = qp->rx_info + 1;
 	/* Due to house-keeping, there must be at least 2 buffs */
 	qp->tx_max_frame = min(transport_mtu + sizeof(struct ntb_payload_header),
 	    tx_size / 2);
@@ -608,7 +679,8 @@ ntb_transport_init_queue(struct ntb_transport_ctx *nt, unsigned int qp_num)
 	mtx_init(&qp->ntb_rx_pend_q_lock, "ntb rx pend q", NULL, MTX_SPIN);
 	mtx_init(&qp->ntb_rx_free_q_lock, "ntb rx free q", NULL, MTX_SPIN);
 	mtx_init(&qp->ntb_tx_free_q_lock, "ntb tx free q", NULL, MTX_SPIN);
-	TASK_INIT(&qp->rx_completion_task, 0, ntb_rx_completion_task, qp);
+	TASK_INIT(&qp->rx_completion_task, 0, ntb_complete_rxc, qp);
+	TASK_INIT(&qp->rxc_db_work, 0, ntb_transport_rxc_db, qp);
 
 	STAILQ_INIT(&qp->rx_pend_q);
 	STAILQ_INIT(&qp->rx_free_q);
@@ -625,7 +697,14 @@ ntb_transport_free_queue(struct ntb_transport_qp *qp)
 
 	callout_drain(&qp->link_work);
 
-	ntb_unregister_db_callback(qp->ntb, qp->qp_num);
+	ntb_db_set_mask(qp->ntb, 1ull << qp->qp_num);
+	taskqueue_drain(taskqueue_swi, &qp->rxc_db_work);
+	taskqueue_drain(taskqueue_swi, &qp->rx_completion_task);
+
+	qp->cb_data = NULL;
+	qp->rx_handler = NULL;
+	qp->tx_handler = NULL;
+	qp->event_handler = NULL;
 
 	while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q)))
 		free(entry, M_NTB_IF);
@@ -636,7 +715,7 @@ ntb_transport_free_queue(struct ntb_transport_qp *qp)
 	while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))
 		free(entry, M_NTB_IF);
 
-	set_bit(qp->qp_num, &qp->transport->qp_bitmap);
+	set_bit(qp->qp_num, &qp->transport->qp_bitmap_free);
 }
 
 /**
@@ -654,37 +733,34 @@ ntb_transport_free_queue(struct ntb_transport_qp *qp)
  * RETURNS: pointer to newly created ntb_queue, NULL on error.
  */
 static struct ntb_transport_qp *
-ntb_transport_create_queue(void *data, struct ntb_softc *pdev,
+ntb_transport_create_queue(void *data, struct ntb_softc *ntb,
     const struct ntb_queue_handlers *handlers)
 {
 	struct ntb_queue_entry *entry;
 	struct ntb_transport_qp *qp;
 	struct ntb_transport_ctx *nt;
 	unsigned int free_queue;
-	int rc, i;
+	int i;
 
-	nt = ntb_find_transport(pdev);
-	if (nt == NULL)
-		goto err;
+	nt = ntb_get_ctx(ntb, NULL);
+	KASSERT(nt != NULL, ("bogus"));
 
 	free_queue = ffs_bit(&nt->qp_bitmap);
 	if (free_queue == 0)
-		goto err;
+		return (NULL);
 
 	/* decrement free_queue to make it zero based */
 	free_queue--;
 
-	clear_bit(free_queue, &nt->qp_bitmap);
-
 	qp = &nt->qp_vec[free_queue];
+	clear_bit(1ull << qp->qp_num, &nt->qp_bitmap_free);
 	qp->cb_data = data;
 	qp->rx_handler = handlers->rx_handler;
 	qp->tx_handler = handlers->tx_handler;
 	qp->event_handler = handlers->event_handler;
 
 	for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) {
-		entry = malloc(sizeof(struct ntb_queue_entry), M_NTB_IF,
-		    M_WAITOK|M_ZERO);
+		entry = malloc(sizeof(*entry), M_NTB_IF, M_WAITOK | M_ZERO);
 		entry->cb_data = nt->ifp;
 		entry->buf = NULL;
 		entry->len = transport_mtu;
@@ -692,26 +768,13 @@ ntb_transport_create_queue(void *data, struct ntb_softc *pdev,
 	}
 
 	for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) {
-		entry = malloc(sizeof(struct ntb_queue_entry), M_NTB_IF,
-		    M_WAITOK|M_ZERO);
+		entry = malloc(sizeof(*entry), M_NTB_IF, M_WAITOK | M_ZERO);
 		ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);
 	}
 
-	rc = ntb_register_db_callback(qp->ntb, free_queue, qp,
-				      ntb_transport_rxc_db);
-	if (rc != 0)
-		goto err1;
-
+	ntb_db_clear(ntb, 1ull << qp->qp_num);
+	ntb_db_clear_mask(ntb, 1ull << qp->qp_num);
 	return (qp);
-
-err1:
-	while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))
-		free(entry, M_NTB_IF);
-	while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q)))
-		free(entry, M_NTB_IF);
-	set_bit(free_queue, &nt->qp_bitmap);
-err:
-	return (NULL);
 }
 
 /**
@@ -813,7 +876,7 @@ ntb_process_tx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry)
 		return (0);
 	}
 	CTR2(KTR_NTB, "TX: copying entry %p to offset %p", entry, offset);
-	ntb_tx_copy_task(qp, entry, offset);
+	ntb_memcpy_tx(qp, entry, offset);
 
 	qp->tx_index++;
 	qp->tx_index %= qp->tx_max_entry;
@@ -824,20 +887,31 @@ ntb_process_tx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry)
 }
 
 static void
-ntb_tx_copy_task(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry,
+ntb_memcpy_tx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry,
     void *offset)
 {
 	struct ntb_payload_header *hdr;
 
-	CTR2(KTR_NTB, "TX: copying %d bytes to offset %p", entry->len, offset);
-	if (entry->buf != NULL)
-		m_copydata((struct mbuf *)entry->buf, 0, entry->len, offset);
-
+	/* This piece is from Linux' ntb_async_tx() */
 	hdr = (struct ntb_payload_header *)((char *)offset + qp->tx_max_frame -
 	    sizeof(struct ntb_payload_header));
+	entry->x_hdr = hdr;
 	hdr->len = entry->len; /* TODO: replace with bus_space_write */
 	hdr->ver = qp->tx_pkts; /* TODO: replace with bus_space_write */
-	wmb();
+
+	/* This piece is ntb_memcpy_tx() */
+	CTR2(KTR_NTB, "TX: copying %d bytes to offset %p", entry->len, offset);
+	if (entry->buf != NULL) {
+		m_copydata((struct mbuf *)entry->buf, 0, entry->len, offset);
+
+		/*
+		 * Ensure that the data is fully copied before setting the
+		 * flags
+		 */
+		wmb();
+	}
+
+	/* The rest is ntb_tx_copy_callback() */
 	/* TODO: replace with bus_space_write */
 	hdr->flags = entry->flags | IF_NTB_DESC_DONE_FLAG;
 
@@ -879,8 +953,8 @@ ntb_rx_pendq_full(void *arg)
 	ntb_transport_rxc_db(arg, 0);
 }
 
-static int
-ntb_transport_rxc_db(void *arg, int dummy __unused)
+static void
+ntb_transport_rxc_db(void *arg, int pending __unused)
 {
 	struct ntb_transport_qp *qp = arg;
 	uint64_t i;
@@ -890,9 +964,9 @@ ntb_transport_rxc_db(void *arg, int dummy __unused)
 	 * Limit the number of packets processed in a single interrupt to
 	 * provide fairness to others
 	 */
-	mtx_lock(&qp->transport->rx_lock);
 	CTR0(KTR_NTB, "RX: transport_rx");
-	for (i = 0; i < MIN(qp->rx_max_entry, INT_MAX); i++) {
+	mtx_lock(&qp->transport->rx_lock);
+	for (i = 0; i < qp->rx_max_entry; i++) {
 		rc = ntb_process_rxc(qp);
 		if (rc != 0) {
 			CTR0(KTR_NTB, "RX: process_rxc failed");
@@ -901,7 +975,20 @@ ntb_transport_rxc_db(void *arg, int dummy __unused)
 	}
 	mtx_unlock(&qp->transport->rx_lock);
 
-	return ((int)i);
+	if (i == qp->rx_max_entry)
+		taskqueue_enqueue(taskqueue_swi, &qp->rxc_db_work);
+	else if ((ntb_db_read(qp->ntb) & (1ull << qp->qp_num)) != 0) {
+		/* If db is set, clear it and read it back to commit clear. */
+		ntb_db_clear(qp->ntb, 1ull << qp->qp_num);
+		(void)ntb_db_read(qp->ntb);
+
+		/*
+		 * An interrupt may have arrived between finishing
+		 * ntb_process_rxc and clearing the doorbell bit: there might
+		 * be some more work to do.
+		 */
+		taskqueue_enqueue(taskqueue_swi, &qp->rxc_db_work);
+	}
 }
 
 static int
@@ -918,66 +1005,61 @@ ntb_process_rxc(struct ntb_transport_qp *qp)
 		sizeof(struct ntb_payload_header));
 
 	CTR1(KTR_NTB, "RX: process_rxc rx_index = %u", qp->rx_index);
-	entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q);
-	if (entry == NULL) {
-		qp->rx_err_no_buf++;
-		CTR0(KTR_NTB, "RX: No entries in rx_pend_q");
-		return (ENOMEM);
-	}
-	callout_stop(&qp->rx_full);
-	CTR1(KTR_NTB, "RX: rx entry %p from rx_pend_q", entry);
-
 	if ((hdr->flags & IF_NTB_DESC_DONE_FLAG) == 0) {
-		CTR1(KTR_NTB,
-		    "RX: hdr not done. Returning entry %p to rx_pend_q", entry);
-		ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
+		CTR0(KTR_NTB, "RX: hdr not done");
 		qp->rx_ring_empty++;
 		return (EAGAIN);
 	}
 
-	if (hdr->ver != (uint32_t) qp->rx_pkts) {
-		CTR3(KTR_NTB,"RX: ver != rx_pkts (%x != %lx). "
-		    "Returning entry %p to rx_pend_q", hdr->ver, qp->rx_pkts,
-		    entry);
-		ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
+	if ((hdr->flags & IF_NTB_LINK_DOWN_FLAG) != 0) {
+		CTR0(KTR_NTB, "RX: link down");
+		ntb_qp_link_down(qp);
+		hdr->flags = 0;
+		return (EAGAIN);
+	}
+
+	if (hdr->ver != (uint32_t)qp->rx_pkts) {
+		CTR2(KTR_NTB,"RX: ver != rx_pkts (%x != %lx). "
+		    "Returning entry %p to rx_pend_q", hdr->ver, qp->rx_pkts);
 		qp->rx_err_ver++;
 		return (EIO);
 	}
 
-	if ((hdr->flags & IF_NTB_LINK_DOWN_FLAG) != 0) {
-		ntb_qp_link_down(qp);
-		CTR1(KTR_NTB,
-		    "RX: link down. adding entry %p back to rx_pend_q", entry);
-		ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
-		goto out;
+	entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q);
+	if (entry == NULL) {
+		qp->rx_err_no_buf++;
+		CTR0(KTR_NTB, "RX: No entries in rx_pend_q");
+		return (EAGAIN);
 	}
+	callout_stop(&qp->rx_full);
+	CTR1(KTR_NTB, "RX: rx entry %p from rx_pend_q", entry);
 
-	if (hdr->len <= entry->len) {
-		entry->len = hdr->len;
-		ntb_rx_copy_task(qp, entry, offset);
-	} else {
-		CTR1(KTR_NTB,
-		    "RX: len too long. Returning entry %p to rx_pend_q", entry);
-		ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
+	entry->x_hdr = hdr;
+	entry->index = qp->rx_index;
 
+	if (hdr->len > entry->len) {
+		CTR2(KTR_NTB, "RX: len too long. Wanted %ju got %ju",
+		    (uintmax_t)hdr->len, (uintmax_t)entry->len);
 		qp->rx_err_oflow++;
+
+		entry->len = -EIO;
+		entry->flags |= IF_NTB_DESC_DONE_FLAG;
+
+		ntb_list_add(&qp->ntb_rx_free_q_lock, entry, &qp->rx_free_q);
+		taskqueue_enqueue(taskqueue_swi, &qp->rx_completion_task);
+	} else {
+		qp->rx_bytes += hdr->len;
+		qp->rx_pkts++;
+
+		CTR1(KTR_NTB, "RX: received %ld rx_pkts", qp->rx_pkts);
+
+		entry->len = hdr->len;
+
+		ntb_rx_copy_task(qp, entry, offset);
 	}
 
-	qp->rx_bytes += hdr->len;
-	qp->rx_pkts++;
-	CTR1(KTR_NTB, "RX: received %ld rx_pkts", qp->rx_pkts);
-
-
-out:
-	/* Ensure that the data is globally visible before clearing the flag */
-	wmb();
-	hdr->flags = 0;
-	/* TODO: replace with bus_space_write */
-	qp->rx_info->entry = qp->rx_index;
-
 	qp->rx_index++;
 	qp->rx_index %= qp->rx_max_entry;
-
 	return (0);
 }
 
@@ -995,6 +1077,12 @@ ntb_rx_copy_task(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry,
 
 	entry->buf = (void *)m;
 
+	/* Ensure that the data is globally visible before clearing the flag */
+	wmb();
+	entry->x_hdr->flags = 0;
+	/* TODO: replace with bus_space_write */
+	qp->rx_info->entry = qp->rx_index;
+
 	CTR2(KTR_NTB,
 	    "RX: copied entry %p to mbuf %p. Adding entry to rx_free_q", entry,
 	    m);
@@ -1004,7 +1092,7 @@ ntb_rx_copy_task(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry,
 }
 
 static void
-ntb_rx_completion_task(void *arg, int pending)
+ntb_complete_rxc(void *arg, int pending)
 {
 	struct ntb_transport_qp *qp = arg;
 	struct mbuf *m;
@@ -1033,25 +1121,52 @@ ntb_rx_completion_task(void *arg, int pending)
 	}
 }
 
+static void
+ntb_transport_doorbell_callback(void *data, int vector)
+{
+	struct ntb_transport_ctx *nt = data;
+	struct ntb_transport_qp *qp;
+	struct _qpset db_bits;
+	uint64_t vec_mask;
+	unsigned qp_num;
+
+	BIT_COPY(QP_SETSIZE, &nt->qp_bitmap, &db_bits);
+	BIT_NAND(QP_SETSIZE, &db_bits, &nt->qp_bitmap_free);
+
+	vec_mask = ntb_db_vector_mask(nt->ntb, vector);
+	while (vec_mask != 0) {
+		qp_num = ffsl(vec_mask);
+		/* i386 doesn't have ffsll(), fake it */
+		if (qp_num == 0) {
+			qp_num = ffsl(vec_mask >> 32);
+			KASSERT(qp_num != 0, ("ffs"));
+			qp_num += 32;
+		}
+		qp_num--;
+
+		if (test_bit(qp_num, &db_bits)) {
+			qp = &nt->qp_vec[qp_num];
+			taskqueue_enqueue(taskqueue_swi, &qp->rxc_db_work);
+		}
+
+		vec_mask &= ~(1ull << qp_num);
+	}
+}
+
 /* Link Event handler */
 static void
-ntb_transport_event_callback(void *data, enum ntb_hw_event event)
+ntb_transport_event_callback(void *data)
 {
 	struct ntb_transport_ctx *nt = data;
 
-	switch (event) {
-	case NTB_EVENT_HW_LINK_UP:
+	if (ntb_link_is_up(nt->ntb, NULL, NULL)) {
 		if (bootverbose)
 			device_printf(ntb_get_device(nt->ntb), "HW link up\n");
 		callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt);
-		break;
-	case NTB_EVENT_HW_LINK_DOWN:
+	} else {
 		if (bootverbose)
 			device_printf(ntb_get_device(nt->ntb), "HW link down\n");
 		ntb_transport_link_cleanup(nt);
-		break;
-	default:
-		panic("ntb: Unknown NTB event");
 	}
 }
 
@@ -1062,72 +1177,48 @@ ntb_transport_link_work(void *arg)
 	struct ntb_transport_ctx *nt = arg;
 	struct ntb_softc *ntb = nt->ntb;
 	struct ntb_transport_qp *qp;
-	uint64_t val64;
-	uint32_t val, i, num_mw;
+	uint64_t val64, size;
+	uint32_t val;
+	unsigned i;
 	int rc;
 
-	num_mw = ntb_mw_count(ntb);
-
 	/* send the local info, in the opposite order of the way we read it */
-	for (i = 0; i < num_mw; i++) {
-		rc = ntb_peer_spad_write(ntb, IF_NTB_MW0_SZ_HIGH + (i * 2),
-		    (uint64_t)ntb_get_mw_size(ntb, i) >> 32);
-		if (rc != 0)
-			goto out;
+	for (i = 0; i < nt->mw_count; i++) {
+		size = nt->mw_vec[i].phys_size;
 
-		rc = ntb_peer_spad_write(ntb, IF_NTB_MW0_SZ_LOW + (i * 2),
-		    (uint32_t)ntb_get_mw_size(ntb, i));
-		if (rc != 0)
-			goto out;
+		if (max_mw_size != 0 && size > max_mw_size)
+			size = max_mw_size;
+
+		ntb_peer_spad_write(ntb, IF_NTB_MW0_SZ_HIGH + (i * 2),
+		    size >> 32);
+		ntb_peer_spad_write(ntb, IF_NTB_MW0_SZ_LOW + (i * 2), size);
 	}
 
-	rc = ntb_peer_spad_write(ntb, IF_NTB_NUM_MWS, num_mw);
-	if (rc != 0)
-		goto out;
+	ntb_peer_spad_write(ntb, IF_NTB_NUM_MWS, nt->mw_count);
 
-	rc = ntb_peer_spad_write(ntb, IF_NTB_NUM_QPS, nt->qp_count);
-	if (rc != 0)
-		goto out;
+	ntb_peer_spad_write(ntb, IF_NTB_NUM_QPS, nt->qp_count);
 
-	rc = ntb_peer_spad_write(ntb, IF_NTB_VERSION, NTB_TRANSPORT_VERSION);
-	if (rc != 0)
-		goto out;
+	ntb_peer_spad_write(ntb, IF_NTB_VERSION, NTB_TRANSPORT_VERSION);
 
 	/* Query the remote side for its info */
-	rc = ntb_spad_read(ntb, IF_NTB_VERSION, &val);
-	if (rc != 0)
-		goto out;
-
+	val = 0;
+	ntb_spad_read(ntb, IF_NTB_VERSION, &val);
 	if (val != NTB_TRANSPORT_VERSION)
 		goto out;
 
-	rc = ntb_spad_read(ntb, IF_NTB_NUM_QPS, &val);
-	if (rc != 0)
-		goto out;
-
+	ntb_spad_read(ntb, IF_NTB_NUM_QPS, &val);
 	if (val != nt->qp_count)
 		goto out;
 
-	rc = ntb_spad_read(ntb, IF_NTB_NUM_MWS, &val);
-	if (rc != 0)
+	ntb_spad_read(ntb, IF_NTB_NUM_MWS, &val);
+	if (val != nt->mw_count)
 		goto out;
 
-	if (val != num_mw)
-		goto out;
-
-	for (i = 0; i < num_mw; i++) {
-		rc = ntb_spad_read(ntb, IF_NTB_MW0_SZ_HIGH + (i * 2),
-		    &val);
-		if (rc != 0)
-			goto free_mws;
-
+	for (i = 0; i < nt->mw_count; i++) {
+		ntb_spad_read(ntb, IF_NTB_MW0_SZ_HIGH + (i * 2), &val);
 		val64 = (uint64_t)val << 32;
 
-		rc = ntb_spad_read(ntb, IF_NTB_MW0_SZ_LOW + (i * 2),
-		    &val);
-		if (rc != 0)
-			goto free_mws;
-
+		ntb_spad_read(ntb, IF_NTB_MW0_SZ_LOW + (i * 2), &val);
 		val64 |= val;
 
 		rc = ntb_set_mw(nt, i, val64);
@@ -1151,33 +1242,40 @@ ntb_transport_link_work(void *arg)
 	return;
 
 free_mws:
-	for (i = 0; i < NTB_MAX_NUM_MW; i++)
+	for (i = 0; i < nt->mw_count; i++)
 		ntb_free_mw(nt, i);
 out:
-	if (ntb_link_is_up(ntb))
+	if (ntb_link_is_up(ntb, NULL, NULL))
 		callout_reset(&nt->link_work,
 		    NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_transport_link_work, nt);
 }
 
 static int
-ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw, unsigned int size)
+ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw, unsigned size)
 {
 	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
+	unsigned xlat_size, buff_size;
+	int rc;
+
+	xlat_size = roundup(size, mw->xlat_align_size);
+	buff_size = roundup(size, mw->xlat_align);
 
 	/* No need to re-setup */
-	if (mw->xlat_size == size)
+	if (mw->xlat_size == xlat_size)
 		return (0);
 
-	if (mw->xlat_size != 0)
+	if (mw->buff_size != 0)
 		ntb_free_mw(nt, num_mw);
 
-	/* Alloc memory for receiving data.  Must be 4k aligned */
-	mw->xlat_size = size;
+	/* Alloc memory for receiving data.  Must be aligned */
+	mw->xlat_size = xlat_size;
+	mw->buff_size = buff_size;
 
-	mw->virt_addr = contigmalloc(mw->xlat_size, M_NTB_IF, M_ZERO, 0,
-	    BUS_SPACE_MAXADDR, mw->xlat_size, 0);
+	mw->virt_addr = contigmalloc(mw->buff_size, M_NTB_IF, M_ZERO, 0,
+	    BUS_SPACE_MAXADDR, mw->xlat_align, 0);
 	if (mw->virt_addr == NULL) {
 		mw->xlat_size = 0;
+		mw->buff_size = 0;
 		printf("ntb: Unable to allocate MW buffer of size %zu\n",
 		    mw->xlat_size);
 		return (ENOMEM);
@@ -1190,7 +1288,7 @@ ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw, unsigned int size)
 	 * requested.  XXX: This may not be needed -- brought in for parity
 	 * with the Linux driver.
 	 */
-	if (mw->dma_addr % size != 0) {
+	if (mw->dma_addr % mw->xlat_align != 0) {
 		device_printf(ntb_get_device(nt->ntb),
 		    "DMA memory 0x%jx not aligned to BAR size 0x%x\n",
 		    (uintmax_t)mw->dma_addr, size);
@@ -1199,7 +1297,13 @@ ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw, unsigned int size)
 	}
 
 	/* Notify HW the memory location of the receive buffer */
-	ntb_set_mw_addr(nt->ntb, num_mw, mw->dma_addr);
+	rc = ntb_mw_set_trans(nt->ntb, num_mw, mw->dma_addr, mw->xlat_size);
+	if (rc) {
+		device_printf(ntb_get_device(nt->ntb),
+		    "Unable to set mw%d translation", num_mw);
+		ntb_free_mw(nt, num_mw);
+		return (rc);
+	}
 
 	return (0);
 }
@@ -1212,33 +1316,41 @@ ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw)
 	if (mw->virt_addr == NULL)
 		return;
 
+	ntb_mw_clear_trans(nt->ntb, num_mw);
 	contigfree(mw->virt_addr, mw->xlat_size, M_NTB_IF);
+	mw->xlat_size = 0;
+	mw->buff_size = 0;
 	mw->virt_addr = NULL;
 }
 
-static void
+static int
 ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt, unsigned int qp_num)
 {
 	struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
+	struct ntb_transport_mw *mw;
 	void *offset;
-	unsigned int rx_size, num_qps_mw;
-	uint8_t mw_num, mw_count;
-	unsigned int i;
+	uint64_t i;
+	size_t rx_size;
+	unsigned num_qps_mw, mw_num, mw_count;
 
-	mw_count = ntb_mw_count(nt->ntb);
+	mw_count = nt->mw_count;
 	mw_num = QP_TO_MW(nt, qp_num);
+	mw = &nt->mw_vec[mw_num];
+
+	if (mw->virt_addr == NULL)
+		return (ENOMEM);
 
 	if (nt->qp_count % mw_count && mw_num + 1 < nt->qp_count / mw_count)
 		num_qps_mw = nt->qp_count / mw_count + 1;
 	else
 		num_qps_mw = nt->qp_count / mw_count;
 
-	rx_size = nt->mw_vec[mw_num].xlat_size / num_qps_mw;
-	qp->remote_rx_info = (void *)((uint8_t *)nt->mw_vec[mw_num].virt_addr +
-	    (qp_num / mw_count * rx_size));
+	rx_size = mw->xlat_size / num_qps_mw;
+	qp->rx_buff = (char *)mw->virt_addr + rx_size * qp_num / mw_count;
 	rx_size -= sizeof(struct ntb_rx_info);
 
-	qp->rx_buff = qp->remote_rx_info + 1;
+	qp->remote_rx_info = (void*)((char *)qp->rx_buff + rx_size);
+
 	/* Due to house-keeping, there must be at least 2 buffs */
 	qp->rx_max_frame = min(transport_mtu + sizeof(struct ntb_payload_header),
 	    rx_size / 2);
@@ -1247,7 +1359,7 @@ ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt, unsigned int qp_num)
 
 	qp->remote_rx_info->entry = qp->rx_max_entry - 1;
 
-	/* setup the hdr offsets with 0's */
+	/* Set up the hdr offsets with 0s */
 	for (i = 0; i < qp->rx_max_entry; i++) {
 		offset = (void *)((uint8_t *)qp->rx_buff +
 		    qp->rx_max_frame * (i + 1) -
@@ -1258,6 +1370,8 @@ ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt, unsigned int qp_num)
 	qp->rx_pkts = 0;
 	qp->tx_pkts = 0;
 	qp->tx_index = 0;
+
+	return (0);
 }
 
 static void
@@ -1266,46 +1380,51 @@ ntb_qp_link_work(void *arg)
 	struct ntb_transport_qp *qp = arg;
 	struct ntb_softc *ntb = qp->ntb;
 	struct ntb_transport_ctx *nt = qp->transport;
-	int rc, val;
+	int val;
 
+	ntb_spad_read(ntb, IF_NTB_QP_LINKS, &val);
 
-	rc = ntb_peer_spad_read(ntb, IF_NTB_QP_LINKS, &val);
-	if (rc != 0)
-		return;
-
-	rc = ntb_peer_spad_write(ntb, IF_NTB_QP_LINKS, val | 1 << qp->qp_num);
+	ntb_peer_spad_write(ntb, IF_NTB_QP_LINKS, val | (1ull << qp->qp_num));
 
 	/* query remote spad for qp ready bits */
-	rc = ntb_spad_read(ntb, IF_NTB_QP_LINKS, &val);
+	ntb_peer_spad_read(ntb, IF_NTB_QP_LINKS, &val);
 
 	/* See if the remote side is up */
-	if ((1 << qp->qp_num & val) != 0) {
-		qp->link_is_up = true;
-		if (qp->event_handler != NULL)
-			qp->event_handler(qp->cb_data, NTB_LINK_UP);
+	if ((val & (1ull << qp->qp_num)) != 0) {
 		if (bootverbose)
 			device_printf(ntb_get_device(ntb), "qp link up\n");
-	} else if (nt->link_is_up) {
+		qp->link_is_up = true;
+
+		if (qp->event_handler != NULL)
+			qp->event_handler(qp->cb_data, NTB_LINK_UP);
+
+		taskqueue_enqueue(taskqueue_swi, &qp->rxc_db_work);
+	} else if (nt->link_is_up)
 		callout_reset(&qp->link_work,
 		    NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp);
-	}
 }
 
 /* Link down event*/
 static void
 ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)
 {
-	uint8_t i;
+	struct ntb_transport_qp *qp;
+	struct _qpset qp_bitmap_alloc;
+	unsigned i;
+
+	BIT_COPY(QP_SETSIZE, &nt->qp_bitmap, &qp_bitmap_alloc);
+	BIT_NAND(QP_SETSIZE, &qp_bitmap_alloc, &nt->qp_bitmap_free);
 
 	/* Pass along the info to any clients */
 	for (i = 0; i < nt->qp_count; i++)
-		if (!test_bit(i, &nt->qp_bitmap))
-			ntb_qp_link_down(&nt->qp_vec[i]);
+		if (test_bit(i, &qp_bitmap_alloc)) {
+			qp = &nt->qp_vec[i];
+			ntb_qp_link_cleanup(qp);
+			callout_drain(&qp->link_work);
+		}
 
 	if (!nt->link_is_up)
 		callout_drain(&nt->link_work);
-	else
-		nt->link_is_up = false;
 
 	/*
 	 * The scratchpad registers keep the values if the remote side
@@ -1334,11 +1453,11 @@ ntb_qp_link_cleanup(struct ntb_transport_qp *qp)
 		return;
 	}
 
+	qp->link_is_up = false;
+
 	if (qp->event_handler != NULL)
 		qp->event_handler(qp->cb_data, NTB_LINK_DOWN);
 
-	qp->link_is_up = false;
-
 	if (nt->link_is_up)
 		callout_reset(&qp->link_work,
 		    NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp);
@@ -1356,25 +1475,22 @@ ntb_qp_link_cleanup(struct ntb_transport_qp *qp)
 static void
 ntb_transport_link_down(struct ntb_transport_qp *qp)
 {
-	int rc, val;
+	uint32_t val;
 
 	if (qp == NULL)
 		return;
 
 	qp->client_ready = false;
 
-	rc = ntb_peer_spad_read(qp->ntb, IF_NTB_QP_LINKS, &val);
-	if (rc != 0)
-		return;
+	ntb_spad_read(qp->ntb, IF_NTB_QP_LINKS, &val);
 
-	rc = ntb_peer_spad_write(qp->ntb, IF_NTB_QP_LINKS,
+	ntb_peer_spad_write(qp->ntb, IF_NTB_QP_LINKS,
 	   val & ~(1 << qp->qp_num));
 
 	if (qp->link_is_up)
 		ntb_send_link_down(qp);
 	else
 		callout_drain(&qp->link_work);
-
 }
 
 static void
diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c
index cee2424db8d4..9a3e189b974d 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.c
@@ -103,8 +103,6 @@ struct ntb_pci_bar_info {
 	/* Configuration register offsets */
 	uint32_t		psz_off;
 	uint32_t		ssz_off;
-	uint32_t		sbarbase_off;
-	uint32_t		sbarlmt_off;
 	uint32_t		pbarxlat_off;
 };
 
@@ -114,13 +112,9 @@ struct ntb_int_info {
 	void		*tag;
 };
 
-struct ntb_db_cb {
-	ntb_db_callback		callback;
-	unsigned int		db_num;
-	void			*data;
+struct ntb_vec {
 	struct ntb_softc	*ntb;
-	struct callout		irq_work;
-	bool			reserved;
+	uint32_t		num;
 };
 
 struct ntb_reg {
@@ -137,9 +131,18 @@ struct ntb_alt_reg {
 };
 
 struct ntb_xlat_reg {
-	uint64_t	bar0_base;
-	uint64_t	bar2_xlat;
-	uint64_t	bar2_limit;
+	uint32_t	bar0_base;
+	uint32_t	bar2_base;
+	uint32_t	bar4_base;
+	uint32_t	bar5_base;
+
+	uint32_t	bar2_xlat;
+	uint32_t	bar4_xlat;
+	uint32_t	bar5_xlat;
+
+	uint32_t	bar2_limit;
+	uint32_t	bar4_limit;
+	uint32_t	bar5_limit;
 };
 
 struct ntb_b2b_addr {
@@ -162,10 +165,13 @@ struct ntb_softc {
 	struct callout		heartbeat_timer;
 	struct callout		lr_timer;
 
-	void			*ntb_transport;
-	ntb_event_callback	event_cb;
-	struct ntb_db_cb	*db_cb;
-	uint8_t			max_cbs;
+	void			*ntb_ctx;
+	const struct ntb_ctx_ops *ctx_ops;
+	struct ntb_vec		*msix_vec;
+#define CTX_LOCK(sc)		mtx_lock_spin(&(sc)->ctx_lock)
+#define CTX_UNLOCK(sc)		mtx_unlock_spin(&(sc)->ctx_lock)
+#define CTX_ASSERT(sc,f)	mtx_assert(&(sc)->ctx_lock, (f))
+	struct mtx		ctx_lock;
 
 	struct {
 		uint32_t ldb;
@@ -178,7 +184,6 @@ struct ntb_softc {
 	uint32_t ppd;
 	uint8_t conn_type;
 	uint8_t dev_type;
-	uint8_t link_status;
 	uint8_t link_width;
 	uint8_t link_speed;
 
@@ -194,18 +199,18 @@ struct ntb_softc {
 	uint8_t				db_vec_count;
 	uint8_t				db_vec_shift;
 
-	/* Protects local DB mask and (h). */
-#define HW_LOCK(sc)	mtx_lock_spin(&(sc)->db_mask_lock)
-#define HW_UNLOCK(sc)	mtx_unlock_spin(&(sc)->db_mask_lock)
-#define HW_ASSERT(sc,f)	mtx_assert(&(sc)->db_mask_lock, (f))
+	/* Protects local db_mask. */
+#define DB_MASK_LOCK(sc)	mtx_lock_spin(&(sc)->db_mask_lock)
+#define DB_MASK_UNLOCK(sc)	mtx_unlock_spin(&(sc)->db_mask_lock)
+#define DB_MASK_ASSERT(sc,f)	mtx_assert(&(sc)->db_mask_lock, (f))
 	struct mtx			db_mask_lock;
 
-	uint32_t			ntb_ctl;	/* (h) - SOC only */
-	uint32_t			lnk_sta;	/* (h) - SOC only */
+	uint32_t			ntb_ctl;
+	uint32_t			lnk_sta;
 
 	uint64_t			db_valid_mask;
 	uint64_t			db_link_mask;
-	uint64_t			db_mask;	/* (h) */
+	uint64_t			db_mask;
 
 	int				last_ts;	/* ticks @ last irq */
 
@@ -254,6 +259,9 @@ static int ntb_probe(device_t device);
 static int ntb_attach(device_t device);
 static int ntb_detach(device_t device);
 static inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw);
+static inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar);
+static inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar,
+    uint32_t *base, uint32_t *xlat, uint32_t *lmt);
 static int ntb_map_pci_bars(struct ntb_softc *ntb);
 static void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *);
 static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar);
@@ -266,17 +274,13 @@ static int ntb_setup_legacy_interrupt(struct ntb_softc *ntb);
 static int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors);
 static void ntb_teardown_interrupts(struct ntb_softc *ntb);
 static inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector);
-static void handle_irq(void *arg);
-static void ntb_handle_legacy_interrupt(void *arg);
-static void ntb_irq_work(void *arg);
-static inline uint64_t ntb_db_read(struct ntb_softc *, uint64_t regoff);
-static inline void ntb_db_write(struct ntb_softc *, uint64_t regoff, uint64_t val);
-static inline void mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx);
-static inline void unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx);
-static inline void ntb_db_set_mask(struct ntb_softc *, uint64_t bits);
-static inline void ntb_db_clear_mask(struct ntb_softc *, uint64_t bits);
-static int ntb_create_callbacks(struct ntb_softc *ntb, uint32_t num_vectors);
-static void ntb_free_callbacks(struct ntb_softc *ntb);
+static void ntb_interrupt(struct ntb_softc *, uint32_t vec);
+static void ndev_vec_isr(void *arg);
+static void ndev_irq_isr(void *arg);
+static inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff);
+static inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t val);
+static int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors);
+static void ntb_free_msix_vec(struct ntb_softc *ntb);
 static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id);
 static void ntb_detect_max_mw(struct ntb_softc *ntb);
 static int ntb_detect_xeon(struct ntb_softc *ntb);
@@ -293,12 +297,14 @@ static void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr,
     enum ntb_bar idx);
 static int xeon_setup_b2b_mw(struct ntb_softc *,
     const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr);
+static inline bool link_is_up(struct ntb_softc *ntb);
+static inline bool soc_link_is_err(struct ntb_softc *ntb);
+static inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *);
+static inline enum ntb_width ntb_link_sta_width(struct ntb_softc *);
 static void soc_link_hb(void *arg);
-static void ntb_handle_link_event(struct ntb_softc *ntb, int link_state);
-static void ntb_link_disable(struct ntb_softc *ntb);
-static void ntb_link_enable(struct ntb_softc *ntb);
+static void ntb_db_event(struct ntb_softc *ntb, uint32_t vec);
 static void recover_soc_link(void *arg);
-static int ntb_poll_link(struct ntb_softc *ntb);
+static bool ntb_poll_link(struct ntb_softc *ntb);
 static void save_bar_parameters(struct ntb_pci_bar_info *bar);
 
 static struct ntb_hw_info pci_ids[] = {
@@ -338,9 +344,15 @@ static const struct ntb_xlat_reg soc_sec_xlat = {
 #if 0
 	/* "FIXME" says the Linux driver. */
 	.bar0_base = SOC_SBAR0BASE_OFFSET,
+	.bar2_base = SOC_SBAR2BASE_OFFSET,
+	.bar4_base = SOC_SBAR4BASE_OFFSET,
+
 	.bar2_limit = SOC_SBAR2LMT_OFFSET,
+	.bar4_limit = SOC_SBAR4LMT_OFFSET,
 #endif
+
 	.bar2_xlat = SOC_SBAR2XLAT_OFFSET,
+	.bar4_xlat = SOC_SBAR4XLAT_OFFSET,
 };
 
 static const struct ntb_reg xeon_reg = {
@@ -357,8 +369,17 @@ static const struct ntb_alt_reg xeon_b2b_reg = {
 
 static const struct ntb_xlat_reg xeon_sec_xlat = {
 	.bar0_base = XEON_SBAR0BASE_OFFSET,
+	.bar2_base = XEON_SBAR2BASE_OFFSET,
+	.bar4_base = XEON_SBAR4BASE_OFFSET,
+	.bar5_base = XEON_SBAR5BASE_OFFSET,
+
 	.bar2_limit = XEON_SBAR2LMT_OFFSET,
+	.bar4_limit = XEON_SBAR4LMT_OFFSET,
+	.bar5_limit = XEON_SBAR5LMT_OFFSET,
+
 	.bar2_xlat = XEON_SBAR2XLAT_OFFSET,
+	.bar4_xlat = XEON_SBAR4XLAT_OFFSET,
+	.bar5_xlat = XEON_SBAR5XLAT_OFFSET,
 };
 
 static const struct ntb_b2b_addr xeon_b2b_usd_addr = {
@@ -437,6 +458,7 @@ ntb_attach(device_t device)
 	callout_init(&ntb->heartbeat_timer, 1);
 	callout_init(&ntb->lr_timer, 1);
 	mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN);
+	mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_SPIN);
 
 	if (ntb->type == NTB_SOC)
 		error = ntb_detect_soc(ntb);
@@ -483,6 +505,7 @@ ntb_detach(device_t device)
 	ntb_teardown_interrupts(ntb);
 
 	mtx_destroy(&ntb->db_mask_lock);
+	mtx_destroy(&ntb->ctx_lock);
 
 	/*
 	 * Redetect total MWs so we unmap properly -- in case we lowered the
@@ -504,10 +527,56 @@ ntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw)
 	KASSERT(mw < ntb->mw_count ||
 	    (mw != B2B_MW_DISABLED && mw == ntb->b2b_mw_idx),
 	    ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count));
+	KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw"));
 
 	return (ntb->reg->mw_bar[mw]);
 }
 
+static inline bool
+bar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar)
+{
+	/* XXX This assertion could be stronger. */
+	KASSERT(bar < NTB_MAX_BARS, ("bogus bar"));
+	return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(NTB_SPLIT_BAR));
+}
+
+static inline void
+bar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base,
+    uint32_t *xlat, uint32_t *lmt)
+{
+	uint32_t basev, lmtv, xlatv;
+
+	switch (bar) {
+	case NTB_B2B_BAR_1:
+		basev = ntb->xlat_reg->bar2_base;
+		lmtv = ntb->xlat_reg->bar2_limit;
+		xlatv = ntb->xlat_reg->bar2_xlat;
+		break;
+	case NTB_B2B_BAR_2:
+		basev = ntb->xlat_reg->bar4_base;
+		lmtv = ntb->xlat_reg->bar4_limit;
+		xlatv = ntb->xlat_reg->bar4_xlat;
+		break;
+	case NTB_B2B_BAR_3:
+		basev = ntb->xlat_reg->bar5_base;
+		lmtv = ntb->xlat_reg->bar5_limit;
+		xlatv = ntb->xlat_reg->bar5_xlat;
+		break;
+	default:
+		KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS,
+		    ("bad bar"));
+		basev = lmtv = xlatv = 0;
+		break;
+	}
+
+	if (base != NULL)
+		*base = basev;
+	if (xlat != NULL)
+		*xlat = xlatv;
+	if (lmt != NULL)
+		*lmt = lmtv;
+}
+
 static int
 ntb_map_pci_bars(struct ntb_softc *ntb)
 {
@@ -524,8 +593,6 @@ ntb_map_pci_bars(struct ntb_softc *ntb)
 		goto out;
 	ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET;
 	ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET;
-	ntb->bar_info[NTB_B2B_BAR_1].sbarbase_off = XEON_SBAR2BASE_OFFSET;
-	ntb->bar_info[NTB_B2B_BAR_1].sbarlmt_off = XEON_SBAR2LMT_OFFSET;
 	ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET;
 
 	ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4);
@@ -536,8 +603,6 @@ ntb_map_pci_bars(struct ntb_softc *ntb)
 		rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]);
 	ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET;
 	ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET;
-	ntb->bar_info[NTB_B2B_BAR_2].sbarbase_off = XEON_SBAR4BASE_OFFSET;
-	ntb->bar_info[NTB_B2B_BAR_2].sbarlmt_off = XEON_SBAR4LMT_OFFSET;
 	ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET;
 
 	if (!HAS_FEATURE(NTB_SPLIT_BAR))
@@ -550,8 +615,6 @@ ntb_map_pci_bars(struct ntb_softc *ntb)
 		rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]);
 	ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET;
 	ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET;
-	ntb->bar_info[NTB_B2B_BAR_3].sbarbase_off = XEON_SBAR5BASE_OFFSET;
-	ntb->bar_info[NTB_B2B_BAR_3].sbarlmt_off = XEON_SBAR5LMT_OFFSET;
 	ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET;
 
 out:
@@ -677,8 +740,8 @@ ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors)
 		ntb->int_info[i].tag = NULL;
 		ntb->allocated_interrupts++;
 		rc = bus_setup_intr(ntb->device, ntb->int_info[i].res,
-		    INTR_MPSAFE | INTR_TYPE_MISC, NULL, handle_irq,
-		    &ntb->db_cb[i], &ntb->int_info[i].tag);
+		    INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr,
+		    &ntb->msix_vec[i], &ntb->int_info[i].tag);
 		if (rc != 0) {
 			device_printf(ntb->device, "bus_setup_intr failed\n");
 			return (ENXIO);
@@ -736,20 +799,15 @@ static int
 ntb_init_isr(struct ntb_softc *ntb)
 {
 	uint32_t desired_vectors, num_vectors;
-	uint64_t mask;
 	int rc;
 
 	ntb->allocated_interrupts = 0;
 	ntb->last_ts = ticks;
 
 	/*
-	 * On SOC, disable all interrupts.  On XEON, disable all but Link
-	 * Interrupt.  The rest will be unmasked as callbacks are registered.
+	 * Mask all doorbell interrupts.
 	 */
-	mask = ntb->db_valid_mask;
-	if (ntb->type == NTB_XEON)
-		mask &= ~ntb->db_link_mask;
-	ntb_db_set_mask(ntb, mask);
+	ntb_db_set_mask(ntb, ntb->db_valid_mask);
 
 	num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device),
 	    ntb->db_count);
@@ -774,31 +832,17 @@ ntb_init_isr(struct ntb_softc *ntb)
 		num_vectors = 1;
 
 	if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) {
-		/*
-		 * If allocating MSI-X interrupts failed and we're forced to
-		 * use legacy INTx anyway, the only limit on individual
-		 * callbacks is the number of doorbell bits.
-		 */
 		ntb->db_vec_count = 1;
 		ntb->db_vec_shift = ntb->db_count;
-		ntb_create_callbacks(ntb, ntb->db_count);
 		rc = ntb_setup_legacy_interrupt(ntb);
 	} else {
-		ntb_create_callbacks(ntb, num_vectors);
+		ntb_create_msix_vec(ntb, num_vectors);
 		rc = ntb_setup_msix(ntb, num_vectors);
-		if (rc == 0 && ntb->type == NTB_XEON) {
-			/*
-			 * Prevent consumers from registering callbacks on the link event irq
-			 * slot, from which they will never be called back.
-			 */
-			ntb->db_cb[num_vectors - 1].reserved = true;
-			ntb->max_cbs--;
-		}
 	}
 	if (rc != 0) {
 		device_printf(ntb->device,
 		    "Error allocating interrupts: %d\n", rc);
-		ntb_free_callbacks(ntb);
+		ntb_free_msix_vec(ntb);
 	}
 
 	return (rc);
@@ -821,7 +865,7 @@ ntb_setup_legacy_interrupt(struct ntb_softc *ntb)
 	ntb->allocated_interrupts = 1;
 
 	rc = bus_setup_intr(ntb->device, ntb->int_info[0].res,
-	    INTR_MPSAFE | INTR_TYPE_MISC, NULL, ntb_handle_legacy_interrupt,
+	    INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr,
 	    ntb, &ntb->int_info[0].tag);
 	if (rc != 0) {
 		device_printf(ntb->device, "bus_setup_intr failed\n");
@@ -848,7 +892,7 @@ ntb_teardown_interrupts(struct ntb_softc *ntb)
 			    rman_get_rid(current_int->res), current_int->res);
 	}
 
-	ntb_free_callbacks(ntb);
+	ntb_free_msix_vec(ntb);
 	pci_release_msi(ntb->device);
 }
 
@@ -857,7 +901,7 @@ ntb_teardown_interrupts(struct ntb_softc *ntb)
  * out to make code clearer.
  */
 static inline uint64_t
-ntb_db_read(struct ntb_softc *ntb, uint64_t regoff)
+db_ioread(struct ntb_softc *ntb, uint64_t regoff)
 {
 
 	if (ntb->type == NTB_SOC)
@@ -869,7 +913,7 @@ ntb_db_read(struct ntb_softc *ntb, uint64_t regoff)
 }
 
 static inline void
-ntb_db_write(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)
+db_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)
 {
 
 	KASSERT((val & ~ntb->db_valid_mask) == 0,
@@ -878,7 +922,7 @@ ntb_db_write(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)
 	     (uintmax_t)ntb->db_valid_mask));
 
 	if (regoff == ntb->reg_ofs.ldb_mask)
-		HW_ASSERT(ntb, MA_OWNED);
+		DB_MASK_ASSERT(ntb, MA_OWNED);
 
 	if (ntb->type == NTB_SOC) {
 		ntb_reg_write(8, regoff, val);
@@ -889,17 +933,17 @@ ntb_db_write(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)
 	ntb_reg_write(2, regoff, (uint16_t)val);
 }
 
-static inline void
+void
 ntb_db_set_mask(struct ntb_softc *ntb, uint64_t bits)
 {
 
-	HW_LOCK(ntb);
+	DB_MASK_LOCK(ntb);
 	ntb->db_mask |= bits;
-	ntb_db_write(ntb, ntb->reg_ofs.ldb_mask, ntb->db_mask);
-	HW_UNLOCK(ntb);
+	db_iowrite(ntb, ntb->reg_ofs.ldb_mask, ntb->db_mask);
+	DB_MASK_UNLOCK(ntb);
 }
 
-static inline void
+void
 ntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits)
 {
 
@@ -908,28 +952,29 @@ ntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits)
 	     (uintmax_t)(bits & ~ntb->db_valid_mask),
 	     (uintmax_t)ntb->db_valid_mask));
 
-	HW_LOCK(ntb);
+	DB_MASK_LOCK(ntb);
 	ntb->db_mask &= ~bits;
-	ntb_db_write(ntb, ntb->reg_ofs.ldb_mask, ntb->db_mask);
-	HW_UNLOCK(ntb);
+	db_iowrite(ntb, ntb->reg_ofs.ldb_mask, ntb->db_mask);
+	DB_MASK_UNLOCK(ntb);
 }
 
-static inline void
-mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx)
+uint64_t
+ntb_db_read(struct ntb_softc *ntb)
 {
-	uint64_t mask;
 
-	mask = 1ull << (idx * ntb->db_vec_shift);
-	ntb_db_set_mask(ntb, mask);
+	return (db_ioread(ntb, ntb->reg_ofs.ldb));
 }
 
-static inline void
-unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx)
+void
+ntb_db_clear(struct ntb_softc *ntb, uint64_t bits)
 {
-	uint64_t mask;
 
-	mask = 1ull << (idx * ntb->db_vec_shift);
-	ntb_db_clear_mask(ntb, mask);
+	KASSERT((bits & ~ntb->db_valid_mask) == 0,
+	    ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__,
+	     (uintmax_t)(bits & ~ntb->db_valid_mask),
+	     (uintmax_t)ntb->db_valid_mask));
+
+	db_iowrite(ntb, ntb->reg_ofs.ldb, bits);
 }
 
 static inline uint64_t
@@ -943,79 +988,61 @@ ntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector)
 }
 
 static void
-handle_irq(void *arg)
+ntb_interrupt(struct ntb_softc *ntb, uint32_t vec)
 {
-	struct ntb_db_cb *db_cb = arg;
-	struct ntb_softc *ntb = db_cb->ntb;
 	uint64_t vec_mask;
-	int rc;
 
 	ntb->last_ts = ticks;
-	vec_mask = ntb_vec_mask(ntb, db_cb->db_num);
+	vec_mask = ntb_vec_mask(ntb, vec);
 
 	if ((vec_mask & ntb->db_link_mask) != 0) {
-		rc = ntb_poll_link(ntb);
-		if (rc != 0)
-			device_printf(ntb->device,
-			    "Error determining link status\n");
+		if (ntb_poll_link(ntb))
+			ntb_link_event(ntb);
 	}
 
-	if (db_cb->callback != NULL) {
-		KASSERT(!db_cb->reserved, ("user callback on link event cb"));
-		mask_ldb_interrupt(ntb, db_cb->db_num);
-	}
-
-	ntb_db_write(ntb, ntb->reg_ofs.ldb, vec_mask);
-
-	if (db_cb->callback != NULL)
-		callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb);
+	if ((vec_mask & ntb->db_valid_mask) != 0)
+		ntb_db_event(ntb, vec);
 }
 
 static void
-ntb_handle_legacy_interrupt(void *arg)
+ndev_vec_isr(void *arg)
 {
-	struct ntb_softc *ntb = arg;
-	unsigned int i;
-	uint64_t ldb;
+	struct ntb_vec *nvec = arg;
 
-	ldb = ntb_db_read(ntb, ntb->reg_ofs.ldb);
-	while (ldb != 0) {
-		i = ffs(ldb);
-		ldb &= ldb - 1;
-		handle_irq(&ntb->db_cb[i]);
-	}
+	ntb_interrupt(nvec->ntb, nvec->num);
+}
+
+static void
+ndev_irq_isr(void *arg)
+{
+	/* If we couldn't set up MSI-X, we only have the one vector. */
+	ntb_interrupt(arg, 0);
 }
 
 static int
-ntb_create_callbacks(struct ntb_softc *ntb, uint32_t num_vectors)
+ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors)
 {
 	uint32_t i;
 
-	ntb->max_cbs = num_vectors;
-	ntb->db_cb = malloc(num_vectors * sizeof(*ntb->db_cb), M_NTB,
+	ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB,
 	    M_ZERO | M_WAITOK);
 	for (i = 0; i < num_vectors; i++) {
-		ntb->db_cb[i].db_num = i;
-		ntb->db_cb[i].ntb = ntb;
+		ntb->msix_vec[i].num = i;
+		ntb->msix_vec[i].ntb = ntb;
 	}
 
 	return (0);
 }
 
 static void
-ntb_free_callbacks(struct ntb_softc *ntb)
+ntb_free_msix_vec(struct ntb_softc *ntb)
 {
-	uint8_t i;
 
-	if (ntb->db_cb == NULL)
+	if (ntb->msix_vec == NULL)
 		return;
 
-	for (i = 0; i < ntb->max_cbs; i++)
-		ntb_unregister_db_callback(ntb, i);
-
-	free(ntb->db_cb, M_NTB);
-	ntb->db_cb = NULL;
-	ntb->max_cbs = 0;
+	free(ntb->msix_vec, M_NTB);
+	ntb->msix_vec = NULL;
 }
 
 static struct ntb_hw_info *
@@ -1179,7 +1206,7 @@ ntb_xeon_init_dev(struct ntb_softc *ntb)
 	    PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
 
 	/* Enable link training */
-	ntb_link_enable(ntb);
+	ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
 
 	return (0);
 }
@@ -1220,7 +1247,7 @@ ntb_soc_init_dev(struct ntb_softc *ntb)
 	    PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
 
 	/* Initiate PCI-E link training */
-	ntb_link_enable(ntb);
+	ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
 
 	callout_reset(&ntb->heartbeat_timer, 0, soc_link_hb, ntb);
 
@@ -1288,29 +1315,33 @@ xeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx,
 }
 
 static void
-xeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t base_addr,
+xeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr,
     enum ntb_bar idx, enum ntb_bar regbar)
 {
-	struct ntb_pci_bar_info *bar;
-	vm_paddr_t bar_addr;
+	uint64_t reg_val;
+	uint32_t base_reg, lmt_reg;
 
-	bar = &ntb->bar_info[idx];
-	bar_addr = base_addr + ((idx == regbar) ? ntb->b2b_off : 0);
+	bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg);
+	if (idx == regbar)
+		bar_addr += ntb->b2b_off;
 
-	if (HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) {
-		ntb_reg_write(4, bar->sbarbase_off, bar_addr);
-		ntb_reg_write(4, bar->sbarlmt_off, bar_addr);
-		bar_addr = ntb_reg_read(4, bar->sbarbase_off);
-		(void)bar_addr;
-		bar_addr = ntb_reg_read(4, bar->sbarlmt_off);
+	if (!bar_is_64bit(ntb, idx)) {
+		ntb_reg_write(4, base_reg, bar_addr);
+		reg_val = ntb_reg_read(4, base_reg);
+		(void)reg_val;
+
+		ntb_reg_write(4, lmt_reg, bar_addr);
+		reg_val = ntb_reg_read(4, lmt_reg);
+		(void)reg_val;
 	} else {
-		ntb_reg_write(8, bar->sbarbase_off, bar_addr);
-		ntb_reg_write(8, bar->sbarlmt_off, bar_addr);
-		bar_addr = ntb_reg_read(8, bar->sbarbase_off);
-		(void)bar_addr;
-		bar_addr = ntb_reg_read(8, bar->sbarlmt_off);
+		ntb_reg_write(8, base_reg, bar_addr);
+		reg_val = ntb_reg_read(8, base_reg);
+		(void)reg_val;
+
+		ntb_reg_write(8, lmt_reg, bar_addr);
+		reg_val = ntb_reg_read(8, lmt_reg);
+		(void)reg_val;
 	}
-	(void)bar_addr;
 }
 
 static void
@@ -1444,41 +1475,62 @@ xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr,
 	return (0);
 }
 
+static inline bool
+link_is_up(struct ntb_softc *ntb)
+{
+
+	if (ntb->type == NTB_XEON)
+		return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0);
+
+	KASSERT(ntb->type == NTB_SOC, ("ntb type"));
+	return ((ntb->ntb_ctl & SOC_CNTL_LINK_DOWN) == 0);
+}
+
+static inline bool
+soc_link_is_err(struct ntb_softc *ntb)
+{
+	uint32_t status;
+
+	KASSERT(ntb->type == NTB_SOC, ("ntb type"));
+
+	status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET);
+	if ((status & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0)
+		return (true);
+
+	status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET);
+	return ((status & SOC_IBIST_ERR_OFLOW) != 0);
+}
+
 /* SOC does not have link status interrupt, poll on that platform */
 static void
 soc_link_hb(void *arg)
 {
 	struct ntb_softc *ntb = arg;
-	uint32_t status32;
-	int rc;
+	sbintime_t timo, poll_ts;
+
+	timo = NTB_HB_TIMEOUT * hz;
+	poll_ts = ntb->last_ts + timo;
 
 	/*
 	 * Delay polling the link status if an interrupt was received, unless
 	 * the cached link status says the link is down.
 	 */
-	if ((long)ticks - ((long)ntb->last_ts + NTB_HB_TIMEOUT * hz) < 0 &&
-	    (ntb->ntb_ctl & SOC_CNTL_LINK_DOWN) == 0)
+	if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) {
+		timo = poll_ts - ticks;
 		goto out;
+	}
 
+	if (ntb_poll_link(ntb))
+		ntb_link_event(ntb);
 
-	rc = ntb_poll_link(ntb);
-	if (rc != 0)
-		device_printf(ntb->device,
-		    "Error determining link status\n");
-
-	/* Check to see if a link error is the cause of the link down */
-	if (ntb->link_status == NTB_LINK_DOWN) {
-		status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET);
-		if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) {
-			callout_reset(&ntb->lr_timer, 0, recover_soc_link,
-			    ntb);
-			return;
-		}
+	if (!link_is_up(ntb) && soc_link_is_err(ntb)) {
+		/* Link is down with error, proceed with recovery */
+		callout_reset(&ntb->lr_timer, 0, recover_soc_link, ntb);
+		return;
 	}
 
 out:
-	callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, soc_link_hb,
-	    ntb);
+	callout_reset(&ntb->heartbeat_timer, timo, soc_link_hb, ntb);
 }
 
 static void
@@ -1520,58 +1572,141 @@ soc_perform_link_restart(struct ntb_softc *ntb)
 	ntb_reg_write(4, SOC_LTSSMSTATEJMP_OFFSET, status);
 }
 
-static void
-ntb_handle_link_event(struct ntb_softc *ntb, int link_state)
+/*
+ * ntb_set_ctx() - associate a driver context with an ntb device
+ * @ntb:        NTB device context
+ * @ctx:        Driver context
+ * @ctx_ops:    Driver context operations
+ *
+ * Associate a driver context and operations with a ntb device.  The context is
+ * provided by the client driver, and the driver may associate a different
+ * context with each ntb device.
+ *
+ * Return: Zero if the context is associated, otherwise an error number.
+ */
+int
+ntb_set_ctx(struct ntb_softc *ntb, void *ctx, const struct ntb_ctx_ops *ops)
 {
-	enum ntb_hw_event event;
-	uint16_t status;
 
-	if (ntb->link_status == link_state)
-		return;
+	if (ctx == NULL || ops == NULL)
+		return (EINVAL);
+	if (ntb->ctx_ops != NULL)
+		return (EINVAL);
 
-	if (link_state == NTB_LINK_UP) {
-		device_printf(ntb->device, "Link Up\n");
-		ntb->link_status = NTB_LINK_UP;
-		event = NTB_EVENT_HW_LINK_UP;
-
-		if (ntb->type == NTB_SOC ||
-		    ntb->conn_type == NTB_CONN_TRANSPARENT)
-			status = ntb_reg_read(2, ntb->reg->lnk_sta);
-		else
-			status = pci_read_config(ntb->device,
-			    XEON_LINK_STATUS_OFFSET, 2);
-		ntb->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4;
-		ntb->link_speed = (status & NTB_LINK_SPEED_MASK);
-		device_printf(ntb->device, "Link Width %d, Link Speed %d\n",
-		    ntb->link_width, ntb->link_speed);
-		callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz,
-		    soc_link_hb, ntb);
-	} else {
-		device_printf(ntb->device, "Link Down\n");
-		ntb->link_status = NTB_LINK_DOWN;
-		event = NTB_EVENT_HW_LINK_DOWN;
-		/* Do not modify link width/speed, we need it in link recovery */
+	CTX_LOCK(ntb);
+	if (ntb->ctx_ops != NULL) {
+		CTX_UNLOCK(ntb);
+		return (EINVAL);
 	}
+	ntb->ntb_ctx = ctx;
+	ntb->ctx_ops = ops;
+	CTX_UNLOCK(ntb);
 
-	/* notify the upper layer if we have an event change */
-	if (ntb->event_cb != NULL)
-		ntb->event_cb(ntb->ntb_transport, event);
+	return (0);
 }
 
+/*
+ * It is expected that this will only be used from contexts where the ctx_lock
+ * is not needed to protect ntb_ctx lifetime.
+ */
+void *
+ntb_get_ctx(struct ntb_softc *ntb, const struct ntb_ctx_ops **ops)
+{
+
+	KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus"));
+	if (ops != NULL)
+		*ops = ntb->ctx_ops;
+	return (ntb->ntb_ctx);
+}
+
+/*
+ * ntb_clear_ctx() - disassociate any driver context from an ntb device
+ * @ntb:        NTB device context
+ *
+ * Clear any association that may exist between a driver context and the ntb
+ * device.
+ */
+void
+ntb_clear_ctx(struct ntb_softc *ntb)
+{
+
+	CTX_LOCK(ntb);
+	ntb->ntb_ctx = NULL;
+	ntb->ctx_ops = NULL;
+	CTX_UNLOCK(ntb);
+}
+
+/*
+ * ntb_link_event() - notify driver context of a change in link status
+ * @ntb:        NTB device context
+ *
+ * Notify the driver context that the link status may have changed.  The driver
+ * should call ntb_link_is_up() to get the current status.
+ */
+void
+ntb_link_event(struct ntb_softc *ntb)
+{
+
+	CTX_LOCK(ntb);
+	if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL)
+		ntb->ctx_ops->link_event(ntb->ntb_ctx);
+	CTX_UNLOCK(ntb);
+}
+
+/*
+ * ntb_db_event() - notify driver context of a doorbell event
+ * @ntb:        NTB device context
+ * @vector:     Interrupt vector number
+ *
+ * Notify the driver context of a doorbell event.  If hardware supports
+ * multiple interrupt vectors for doorbells, the vector number indicates which
+ * vector received the interrupt.  The vector number is relative to the first
+ * vector used for doorbells, starting at zero, and must be less than
+ * ntb_db_vector_count().  The driver may call ntb_db_read() to check which
+ * doorbell bits need service, and ntb_db_vector_mask() to determine which of
+ * those bits are associated with the vector number.
+ */
 static void
-ntb_link_enable(struct ntb_softc *ntb)
+ntb_db_event(struct ntb_softc *ntb, uint32_t vec)
+{
+
+	CTX_LOCK(ntb);
+	if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL)
+		ntb->ctx_ops->db_event(ntb->ntb_ctx, vec);
+	CTX_UNLOCK(ntb);
+}
+
+/*
+ * ntb_link_enable() - enable the link on the secondary side of the ntb
+ * @ntb:        NTB device context
+ * @max_speed:  The maximum link speed expressed as PCIe generation number[0]
+ * @max_width:  The maximum link width expressed as the number of PCIe lanes[0]
+ *
+ * Enable the link on the secondary side of the ntb.  This can only be done
+ * from the primary side of the ntb in primary or b2b topology.  The ntb device
+ * should train the link to its maximum speed and width, or the requested speed
+ * and width, whichever is smaller, if supported.
+ *
+ * Return: Zero on success, otherwise an error number.
+ *
+ * [0]: Only NTB_SPEED_AUTO and NTB_WIDTH_AUTO are valid inputs; other speed
+ *      and width input will be ignored.
+ */
+int
+ntb_link_enable(struct ntb_softc *ntb, enum ntb_speed s __unused,
+    enum ntb_width w __unused)
 {
 	uint32_t cntl;
 
 	if (ntb->type == NTB_SOC) {
 		pci_write_config(ntb->device, NTB_PPD_OFFSET,
 		    ntb->ppd | SOC_PPD_INIT_LINK, 4);
-		return;
+		return (0);
 	}
 
 	if (ntb->conn_type == NTB_CONN_TRANSPARENT) {
-		ntb_handle_link_event(ntb, NTB_LINK_UP);
-		return;
+		ntb_link_event(ntb);
+		return (0);
 	}
 
 	cntl = ntb_reg_read(4, ntb->reg->ntb_ctl);
@@ -1581,16 +1716,29 @@ ntb_link_enable(struct ntb_softc *ntb)
 	if (HAS_FEATURE(NTB_SPLIT_BAR))
 		cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP;
 	ntb_reg_write(4, ntb->reg->ntb_ctl, cntl);
+	return (0);
 }
 
-static void
+/*
+ * ntb_link_disable() - disable the link on the secondary side of the ntb
+ * @ntb:        NTB device context
+ *
+ * Disable the link on the secondary side of the ntb.  This can only be done
+ * from the primary side of the ntb in primary or b2b topology.  The ntb device
+ * should disable the link.  Returning from this call must indicate that a
+ * barrier has passed, though with no more writes may pass in either direction
+ * across the link, except if this call returns an error number.
+ *
+ * Return: Zero on success, otherwise an error number.
+ */
+int
 ntb_link_disable(struct ntb_softc *ntb)
 {
 	uint32_t cntl;
 
 	if (ntb->conn_type == NTB_CONN_TRANSPARENT) {
-		ntb_handle_link_event(ntb, NTB_LINK_DOWN);
-		return;
+		ntb_link_event(ntb);
+		return (0);
 	}
 
 	cntl = ntb_reg_read(4, ntb->reg->ntb_ctl);
@@ -1600,6 +1748,7 @@ ntb_link_disable(struct ntb_softc *ntb)
 		cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP);
 	cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK;
 	ntb_reg_write(4, ntb->reg->ntb_ctl, cntl);
+	return (0);
 }
 
 static void
@@ -1648,218 +1797,56 @@ retry:
 	    ntb);
 }
 
-static int
+/*
+ * Polls the HW link status register(s); returns true if something has changed.
+ */
+static bool
 ntb_poll_link(struct ntb_softc *ntb)
 {
-	int link_state;
 	uint32_t ntb_cntl;
-	uint16_t status;
+	uint16_t reg_val;
 
 	if (ntb->type == NTB_SOC) {
-		HW_LOCK(ntb);
 		ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl);
-		if (ntb_cntl == ntb->ntb_ctl) {
-			HW_UNLOCK(ntb);
-			return (0);
-		}
+		if (ntb_cntl == ntb->ntb_ctl)
+			return (false);
+
 		ntb->ntb_ctl = ntb_cntl;
 		ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta);
-		HW_UNLOCK(ntb);
-
-		if ((ntb_cntl & SOC_CNTL_LINK_DOWN) != 0)
-			link_state = NTB_LINK_DOWN;
-		else
-			link_state = NTB_LINK_UP;
 	} else {
-		status = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2);
-		if (status == ntb->lnk_sta)
-			return (0);
-		ntb->lnk_sta = status;
+		db_iowrite(ntb, ntb->reg_ofs.ldb, ntb->db_link_mask);
 
-		if ((status & NTB_LINK_STATUS_ACTIVE) != 0)
-			link_state = NTB_LINK_UP;
-		else
-			link_state = NTB_LINK_DOWN;
+		reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2);
+		if (reg_val == ntb->lnk_sta)
+			return (false);
+
+		ntb->lnk_sta = reg_val;
 	}
-
-	ntb_handle_link_event(ntb, link_state);
-	return (0);
+	return (true);
 }
 
-static void
-ntb_irq_work(void *arg)
+static inline enum ntb_speed
+ntb_link_sta_speed(struct ntb_softc *ntb)
 {
-	struct ntb_db_cb *db_cb = arg;
-	struct ntb_softc *ntb;
-	int rc;
 
-	rc = db_cb->callback(db_cb->data, db_cb->db_num);
-	/* Poll if forward progress was made. */
-	if (rc != 0) {
-		callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb);
-		return;
-	}
+	if (!link_is_up(ntb))
+		return (NTB_SPEED_NONE);
+	return (ntb->lnk_sta & NTB_LINK_SPEED_MASK);
+}
 
-	/* Unmask interrupt if no progress was made. */
-	ntb = db_cb->ntb;
-	unmask_ldb_interrupt(ntb, db_cb->db_num);
+static inline enum ntb_width
+ntb_link_sta_width(struct ntb_softc *ntb)
+{
+
+	if (!link_is_up(ntb))
+		return (NTB_WIDTH_NONE);
+	return (NTB_LNK_STA_WIDTH(ntb->lnk_sta));
 }
 
 /*
  * Public API to the rest of the OS
  */
 
-/**
- * ntb_register_event_callback() - register event callback
- * @ntb: pointer to ntb_softc instance
- * @func: callback function to register
- *
- * This function registers a callback for any HW driver events such as link
- * up/down, power management notices and etc.
- *
- * RETURNS: An appropriate ERRNO error value on error, or zero for success.
- */
-int
-ntb_register_event_callback(struct ntb_softc *ntb, ntb_event_callback func)
-{
-
-	if (ntb->event_cb != NULL)
-		return (EINVAL);
-
-	ntb->event_cb = func;
-
-	return (0);
-}
-
-/**
- * ntb_unregister_event_callback() - unregisters the event callback
- * @ntb: pointer to ntb_softc instance
- *
- * This function unregisters the existing callback from transport
- */
-void
-ntb_unregister_event_callback(struct ntb_softc *ntb)
-{
-
-	ntb->event_cb = NULL;
-}
-
-/**
- * ntb_register_db_callback() - register a callback for doorbell interrupt
- * @ntb: pointer to ntb_softc instance
- * @idx: doorbell index to register callback, zero based
- * @data: pointer to be returned to caller with every callback
- * @func: callback function to register
- *
- * This function registers a callback function for the doorbell interrupt
- * on the primary side. The function will unmask the doorbell as well to
- * allow interrupt.
- *
- * RETURNS: An appropriate ERRNO error value on error, or zero for success.
- */
-int
-ntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data,
-    ntb_db_callback func)
-{
-	struct ntb_db_cb *db_cb = &ntb->db_cb[idx];
-
-	if (idx >= ntb->max_cbs || db_cb->callback != NULL || db_cb->reserved) {
-		device_printf(ntb->device, "Invalid Index.\n");
-		return (EINVAL);
-	}
-
-	db_cb->callback = func;
-	db_cb->data = data;
-	callout_init(&db_cb->irq_work, 1);
-
-	unmask_ldb_interrupt(ntb, idx);
-
-	return (0);
-}
-
-/**
- * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt
- * @ntb: pointer to ntb_softc instance
- * @idx: doorbell index to register callback, zero based
- *
- * This function unregisters a callback function for the doorbell interrupt
- * on the primary side. The function will also mask the said doorbell.
- */
-void
-ntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx)
-{
-
-	if (idx >= ntb->max_cbs || ntb->db_cb[idx].callback == NULL)
-		return;
-
-	mask_ldb_interrupt(ntb, idx);
-
-	callout_drain(&ntb->db_cb[idx].irq_work);
-	ntb->db_cb[idx].callback = NULL;
-}
-
-/**
- * ntb_find_transport() - find the transport pointer
- * @transport: pointer to pci device
- *
- * Given the pci device pointer, return the transport pointer passed in when
- * the transport attached when it was inited.
- *
- * RETURNS: pointer to transport.
- */
-void *
-ntb_find_transport(struct ntb_softc *ntb)
-{
-
-	return (ntb->ntb_transport);
-}
-
-/**
- * ntb_register_transport() - Register NTB transport with NTB HW driver
- * @transport: transport identifier
- *
- * This function allows a transport to reserve the hardware driver for
- * NTB usage.
- *
- * RETURNS: pointer to ntb_softc, NULL on error.
- */
-struct ntb_softc *
-ntb_register_transport(struct ntb_softc *ntb, void *transport)
-{
-
-	/*
-	 * TODO: when we have more than one transport, we will need to rewrite
-	 * this to prevent race conditions
-	 */
-	if (ntb->ntb_transport != NULL)
-		return (NULL);
-
-	ntb->ntb_transport = transport;
-	return (ntb);
-}
-
-/**
- * ntb_unregister_transport() - Unregister the transport with the NTB HW driver
- * @ntb - ntb_softc of the transport to be freed
- *
- * This function unregisters the transport from the HW driver and performs any
- * necessary cleanups.
- */
-void
-ntb_unregister_transport(struct ntb_softc *ntb)
-{
-	uint8_t i;
-
-	if (ntb->ntb_transport == NULL)
-		return;
-
-	for (i = 0; i < ntb->max_cbs; i++)
-		ntb_unregister_db_callback(ntb, i);
-
-	ntb_unregister_event_callback(ntb);
-	ntb->ntb_transport = NULL;
-}
-
 /**
  * ntb_get_max_spads() - get the total scratch regs usable
  * @ntb: pointer to ntb_softc instance
@@ -1876,13 +1863,6 @@ ntb_get_max_spads(struct ntb_softc *ntb)
 	return (ntb->spad_count);
 }
 
-uint8_t
-ntb_get_max_cbs(struct ntb_softc *ntb)
-{
-
-	return (ntb->max_cbs);
-}
-
 uint8_t
 ntb_mw_count(struct ntb_softc *ntb)
 {
@@ -1988,89 +1968,153 @@ ntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
 	return (0);
 }
 
-/**
- * ntb_get_mw_vbase() - get virtual addr for the NTB memory window
- * @ntb: pointer to ntb_softc instance
- * @mw: memory window number
+/*
+ * ntb_mw_get_range() - get the range of a memory window
+ * @ntb:        NTB device context
+ * @idx:        Memory window number
+ * @base:       OUT - the base address for mapping the memory window
+ * @size:       OUT - the size for mapping the memory window
+ * @align:      OUT - the base alignment for translating the memory window
+ * @align_size: OUT - the size alignment for translating the memory window
  *
- * This function provides the base virtual address of the memory window
- * specified.
+ * Get the range of a memory window.  NULL may be given for any output
+ * parameter if the value is not needed.  The base and size may be used for
+ * mapping the memory window, to access the peer memory.  The alignment and
+ * size may be used for translating the memory window, for the peer to access
+ * memory on the local system.
  *
- * RETURNS: pointer to virtual address, or NULL on error.
+ * Return: Zero on success, otherwise an error number.
  */
-void *
-ntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw)
+int
+ntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base,
+    void **vbase, size_t *size, size_t *align, size_t *align_size)
 {
+	struct ntb_pci_bar_info *bar;
+	size_t bar_b2b_off;
 
-	if (mw >= ntb_mw_count(ntb))
-		return (NULL);
+	if (mw_idx >= ntb_mw_count(ntb))
+		return (EINVAL);
 
-	return (ntb->bar_info[ntb_mw_to_bar(ntb, mw)].vbase);
-}
-
-bus_addr_t
-ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw)
-{
-
-	if (mw >= ntb_mw_count(ntb))
-		return (0);
-
-	return (ntb->bar_info[ntb_mw_to_bar(ntb, mw)].pbase);
-}
-
-/**
- * ntb_get_mw_size() - return size of NTB memory window
- * @ntb: pointer to ntb_softc instance
- * @mw: memory window number
- *
- * This function provides the physical size of the memory window specified
- *
- * RETURNS: the size of the memory window or zero on error
- */
-u_long
-ntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw)
-{
-
-	if (mw >= ntb_mw_count(ntb))
-		return (0);
-
-	return (ntb->bar_info[ntb_mw_to_bar(ntb, mw)].size);
-}
-
-/**
- * ntb_set_mw_addr - set the memory window address
- * @ntb: pointer to ntb_softc instance
- * @mw: memory window number
- * @addr: base address for data
- *
- * This function sets the base physical address of the memory window.  This
- * memory address is where data from the remote system will be transfered into
- * or out of depending on how the transport is configured.
- */
-void
-ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr)
-{
-
-	if (mw >= ntb_mw_count(ntb))
-		return;
-
-	switch (ntb_mw_to_bar(ntb, mw)) {
-	case NTB_B2B_BAR_1:
-		ntb_reg_write(8, ntb->xlat_reg->bar2_xlat, addr);
-		break;
-	case NTB_B2B_BAR_2:
-		if (HAS_FEATURE(NTB_SPLIT_BAR))
-			ntb_reg_write(4, ntb->reg_ofs.bar4_xlat, addr);
-		else
-			ntb_reg_write(8, ntb->reg_ofs.bar4_xlat, addr);
-		break;
-	case NTB_B2B_BAR_3:
-		ntb_reg_write(4, ntb->reg_ofs.bar5_xlat, addr);
-		break;
-	default:
-		KASSERT(false, ("invalid BAR"));
-		break;
+	bar = &ntb->bar_info[ntb_mw_to_bar(ntb, mw_idx)];
+	bar_b2b_off = 0;
+	if (mw_idx == ntb->b2b_mw_idx) {
+		KASSERT(ntb->b2b_off != 0,
+		    ("user shouldn't get non-shared b2b mw"));
+		bar_b2b_off = ntb->b2b_off;
 	}
+
+	if (base != NULL)
+		*base = bar->pbase + bar_b2b_off;
+	if (vbase != NULL)
+		*vbase = (char *)bar->vbase + bar_b2b_off;
+	if (size != NULL)
+		*size = bar->size - bar_b2b_off;
+	if (align != NULL)
+		*align = bar->size;
+	if (align_size != NULL)
+		*align_size = 1;
+	return (0);
+}
+
+/*
+ * ntb_mw_set_trans() - set the translation of a memory window
+ * @ntb:        NTB device context
+ * @idx:        Memory window number
+ * @addr:       The dma address local memory to expose to the peer
+ * @size:       The size of the local memory to expose to the peer
+ *
+ * Set the translation of a memory window.  The peer may access local memory
+ * through the window starting at the address, up to the size.  The address
+ * must be aligned to the alignment specified by ntb_mw_get_range().  The size
+ * must be aligned to the size alignment specified by ntb_mw_get_range().
+ *
+ * Return: Zero on success, otherwise an error number.
+ */
+int
+ntb_mw_set_trans(struct ntb_softc *ntb, unsigned idx, bus_addr_t addr,
+    size_t size)
+{
+	struct ntb_pci_bar_info *bar;
+	uint64_t base, limit, reg_val;
+	size_t bar_size, mw_size;
+	uint32_t base_reg, xlat_reg, limit_reg;
+	enum ntb_bar bar_num;
+
+	if (idx >= ntb_mw_count(ntb))
+		return (EINVAL);
+
+	bar_num = ntb_mw_to_bar(ntb, idx);
+	bar = &ntb->bar_info[bar_num];
+
+	bar_size = bar->size;
+	if (idx == ntb->b2b_mw_idx)
+		mw_size = bar_size - ntb->b2b_off;
+	else
+		mw_size = bar_size;
+
+	/* Hardware requires that addr is aligned to bar size */
+	if ((addr & (bar_size - 1)) != 0)
+		return (EINVAL);
+
+	if (size > mw_size)
+		return (EINVAL);
+
+	bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg);
+
+	limit = 0;
+	if (bar_is_64bit(ntb, bar_num)) {
+		base = ntb_reg_read(8, base_reg);
+
+		if (limit_reg != 0 && size != mw_size)
+			limit = base + size;
+
+		/* Set and verify translation address */
+		ntb_reg_write(8, xlat_reg, addr);
+		reg_val = ntb_reg_read(8, xlat_reg);
+		if (reg_val != addr) {
+			ntb_reg_write(8, xlat_reg, 0);
+			return (EIO);
+		}
+
+		/* Set and verify the limit */
+		ntb_reg_write(8, limit_reg, limit);
+		reg_val = ntb_reg_read(8, limit_reg);
+		if (reg_val != limit) {
+			ntb_reg_write(8, limit_reg, base);
+			ntb_reg_write(8, xlat_reg, 0);
+			return (EIO);
+		}
+	} else {
+		/* Configure 32-bit (split) BAR MW */
+
+		if ((addr & ~UINT32_MAX) != 0)
+			return (EINVAL);
+		if (((addr + size) & ~UINT32_MAX) != 0)
+			return (EINVAL);
+
+		base = ntb_reg_read(4, base_reg);
+
+		if (limit_reg != 0 && size != mw_size)
+			limit = base + size;
+
+		/* Set and verify translation address */
+		ntb_reg_write(4, xlat_reg, addr);
+		reg_val = ntb_reg_read(4, xlat_reg);
+		if (reg_val != addr) {
+			ntb_reg_write(4, xlat_reg, 0);
+			return (EIO);
+		}
+
+		/* Set and verify the limit */
+		ntb_reg_write(4, limit_reg, limit);
+		reg_val = ntb_reg_read(4, limit_reg);
+		if (reg_val != limit) {
+			ntb_reg_write(4, limit_reg, base);
+			ntb_reg_write(4, xlat_reg, 0);
+			return (EIO);
+		}
+	}
+	return (0);
 }
 
 /**
@@ -2090,7 +2134,7 @@ ntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit)
 		return;
 	}
 
-	ntb_db_write(ntb, ntb->peer_reg->db_bell, bit);
+	db_iowrite(ntb, ntb->peer_reg->db_bell, bit);
 }
 
 /*
@@ -2132,18 +2176,23 @@ ntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out)
 }
 
 /**
- * ntb_link_is_up() - return the hardware link status
- * @ndev: pointer to ntb_device instance
- *
- * Returns true if the hardware is connected to the remote system
+ * ntb_link_is_up() - get the current ntb link state
+ * @ntb:        NTB device context
+ * @speed:      OUT - The link speed expressed as PCIe generation number
+ * @width:      OUT - The link width expressed as the number of PCIe lanes
  *
  * RETURNS: true or false based on the hardware link state
  */
 bool
-ntb_link_is_up(struct ntb_softc *ntb)
+ntb_link_is_up(struct ntb_softc *ntb, enum ntb_speed *speed,
+    enum ntb_width *width)
 {
 
-	return (ntb->link_status == NTB_LINK_UP);
+	if (speed != NULL)
+		*speed = ntb_link_sta_speed(ntb);
+	if (width != NULL)
+		*width = ntb_link_sta_width(ntb);
+	return (link_is_up(ntb));
 }
 
 static void
diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.h b/sys/dev/ntb/ntb_hw/ntb_hw.h
index 05a5f90b5837..c28596dc7fa9 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.h
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.h
@@ -34,36 +34,53 @@ struct ntb_softc;
 
 #define NTB_MAX_NUM_MW	3
 
-enum ntb_link_event {
-	NTB_LINK_DOWN = 0,
-	NTB_LINK_UP,
+enum ntb_speed {
+	NTB_SPEED_AUTO = -1,
+	NTB_SPEED_NONE = 0,
+	NTB_SPEED_GEN1 = 1,
+	NTB_SPEED_GEN2 = 2,
+	NTB_SPEED_GEN3 = 3,
 };
 
-enum ntb_hw_event {
-	NTB_EVENT_SW_EVENT0 = 0,
-	NTB_EVENT_SW_EVENT1,
-	NTB_EVENT_SW_EVENT2,
-	NTB_EVENT_HW_ERROR,
-	NTB_EVENT_HW_LINK_UP,
-	NTB_EVENT_HW_LINK_DOWN,
+enum ntb_width {
+	NTB_WIDTH_AUTO = -1,
+	NTB_WIDTH_NONE = 0,
+	NTB_WIDTH_1 = 1,
+	NTB_WIDTH_2 = 2,
+	NTB_WIDTH_4 = 4,
+	NTB_WIDTH_8 = 8,
+	NTB_WIDTH_12 = 12,
+	NTB_WIDTH_16 = 16,
+	NTB_WIDTH_32 = 32,
 };
 
 SYSCTL_DECL(_hw_ntb);
 
-typedef int (*ntb_db_callback)(void *data, int db_num);
-typedef void (*ntb_event_callback)(void *data, enum ntb_hw_event event);
+typedef void (*ntb_db_callback)(void *data, int vector);
+typedef void (*ntb_event_callback)(void *data);
+
+struct ntb_ctx_ops {
+	ntb_event_callback	link_event;
+	ntb_db_callback		db_event;
+};
+
+device_t ntb_get_device(struct ntb_softc *);
+
+bool ntb_link_is_up(struct ntb_softc *, enum ntb_speed *, enum ntb_width *);
+void ntb_link_event(struct ntb_softc *);
+int ntb_link_enable(struct ntb_softc *, enum ntb_speed, enum ntb_width);
+int ntb_link_disable(struct ntb_softc *);
+
+int ntb_set_ctx(struct ntb_softc *, void *, const struct ntb_ctx_ops *);
+void *ntb_get_ctx(struct ntb_softc *, const struct ntb_ctx_ops **);
+void ntb_clear_ctx(struct ntb_softc *);
+
+uint8_t ntb_mw_count(struct ntb_softc *);
+int ntb_mw_get_range(struct ntb_softc *, unsigned mw_idx, vm_paddr_t *base,
+    void **vbase, size_t *size, size_t *align, size_t *align_size);
+int ntb_mw_set_trans(struct ntb_softc *, unsigned mw_idx, bus_addr_t, size_t);
+int ntb_mw_clear_trans(struct ntb_softc *, unsigned mw_idx);
 
-int ntb_register_event_callback(struct ntb_softc *ntb, ntb_event_callback func);
-void ntb_unregister_event_callback(struct ntb_softc *ntb);
-int ntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx,
-    void *data, ntb_db_callback func);
-void ntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx);
-void *ntb_find_transport(struct ntb_softc *ntb);
-struct ntb_softc *ntb_register_transport(struct ntb_softc *ntb,
-    void *transport);
-void ntb_unregister_transport(struct ntb_softc *ntb);
-uint8_t ntb_get_max_cbs(struct ntb_softc *ntb);
-uint8_t ntb_mw_count(struct ntb_softc *ntb);
 uint8_t ntb_get_max_spads(struct ntb_softc *ntb);
 int ntb_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val);
 int ntb_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val);
@@ -71,14 +88,16 @@ int ntb_peer_spad_write(struct ntb_softc *ntb, unsigned int idx,
     uint32_t val);
 int ntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx,
     uint32_t *val);
-void *ntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw);
-bus_addr_t ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw);
-u_long ntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw);
-void ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr);
-void ntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit);
-bus_addr_t ntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out);
-bool ntb_link_is_up(struct ntb_softc *ntb);
-device_t ntb_get_device(struct ntb_softc *ntb);
+
+uint64_t ntb_db_valid_mask(struct ntb_softc *);
+bus_addr_t ntb_get_peer_db_addr(struct ntb_softc *, vm_size_t *sz_out);
+
+void ntb_db_clear(struct ntb_softc *, uint64_t bits);
+void ntb_db_clear_mask(struct ntb_softc *, uint64_t bits);
+uint64_t ntb_db_read(struct ntb_softc *);
+void ntb_db_set_mask(struct ntb_softc *, uint64_t bits);
+uint64_t ntb_db_vector_mask(struct ntb_softc *, int vector);
+void ntb_peer_db_set(struct ntb_softc *, uint64_t bits);
 
 /* Hardware owns the low 32 bits of features. */
 #define NTB_BAR_SIZE_4K		(1 << 0)
diff --git a/sys/dev/ntb/ntb_hw/ntb_regs.h b/sys/dev/ntb/ntb_hw/ntb_regs.h
index 2daf681fd918..22d24227c44b 100644
--- a/sys/dev/ntb/ntb_hw/ntb_regs.h
+++ b/sys/dev/ntb/ntb_hw/ntb_regs.h
@@ -33,6 +33,7 @@
 #define NTB_LINK_STATUS_ACTIVE	0x2000
 #define NTB_LINK_SPEED_MASK	0x000f
 #define NTB_LINK_WIDTH_MASK	0x03f0
+#define NTB_LNK_STA_WIDTH(sta)	(((sta) & NTB_LINK_WIDTH_MASK) >> 4)
 
 #define XEON_SNB_MW_COUNT	2
 #define XEON_HSX_SPLIT_MW_COUNT	3

From d9b26080f66dd6280463f8a250fec9578a516ccd Mon Sep 17 00:00:00 2001
From: ian <ian@FreeBSD.org>
Date: Sun, 18 Oct 2015 20:32:37 +0000
Subject: [PATCH 361/380] Include "opt_platform.h" early so that the FDT option
 is visible as needed.

---
 sys/arm/annapurna/alpine/common.c | 2 ++
 sys/arm/qemu/virt_common.c        | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/sys/arm/annapurna/alpine/common.c b/sys/arm/annapurna/alpine/common.c
index a0fade09128e..774bf00aceca 100644
--- a/sys/arm/annapurna/alpine/common.c
+++ b/sys/arm/annapurna/alpine/common.c
@@ -29,6 +29,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_platform.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bus.h>
diff --git a/sys/arm/qemu/virt_common.c b/sys/arm/qemu/virt_common.c
index fc022473e6ea..0f407243cff4 100644
--- a/sys/arm/qemu/virt_common.c
+++ b/sys/arm/qemu/virt_common.c
@@ -28,6 +28,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_platform.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 

From 7b5d098c90f0f2e5151f5a80014ba3932988a132 Mon Sep 17 00:00:00 2001
From: ian <ian@FreeBSD.org>
Date: Sun, 18 Oct 2015 20:37:10 +0000
Subject: [PATCH 362/380] Only decode fdt data which belongs to the GIC
 controller.

The interrupts-extended property is a list of controller-specific
interrupt tuples for more than one controller.  The decode routine of
every PIC gets called in the pre-INTRNG code (nexus doesn't know which
device instance belongs to which fdt node), so the GIC code has to
check each FDT node it is asked to decode to ensure it is the owner.

Because in the pre-INTRNG world there can only be one instance of a GIC,
it's safe to cache the results of a positive lookup in a static variable
to avoid the expensive lookups on subsequent calls.

Submitted by:	Svatopluk Kraus <onwahe@gmail.com>
Differential Revision:	https://reviews.freebsd.org/D2345
---
 sys/arm/arm/gic.c      | 17 ++++++++++++++++-
 sys/arm/include/intr.h |  3 +--
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/sys/arm/arm/gic.c b/sys/arm/arm/gic.c
index 9ceac4865ea3..a1ce111f9f14 100644
--- a/sys/arm/arm/gic.c
+++ b/sys/arm/arm/gic.c
@@ -36,6 +36,8 @@ __FBSDID("$FreeBSD$");
 
 #include "opt_platform.h"
 
+#include "opt_platform.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bus.h>
@@ -288,10 +290,23 @@ arm_gic_init_secondary(device_t dev)
  
 #ifndef ARM_INTRNG
 int
-gic_decode_fdt(uint32_t iparent, uint32_t *intr, int *interrupt,
+gic_decode_fdt(phandle_t iparent, pcell_t *intr, int *interrupt,
     int *trig, int *pol)
 {
 	static u_int num_intr_cells;
+	static phandle_t self;
+	struct ofw_compat_data *ocd;
+
+	if (self == 0) {
+		for (ocd = compat_data; ocd->ocd_str != NULL; ocd++) {
+			if (fdt_is_compatible(iparent, ocd->ocd_str)) {
+				self = iparent;
+				break;
+			}
+		}
+	}
+	if (self != iparent)
+		return (ENXIO);
 
 	if (num_intr_cells == 0) {
 		if (OF_searchencprop(OF_node_from_xref(iparent),
diff --git a/sys/arm/include/intr.h b/sys/arm/include/intr.h
index ed075dfd39c1..2bb479acb7f7 100644
--- a/sys/arm/include/intr.h
+++ b/sys/arm/include/intr.h
@@ -179,10 +179,9 @@ extern int (*arm_config_irq)(int irq, enum intr_trigger trig,
     enum intr_polarity pol);
 
 void arm_pic_init_secondary(void);
-int  gic_decode_fdt(uint32_t iparentnode, uint32_t *intrcells, int *interrupt,
-    int *trig, int *pol);
 
 #ifdef FDT
+int gic_decode_fdt(phandle_t, pcell_t *, int *, int *, int *);
 int arm_fdt_map_irq(phandle_t, pcell_t *, int);
 #endif
 

From 5b831e248abfef6965ad7c7d2addfda8be7994c5 Mon Sep 17 00:00:00 2001
From: zbb <zbb@FreeBSD.org>
Date: Sun, 18 Oct 2015 21:39:15 +0000
Subject: [PATCH 363/380] Raw import of ThunderX VNIC networking driver
 components

This import brings following components of the Linux driver:
- Thunder BGX (programmable MAC)
- Physical Function driver
- Virtual Function driver
- Headers

Revision:            1.0
Obtained from:       Cavium
License information: Cavium provided these files under BSD license
---
 sys/dev/vnic/nic.h          |  539 +++++++++++
 sys/dev/vnic/nic_main.c     | 1192 +++++++++++++++++++++++++
 sys/dev/vnic/nic_reg.h      |  234 +++++
 sys/dev/vnic/nicvf_main.c   | 1667 +++++++++++++++++++++++++++++++++++
 sys/dev/vnic/nicvf_queues.c | 1467 ++++++++++++++++++++++++++++++
 sys/dev/vnic/nicvf_queues.h |  366 ++++++++
 sys/dev/vnic/q_struct.h     |  719 +++++++++++++++
 sys/dev/vnic/thunder_bgx.c  | 1212 +++++++++++++++++++++++++
 sys/dev/vnic/thunder_bgx.h  |  248 ++++++
 9 files changed, 7644 insertions(+)
 create mode 100644 sys/dev/vnic/nic.h
 create mode 100644 sys/dev/vnic/nic_main.c
 create mode 100644 sys/dev/vnic/nic_reg.h
 create mode 100644 sys/dev/vnic/nicvf_main.c
 create mode 100644 sys/dev/vnic/nicvf_queues.c
 create mode 100644 sys/dev/vnic/nicvf_queues.h
 create mode 100644 sys/dev/vnic/q_struct.h
 create mode 100644 sys/dev/vnic/thunder_bgx.c
 create mode 100644 sys/dev/vnic/thunder_bgx.h

diff --git a/sys/dev/vnic/nic.h b/sys/dev/vnic/nic.h
new file mode 100644
index 000000000000..3023deabd05c
--- /dev/null
+++ b/sys/dev/vnic/nic.h
@@ -0,0 +1,539 @@
+/*
+ * Copyright (C) 2015 Cavium Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ *
+ */
+
+#ifndef NIC_H
+#define	NIC_H
+
+#include <linux/netdevice.h>
+#include <linux/interrupt.h>
+#include <linux/pci.h>
+#include "thunder_bgx.h"
+
+/* PCI device IDs */
+#define	PCI_DEVICE_ID_THUNDER_NIC_PF		0xA01E
+#define	PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF	0x0011
+#define	PCI_DEVICE_ID_THUNDER_NIC_VF		0xA034
+#define	PCI_DEVICE_ID_THUNDER_BGX		0xA026
+
+/* PCI BAR nos */
+#define	PCI_CFG_REG_BAR_NUM		0
+#define	PCI_MSIX_REG_BAR_NUM		4
+
+/* NIC SRIOV VF count */
+#define	MAX_NUM_VFS_SUPPORTED		128
+#define	DEFAULT_NUM_VF_ENABLED		8
+
+#define	NIC_TNS_BYPASS_MODE		0
+#define	NIC_TNS_MODE			1
+
+/* NIC priv flags */
+#define	NIC_SRIOV_ENABLED		BIT(0)
+#define	NIC_TNS_ENABLED			BIT(1)
+
+/* VNIC HW optimiation features */
+#define VNIC_RSS_SUPPORT
+#define VNIC_MULTI_QSET_SUPPORT
+
+/* Min/Max packet size */
+#define	NIC_HW_MIN_FRS			64
+#define	NIC_HW_MAX_FRS			9200 /* 9216 max packet including FCS */
+
+/* Max pkinds */
+#define	NIC_MAX_PKIND			16
+
+/* Rx Channels */
+/* Receive channel configuration in TNS bypass mode
+ * Below is configuration in TNS bypass mode
+ * BGX0-LMAC0-CHAN0 - VNIC CHAN0
+ * BGX0-LMAC1-CHAN0 - VNIC CHAN16
+ * ...
+ * BGX1-LMAC0-CHAN0 - VNIC CHAN128
+ * ...
+ * BGX1-LMAC3-CHAN0 - VNIC CHAN174
+ */
+#define	NIC_INTF_COUNT			2  /* Interfaces btw VNIC and TNS/BGX */
+#define	NIC_CHANS_PER_INF		128
+#define	NIC_MAX_CHANS			(NIC_INTF_COUNT * NIC_CHANS_PER_INF)
+#define	NIC_CPI_COUNT			2048 /* No of channel parse indices */
+
+/* TNS bypass mode: 1-1 mapping between VNIC and BGX:LMAC */
+#define NIC_MAX_BGX			MAX_BGX_PER_CN88XX
+#define	NIC_CPI_PER_BGX			(NIC_CPI_COUNT / NIC_MAX_BGX)
+#define	NIC_MAX_CPI_PER_LMAC		64 /* Max when CPI_ALG is IP diffserv */
+#define	NIC_RSSI_PER_BGX		(NIC_RSSI_COUNT / NIC_MAX_BGX)
+
+/* Tx scheduling */
+#define	NIC_MAX_TL4			1024
+#define	NIC_MAX_TL4_SHAPERS		256 /* 1 shaper for 4 TL4s */
+#define	NIC_MAX_TL3			256
+#define	NIC_MAX_TL3_SHAPERS		64  /* 1 shaper for 4 TL3s */
+#define	NIC_MAX_TL2			64
+#define	NIC_MAX_TL2_SHAPERS		2  /* 1 shaper for 32 TL2s */
+#define	NIC_MAX_TL1			2
+
+/* TNS bypass mode */
+#define	NIC_TL2_PER_BGX			32
+#define	NIC_TL4_PER_BGX			(NIC_MAX_TL4 / NIC_MAX_BGX)
+#define	NIC_TL4_PER_LMAC		(NIC_MAX_TL4 / NIC_CHANS_PER_INF)
+
+/* NIC VF Interrupts */
+#define	NICVF_INTR_CQ			0
+#define	NICVF_INTR_SQ			1
+#define	NICVF_INTR_RBDR			2
+#define	NICVF_INTR_PKT_DROP		3
+#define	NICVF_INTR_TCP_TIMER		4
+#define	NICVF_INTR_MBOX			5
+#define	NICVF_INTR_QS_ERR		6
+
+#define	NICVF_INTR_CQ_SHIFT		0
+#define	NICVF_INTR_SQ_SHIFT		8
+#define	NICVF_INTR_RBDR_SHIFT		16
+#define	NICVF_INTR_PKT_DROP_SHIFT	20
+#define	NICVF_INTR_TCP_TIMER_SHIFT	21
+#define	NICVF_INTR_MBOX_SHIFT		22
+#define	NICVF_INTR_QS_ERR_SHIFT		23
+
+#define	NICVF_INTR_CQ_MASK		(0xFF << NICVF_INTR_CQ_SHIFT)
+#define	NICVF_INTR_SQ_MASK		(0xFF << NICVF_INTR_SQ_SHIFT)
+#define	NICVF_INTR_RBDR_MASK		(0x03 << NICVF_INTR_RBDR_SHIFT)
+#define	NICVF_INTR_PKT_DROP_MASK	BIT(NICVF_INTR_PKT_DROP_SHIFT)
+#define	NICVF_INTR_TCP_TIMER_MASK	BIT(NICVF_INTR_TCP_TIMER_SHIFT)
+#define	NICVF_INTR_MBOX_MASK		BIT(NICVF_INTR_MBOX_SHIFT)
+#define	NICVF_INTR_QS_ERR_MASK		BIT(NICVF_INTR_QS_ERR_SHIFT)
+
+/* MSI-X interrupts */
+#define	NIC_PF_MSIX_VECTORS		10
+#define	NIC_VF_MSIX_VECTORS		20
+
+#define NIC_PF_INTR_ID_ECC0_SBE		0
+#define NIC_PF_INTR_ID_ECC0_DBE		1
+#define NIC_PF_INTR_ID_ECC1_SBE		2
+#define NIC_PF_INTR_ID_ECC1_DBE		3
+#define NIC_PF_INTR_ID_ECC2_SBE		4
+#define NIC_PF_INTR_ID_ECC2_DBE		5
+#define NIC_PF_INTR_ID_ECC3_SBE		6
+#define NIC_PF_INTR_ID_ECC3_DBE		7
+#define NIC_PF_INTR_ID_MBOX0		8
+#define NIC_PF_INTR_ID_MBOX1		9
+
+/* Global timer for CQ timer thresh interrupts
+ * Calculated for SCLK of 700Mhz
+ * value written should be a 1/16th of what is expected
+ *
+ * 1 tick per 0.05usec = value of 2.2
+ * This 10% would be covered in CQ timer thresh value
+ */
+#define NICPF_CLK_PER_INT_TICK		2
+
+/* Time to wait before we decide that a SQ is stuck.
+ *
+ * Since both pkt rx and tx notifications are done with same CQ,
+ * when packets are being received at very high rate (eg: L2 forwarding)
+ * then freeing transmitted skbs will be delayed and watchdog
+ * will kick in, resetting interface. Hence keeping this value high.
+ */
+#define	NICVF_TX_TIMEOUT		(50 * HZ)
+
+struct nicvf_cq_poll {
+	struct  nicvf *nicvf;
+	u8	cq_idx;		/* Completion queue index */
+	struct	napi_struct napi;
+};
+
+#define	NIC_RSSI_COUNT			4096 /* Total no of RSS indices */
+#define NIC_MAX_RSS_HASH_BITS		8
+#define NIC_MAX_RSS_IDR_TBL_SIZE	(1 << NIC_MAX_RSS_HASH_BITS)
+#define RSS_HASH_KEY_SIZE		5 /* 320 bit key */
+
+#ifdef VNIC_RSS_SUPPORT
+struct nicvf_rss_info {
+	bool enable;
+#define	RSS_L2_EXTENDED_HASH_ENA	BIT(0)
+#define	RSS_IP_HASH_ENA			BIT(1)
+#define	RSS_TCP_HASH_ENA		BIT(2)
+#define	RSS_TCP_SYN_DIS			BIT(3)
+#define	RSS_UDP_HASH_ENA		BIT(4)
+#define RSS_L4_EXTENDED_HASH_ENA	BIT(5)
+#define	RSS_ROCE_ENA			BIT(6)
+#define	RSS_L3_BI_DIRECTION_ENA		BIT(7)
+#define	RSS_L4_BI_DIRECTION_ENA		BIT(8)
+	u64 cfg;
+	u8  hash_bits;
+	u16 rss_size;
+	u8  ind_tbl[NIC_MAX_RSS_IDR_TBL_SIZE];
+	u64 key[RSS_HASH_KEY_SIZE];
+} ____cacheline_aligned_in_smp;
+#endif
+
+enum rx_stats_reg_offset {
+	RX_OCTS = 0x0,
+	RX_UCAST = 0x1,
+	RX_BCAST = 0x2,
+	RX_MCAST = 0x3,
+	RX_RED = 0x4,
+	RX_RED_OCTS = 0x5,
+	RX_ORUN = 0x6,
+	RX_ORUN_OCTS = 0x7,
+	RX_FCS = 0x8,
+	RX_L2ERR = 0x9,
+	RX_DRP_BCAST = 0xa,
+	RX_DRP_MCAST = 0xb,
+	RX_DRP_L3BCAST = 0xc,
+	RX_DRP_L3MCAST = 0xd,
+	RX_STATS_ENUM_LAST,
+};
+
+enum tx_stats_reg_offset {
+	TX_OCTS = 0x0,
+	TX_UCAST = 0x1,
+	TX_BCAST = 0x2,
+	TX_MCAST = 0x3,
+	TX_DROP = 0x4,
+	TX_STATS_ENUM_LAST,
+};
+
+struct nicvf_hw_stats {
+	u64 rx_bytes;
+	u64 rx_ucast_frames;
+	u64 rx_bcast_frames;
+	u64 rx_mcast_frames;
+	u64 rx_fcs_errors;
+	u64 rx_l2_errors;
+	u64 rx_drop_red;
+	u64 rx_drop_red_bytes;
+	u64 rx_drop_overrun;
+	u64 rx_drop_overrun_bytes;
+	u64 rx_drop_bcast;
+	u64 rx_drop_mcast;
+	u64 rx_drop_l3_bcast;
+	u64 rx_drop_l3_mcast;
+	u64 rx_bgx_truncated_pkts;
+	u64 rx_jabber_errs;
+	u64 rx_fcs_errs;
+	u64 rx_bgx_errs;
+	u64 rx_prel2_errs;
+	u64 rx_l2_hdr_malformed;
+	u64 rx_oversize;
+	u64 rx_undersize;
+	u64 rx_l2_len_mismatch;
+	u64 rx_l2_pclp;
+	u64 rx_ip_ver_errs;
+	u64 rx_ip_csum_errs;
+	u64 rx_ip_hdr_malformed;
+	u64 rx_ip_payload_malformed;
+	u64 rx_ip_ttl_errs;
+	u64 rx_l3_pclp;
+	u64 rx_l4_malformed;
+	u64 rx_l4_csum_errs;
+	u64 rx_udp_len_errs;
+	u64 rx_l4_port_errs;
+	u64 rx_tcp_flag_errs;
+	u64 rx_tcp_offset_errs;
+	u64 rx_l4_pclp;
+	u64 rx_truncated_pkts;
+
+	u64 tx_bytes_ok;
+	u64 tx_ucast_frames_ok;
+	u64 tx_bcast_frames_ok;
+	u64 tx_mcast_frames_ok;
+	u64 tx_drops;
+};
+
+struct nicvf_drv_stats {
+	/* Rx */
+	u64 rx_frames_ok;
+	u64 rx_frames_64;
+	u64 rx_frames_127;
+	u64 rx_frames_255;
+	u64 rx_frames_511;
+	u64 rx_frames_1023;
+	u64 rx_frames_1518;
+	u64 rx_frames_jumbo;
+	u64 rx_drops;
+
+	/* Tx */
+	u64 tx_frames_ok;
+	u64 tx_drops;
+	u64 tx_tso;
+	u64 txq_stop;
+	u64 txq_wake;
+};
+
+struct nicvf {
+	struct nicvf		*pnicvf;
+	struct net_device	*netdev;
+	struct pci_dev		*pdev;
+	u8			vf_id;
+	u8			node;
+	bool			tns_mode:1;
+	bool                    sqs_mode:1;
+	bool			loopback_supported:1;
+	u16			mtu;
+	struct queue_set	*qs;
+#ifdef VNIC_MULTI_QSET_SUPPORT
+#define	MAX_SQS_PER_VF_SINGLE_NODE		5
+#define	MAX_SQS_PER_VF				11
+	u8			sqs_id;
+	u8			sqs_count; /* Secondary Qset count */
+	struct nicvf		*snicvf[MAX_SQS_PER_VF];
+#endif
+	u8			rx_queues;
+	u8			tx_queues;
+	u8			max_queues;
+	void __iomem		*reg_base;
+	bool			link_up;
+	u8			duplex;
+	u32			speed;
+	struct page		*rb_page;
+	u32			rb_page_offset;
+	bool			rb_alloc_fail;
+	bool			rb_work_scheduled;
+	struct delayed_work	rbdr_work;
+	struct tasklet_struct	rbdr_task;
+	struct tasklet_struct	qs_err_task;
+	struct tasklet_struct	cq_task;
+	struct nicvf_cq_poll	*napi[8];
+#ifdef VNIC_RSS_SUPPORT
+	struct nicvf_rss_info	rss_info;
+#endif
+	u8			cpi_alg;
+	/* Interrupt coalescing settings */
+	u32			cq_coalesce_usecs;
+
+	u32			msg_enable;
+	struct nicvf_hw_stats   hw_stats;
+	struct nicvf_drv_stats  drv_stats;
+	struct bgx_stats	bgx_stats;
+	struct work_struct	reset_task;
+
+	/* MSI-X  */
+	bool			msix_enabled;
+	u8			num_vec;
+	struct msix_entry	msix_entries[NIC_VF_MSIX_VECTORS];
+	char			irq_name[NIC_VF_MSIX_VECTORS][20];
+	bool			irq_allocated[NIC_VF_MSIX_VECTORS];
+
+	/* VF <-> PF mailbox communication */
+	bool			pf_acked;
+	bool			pf_nacked;
+} ____cacheline_aligned_in_smp;
+
+/* PF <--> VF Mailbox communication
+ * Eight 64bit registers are shared between PF and VF.
+ * Separate set for each VF.
+ * Writing '1' into last register mbx7 means end of message.
+ */
+
+/* PF <--> VF mailbox communication */
+#define	NIC_PF_VF_MAILBOX_SIZE		2
+#define	NIC_MBOX_MSG_TIMEOUT		2000 /* ms */
+
+/* Mailbox message types */
+#define	NIC_MBOX_MSG_READY		0x01	/* Is PF ready to rcv msgs */
+#define	NIC_MBOX_MSG_ACK		0x02	/* ACK the message received */
+#define	NIC_MBOX_MSG_NACK		0x03	/* NACK the message received */
+#define	NIC_MBOX_MSG_QS_CFG		0x04	/* Configure Qset */
+#define	NIC_MBOX_MSG_RQ_CFG		0x05	/* Configure receive queue */
+#define	NIC_MBOX_MSG_SQ_CFG		0x06	/* Configure Send queue */
+#define	NIC_MBOX_MSG_RQ_DROP_CFG	0x07	/* Configure receive queue */
+#define	NIC_MBOX_MSG_SET_MAC		0x08	/* Add MAC ID to DMAC filter */
+#define	NIC_MBOX_MSG_SET_MAX_FRS	0x09	/* Set max frame size */
+#define	NIC_MBOX_MSG_CPI_CFG		0x0A	/* Config CPI, RSSI */
+#define	NIC_MBOX_MSG_RSS_SIZE		0x0B	/* Get RSS indir_tbl size */
+#define	NIC_MBOX_MSG_RSS_CFG		0x0C	/* Config RSS table */
+#define	NIC_MBOX_MSG_RSS_CFG_CONT	0x0D	/* RSS config continuation */
+#define	NIC_MBOX_MSG_RQ_BP_CFG		0x0E	/* RQ backpressure config */
+#define	NIC_MBOX_MSG_RQ_SW_SYNC		0x0F	/* Flush inflight pkts to RQ */
+#define	NIC_MBOX_MSG_BGX_STATS		0x10	/* Get stats from BGX */
+#define	NIC_MBOX_MSG_BGX_LINK_CHANGE	0x11	/* BGX:LMAC link status */
+#define	NIC_MBOX_MSG_ALLOC_SQS		0x12	/* Allocate secondary Qset */
+#define	NIC_MBOX_MSG_NICVF_PTR		0x13	/* Send nicvf ptr to PF */
+#define	NIC_MBOX_MSG_PNICVF_PTR		0x14	/* Get primary qset nicvf ptr */
+#define	NIC_MBOX_MSG_SNICVF_PTR		0x15	/* Send sqet nicvf ptr to PVF */
+#define	NIC_MBOX_MSG_LOOPBACK		0x16	/* Set interface in loopback */
+#define	NIC_MBOX_MSG_CFG_DONE		0xF0	/* VF configuration done */
+#define	NIC_MBOX_MSG_SHUTDOWN		0xF1	/* VF is being shutdown */
+
+struct nic_cfg_msg {
+	u8    msg;
+	u8    vf_id;
+	u8    node_id;
+	bool  tns_mode:1;
+	bool  sqs_mode:1;
+	bool  loopback_supported:1;
+	u8    mac_addr[ETH_ALEN];
+};
+
+/* Qset configuration */
+struct qs_cfg_msg {
+	u8    msg;
+	u8    num;
+	u8    sqs_count;
+	u64   cfg;
+};
+
+/* Receive queue configuration */
+struct rq_cfg_msg {
+	u8    msg;
+	u8    qs_num;
+	u8    rq_num;
+	u64   cfg;
+};
+
+/* Send queue configuration */
+struct sq_cfg_msg {
+	u8    msg;
+	u8    qs_num;
+	u8    sq_num;
+	bool  sqs_mode;
+	u64   cfg;
+};
+
+/* Set VF's MAC address */
+struct set_mac_msg {
+	u8    msg;
+	u8    vf_id;
+	u8    mac_addr[ETH_ALEN];
+};
+
+/* Set Maximum frame size */
+struct set_frs_msg {
+	u8    msg;
+	u8    vf_id;
+	u16   max_frs;
+};
+
+/* Set CPI algorithm type */
+struct cpi_cfg_msg {
+	u8    msg;
+	u8    vf_id;
+	u8    rq_cnt;
+	u8    cpi_alg;
+};
+
+/* Get RSS table size */
+struct rss_sz_msg {
+	u8    msg;
+	u8    vf_id;
+	u16   ind_tbl_size;
+};
+
+/* Set RSS configuration */
+struct rss_cfg_msg {
+	u8    msg;
+	u8    vf_id;
+	u8    hash_bits;
+	u8    tbl_len;
+	u8    tbl_offset;
+#define RSS_IND_TBL_LEN_PER_MBX_MSG	8
+	u8    ind_tbl[RSS_IND_TBL_LEN_PER_MBX_MSG];
+};
+
+struct bgx_stats_msg {
+	u8    msg;
+	u8    vf_id;
+	u8    rx;
+	u8    idx;
+	u64   stats;
+};
+
+/* Physical interface link status */
+struct bgx_link_status {
+	u8    msg;
+	u8    link_up;
+	u8    duplex;
+	u32   speed;
+};
+
+#ifdef VNIC_MULTI_QSET_SUPPORT
+/* Get Extra Qset IDs */
+struct sqs_alloc {
+	u8    msg;
+	u8    vf_id;
+	u8    qs_count;
+};
+
+struct nicvf_ptr {
+	u8    msg;
+	u8    vf_id;
+	bool  sqs_mode;
+	u8    sqs_id;
+	u64   nicvf;
+};
+#endif
+
+/* Set interface in loopback mode */
+struct set_loopback {
+	u8    msg;
+	u8    vf_id;
+	bool  enable;
+};
+
+/* 128 bit shared memory between PF and each VF */
+union nic_mbx {
+	struct { u8 msg; }	msg;
+	struct nic_cfg_msg	nic_cfg;
+	struct qs_cfg_msg	qs;
+	struct rq_cfg_msg	rq;
+	struct sq_cfg_msg	sq;
+	struct set_mac_msg	mac;
+	struct set_frs_msg	frs;
+	struct cpi_cfg_msg	cpi_cfg;
+	struct rss_sz_msg	rss_size;
+	struct rss_cfg_msg	rss_cfg;
+	struct bgx_stats_msg    bgx_stats;
+	struct bgx_link_status  link_status;
+#ifdef VNIC_MULTI_QSET_SUPPORT
+	struct sqs_alloc        sqs_alloc;
+	struct nicvf_ptr	nicvf;
+#endif
+	struct set_loopback	lbk;
+};
+
+#define NIC_NODE_ID_MASK	0x03
+#define NIC_NODE_ID_SHIFT	44
+
+static inline int nic_get_node_id(struct pci_dev *pdev)
+{
+	u64 addr = pci_resource_start(pdev, PCI_CFG_REG_BAR_NUM);
+	return ((addr >> NIC_NODE_ID_SHIFT) & NIC_NODE_ID_MASK);
+}
+
+int nicvf_set_real_num_queues(struct net_device *netdev,
+			      int tx_queues, int rx_queues);
+int nicvf_open(struct net_device *netdev);
+int nicvf_stop(struct net_device *netdev);
+int nicvf_send_msg_to_pf(struct nicvf *vf, union nic_mbx *mbx);
+void nicvf_config_rss(struct nicvf *nic);
+void nicvf_set_rss_key(struct nicvf *nic);
+void nicvf_set_ethtool_ops(struct net_device *netdev);
+void nicvf_update_stats(struct nicvf *nic);
+void nicvf_update_lmac_stats(struct nicvf *nic);
+
+#endif /* NIC_H */
diff --git a/sys/dev/vnic/nic_main.c b/sys/dev/vnic/nic_main.c
new file mode 100644
index 000000000000..de2f22ce0daa
--- /dev/null
+++ b/sys/dev/vnic/nic_main.c
@@ -0,0 +1,1192 @@
+/*
+ * Copyright (C) 2015 Cavium Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/interrupt.h>
+#include <linux/pci.h>
+#include <linux/etherdevice.h>
+#include <linux/of.h>
+
+#include "nic_reg.h"
+#include "nic.h"
+#include "q_struct.h"
+#include "thunder_bgx.h"
+
+#define DRV_NAME	"thunder-nic"
+#define DRV_VERSION	"1.0"
+
+struct nicpf {
+	struct pci_dev		*pdev;
+	u8			rev_id;
+	u8			node;
+	unsigned int		flags;
+	u8			num_vf_en;      /* No of VF enabled */
+	bool			vf_enabled[MAX_NUM_VFS_SUPPORTED];
+	void __iomem		*reg_base;       /* Register start address */
+#ifdef VNIC_MULTI_QSET_SUPPORT
+	u8			num_sqs_en;	/* Secondary qsets enabled */
+	u64			nicvf[MAX_NUM_VFS_SUPPORTED];
+	u8			vf_sqs[MAX_NUM_VFS_SUPPORTED][MAX_SQS_PER_VF];
+	u8			pqs_vf[MAX_NUM_VFS_SUPPORTED];
+	bool			sqs_used[MAX_NUM_VFS_SUPPORTED];
+#endif
+	struct pkind_cfg	pkind;
+#define	NIC_SET_VF_LMAC_MAP(bgx, lmac)	(((bgx & 0xF) << 4) | (lmac & 0xF))
+#define	NIC_GET_BGX_FROM_VF_LMAC_MAP(map)	((map >> 4) & 0xF)
+#define	NIC_GET_LMAC_FROM_VF_LMAC_MAP(map)	(map & 0xF)
+	u8			vf_lmac_map[MAX_LMAC];
+	struct delayed_work     dwork;
+	struct workqueue_struct *check_link;
+	u8			link[MAX_LMAC];
+	u8			duplex[MAX_LMAC];
+	u32			speed[MAX_LMAC];
+	u16			cpi_base[MAX_NUM_VFS_SUPPORTED];
+	u16			rss_ind_tbl_size;
+	bool			mbx_lock[MAX_NUM_VFS_SUPPORTED];
+
+	/* MSI-X */
+	bool			msix_enabled;
+	u8			num_vec;
+	struct msix_entry	msix_entries[NIC_PF_MSIX_VECTORS];
+	bool			irq_allocated[NIC_PF_MSIX_VECTORS];
+};
+
+/* Supported devices */
+static const struct pci_device_id nic_id_table[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_NIC_PF) },
+	{ 0, }  /* end of table */
+};
+
+MODULE_AUTHOR("Sunil Goutham");
+MODULE_DESCRIPTION("Cavium Thunder NIC Physical Function Driver");
+MODULE_VERSION(DRV_VERSION);
+MODULE_DEVICE_TABLE(pci, nic_id_table);
+
+/* The Cavium ThunderX network controller can *only* be found in SoCs
+ * containing the ThunderX ARM64 CPU implementation.  All accesses to the device
+ * registers on this platform are implicitly strongly ordered with respect
+ * to memory accesses. So writeq_relaxed() and readq_relaxed() are safe to use
+ * with no memory barriers in this driver.  The readq()/writeq() functions add
+ * explicit ordering operation which in this case are redundant, and only
+ * add overhead.
+ */
+
+/* Register read/write APIs */
+static void nic_reg_write(struct nicpf *nic, u64 offset, u64 val)
+{
+	writeq_relaxed(val, nic->reg_base + offset);
+}
+
+static u64 nic_reg_read(struct nicpf *nic, u64 offset)
+{
+	return readq_relaxed(nic->reg_base + offset);
+}
+
+/* PF -> VF mailbox communication APIs */
+static void nic_enable_mbx_intr(struct nicpf *nic)
+{
+	/* Enable mailbox interrupt for all 128 VFs */
+	nic_reg_write(nic, NIC_PF_MAILBOX_ENA_W1S, ~0ull);
+	nic_reg_write(nic, NIC_PF_MAILBOX_ENA_W1S + sizeof(u64), ~0ull);
+}
+
+static void nic_clear_mbx_intr(struct nicpf *nic, int vf, int mbx_reg)
+{
+	nic_reg_write(nic, NIC_PF_MAILBOX_INT + (mbx_reg << 3), BIT_ULL(vf));
+}
+
+static u64 nic_get_mbx_addr(int vf)
+{
+	return NIC_PF_VF_0_127_MAILBOX_0_1 + (vf << NIC_VF_NUM_SHIFT);
+}
+
+/* Send a mailbox message to VF
+ * @vf: vf to which this message to be sent
+ * @mbx: Message to be sent
+ */
+static void nic_send_msg_to_vf(struct nicpf *nic, int vf, union nic_mbx *mbx)
+{
+	void __iomem *mbx_addr = nic->reg_base + nic_get_mbx_addr(vf);
+	u64 *msg = (u64 *)mbx;
+
+	/* In first revision HW, mbox interrupt is triggerred
+	 * when PF writes to MBOX(1), in next revisions when
+	 * PF writes to MBOX(0)
+	 */
+	if (nic->rev_id == 0) {
+		/* see the comment for nic_reg_write()/nic_reg_read()
+		 * functions above
+		 */
+		writeq_relaxed(msg[0], mbx_addr);
+		writeq_relaxed(msg[1], mbx_addr + 8);
+	} else {
+		writeq_relaxed(msg[1], mbx_addr + 8);
+		writeq_relaxed(msg[0], mbx_addr);
+	}
+}
+
+/* Responds to VF's READY message with VF's
+ * ID, node, MAC address e.t.c
+ * @vf: VF which sent READY message
+ */
+static void nic_mbx_send_ready(struct nicpf *nic, int vf)
+{
+	union nic_mbx mbx = {};
+	int bgx_idx, lmac;
+	const char *mac;
+
+	mbx.nic_cfg.msg = NIC_MBOX_MSG_READY;
+	mbx.nic_cfg.vf_id = vf;
+
+	if (nic->flags & NIC_TNS_ENABLED)
+		mbx.nic_cfg.tns_mode = NIC_TNS_MODE;
+	else
+		mbx.nic_cfg.tns_mode = NIC_TNS_BYPASS_MODE;
+
+	if (vf < MAX_LMAC) {
+		bgx_idx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]);
+		lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]);
+
+		mac = bgx_get_lmac_mac(nic->node, bgx_idx, lmac);
+		if (mac)
+			ether_addr_copy((u8 *)&mbx.nic_cfg.mac_addr, mac);
+	}
+#ifdef VNIC_MULTI_QSET_SUPPORT
+	mbx.nic_cfg.sqs_mode = (vf >= nic->num_vf_en) ? true : false;
+#endif
+	mbx.nic_cfg.node_id = nic->node;
+
+	mbx.nic_cfg.loopback_supported = vf < MAX_LMAC;
+
+	nic_send_msg_to_vf(nic, vf, &mbx);
+}
+
+/* ACKs VF's mailbox message
+ * @vf: VF to which ACK to be sent
+ */
+static void nic_mbx_send_ack(struct nicpf *nic, int vf)
+{
+	union nic_mbx mbx = {};
+
+	mbx.msg.msg = NIC_MBOX_MSG_ACK;
+	nic_send_msg_to_vf(nic, vf, &mbx);
+}
+
+/* NACKs VF's mailbox message that PF is not able to
+ * complete the action
+ * @vf: VF to which ACK to be sent
+ */
+static void nic_mbx_send_nack(struct nicpf *nic, int vf)
+{
+	union nic_mbx mbx = {};
+
+	mbx.msg.msg = NIC_MBOX_MSG_NACK;
+	nic_send_msg_to_vf(nic, vf, &mbx);
+}
+
+/* Flush all in flight receive packets to memory and
+ * bring down an active RQ
+ */
+static int nic_rcv_queue_sw_sync(struct nicpf *nic)
+{
+	u16 timeout = ~0x00;
+
+	nic_reg_write(nic, NIC_PF_SW_SYNC_RX, 0x01);
+	/* Wait till sync cycle is finished */
+	while (timeout) {
+		if (nic_reg_read(nic, NIC_PF_SW_SYNC_RX_DONE) & 0x1)
+			break;
+		timeout--;
+	}
+	nic_reg_write(nic, NIC_PF_SW_SYNC_RX, 0x00);
+	if (!timeout) {
+		dev_err(&nic->pdev->dev, "Receive queue software sync failed");
+		return 1;
+	}
+	return 0;
+}
+
+/* Get BGX Rx/Tx stats and respond to VF's request */
+static void nic_get_bgx_stats(struct nicpf *nic, struct bgx_stats_msg *bgx)
+{
+	int bgx_idx, lmac;
+	union nic_mbx mbx = {};
+
+	bgx_idx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[bgx->vf_id]);
+	lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[bgx->vf_id]);
+
+	mbx.bgx_stats.msg = NIC_MBOX_MSG_BGX_STATS;
+	mbx.bgx_stats.vf_id = bgx->vf_id;
+	mbx.bgx_stats.rx = bgx->rx;
+	mbx.bgx_stats.idx = bgx->idx;
+	if (bgx->rx)
+		mbx.bgx_stats.stats = bgx_get_rx_stats(nic->node, bgx_idx,
+							    lmac, bgx->idx);
+	else
+		mbx.bgx_stats.stats = bgx_get_tx_stats(nic->node, bgx_idx,
+							    lmac, bgx->idx);
+	nic_send_msg_to_vf(nic, bgx->vf_id, &mbx);
+}
+
+/* Update hardware min/max frame size */
+static int nic_update_hw_frs(struct nicpf *nic, int new_frs, int vf)
+{
+	if ((new_frs > NIC_HW_MAX_FRS) || (new_frs < NIC_HW_MIN_FRS)) {
+		dev_err(&nic->pdev->dev,
+			"Invalid MTU setting from VF%d rejected, should be between %d and %d\n",
+			   vf, NIC_HW_MIN_FRS, NIC_HW_MAX_FRS);
+		return 1;
+	}
+	new_frs += ETH_HLEN;
+	if (new_frs <= nic->pkind.maxlen)
+		return 0;
+
+	nic->pkind.maxlen = new_frs;
+	nic_reg_write(nic, NIC_PF_PKIND_0_15_CFG, *(u64 *)&nic->pkind);
+	return 0;
+}
+
+/* Set minimum transmit packet size */
+static void nic_set_tx_pkt_pad(struct nicpf *nic, int size)
+{
+	int lmac;
+	u64 lmac_cfg;
+
+	/* Max value that can be set is 60 */
+	if (size > 60)
+		size = 60;
+
+	for (lmac = 0; lmac < (MAX_BGX_PER_CN88XX * MAX_LMAC_PER_BGX); lmac++) {
+		lmac_cfg = nic_reg_read(nic, NIC_PF_LMAC_0_7_CFG | (lmac << 3));
+		lmac_cfg &= ~(0xF << 2);
+		lmac_cfg |= ((size / 4) << 2);
+		nic_reg_write(nic, NIC_PF_LMAC_0_7_CFG | (lmac << 3), lmac_cfg);
+	}
+}
+
+/* Function to check number of LMACs present and set VF::LMAC mapping.
+ * Mapping will be used while initializing channels.
+ */
+static void nic_set_lmac_vf_mapping(struct nicpf *nic)
+{
+	unsigned bgx_map = bgx_get_map(nic->node);
+	int bgx, next_bgx_lmac = 0;
+	int lmac, lmac_cnt = 0;
+	u64 lmac_credit;
+
+	nic->num_vf_en = 0;
+	if (nic->flags & NIC_TNS_ENABLED) {
+		nic->num_vf_en = DEFAULT_NUM_VF_ENABLED;
+		return;
+	}
+
+	for (bgx = 0; bgx < NIC_MAX_BGX; bgx++) {
+		if (!(bgx_map & (1 << bgx)))
+			continue;
+		lmac_cnt = bgx_get_lmac_count(nic->node, bgx);
+		for (lmac = 0; lmac < lmac_cnt; lmac++)
+			nic->vf_lmac_map[next_bgx_lmac++] =
+						NIC_SET_VF_LMAC_MAP(bgx, lmac);
+		nic->num_vf_en += lmac_cnt;
+
+		/* Program LMAC credits */
+		lmac_credit = (1ull << 1); /* channel credit enable */
+		lmac_credit |= (0x1ff << 2); /* Max outstanding pkt count */
+		/* 48KB BGX Tx buffer size, each unit is of size 16bytes */
+		lmac_credit |= (((((48 * 1024) / lmac_cnt) -
+				NIC_HW_MAX_FRS) / 16) << 12);
+		lmac = bgx * MAX_LMAC_PER_BGX;
+		for (; lmac < lmac_cnt + (bgx * MAX_LMAC_PER_BGX); lmac++)
+			nic_reg_write(nic,
+				      NIC_PF_LMAC_0_7_CREDIT + (lmac * 8),
+				      lmac_credit);
+	}
+}
+
+#define TNS_PORT0_BLOCK 6
+#define TNS_PORT1_BLOCK 7
+#define BGX0_BLOCK 8
+#define BGX1_BLOCK 9
+
+static void nic_init_hw(struct nicpf *nic)
+{
+	int i;
+
+	/* Reset NIC, in case the driver is repeatedly inserted and removed */
+	nic_reg_write(nic, NIC_PF_SOFT_RESET, 1);
+
+	/* Enable NIC HW block */
+	nic_reg_write(nic, NIC_PF_CFG, 0x3);
+
+	/* Enable backpressure */
+	nic_reg_write(nic, NIC_PF_BP_CFG, (1ULL << 6) | 0x03);
+
+	if (nic->flags & NIC_TNS_ENABLED) {
+		nic_reg_write(nic, NIC_PF_INTF_0_1_SEND_CFG,
+			      (NIC_TNS_MODE << 7) | TNS_PORT0_BLOCK);
+		nic_reg_write(nic, NIC_PF_INTF_0_1_SEND_CFG | (1 << 8),
+			      (NIC_TNS_MODE << 7) | TNS_PORT1_BLOCK);
+		nic_reg_write(nic, NIC_PF_INTF_0_1_BP_CFG,
+			      (1ULL << 63) | TNS_PORT0_BLOCK);
+		nic_reg_write(nic, NIC_PF_INTF_0_1_BP_CFG + (1 << 8),
+			      (1ULL << 63) | TNS_PORT1_BLOCK);
+
+	} else {
+		/* Disable TNS mode on both interfaces */
+		nic_reg_write(nic, NIC_PF_INTF_0_1_SEND_CFG,
+			      (NIC_TNS_BYPASS_MODE << 7) | BGX0_BLOCK);
+		nic_reg_write(nic, NIC_PF_INTF_0_1_SEND_CFG | (1 << 8),
+			      (NIC_TNS_BYPASS_MODE << 7) | BGX1_BLOCK);
+		nic_reg_write(nic, NIC_PF_INTF_0_1_BP_CFG,
+			      (1ULL << 63) | BGX0_BLOCK);
+		nic_reg_write(nic, NIC_PF_INTF_0_1_BP_CFG + (1 << 8),
+			      (1ULL << 63) | BGX1_BLOCK);
+	}
+
+	/* PKIND configuration */
+	nic->pkind.minlen = 0;
+	nic->pkind.maxlen = NIC_HW_MAX_FRS + ETH_HLEN;
+	nic->pkind.lenerr_en = 1;
+	nic->pkind.rx_hdr = 0;
+	nic->pkind.hdr_sl = 0;
+
+	for (i = 0; i < NIC_MAX_PKIND; i++)
+		nic_reg_write(nic, NIC_PF_PKIND_0_15_CFG | (i << 3),
+			      *(u64 *)&nic->pkind);
+
+	nic_set_tx_pkt_pad(nic, NIC_HW_MIN_FRS);
+
+	/* Timer config */
+	nic_reg_write(nic, NIC_PF_INTR_TIMER_CFG, NICPF_CLK_PER_INT_TICK);
+
+	/* Enable VLAN ethertype matching and stripping */
+	nic_reg_write(nic, NIC_PF_RX_ETYPE_0_7,
+		      (2 << 19) | (ETYPE_ALG_VLAN_STRIP << 16) | ETH_P_8021Q);
+}
+
+/* Channel parse index configuration */
+static void nic_config_cpi(struct nicpf *nic, struct cpi_cfg_msg *cfg)
+{
+	u32 vnic, bgx, lmac, chan;
+	u32 padd, cpi_count = 0;
+	u64 cpi_base, cpi, rssi_base, rssi;
+	u8  qset, rq_idx = 0;
+
+	vnic = cfg->vf_id;
+	bgx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vnic]);
+	lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vnic]);
+
+	chan = (lmac * MAX_BGX_CHANS_PER_LMAC) + (bgx * NIC_CHANS_PER_INF);
+	cpi_base = (lmac * NIC_MAX_CPI_PER_LMAC) + (bgx * NIC_CPI_PER_BGX);
+	rssi_base = (lmac * nic->rss_ind_tbl_size) + (bgx * NIC_RSSI_PER_BGX);
+
+	/* Rx channel configuration */
+	nic_reg_write(nic, NIC_PF_CHAN_0_255_RX_BP_CFG | (chan << 3),
+		      (1ull << 63) | (vnic << 0));
+	nic_reg_write(nic, NIC_PF_CHAN_0_255_RX_CFG | (chan << 3),
+		      ((u64)cfg->cpi_alg << 62) | (cpi_base << 48));
+
+	if (cfg->cpi_alg == CPI_ALG_NONE)
+		cpi_count = 1;
+	else if (cfg->cpi_alg == CPI_ALG_VLAN) /* 3 bits of PCP */
+		cpi_count = 8;
+	else if (cfg->cpi_alg == CPI_ALG_VLAN16) /* 3 bits PCP + DEI */
+		cpi_count = 16;
+	else if (cfg->cpi_alg == CPI_ALG_DIFF) /* 6bits DSCP */
+		cpi_count = NIC_MAX_CPI_PER_LMAC;
+
+	/* RSS Qset, Qidx mapping */
+	qset = cfg->vf_id;
+	rssi = rssi_base;
+	for (; rssi < (rssi_base + cfg->rq_cnt); rssi++) {
+		nic_reg_write(nic, NIC_PF_RSSI_0_4097_RQ | (rssi << 3),
+			      (qset << 3) | rq_idx);
+		rq_idx++;
+	}
+
+	rssi = 0;
+	cpi = cpi_base;
+	for (; cpi < (cpi_base + cpi_count); cpi++) {
+		/* Determine port to channel adder */
+		if (cfg->cpi_alg != CPI_ALG_DIFF)
+			padd = cpi % cpi_count;
+		else
+			padd = cpi % 8; /* 3 bits CS out of 6bits DSCP */
+
+		/* Leave RSS_SIZE as '0' to disable RSS */
+		nic_reg_write(nic, NIC_PF_CPI_0_2047_CFG | (cpi << 3),
+			      (vnic << 24) | (padd << 16) | (rssi_base + rssi));
+
+		if ((rssi + 1) >= cfg->rq_cnt)
+			continue;
+
+		if (cfg->cpi_alg == CPI_ALG_VLAN)
+			rssi++;
+		else if (cfg->cpi_alg == CPI_ALG_VLAN16)
+			rssi = ((cpi - cpi_base) & 0xe) >> 1;
+		else if (cfg->cpi_alg == CPI_ALG_DIFF)
+			rssi = ((cpi - cpi_base) & 0x38) >> 3;
+	}
+	nic->cpi_base[cfg->vf_id] = cpi_base;
+}
+
+#ifdef VNIC_RSS_SUPPORT
+/* Responsds to VF with its RSS indirection table size */
+static void nic_send_rss_size(struct nicpf *nic, int vf)
+{
+	union nic_mbx mbx = {};
+	u64  *msg;
+
+	msg = (u64 *)&mbx;
+
+	mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE;
+	mbx.rss_size.ind_tbl_size = nic->rss_ind_tbl_size;
+	nic_send_msg_to_vf(nic, vf, &mbx);
+}
+
+/* Receive side scaling configuration
+ * configure:
+ * - RSS index
+ * - indir table i.e hash::RQ mapping
+ * - no of hash bits to consider
+ */
+static void nic_config_rss(struct nicpf *nic, struct rss_cfg_msg *cfg)
+{
+	u8  qset, idx = 0;
+	u64 cpi_cfg, cpi_base, rssi_base, rssi;
+
+	cpi_base = nic->cpi_base[cfg->vf_id];
+	cpi_cfg = nic_reg_read(nic, NIC_PF_CPI_0_2047_CFG | (cpi_base << 3));
+	rssi_base = (cpi_cfg & 0x0FFF) + cfg->tbl_offset;
+
+	rssi = rssi_base;
+	qset = cfg->vf_id;
+
+	for (; rssi < (rssi_base + cfg->tbl_len); rssi++) {
+#ifdef VNIC_MULTI_QSET_SUPPORT
+		u8 svf = cfg->ind_tbl[idx] >> 3;
+
+		if (svf)
+			qset = nic->vf_sqs[cfg->vf_id][svf - 1];
+		else
+			qset = cfg->vf_id;
+#endif
+		nic_reg_write(nic, NIC_PF_RSSI_0_4097_RQ | (rssi << 3),
+			      (qset << 3) | (cfg->ind_tbl[idx] & 0x7));
+		idx++;
+	}
+
+	cpi_cfg &= ~(0xFULL << 20);
+	cpi_cfg |= (cfg->hash_bits << 20);
+	nic_reg_write(nic, NIC_PF_CPI_0_2047_CFG | (cpi_base << 3), cpi_cfg);
+}
+#endif
+
+/* 4 level transmit side scheduler configutation
+ * for TNS bypass mode
+ *
+ * Sample configuration for SQ0
+ * VNIC0-SQ0 -> TL4(0)   -> TL3[0]   -> TL2[0]  -> TL1[0] -> BGX0
+ * VNIC1-SQ0 -> TL4(8)   -> TL3[2]   -> TL2[0]  -> TL1[0] -> BGX0
+ * VNIC2-SQ0 -> TL4(16)  -> TL3[4]   -> TL2[1]  -> TL1[0] -> BGX0
+ * VNIC3-SQ0 -> TL4(24)  -> TL3[6]   -> TL2[1]  -> TL1[0] -> BGX0
+ * VNIC4-SQ0 -> TL4(512) -> TL3[128] -> TL2[32] -> TL1[1] -> BGX1
+ * VNIC5-SQ0 -> TL4(520) -> TL3[130] -> TL2[32] -> TL1[1] -> BGX1
+ * VNIC6-SQ0 -> TL4(528) -> TL3[132] -> TL2[33] -> TL1[1] -> BGX1
+ * VNIC7-SQ0 -> TL4(536) -> TL3[134] -> TL2[33] -> TL1[1] -> BGX1
+ */
+static void nic_tx_channel_cfg(struct nicpf *nic, u8 vnic,
+			       struct sq_cfg_msg *sq)
+{
+	u32 bgx, lmac, chan;
+	u32 tl2, tl3, tl4;
+	u32 rr_quantum;
+	u8 sq_idx = sq->sq_num;
+	u8 pqs_vnic;
+
+#ifdef VNIC_MULTI_QSET_SUPPORT
+	if (sq->sqs_mode)
+		pqs_vnic = nic->pqs_vf[vnic];
+	else
+#endif
+		pqs_vnic = vnic;
+
+	bgx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[pqs_vnic]);
+	lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[pqs_vnic]);
+
+	/* 24 bytes for FCS, IPG and preamble */
+	rr_quantum = ((NIC_HW_MAX_FRS + 24) / 4);
+
+	tl4 = (lmac * NIC_TL4_PER_LMAC) + (bgx * NIC_TL4_PER_BGX);
+	tl4 += sq_idx;
+#ifdef VNIC_MULTI_QSET_SUPPORT
+	if (sq->sqs_mode)
+		tl4 += vnic * 8;
+#endif
+
+	tl3 = tl4 / (NIC_MAX_TL4 / NIC_MAX_TL3);
+	nic_reg_write(nic, NIC_PF_QSET_0_127_SQ_0_7_CFG2 |
+		      ((u64)vnic << NIC_QS_ID_SHIFT) |
+		      ((u32)sq_idx << NIC_Q_NUM_SHIFT), tl4);
+	nic_reg_write(nic, NIC_PF_TL4_0_1023_CFG | (tl4 << 3),
+		      ((u64)vnic << 27) | ((u32)sq_idx << 24) | rr_quantum);
+
+	nic_reg_write(nic, NIC_PF_TL3_0_255_CFG | (tl3 << 3), rr_quantum);
+	chan = (lmac * MAX_BGX_CHANS_PER_LMAC) + (bgx * NIC_CHANS_PER_INF);
+	nic_reg_write(nic, NIC_PF_TL3_0_255_CHAN | (tl3 << 3), chan);
+	/* Enable backpressure on the channel */
+	nic_reg_write(nic, NIC_PF_CHAN_0_255_TX_CFG | (chan << 3), 1);
+
+	tl2 = tl3 >> 2;
+	nic_reg_write(nic, NIC_PF_TL3A_0_63_CFG | (tl2 << 3), tl2);
+	nic_reg_write(nic, NIC_PF_TL2_0_63_CFG | (tl2 << 3), rr_quantum);
+	/* No priorities as of now */
+	nic_reg_write(nic, NIC_PF_TL2_0_63_PRI | (tl2 << 3), 0x00);
+}
+
+#ifdef VNIC_MULTI_QSET_SUPPORT
+/* Send primary nicvf pointer to secondary QS's VF */
+static void nic_send_pnicvf(struct nicpf *nic, int sqs)
+{
+	union nic_mbx mbx = {};
+
+	mbx.nicvf.msg = NIC_MBOX_MSG_PNICVF_PTR;
+	mbx.nicvf.nicvf = nic->nicvf[nic->pqs_vf[sqs]];
+	nic_send_msg_to_vf(nic, sqs, &mbx);
+}
+
+/* Send SQS's nicvf pointer to primary QS's VF */
+static void nic_send_snicvf(struct nicpf *nic, struct nicvf_ptr *nicvf)
+{
+	union nic_mbx mbx = {};
+	int sqs_id = nic->vf_sqs[nicvf->vf_id][nicvf->sqs_id];
+
+	mbx.nicvf.msg = NIC_MBOX_MSG_SNICVF_PTR;
+	mbx.nicvf.sqs_id = nicvf->sqs_id;
+	mbx.nicvf.nicvf = nic->nicvf[sqs_id];
+	nic_send_msg_to_vf(nic, nicvf->vf_id, &mbx);
+}
+
+/* Find next available Qset that can be assigned as a
+ * secondary Qset to a VF.
+ */
+static int nic_nxt_avail_sqs(struct nicpf *nic)
+{
+	int sqs;
+
+	for (sqs = 0; sqs < nic->num_sqs_en; sqs++) {
+		if (!nic->sqs_used[sqs])
+			nic->sqs_used[sqs] = true;
+		else
+			continue;
+		return sqs + nic->num_vf_en;
+	}
+	return -1;
+}
+
+/* Allocate additional Qsets for requested VF */
+static void nic_alloc_sqs(struct nicpf *nic, struct sqs_alloc *sqs)
+{
+	union nic_mbx mbx = {};
+	int idx, alloc_qs = 0;
+	int sqs_id;
+
+	if (!nic->num_sqs_en)
+		goto send_mbox;
+
+	for (idx = 0; idx < sqs->qs_count; idx++) {
+		sqs_id = nic_nxt_avail_sqs(nic);
+		if (sqs_id < 0)
+			break;
+		nic->vf_sqs[sqs->vf_id][idx] = sqs_id;
+		nic->pqs_vf[sqs_id] = sqs->vf_id;
+		alloc_qs++;
+	}
+
+send_mbox:
+	mbx.sqs_alloc.msg = NIC_MBOX_MSG_ALLOC_SQS;
+	mbx.sqs_alloc.vf_id = sqs->vf_id;
+	mbx.sqs_alloc.qs_count = alloc_qs;
+	nic_send_msg_to_vf(nic, sqs->vf_id, &mbx);
+}
+#endif
+
+static int nic_config_loopback(struct nicpf *nic, struct set_loopback *lbk)
+{
+	int bgx_idx, lmac_idx;
+
+	if (lbk->vf_id > MAX_LMAC)
+		return -1;
+
+	bgx_idx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[lbk->vf_id]);
+	lmac_idx = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[lbk->vf_id]);
+
+	bgx_lmac_internal_loopback(nic->node, bgx_idx, lmac_idx, lbk->enable);
+
+	return 0;
+}
+
+/* Interrupt handler to handle mailbox messages from VFs */
+static void nic_handle_mbx_intr(struct nicpf *nic, int vf)
+{
+	union nic_mbx mbx = {};
+	u64 *mbx_data;
+	u64 mbx_addr;
+	u64 reg_addr;
+	u64 cfg;
+	int bgx, lmac;
+	int i;
+	int ret = 0;
+
+	nic->mbx_lock[vf] = true;
+
+	mbx_addr = nic_get_mbx_addr(vf);
+	mbx_data = (u64 *)&mbx;
+
+	for (i = 0; i < NIC_PF_VF_MAILBOX_SIZE; i++) {
+		*mbx_data = nic_reg_read(nic, mbx_addr);
+		mbx_data++;
+		mbx_addr += sizeof(u64);
+	}
+
+	dev_dbg(&nic->pdev->dev, "%s: Mailbox msg %d from VF%d\n",
+		__func__, mbx.msg.msg, vf);
+	switch (mbx.msg.msg) {
+	case NIC_MBOX_MSG_READY:
+		nic_mbx_send_ready(nic, vf);
+		if (vf < MAX_LMAC) {
+			nic->link[vf] = 0;
+			nic->duplex[vf] = 0;
+			nic->speed[vf] = 0;
+		}
+		ret = 1;
+		break;
+	case NIC_MBOX_MSG_QS_CFG:
+		reg_addr = NIC_PF_QSET_0_127_CFG |
+			   (mbx.qs.num << NIC_QS_ID_SHIFT);
+		cfg = mbx.qs.cfg;
+#ifdef VNIC_MULTI_QSET_SUPPORT
+		/* Check if its a secondary Qset */
+		if (vf >= nic->num_vf_en) {
+			cfg = cfg & (~0x7FULL);
+			/* Assign this Qset to primary Qset's VF */
+			cfg |= nic->pqs_vf[vf];
+		}
+#endif
+		nic_reg_write(nic, reg_addr, cfg);
+		break;
+	case NIC_MBOX_MSG_RQ_CFG:
+		reg_addr = NIC_PF_QSET_0_127_RQ_0_7_CFG |
+			   (mbx.rq.qs_num << NIC_QS_ID_SHIFT) |
+			   (mbx.rq.rq_num << NIC_Q_NUM_SHIFT);
+		nic_reg_write(nic, reg_addr, mbx.rq.cfg);
+		break;
+	case NIC_MBOX_MSG_RQ_BP_CFG:
+		reg_addr = NIC_PF_QSET_0_127_RQ_0_7_BP_CFG |
+			   (mbx.rq.qs_num << NIC_QS_ID_SHIFT) |
+			   (mbx.rq.rq_num << NIC_Q_NUM_SHIFT);
+		nic_reg_write(nic, reg_addr, mbx.rq.cfg);
+		break;
+	case NIC_MBOX_MSG_RQ_SW_SYNC:
+		ret = nic_rcv_queue_sw_sync(nic);
+		break;
+	case NIC_MBOX_MSG_RQ_DROP_CFG:
+		reg_addr = NIC_PF_QSET_0_127_RQ_0_7_DROP_CFG |
+			   (mbx.rq.qs_num << NIC_QS_ID_SHIFT) |
+			   (mbx.rq.rq_num << NIC_Q_NUM_SHIFT);
+		nic_reg_write(nic, reg_addr, mbx.rq.cfg);
+		break;
+	case NIC_MBOX_MSG_SQ_CFG:
+		reg_addr = NIC_PF_QSET_0_127_SQ_0_7_CFG |
+			   (mbx.sq.qs_num << NIC_QS_ID_SHIFT) |
+			   (mbx.sq.sq_num << NIC_Q_NUM_SHIFT);
+		nic_reg_write(nic, reg_addr, mbx.sq.cfg);
+		nic_tx_channel_cfg(nic, mbx.qs.num, &mbx.sq);
+		break;
+	case NIC_MBOX_MSG_SET_MAC:
+#ifdef VNIC_MULTI_QSET_SUPPORT
+		if (vf >= nic->num_vf_en)
+			break;
+#endif
+		lmac = mbx.mac.vf_id;
+		bgx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[lmac]);
+		lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[lmac]);
+		bgx_set_lmac_mac(nic->node, bgx, lmac, mbx.mac.mac_addr);
+		break;
+	case NIC_MBOX_MSG_SET_MAX_FRS:
+		ret = nic_update_hw_frs(nic, mbx.frs.max_frs,
+					mbx.frs.vf_id);
+		break;
+	case NIC_MBOX_MSG_CPI_CFG:
+		nic_config_cpi(nic, &mbx.cpi_cfg);
+		break;
+#ifdef VNIC_RSS_SUPPORT
+	case NIC_MBOX_MSG_RSS_SIZE:
+		nic_send_rss_size(nic, vf);
+		goto unlock;
+	case NIC_MBOX_MSG_RSS_CFG:
+	case NIC_MBOX_MSG_RSS_CFG_CONT:
+		nic_config_rss(nic, &mbx.rss_cfg);
+		break;
+#endif
+	case NIC_MBOX_MSG_CFG_DONE:
+		/* Last message of VF config msg sequence */
+		nic->vf_enabled[vf] = true;
+		goto unlock;
+	case NIC_MBOX_MSG_SHUTDOWN:
+		/* First msg in VF teardown sequence */
+		nic->vf_enabled[vf] = false;
+#ifdef VNIC_MULTI_QSET_SUPPORT
+		if (vf >= nic->num_vf_en)
+			nic->sqs_used[vf - nic->num_vf_en] = false;
+		nic->pqs_vf[vf] = 0;
+#endif
+		break;
+#ifdef VNIC_MULTI_QSET_SUPPORT
+	case NIC_MBOX_MSG_ALLOC_SQS:
+		nic_alloc_sqs(nic, &mbx.sqs_alloc);
+		goto unlock;
+	case NIC_MBOX_MSG_NICVF_PTR:
+		nic->nicvf[vf] = mbx.nicvf.nicvf;
+		break;
+	case NIC_MBOX_MSG_PNICVF_PTR:
+		nic_send_pnicvf(nic, vf);
+		goto unlock;
+	case NIC_MBOX_MSG_SNICVF_PTR:
+		nic_send_snicvf(nic, &mbx.nicvf);
+		goto unlock;
+#endif
+	case NIC_MBOX_MSG_BGX_STATS:
+		nic_get_bgx_stats(nic, &mbx.bgx_stats);
+		goto unlock;
+	case NIC_MBOX_MSG_LOOPBACK:
+		ret = nic_config_loopback(nic, &mbx.lbk);
+		break;
+	default:
+		dev_err(&nic->pdev->dev,
+			"Invalid msg from VF%d, msg 0x%x\n", vf, mbx.msg.msg);
+		break;
+	}
+
+	if (!ret)
+		nic_mbx_send_ack(nic, vf);
+	else if (mbx.msg.msg != NIC_MBOX_MSG_READY)
+		nic_mbx_send_nack(nic, vf);
+unlock:
+	nic->mbx_lock[vf] = false;
+}
+
+static void nic_mbx_intr_handler (struct nicpf *nic, int mbx)
+{
+	u64 intr;
+	u8  vf, vf_per_mbx_reg = 64;
+
+	intr = nic_reg_read(nic, NIC_PF_MAILBOX_INT + (mbx << 3));
+	dev_dbg(&nic->pdev->dev, "PF interrupt Mbox%d 0x%llx\n", mbx, intr);
+	for (vf = 0; vf < vf_per_mbx_reg; vf++) {
+		if (intr & (1ULL << vf)) {
+			dev_dbg(&nic->pdev->dev, "Intr from VF %d\n",
+				vf + (mbx * vf_per_mbx_reg));
+
+			nic_handle_mbx_intr(nic, vf + (mbx * vf_per_mbx_reg));
+			nic_clear_mbx_intr(nic, vf, mbx);
+		}
+	}
+}
+
+static irqreturn_t nic_mbx0_intr_handler (int irq, void *nic_irq)
+{
+	struct nicpf *nic = (struct nicpf *)nic_irq;
+
+	nic_mbx_intr_handler(nic, 0);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t nic_mbx1_intr_handler (int irq, void *nic_irq)
+{
+	struct nicpf *nic = (struct nicpf *)nic_irq;
+
+	nic_mbx_intr_handler(nic, 1);
+
+	return IRQ_HANDLED;
+}
+
+static int nic_enable_msix(struct nicpf *nic)
+{
+	int i, ret;
+
+	nic->num_vec = NIC_PF_MSIX_VECTORS;
+
+	for (i = 0; i < nic->num_vec; i++)
+		nic->msix_entries[i].entry = i;
+
+	ret = pci_enable_msix(nic->pdev, nic->msix_entries, nic->num_vec);
+	if (ret) {
+		dev_err(&nic->pdev->dev,
+			"Request for #%d msix vectors failed\n",
+			   nic->num_vec);
+		return ret;
+	}
+
+	nic->msix_enabled = 1;
+	return 0;
+}
+
+static void nic_disable_msix(struct nicpf *nic)
+{
+	if (nic->msix_enabled) {
+		pci_disable_msix(nic->pdev);
+		nic->msix_enabled = 0;
+		nic->num_vec = 0;
+	}
+}
+
+static void nic_free_all_interrupts(struct nicpf *nic)
+{
+	int irq;
+
+	for (irq = 0; irq < nic->num_vec; irq++) {
+		if (nic->irq_allocated[irq])
+			free_irq(nic->msix_entries[irq].vector, nic);
+		nic->irq_allocated[irq] = false;
+	}
+}
+
+static int nic_register_interrupts(struct nicpf *nic)
+{
+	int ret;
+
+	/* Enable MSI-X */
+	ret = nic_enable_msix(nic);
+	if (ret)
+		return ret;
+
+	/* Register mailbox interrupt handlers */
+	ret = request_irq(nic->msix_entries[NIC_PF_INTR_ID_MBOX0].vector,
+			  nic_mbx0_intr_handler, 0, "NIC Mbox0", nic);
+	if (ret)
+		goto fail;
+
+	nic->irq_allocated[NIC_PF_INTR_ID_MBOX0] = true;
+
+	ret = request_irq(nic->msix_entries[NIC_PF_INTR_ID_MBOX1].vector,
+			  nic_mbx1_intr_handler, 0, "NIC Mbox1", nic);
+	if (ret)
+		goto fail;
+
+	nic->irq_allocated[NIC_PF_INTR_ID_MBOX1] = true;
+
+	/* Enable mailbox interrupt */
+	nic_enable_mbx_intr(nic);
+	return 0;
+
+fail:
+	dev_err(&nic->pdev->dev, "Request irq failed\n");
+	nic_free_all_interrupts(nic);
+	return ret;
+}
+
+static void nic_unregister_interrupts(struct nicpf *nic)
+{
+	nic_free_all_interrupts(nic);
+	nic_disable_msix(nic);
+}
+
+#ifdef VNIC_MULTI_QSET_SUPPORT
+int nic_num_sqs_en(struct nicpf *nic, int vf_en)
+{
+	int pos = 0, sqs_per_vf = MAX_SQS_PER_VF_SINGLE_NODE;
+	u16 total_vf;
+
+	/* Check if its a multi-node environment */
+	if (nr_node_ids > 1)
+		sqs_per_vf = MAX_SQS_PER_VF;
+
+	pos = pci_find_ext_capability(nic->pdev, PCI_EXT_CAP_ID_SRIOV);
+	pci_read_config_word(nic->pdev, (pos + PCI_SRIOV_TOTAL_VF), &total_vf);
+	return min(total_vf - vf_en, vf_en * sqs_per_vf);
+}
+#endif
+
+int nic_sriov_configure(struct pci_dev *pdev, int num_vfs_requested)
+{
+	struct nicpf *nic = pci_get_drvdata(pdev);
+	int vf_en;
+	int err;
+
+	if (nic->num_vf_en == num_vfs_requested)
+		return num_vfs_requested;
+
+	if (nic->flags & NIC_SRIOV_ENABLED) {
+		nic->flags &= ~NIC_SRIOV_ENABLED;
+	}
+
+	nic->num_vf_en = 0;
+	if (num_vfs_requested > MAX_NUM_VFS_SUPPORTED ||
+	    num_vfs_requested < 0)
+		return -EINVAL;
+
+	if (num_vfs_requested) {
+		vf_en = num_vfs_requested;
+#ifdef VNIC_MULTI_QSET_SUPPORT
+		nic->num_sqs_en = nic_num_sqs_en(nic, num_vfs_requested);
+		vf_en += nic->num_sqs_en;
+#endif
+		nic->num_vf_en = num_vfs_requested;
+		nic->flags |= NIC_SRIOV_ENABLED;
+	}
+
+	return num_vfs_requested;
+}
+
+static int nic_sriov_init(struct pci_dev *pdev, struct nicpf *nic)
+{
+	int pos = 0;
+	int vf_en;
+	int err;
+	u16 total_vf_cnt;
+
+	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
+	if (!pos) {
+		dev_err(&pdev->dev, "SRIOV capability is not found in PCIe config space\n");
+		return -ENODEV;
+	}
+
+	pci_read_config_word(pdev, (pos + PCI_SRIOV_TOTAL_VF), &total_vf_cnt);
+	if (total_vf_cnt < nic->num_vf_en)
+		nic->num_vf_en = total_vf_cnt;
+
+	if (!total_vf_cnt)
+		return 0;
+
+	vf_en = nic->num_vf_en;
+#ifdef VNIC_MULTI_QSET_SUPPORT
+	nic->num_sqs_en = nic_num_sqs_en(nic, nic->num_vf_en);
+	vf_en += nic->num_sqs_en;
+#endif
+
+	if (err) {
+		dev_err(&pdev->dev, "SRIOV enable failed, num VF is %d\n",
+			vf_en);
+		nic->num_vf_en = 0;
+		return err;
+	}
+
+	dev_info(&pdev->dev, "SRIOV enabled, number of VF available %d\n",
+		 vf_en);
+
+	nic->flags |= NIC_SRIOV_ENABLED;
+	return 0;
+}
+
+/* Poll for BGX LMAC link status and update corresponding VF
+ * if there is a change, valid only if internal L2 switch
+ * is not present otherwise VF link is always treated as up
+ */
+static void nic_poll_for_link(struct work_struct *work)
+{
+	union nic_mbx mbx = {};
+	struct nicpf *nic;
+	struct bgx_link_status link;
+	u8 vf, bgx, lmac;
+
+	nic = container_of(work, struct nicpf, dwork.work);
+
+	mbx.link_status.msg = NIC_MBOX_MSG_BGX_LINK_CHANGE;
+
+	for (vf = 0; vf < nic->num_vf_en; vf++) {
+		/* Poll only if VF is UP */
+		if (!nic->vf_enabled[vf])
+			continue;
+
+		/* Get BGX, LMAC indices for the VF */
+		bgx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]);
+		lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]);
+		/* Get interface link status */
+		bgx_get_lmac_link_state(nic->node, bgx, lmac, &link);
+
+		/* Inform VF only if link status changed */
+		if (nic->link[vf] == link.link_up)
+			continue;
+
+		if (!nic->mbx_lock[vf]) {
+			nic->link[vf] = link.link_up;
+			nic->duplex[vf] = link.duplex;
+			nic->speed[vf] = link.speed;
+
+			/* Send a mbox message to VF with current link status */
+			mbx.link_status.link_up = link.link_up;
+			mbx.link_status.duplex = link.duplex;
+			mbx.link_status.speed = link.speed;
+			nic_send_msg_to_vf(nic, vf, &mbx);
+		}
+	}
+	queue_delayed_work(nic->check_link, &nic->dwork, HZ * 2);
+}
+
+static int nic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+	struct device *dev = &pdev->dev;
+	struct nicpf *nic;
+	int    err;
+
+	BUILD_BUG_ON(sizeof(union nic_mbx) > 16);
+
+	nic = devm_kzalloc(dev, sizeof(*nic), GFP_KERNEL);
+	if (!nic)
+		return -ENOMEM;
+
+	pci_set_drvdata(pdev, nic);
+
+	nic->pdev = pdev;
+
+	err = pci_enable_device(pdev);
+	if (err) {
+		dev_err(dev, "Failed to enable PCI device\n");
+		pci_set_drvdata(pdev, NULL);
+		return err;
+	}
+
+	err = pci_request_regions(pdev, DRV_NAME);
+	if (err) {
+		dev_err(dev, "PCI request regions failed 0x%x\n", err);
+		goto err_disable_device;
+	}
+
+	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
+	if (err) {
+		dev_err(dev, "Unable to get usable DMA configuration\n");
+		goto err_release_regions;
+	}
+
+	err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
+	if (err) {
+		dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n");
+		goto err_release_regions;
+	}
+
+	/* MAP PF's configuration registers */
+	nic->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0);
+	if (!nic->reg_base) {
+		dev_err(dev, "Cannot map config register space, aborting\n");
+		err = -ENOMEM;
+		goto err_release_regions;
+	}
+
+	pci_read_config_byte(pdev, PCI_REVISION_ID, &nic->rev_id);
+
+	nic->node = nic_get_node_id(pdev);
+
+	/* By default set NIC in TNS bypass mode */
+	nic->flags &= ~NIC_TNS_ENABLED;
+	nic_set_lmac_vf_mapping(nic);
+
+	/* Initialize hardware */
+	nic_init_hw(nic);
+
+	/* Set RSS TBL size for each VF */
+	nic->rss_ind_tbl_size = NIC_MAX_RSS_IDR_TBL_SIZE;
+
+	/* Register interrupts */
+	err = nic_register_interrupts(nic);
+	if (err)
+		goto err_release_regions;
+
+	/* Configure SRIOV */
+	err = nic_sriov_init(pdev, nic);
+	if (err)
+		goto err_unregister_interrupts;
+
+	if (nic->flags & NIC_TNS_ENABLED)
+		return 0;
+
+	/* Register a physical link status poll fn() */
+	nic->check_link = alloc_workqueue("check_link_status",
+					  WQ_UNBOUND | WQ_MEM_RECLAIM, 1);
+	if (!nic->check_link) {
+		err = -ENOMEM;
+		goto err_disable_sriov;
+	}
+
+	INIT_DELAYED_WORK(&nic->dwork, nic_poll_for_link);
+	queue_delayed_work(nic->check_link, &nic->dwork, 0);
+
+	return 0;
+
+err_disable_sriov:
+err_unregister_interrupts:
+	nic_unregister_interrupts(nic);
+err_release_regions:
+	pci_release_regions(pdev);
+err_disable_device:
+	pci_disable_device(pdev);
+	pci_set_drvdata(pdev, NULL);
+	return err;
+}
+
+static void nic_remove(struct pci_dev *pdev)
+{
+	struct nicpf *nic = pci_get_drvdata(pdev);
+
+	if (nic->check_link) {
+		/* Destroy work Queue */
+		cancel_delayed_work(&nic->dwork);
+		flush_workqueue(nic->check_link);
+		destroy_workqueue(nic->check_link);
+	}
+
+	nic_unregister_interrupts(nic);
+	pci_release_regions(pdev);
+	pci_disable_device(pdev);
+	pci_set_drvdata(pdev, NULL);
+}
+
+static struct pci_driver nic_driver = {
+	.name = DRV_NAME,
+	.id_table = nic_id_table,
+	.probe = nic_probe,
+	.remove = nic_remove,
+	.sriov_configure = nic_sriov_configure,
+};
+
+static int __init nic_init_module(void)
+{
+	pr_info("%s, ver %s\n", DRV_NAME, DRV_VERSION);
+
+	return pci_register_driver(&nic_driver);
+}
+
+static void __exit nic_cleanup_module(void)
+{
+	pci_unregister_driver(&nic_driver);
+}
+
+module_init(nic_init_module);
+module_exit(nic_cleanup_module);
diff --git a/sys/dev/vnic/nic_reg.h b/sys/dev/vnic/nic_reg.h
new file mode 100644
index 000000000000..b66f7e38633f
--- /dev/null
+++ b/sys/dev/vnic/nic_reg.h
@@ -0,0 +1,234 @@
+/*
+ * Copyright (C) 2015 Cavium Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ *
+ */
+
+#ifndef NIC_REG_H
+#define NIC_REG_H
+
+#define   NIC_PF_REG_COUNT			29573
+#define   NIC_VF_REG_COUNT			249
+
+/* Physical function register offsets */
+#define   NIC_PF_CFG				(0x0000)
+#define   NIC_PF_STATUS				(0x0010)
+#define   NIC_PF_INTR_TIMER_CFG			(0x0030)
+#define   NIC_PF_BIST_STATUS			(0x0040)
+#define   NIC_PF_SOFT_RESET			(0x0050)
+#define   NIC_PF_TCP_TIMER			(0x0060)
+#define   NIC_PF_BP_CFG				(0x0080)
+#define   NIC_PF_RRM_CFG			(0x0088)
+#define   NIC_PF_CQM_CF				(0x00A0)
+#define   NIC_PF_CNM_CF				(0x00A8)
+#define   NIC_PF_CNM_STATUS			(0x00B0)
+#define   NIC_PF_CQ_AVG_CFG			(0x00C0)
+#define   NIC_PF_RRM_AVG_CFG			(0x00C8)
+#define   NIC_PF_INTF_0_1_SEND_CFG		(0x0200)
+#define   NIC_PF_INTF_0_1_BP_CFG		(0x0208)
+#define   NIC_PF_INTF_0_1_BP_DIS_0_1		(0x0210)
+#define   NIC_PF_INTF_0_1_BP_SW_0_1		(0x0220)
+#define   NIC_PF_RBDR_BP_STATE_0_3		(0x0240)
+#define   NIC_PF_MAILBOX_INT			(0x0410)
+#define   NIC_PF_MAILBOX_INT_W1S		(0x0430)
+#define   NIC_PF_MAILBOX_ENA_W1C		(0x0450)
+#define   NIC_PF_MAILBOX_ENA_W1S		(0x0470)
+#define   NIC_PF_RX_ETYPE_0_7			(0x0500)
+#define   NIC_PF_PKIND_0_15_CFG			(0x0600)
+#define   NIC_PF_ECC0_FLIP0			(0x1000)
+#define   NIC_PF_ECC1_FLIP0			(0x1008)
+#define   NIC_PF_ECC2_FLIP0			(0x1010)
+#define   NIC_PF_ECC3_FLIP0			(0x1018)
+#define   NIC_PF_ECC0_FLIP1			(0x1080)
+#define   NIC_PF_ECC1_FLIP1			(0x1088)
+#define   NIC_PF_ECC2_FLIP1			(0x1090)
+#define   NIC_PF_ECC3_FLIP1			(0x1098)
+#define   NIC_PF_ECC0_CDIS			(0x1100)
+#define   NIC_PF_ECC1_CDIS			(0x1108)
+#define   NIC_PF_ECC2_CDIS			(0x1110)
+#define   NIC_PF_ECC3_CDIS			(0x1118)
+#define   NIC_PF_BIST0_STATUS			(0x1280)
+#define   NIC_PF_BIST1_STATUS			(0x1288)
+#define   NIC_PF_BIST2_STATUS			(0x1290)
+#define   NIC_PF_BIST3_STATUS			(0x1298)
+#define   NIC_PF_ECC0_SBE_INT			(0x2000)
+#define   NIC_PF_ECC0_SBE_INT_W1S		(0x2008)
+#define   NIC_PF_ECC0_SBE_ENA_W1C		(0x2010)
+#define   NIC_PF_ECC0_SBE_ENA_W1S		(0x2018)
+#define   NIC_PF_ECC0_DBE_INT			(0x2100)
+#define   NIC_PF_ECC0_DBE_INT_W1S		(0x2108)
+#define   NIC_PF_ECC0_DBE_ENA_W1C		(0x2110)
+#define   NIC_PF_ECC0_DBE_ENA_W1S		(0x2118)
+#define   NIC_PF_ECC1_SBE_INT			(0x2200)
+#define   NIC_PF_ECC1_SBE_INT_W1S		(0x2208)
+#define   NIC_PF_ECC1_SBE_ENA_W1C		(0x2210)
+#define   NIC_PF_ECC1_SBE_ENA_W1S		(0x2218)
+#define   NIC_PF_ECC1_DBE_INT			(0x2300)
+#define   NIC_PF_ECC1_DBE_INT_W1S		(0x2308)
+#define   NIC_PF_ECC1_DBE_ENA_W1C		(0x2310)
+#define   NIC_PF_ECC1_DBE_ENA_W1S		(0x2318)
+#define   NIC_PF_ECC2_SBE_INT			(0x2400)
+#define   NIC_PF_ECC2_SBE_INT_W1S		(0x2408)
+#define   NIC_PF_ECC2_SBE_ENA_W1C		(0x2410)
+#define   NIC_PF_ECC2_SBE_ENA_W1S		(0x2418)
+#define   NIC_PF_ECC2_DBE_INT			(0x2500)
+#define   NIC_PF_ECC2_DBE_INT_W1S		(0x2508)
+#define   NIC_PF_ECC2_DBE_ENA_W1C		(0x2510)
+#define   NIC_PF_ECC2_DBE_ENA_W1S		(0x2518)
+#define   NIC_PF_ECC3_SBE_INT			(0x2600)
+#define   NIC_PF_ECC3_SBE_INT_W1S		(0x2608)
+#define   NIC_PF_ECC3_SBE_ENA_W1C		(0x2610)
+#define   NIC_PF_ECC3_SBE_ENA_W1S		(0x2618)
+#define   NIC_PF_ECC3_DBE_INT			(0x2700)
+#define   NIC_PF_ECC3_DBE_INT_W1S		(0x2708)
+#define   NIC_PF_ECC3_DBE_ENA_W1C		(0x2710)
+#define   NIC_PF_ECC3_DBE_ENA_W1S		(0x2718)
+#define   NIC_PF_CPI_0_2047_CFG			(0x200000)
+#define   NIC_PF_RSSI_0_4097_RQ			(0x220000)
+#define   NIC_PF_LMAC_0_7_CFG			(0x240000)
+#define   NIC_PF_LMAC_0_7_SW_XOFF		(0x242000)
+#define   NIC_PF_LMAC_0_7_CREDIT		(0x244000)
+#define   NIC_PF_CHAN_0_255_TX_CFG		(0x400000)
+#define   NIC_PF_CHAN_0_255_RX_CFG		(0x420000)
+#define   NIC_PF_CHAN_0_255_SW_XOFF		(0x440000)
+#define   NIC_PF_CHAN_0_255_CREDIT		(0x460000)
+#define   NIC_PF_CHAN_0_255_RX_BP_CFG		(0x480000)
+#define   NIC_PF_SW_SYNC_RX			(0x490000)
+#define   NIC_PF_SW_SYNC_RX_DONE		(0x490008)
+#define   NIC_PF_TL2_0_63_CFG			(0x500000)
+#define   NIC_PF_TL2_0_63_PRI			(0x520000)
+#define   NIC_PF_TL2_0_63_SH_STATUS		(0x580000)
+#define   NIC_PF_TL3A_0_63_CFG			(0x5F0000)
+#define   NIC_PF_TL3_0_255_CFG			(0x600000)
+#define   NIC_PF_TL3_0_255_CHAN			(0x620000)
+#define   NIC_PF_TL3_0_255_PIR			(0x640000)
+#define   NIC_PF_TL3_0_255_SW_XOFF		(0x660000)
+#define   NIC_PF_TL3_0_255_CNM_RATE		(0x680000)
+#define   NIC_PF_TL3_0_255_SH_STATUS		(0x6A0000)
+#define   NIC_PF_TL4A_0_255_CFG			(0x6F0000)
+#define   NIC_PF_TL4_0_1023_CFG			(0x800000)
+#define   NIC_PF_TL4_0_1023_SW_XOFF		(0x820000)
+#define   NIC_PF_TL4_0_1023_SH_STATUS		(0x840000)
+#define   NIC_PF_TL4A_0_1023_CNM_RATE		(0x880000)
+#define   NIC_PF_TL4A_0_1023_CNM_STATUS		(0x8A0000)
+#define   NIC_PF_VF_0_127_MAILBOX_0_1		(0x20002030)
+#define   NIC_PF_VNIC_0_127_TX_STAT_0_4		(0x20004000)
+#define   NIC_PF_VNIC_0_127_RX_STAT_0_13	(0x20004100)
+#define   NIC_PF_QSET_0_127_LOCK_0_15		(0x20006000)
+#define   NIC_PF_QSET_0_127_CFG			(0x20010000)
+#define   NIC_PF_QSET_0_127_RQ_0_7_CFG		(0x20010400)
+#define   NIC_PF_QSET_0_127_RQ_0_7_DROP_CFG	(0x20010420)
+#define   NIC_PF_QSET_0_127_RQ_0_7_BP_CFG	(0x20010500)
+#define   NIC_PF_QSET_0_127_RQ_0_7_STAT_0_1	(0x20010600)
+#define   NIC_PF_QSET_0_127_SQ_0_7_CFG		(0x20010C00)
+#define   NIC_PF_QSET_0_127_SQ_0_7_CFG2		(0x20010C08)
+#define   NIC_PF_QSET_0_127_SQ_0_7_STAT_0_1	(0x20010D00)
+
+#define   NIC_PF_MSIX_VEC_0_18_ADDR		(0x000000)
+#define   NIC_PF_MSIX_VEC_0_CTL			(0x000008)
+#define   NIC_PF_MSIX_PBA_0			(0x0F0000)
+
+/* Virtual function register offsets */
+#define   NIC_VNIC_CFG				(0x000020)
+#define   NIC_VF_PF_MAILBOX_0_1			(0x000130)
+#define   NIC_VF_INT				(0x000200)
+#define   NIC_VF_INT_W1S			(0x000220)
+#define   NIC_VF_ENA_W1C			(0x000240)
+#define   NIC_VF_ENA_W1S			(0x000260)
+
+#define   NIC_VNIC_RSS_CFG			(0x0020E0)
+#define   NIC_VNIC_RSS_KEY_0_4			(0x002200)
+#define   NIC_VNIC_TX_STAT_0_4			(0x004000)
+#define   NIC_VNIC_RX_STAT_0_13			(0x004100)
+#define   NIC_QSET_RQ_GEN_CFG			(0x010010)
+
+#define   NIC_QSET_CQ_0_7_CFG			(0x010400)
+#define   NIC_QSET_CQ_0_7_CFG2			(0x010408)
+#define   NIC_QSET_CQ_0_7_THRESH		(0x010410)
+#define   NIC_QSET_CQ_0_7_BASE			(0x010420)
+#define   NIC_QSET_CQ_0_7_HEAD			(0x010428)
+#define   NIC_QSET_CQ_0_7_TAIL			(0x010430)
+#define   NIC_QSET_CQ_0_7_DOOR			(0x010438)
+#define   NIC_QSET_CQ_0_7_STATUS		(0x010440)
+#define   NIC_QSET_CQ_0_7_STATUS2		(0x010448)
+#define   NIC_QSET_CQ_0_7_DEBUG			(0x010450)
+
+#define   NIC_QSET_RQ_0_7_CFG			(0x010600)
+#define   NIC_QSET_RQ_0_7_STAT_0_1		(0x010700)
+
+#define   NIC_QSET_SQ_0_7_CFG			(0x010800)
+#define   NIC_QSET_SQ_0_7_THRESH		(0x010810)
+#define   NIC_QSET_SQ_0_7_BASE			(0x010820)
+#define   NIC_QSET_SQ_0_7_HEAD			(0x010828)
+#define   NIC_QSET_SQ_0_7_TAIL			(0x010830)
+#define   NIC_QSET_SQ_0_7_DOOR			(0x010838)
+#define   NIC_QSET_SQ_0_7_STATUS		(0x010840)
+#define   NIC_QSET_SQ_0_7_DEBUG			(0x010848)
+#define   NIC_QSET_SQ_0_7_CNM_CHG		(0x010860)
+#define   NIC_QSET_SQ_0_7_STAT_0_1		(0x010900)
+
+#define   NIC_QSET_RBDR_0_1_CFG			(0x010C00)
+#define   NIC_QSET_RBDR_0_1_THRESH		(0x010C10)
+#define   NIC_QSET_RBDR_0_1_BASE		(0x010C20)
+#define   NIC_QSET_RBDR_0_1_HEAD		(0x010C28)
+#define   NIC_QSET_RBDR_0_1_TAIL		(0x010C30)
+#define   NIC_QSET_RBDR_0_1_DOOR		(0x010C38)
+#define   NIC_QSET_RBDR_0_1_STATUS0		(0x010C40)
+#define   NIC_QSET_RBDR_0_1_STATUS1		(0x010C48)
+#define   NIC_QSET_RBDR_0_1_PREFETCH_STATUS	(0x010C50)
+
+#define   NIC_VF_MSIX_VECTOR_0_19_ADDR		(0x000000)
+#define   NIC_VF_MSIX_VECTOR_0_19_CTL		(0x000008)
+#define   NIC_VF_MSIX_PBA			(0x0F0000)
+
+/* Offsets within registers */
+#define   NIC_MSIX_VEC_SHIFT			4
+#define   NIC_Q_NUM_SHIFT			18
+#define   NIC_QS_ID_SHIFT			21
+#define   NIC_VF_NUM_SHIFT			21
+
+/* Port kind configuration register */
+struct pkind_cfg {
+#if defined(__BIG_ENDIAN_BITFIELD)
+	u64 reserved_42_63:22;
+	u64 hdr_sl:5;	/* Header skip length */
+	u64 rx_hdr:3;	/* TNS Receive header present */
+	u64 lenerr_en:1;/* L2 length error check enable */
+	u64 reserved_32_32:1;
+	u64 maxlen:16;	/* Max frame size */
+	u64 minlen:16;	/* Min frame size */
+#elif defined(__LITTLE_ENDIAN_BITFIELD)
+	u64 minlen:16;
+	u64 maxlen:16;
+	u64 reserved_32_32:1;
+	u64 lenerr_en:1;
+	u64 rx_hdr:3;
+	u64 hdr_sl:5;
+	u64 reserved_42_63:22;
+#endif
+};
+
+#endif /* NIC_REG_H */
diff --git a/sys/dev/vnic/nicvf_main.c b/sys/dev/vnic/nicvf_main.c
new file mode 100644
index 000000000000..9e9e0d53a8f0
--- /dev/null
+++ b/sys/dev/vnic/nicvf_main.c
@@ -0,0 +1,1667 @@
+/*
+ * Copyright (C) 2015 Cavium Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/interrupt.h>
+#include <linux/pci.h>
+#include <linux/netdevice.h>
+#include <linux/if_vlan.h>
+#include <linux/etherdevice.h>
+#include <linux/ethtool.h>
+#include <linux/log2.h>
+#include <linux/prefetch.h>
+#include <linux/irq.h>
+
+#include "nic_reg.h"
+#include "nic.h"
+#include "nicvf_queues.h"
+#include "thunder_bgx.h"
+
+#define DRV_NAME	"thunder-nicvf"
+#define DRV_VERSION	"1.0"
+
+/* Supported devices */
+static const struct pci_device_id nicvf_id_table[] = {
+	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM,
+			 PCI_DEVICE_ID_THUNDER_NIC_VF,
+			 PCI_VENDOR_ID_CAVIUM, 0xA11E) },
+	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM,
+			 PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF,
+			 PCI_VENDOR_ID_CAVIUM, 0xA11E) },
+	{ 0, }  /* end of table */
+};
+
+MODULE_AUTHOR("Sunil Goutham");
+MODULE_DESCRIPTION("Cavium Thunder NIC Virtual Function Driver");
+MODULE_VERSION(DRV_VERSION);
+MODULE_DEVICE_TABLE(pci, nicvf_id_table);
+
+static int debug = 0x00;
+module_param(debug, int, 0644);
+MODULE_PARM_DESC(debug, "Debug message level bitmap");
+
+static int cpi_alg = CPI_ALG_NONE;
+module_param(cpi_alg, int, S_IRUGO);
+MODULE_PARM_DESC(cpi_alg,
+		 "PFC algorithm (0=none, 1=VLAN, 2=VLAN16, 3=IP Diffserv)");
+
+static inline u8 nicvf_netdev_qidx(struct nicvf *nic, u8 qidx)
+{
+#ifdef VNIC_MULTI_QSET_SUPPORT
+	if (nic->sqs_mode)
+		return qidx + ((nic->sqs_id + 1) * MAX_CMP_QUEUES_PER_QS);
+	else
+#endif
+		return qidx;
+}
+
+static inline void nicvf_set_rx_frame_cnt(struct nicvf *nic,
+					  struct sk_buff *skb)
+{
+	if (skb->len <= 64)
+		nic->drv_stats.rx_frames_64++;
+	else if (skb->len <= 127)
+		nic->drv_stats.rx_frames_127++;
+	else if (skb->len <= 255)
+		nic->drv_stats.rx_frames_255++;
+	else if (skb->len <= 511)
+		nic->drv_stats.rx_frames_511++;
+	else if (skb->len <= 1023)
+		nic->drv_stats.rx_frames_1023++;
+	else if (skb->len <= 1518)
+		nic->drv_stats.rx_frames_1518++;
+	else
+		nic->drv_stats.rx_frames_jumbo++;
+}
+
+/* The Cavium ThunderX network controller can *only* be found in SoCs
+ * containing the ThunderX ARM64 CPU implementation.  All accesses to the device
+ * registers on this platform are implicitly strongly ordered with respect
+ * to memory accesses. So writeq_relaxed() and readq_relaxed() are safe to use
+ * with no memory barriers in this driver.  The readq()/writeq() functions add
+ * explicit ordering operation which in this case are redundant, and only
+ * add overhead.
+ */
+
+/* Register read/write APIs */
+void nicvf_reg_write(struct nicvf *nic, u64 offset, u64 val)
+{
+	writeq_relaxed(val, nic->reg_base + offset);
+}
+
+u64 nicvf_reg_read(struct nicvf *nic, u64 offset)
+{
+	return readq_relaxed(nic->reg_base + offset);
+}
+
+void nicvf_queue_reg_write(struct nicvf *nic, u64 offset,
+			   u64 qidx, u64 val)
+{
+	void __iomem *addr = nic->reg_base + offset;
+
+	writeq_relaxed(val, addr + (qidx << NIC_Q_NUM_SHIFT));
+}
+
+u64 nicvf_queue_reg_read(struct nicvf *nic, u64 offset, u64 qidx)
+{
+	void __iomem *addr = nic->reg_base + offset;
+
+	return readq_relaxed(addr + (qidx << NIC_Q_NUM_SHIFT));
+}
+
+/* VF -> PF mailbox communication */
+static void nicvf_write_to_mbx(struct nicvf *nic, union nic_mbx *mbx)
+{
+	u64 *msg = (u64 *)mbx;
+
+	nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 0, msg[0]);
+	nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 8, msg[1]);
+}
+
+int nicvf_send_msg_to_pf(struct nicvf *nic, union nic_mbx *mbx)
+{
+	int timeout = NIC_MBOX_MSG_TIMEOUT;
+	int sleep = 10;
+
+	nic->pf_acked = false;
+	nic->pf_nacked = false;
+
+	nicvf_write_to_mbx(nic, mbx);
+
+	/* Wait for previous message to be acked, timeout 2sec */
+	while (!nic->pf_acked) {
+		if (nic->pf_nacked)
+			return -EINVAL;
+		msleep(sleep);
+		if (nic->pf_acked)
+			break;
+		timeout -= sleep;
+		if (!timeout) {
+			netdev_err(nic->netdev,
+				   "PF didn't ack to mbox msg %d from VF%d\n",
+				   (mbx->msg.msg & 0xFF), nic->vf_id);
+			return -EBUSY;
+		}
+	}
+	return 0;
+}
+
+/* Checks if VF is able to comminicate with PF
+* and also gets the VNIC number this VF is associated to.
+*/
+static int nicvf_check_pf_ready(struct nicvf *nic)
+{
+	union nic_mbx mbx = {};
+
+	mbx.msg.msg = NIC_MBOX_MSG_READY;
+	if (nicvf_send_msg_to_pf(nic, &mbx)) {
+		netdev_err(nic->netdev,
+			   "PF didn't respond to READY msg\n");
+		return 0;
+	}
+
+	return 1;
+}
+
+static void nicvf_read_bgx_stats(struct nicvf *nic, struct bgx_stats_msg *bgx)
+{
+	if (bgx->rx)
+		nic->bgx_stats.rx_stats[bgx->idx] = bgx->stats;
+	else
+		nic->bgx_stats.tx_stats[bgx->idx] = bgx->stats;
+}
+
+static void  nicvf_handle_mbx_intr(struct nicvf *nic)
+{
+	union nic_mbx mbx = {};
+	u64 *mbx_data;
+	u64 mbx_addr;
+	int i;
+
+	mbx_addr = NIC_VF_PF_MAILBOX_0_1;
+	mbx_data = (u64 *)&mbx;
+
+	for (i = 0; i < NIC_PF_VF_MAILBOX_SIZE; i++) {
+		*mbx_data = nicvf_reg_read(nic, mbx_addr);
+		mbx_data++;
+		mbx_addr += sizeof(u64);
+	}
+
+	netdev_dbg(nic->netdev, "Mbox message: msg: 0x%x\n", mbx.msg.msg);
+	switch (mbx.msg.msg) {
+	case NIC_MBOX_MSG_READY:
+		nic->pf_acked = true;
+		nic->vf_id = mbx.nic_cfg.vf_id & 0x7F;
+		nic->tns_mode = mbx.nic_cfg.tns_mode & 0x7F;
+		nic->node = mbx.nic_cfg.node_id;
+		ether_addr_copy(nic->netdev->dev_addr, mbx.nic_cfg.mac_addr);
+#ifdef VNIC_MULTI_QSET_SUPPORT
+		nic->sqs_mode = mbx.nic_cfg.sqs_mode;
+#endif
+		nic->loopback_supported = mbx.nic_cfg.loopback_supported;
+		nic->link_up = false;
+		nic->duplex = 0;
+		nic->speed = 0;
+		break;
+	case NIC_MBOX_MSG_ACK:
+		nic->pf_acked = true;
+		break;
+	case NIC_MBOX_MSG_NACK:
+		nic->pf_nacked = true;
+		break;
+#ifdef VNIC_RSS_SUPPORT
+	case NIC_MBOX_MSG_RSS_SIZE:
+		nic->rss_info.rss_size = mbx.rss_size.ind_tbl_size;
+		nic->pf_acked = true;
+		break;
+#endif
+	case NIC_MBOX_MSG_BGX_STATS:
+		nicvf_read_bgx_stats(nic, &mbx.bgx_stats);
+		nic->pf_acked = true;
+		break;
+	case NIC_MBOX_MSG_BGX_LINK_CHANGE:
+		nic->pf_acked = true;
+		nic->link_up = mbx.link_status.link_up;
+		nic->duplex = mbx.link_status.duplex;
+		nic->speed = mbx.link_status.speed;
+		if (nic->link_up) {
+			netdev_info(nic->netdev, "%s: Link is Up %d Mbps %s\n",
+				    nic->netdev->name, nic->speed,
+				    nic->duplex == DUPLEX_FULL ?
+				"Full duplex" : "Half duplex");
+			netif_carrier_on(nic->netdev);
+			netif_tx_start_all_queues(nic->netdev);
+		} else {
+			netdev_info(nic->netdev, "%s: Link is Down\n",
+				    nic->netdev->name);
+			netif_carrier_off(nic->netdev);
+			netif_tx_stop_all_queues(nic->netdev);
+		}
+		break;
+#ifdef VNIC_MULTI_QSET_SUPPORT
+	case NIC_MBOX_MSG_ALLOC_SQS:
+		nic->sqs_count = mbx.sqs_alloc.qs_count;
+		nic->pf_acked = true;
+		break;
+	case NIC_MBOX_MSG_SNICVF_PTR:
+		/* Primary VF: make note of secondary VF's pointer
+		 * to be used while packet transmission.
+		 */
+		nic->snicvf[mbx.nicvf.sqs_id] =
+			(struct nicvf *)mbx.nicvf.nicvf;
+		nic->pf_acked = true;
+		break;
+	case NIC_MBOX_MSG_PNICVF_PTR:
+		/* Secondary VF/Qset: make note of primary VF's pointer
+		 * to be used while packet reception, to handover packet
+		 * to primary VF's netdev.
+		 */
+		nic->pnicvf = (struct nicvf *)mbx.nicvf.nicvf;
+		nic->pf_acked = true;
+		break;
+#endif
+	default:
+		netdev_err(nic->netdev,
+			   "Invalid message from PF, msg 0x%x\n", mbx.msg.msg);
+		break;
+	}
+	nicvf_clear_intr(nic, NICVF_INTR_MBOX, 0);
+}
+
+static int nicvf_hw_set_mac_addr(struct nicvf *nic, struct net_device *netdev)
+{
+	union nic_mbx mbx = {};
+
+	mbx.mac.msg = NIC_MBOX_MSG_SET_MAC;
+	mbx.mac.vf_id = nic->vf_id;
+	ether_addr_copy(mbx.mac.mac_addr, netdev->dev_addr);
+
+	return nicvf_send_msg_to_pf(nic, &mbx);
+}
+
+static void nicvf_config_cpi(struct nicvf *nic)
+{
+	union nic_mbx mbx = {};
+
+	mbx.cpi_cfg.msg = NIC_MBOX_MSG_CPI_CFG;
+	mbx.cpi_cfg.vf_id = nic->vf_id;
+	mbx.cpi_cfg.cpi_alg = nic->cpi_alg;
+	mbx.cpi_cfg.rq_cnt = nic->qs->rq_cnt;
+
+	nicvf_send_msg_to_pf(nic, &mbx);
+}
+
+#ifdef	VNIC_RSS_SUPPORT
+static void nicvf_get_rss_size(struct nicvf *nic)
+{
+	union nic_mbx mbx = {};
+
+	mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE;
+	mbx.rss_size.vf_id = nic->vf_id;
+	nicvf_send_msg_to_pf(nic, &mbx);
+}
+
+void nicvf_config_rss(struct nicvf *nic)
+{
+	union nic_mbx mbx = {};
+	struct nicvf_rss_info *rss = &nic->rss_info;
+	int ind_tbl_len = rss->rss_size;
+	int i, nextq = 0;
+
+	mbx.rss_cfg.vf_id = nic->vf_id;
+	mbx.rss_cfg.hash_bits = rss->hash_bits;
+	while (ind_tbl_len) {
+		mbx.rss_cfg.tbl_offset = nextq;
+		mbx.rss_cfg.tbl_len = min(ind_tbl_len,
+					       RSS_IND_TBL_LEN_PER_MBX_MSG);
+		mbx.rss_cfg.msg = mbx.rss_cfg.tbl_offset ?
+			  NIC_MBOX_MSG_RSS_CFG_CONT : NIC_MBOX_MSG_RSS_CFG;
+
+		for (i = 0; i < mbx.rss_cfg.tbl_len; i++)
+			mbx.rss_cfg.ind_tbl[i] = rss->ind_tbl[nextq++];
+
+		nicvf_send_msg_to_pf(nic, &mbx);
+
+		ind_tbl_len -= mbx.rss_cfg.tbl_len;
+	}
+}
+
+void nicvf_set_rss_key(struct nicvf *nic)
+{
+	struct nicvf_rss_info *rss = &nic->rss_info;
+	u64 key_addr = NIC_VNIC_RSS_KEY_0_4;
+	int idx;
+
+	for (idx = 0; idx < RSS_HASH_KEY_SIZE; idx++) {
+		nicvf_reg_write(nic, key_addr, rss->key[idx]);
+		key_addr += sizeof(u64);
+	}
+}
+
+static int nicvf_rss_init(struct nicvf *nic)
+{
+	struct nicvf_rss_info *rss = &nic->rss_info;
+	int idx;
+
+	nicvf_get_rss_size(nic);
+
+	if (cpi_alg != CPI_ALG_NONE) {
+		rss->enable = false;
+		rss->hash_bits = 0;
+		return 0;
+	}
+
+	rss->enable = true;
+
+	/* Using the HW reset value for now */
+	rss->key[0] = 0xFEED0BADFEED0BADULL;
+	rss->key[1] = 0xFEED0BADFEED0BADULL;
+	rss->key[2] = 0xFEED0BADFEED0BADULL;
+	rss->key[3] = 0xFEED0BADFEED0BADULL;
+	rss->key[4] = 0xFEED0BADFEED0BADULL;
+
+	nicvf_set_rss_key(nic);
+
+	rss->cfg = RSS_IP_HASH_ENA | RSS_TCP_HASH_ENA | RSS_UDP_HASH_ENA;
+	nicvf_reg_write(nic, NIC_VNIC_RSS_CFG, rss->cfg);
+
+	rss->hash_bits =  ilog2(rounddown_pow_of_two(rss->rss_size));
+
+	for (idx = 0; idx < rss->rss_size; idx++)
+		rss->ind_tbl[idx] = ethtool_rxfh_indir_default(idx,
+							       nic->rx_queues);
+	nicvf_config_rss(nic);
+	return 1;
+}
+#endif
+
+#ifdef VNIC_MULTI_QSET_SUPPORT
+/* Request PF to allocate additional Qsets */
+static void nicvf_request_sqs(struct nicvf *nic)
+{
+	union nic_mbx mbx = {};
+	int sqs;
+	int sqs_count = nic->sqs_count;
+	int rx_queues = 0, tx_queues = 0;
+
+	/* Only primary VF should request */
+	if (nic->sqs_mode ||  !nic->sqs_count)
+		return;
+
+	mbx.sqs_alloc.msg = NIC_MBOX_MSG_ALLOC_SQS;
+	mbx.sqs_alloc.vf_id = nic->vf_id;
+	mbx.sqs_alloc.qs_count = nic->sqs_count;
+	if (nicvf_send_msg_to_pf(nic, &mbx)) {
+		/* No response from PF */
+		nic->sqs_count = 0;
+		return;
+	}
+
+	/* Return if no Secondary Qsets available */
+	if (!nic->sqs_count)
+		return;
+
+	if (nic->rx_queues > MAX_RCV_QUEUES_PER_QS)
+		rx_queues = nic->rx_queues - MAX_RCV_QUEUES_PER_QS;
+	if (nic->tx_queues > MAX_SND_QUEUES_PER_QS)
+		tx_queues = nic->tx_queues - MAX_SND_QUEUES_PER_QS;
+
+	/* Set no of Rx/Tx queues in each of the SQsets */
+	for (sqs = 0; sqs < nic->sqs_count; sqs++) {
+		mbx.nicvf.msg = NIC_MBOX_MSG_SNICVF_PTR;
+		mbx.nicvf.vf_id = nic->vf_id;
+		mbx.nicvf.sqs_id = sqs;
+		nicvf_send_msg_to_pf(nic, &mbx);
+
+		nic->snicvf[sqs]->sqs_id = sqs;
+		if (rx_queues > MAX_RCV_QUEUES_PER_QS) {
+			nic->snicvf[sqs]->qs->rq_cnt = MAX_RCV_QUEUES_PER_QS;
+			rx_queues -= MAX_RCV_QUEUES_PER_QS;
+		} else {
+			nic->snicvf[sqs]->qs->rq_cnt = rx_queues;
+			rx_queues = 0;
+		}
+
+		if (tx_queues > MAX_SND_QUEUES_PER_QS) {
+			nic->snicvf[sqs]->qs->sq_cnt = MAX_SND_QUEUES_PER_QS;
+			tx_queues -= MAX_SND_QUEUES_PER_QS;
+		} else {
+			nic->snicvf[sqs]->qs->sq_cnt = tx_queues;
+			tx_queues = 0;
+		}
+
+		nic->snicvf[sqs]->qs->cq_cnt =
+		max(nic->snicvf[sqs]->qs->rq_cnt, nic->snicvf[sqs]->qs->sq_cnt);
+
+		/* Initialize secondary Qset's queues and its interrupts */
+		nicvf_open(nic->snicvf[sqs]->netdev);
+	}
+
+	/* Update stack with actual Rx/Tx queue count allocated */
+	if (sqs_count != nic->sqs_count)
+		nicvf_set_real_num_queues(nic->netdev,
+					  nic->tx_queues, nic->rx_queues);
+}
+
+/* Send this Qset's nicvf pointer to PF.
+ * PF inturn sends primary VF's nicvf struct to secondary Qsets/VFs
+ * so that packets received by these Qsets can use primary VF's netdev
+ */
+static void nicvf_send_vf_struct(struct nicvf *nic)
+{
+	union nic_mbx mbx = {};
+
+	mbx.nicvf.msg = NIC_MBOX_MSG_NICVF_PTR;
+	mbx.nicvf.sqs_mode = nic->sqs_mode;
+	mbx.nicvf.nicvf = (u64)nic;
+	nicvf_send_msg_to_pf(nic, &mbx);
+}
+
+static void nicvf_get_primary_vf_struct(struct nicvf *nic)
+{
+	union nic_mbx mbx = {};
+
+	mbx.nicvf.msg = NIC_MBOX_MSG_PNICVF_PTR;
+	nicvf_send_msg_to_pf(nic, &mbx);
+}
+#endif
+
+int nicvf_set_real_num_queues(struct net_device *netdev,
+			      int tx_queues, int rx_queues)
+{
+	int err = 0;
+
+	err = netif_set_real_num_tx_queues(netdev, tx_queues);
+	if (err) {
+		netdev_err(netdev,
+			   "Failed to set no of Tx queues: %d\n", tx_queues);
+		return err;
+	}
+
+	err = netif_set_real_num_rx_queues(netdev, rx_queues);
+	if (err)
+		netdev_err(netdev,
+			   "Failed to set no of Rx queues: %d\n", rx_queues);
+	return err;
+}
+
+static int nicvf_init_resources(struct nicvf *nic)
+{
+	int err;
+	union nic_mbx mbx = {};
+
+	mbx.msg.msg = NIC_MBOX_MSG_CFG_DONE;
+
+	/* Enable Qset */
+	nicvf_qset_config(nic, true);
+
+	/* Initialize queues and HW for data transfer */
+	err = nicvf_config_data_transfer(nic, true);
+	if (err) {
+		netdev_err(nic->netdev,
+			   "Failed to alloc/config VF's QSet resources\n");
+		return err;
+	}
+
+	/* Send VF config done msg to PF */
+	nicvf_write_to_mbx(nic, &mbx);
+
+	return 0;
+}
+
+static void nicvf_snd_pkt_handler(struct net_device *netdev,
+				  struct cmp_queue *cq,
+				  struct cqe_send_t *cqe_tx, int cqe_type)
+{
+	struct sk_buff *skb = NULL;
+	struct nicvf *nic = netdev_priv(netdev);
+	struct snd_queue *sq;
+	struct sq_hdr_subdesc *hdr;
+
+	sq = &nic->qs->sq[cqe_tx->sq_idx];
+
+	hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, cqe_tx->sqe_ptr);
+	if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER)
+		return;
+
+	netdev_dbg(nic->netdev,
+		   "%s Qset #%d SQ #%d SQ ptr #%d subdesc count %d\n",
+		   __func__, cqe_tx->sq_qs, cqe_tx->sq_idx,
+		   cqe_tx->sqe_ptr, hdr->subdesc_cnt);
+
+	nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1);
+	nicvf_check_cqe_tx_errs(nic, cq, cqe_tx);
+	skb = (struct sk_buff *)sq->skbuff[cqe_tx->sqe_ptr];
+	/* For TSO offloaded packets only one head SKB needs to be freed */
+	if (skb) {
+		prefetch(skb);
+		dev_consume_skb_any(skb);
+		sq->skbuff[cqe_tx->sqe_ptr] = (u64)NULL;
+	}
+}
+
+static inline void nicvf_set_rxhash(struct net_device *netdev,
+				    struct cqe_rx_t *cqe_rx,
+				    struct sk_buff *skb)
+{
+	u8 hash_type;
+	u32 hash;
+
+	if (!(netdev->features & NETIF_F_RXHASH))
+		return;
+
+	switch (cqe_rx->rss_alg) {
+	case RSS_ALG_TCP_IP:
+	case RSS_ALG_UDP_IP:
+		hash_type = PKT_HASH_TYPE_L4;
+		hash = cqe_rx->rss_tag;
+		break;
+	case RSS_ALG_IP:
+		hash_type = PKT_HASH_TYPE_L3;
+		hash = cqe_rx->rss_tag;
+		break;
+	default:
+		hash_type = PKT_HASH_TYPE_NONE;
+		hash = 0;
+	}
+
+	skb_set_hash(skb, hash, hash_type);
+}
+
+static void nicvf_rcv_pkt_handler(struct net_device *netdev,
+				  struct napi_struct *napi,
+				  struct cmp_queue *cq,
+				  struct cqe_rx_t *cqe_rx, int cqe_type)
+{
+	struct sk_buff *skb;
+	struct nicvf *nic = netdev_priv(netdev);
+	int err = 0;
+	int rq_idx;
+
+	rq_idx = nicvf_netdev_qidx(nic, cqe_rx->rq_idx);
+
+#ifdef VNIC_MULTI_QSET_SUPPORT
+	if (nic->sqs_mode) {
+		/* Use primary VF's 'nicvf' struct */
+		nic = nic->pnicvf;
+		netdev = nic->netdev;
+	}
+#endif
+	/* Check for errors */
+	err = nicvf_check_cqe_rx_errs(nic, cq, cqe_rx);
+	if (err && !cqe_rx->rb_cnt)
+		return;
+
+	skb = nicvf_get_rcv_skb(nic, cqe_rx);
+	if (!skb) {
+		netdev_dbg(nic->netdev, "Packet not received\n");
+		return;
+	}
+
+	if (netif_msg_pktdata(nic)) {
+		netdev_info(nic->netdev, "%s: skb 0x%p, len=%d\n", netdev->name,
+			    skb, skb->len);
+		print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 1,
+			       skb->data, skb->len, true);
+	}
+
+	/* If error packet, drop it here */
+	if (err) {
+		dev_kfree_skb_any(skb);
+		return;
+	}
+
+	nicvf_set_rx_frame_cnt(nic, skb);
+
+	nicvf_set_rxhash(netdev, cqe_rx, skb);
+
+	skb_record_rx_queue(skb, rq_idx);
+	if (netdev->hw_features & NETIF_F_RXCSUM) {
+		/* HW by default verifies TCP/UDP/SCTP checksums */
+		skb->ip_summed = CHECKSUM_UNNECESSARY;
+	} else {
+		skb_checksum_none_assert(skb);
+	}
+
+	skb->protocol = eth_type_trans(skb, netdev);
+
+	/* Check for stripped VLAN */
+	if (cqe_rx->vlan_found && cqe_rx->vlan_stripped)
+		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
+				       ntohs(cqe_rx->vlan_tci));
+
+	if (napi && (netdev->features & NETIF_F_GRO))
+		napi_gro_receive(napi, skb);
+	else
+		netif_receive_skb(skb);
+}
+
+static int nicvf_cq_intr_handler(struct net_device *netdev, u8 cq_idx,
+				 struct napi_struct *napi, int budget)
+{
+	int processed_cqe, work_done = 0, tx_done = 0;
+	int cqe_count, cqe_head;
+	struct nicvf *nic = netdev_priv(netdev);
+	struct queue_set *qs = nic->qs;
+	struct cmp_queue *cq = &qs->cq[cq_idx];
+	struct cqe_rx_t *cq_desc;
+	struct netdev_queue *txq;
+
+	spin_lock_bh(&cq->lock);
+loop:
+	processed_cqe = 0;
+	/* Get no of valid CQ entries to process */
+	cqe_count = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS, cq_idx);
+	cqe_count &= CQ_CQE_COUNT;
+	if (!cqe_count)
+		goto done;
+
+	/* Get head of the valid CQ entries */
+	cqe_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD, cq_idx) >> 9;
+	cqe_head &= 0xFFFF;
+
+	netdev_dbg(nic->netdev, "%s CQ%d cqe_count %d cqe_head %d\n",
+		   __func__, cq_idx, cqe_count, cqe_head);
+	while (processed_cqe < cqe_count) {
+		/* Get the CQ descriptor */
+		cq_desc = (struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head);
+		cqe_head++;
+		cqe_head &= (cq->dmem.q_len - 1);
+		/* Initiate prefetch for next descriptor */
+		prefetch((struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head));
+
+		if ((work_done >= budget) && napi &&
+		    (cq_desc->cqe_type != CQE_TYPE_SEND)) {
+			break;
+		}
+
+		netdev_dbg(nic->netdev, "CQ%d cq_desc->cqe_type %d\n",
+			   cq_idx, cq_desc->cqe_type);
+		switch (cq_desc->cqe_type) {
+		case CQE_TYPE_RX:
+			nicvf_rcv_pkt_handler(netdev, napi, cq,
+					      cq_desc, CQE_TYPE_RX);
+			work_done++;
+		break;
+		case CQE_TYPE_SEND:
+			nicvf_snd_pkt_handler(netdev, cq,
+					      (void *)cq_desc, CQE_TYPE_SEND);
+			tx_done++;
+		break;
+		case CQE_TYPE_INVALID:
+		case CQE_TYPE_RX_SPLIT:
+		case CQE_TYPE_RX_TCP:
+		case CQE_TYPE_SEND_PTP:
+			/* Ignore for now */
+		break;
+		}
+		processed_cqe++;
+	}
+	netdev_dbg(nic->netdev,
+		   "%s CQ%d processed_cqe %d work_done %d budget %d\n",
+		   __func__, cq_idx, processed_cqe, work_done, budget);
+
+	/* Ring doorbell to inform H/W to reuse processed CQEs */
+	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_DOOR,
+			      cq_idx, processed_cqe);
+
+	if ((work_done < budget) && napi)
+		goto loop;
+
+done:
+	/* Wakeup TXQ if its stopped earlier due to SQ full */
+	if (tx_done) {
+#ifdef VNIC_MULTI_QSET_SUPPORT
+		netdev = nic->pnicvf->netdev;
+#endif
+		txq = netdev_get_tx_queue(netdev,
+					  nicvf_netdev_qidx(nic, cq_idx));
+#ifdef VNIC_MULTI_QSET_SUPPORT
+		nic = nic->pnicvf;
+#endif
+		if (netif_tx_queue_stopped(txq) && netif_carrier_ok(netdev)) {
+			netif_tx_start_queue(txq);
+			nic->drv_stats.txq_wake++;
+			if (netif_msg_tx_err(nic))
+				netdev_warn(netdev,
+					    "%s: Transmit queue wakeup SQ%d\n",
+					    netdev->name, cq_idx);
+		}
+	}
+
+	spin_unlock_bh(&cq->lock);
+	return work_done;
+}
+
+static int nicvf_poll(struct napi_struct *napi, int budget)
+{
+	u64  cq_head;
+	int  work_done = 0;
+	struct net_device *netdev = napi->dev;
+	struct nicvf *nic = netdev_priv(netdev);
+	struct nicvf_cq_poll *cq;
+
+	cq = container_of(napi, struct nicvf_cq_poll, napi);
+	work_done = nicvf_cq_intr_handler(netdev, cq->cq_idx, napi, budget);
+
+	if (work_done < budget) {
+		/* Slow packet rate, exit polling */
+		napi_complete(napi);
+		/* Re-enable interrupts */
+		cq_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD,
+					       cq->cq_idx);
+		nicvf_clear_intr(nic, NICVF_INTR_CQ, cq->cq_idx);
+		nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_HEAD,
+				      cq->cq_idx, cq_head);
+		nicvf_enable_intr(nic, NICVF_INTR_CQ, cq->cq_idx);
+	}
+	return work_done;
+}
+
+/* Qset error interrupt handler
+ *
+ * As of now only CQ errors are handled
+ */
+static void nicvf_handle_qs_err(unsigned long data)
+{
+	struct nicvf *nic = (struct nicvf *)data;
+	struct queue_set *qs = nic->qs;
+	int qidx;
+	u64 status;
+
+	netif_tx_disable(nic->netdev);
+
+	/* Check if it is CQ err */
+	for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
+		status = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS,
+					      qidx);
+		if (!(status & CQ_ERR_MASK))
+			continue;
+		/* Process already queued CQEs and reconfig CQ */
+		nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
+		nicvf_sq_disable(nic, qidx);
+		nicvf_cq_intr_handler(nic->netdev, qidx, NULL, 0);
+		nicvf_cmp_queue_config(nic, qs, qidx, true);
+		nicvf_sq_free_used_descs(nic->netdev, &qs->sq[qidx], qidx);
+		nicvf_sq_enable(nic, &qs->sq[qidx], qidx);
+
+		nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx);
+	}
+
+	netif_tx_start_all_queues(nic->netdev);
+	/* Re-enable Qset error interrupt */
+	nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0);
+}
+
+static inline void nicvf_dump_intr_status(struct nicvf *nic)
+{
+	if (netif_msg_intr(nic))
+		netdev_info(nic->netdev, "%s: interrupt status 0x%llx\n",
+			    nic->netdev->name, nicvf_reg_read(nic, NIC_VF_INT));
+}
+
+static irqreturn_t nicvf_misc_intr_handler(int irq, void *nicvf_irq)
+{
+	struct nicvf *nic = (struct nicvf *)nicvf_irq;
+	u64 intr;
+
+	nicvf_dump_intr_status(nic);
+
+	intr = nicvf_reg_read(nic, NIC_VF_INT);
+	/* Check for spurious interrupt */
+	if (!(intr & NICVF_INTR_MBOX_MASK))
+		return IRQ_HANDLED;
+
+	nicvf_handle_mbx_intr(nic);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t nicvf_intr_handler(int irq, void *cq_irq)
+{
+	struct nicvf_cq_poll *cq_poll = (struct nicvf_cq_poll *)cq_irq;
+	struct nicvf *nic = cq_poll->nicvf;
+	int qidx = cq_poll->cq_idx;
+
+	nicvf_dump_intr_status(nic);
+
+	/* Disable interrupts */
+	nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
+
+	/* Schedule NAPI */
+	napi_schedule(&cq_poll->napi);
+
+	/* Clear interrupt */
+	nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t nicvf_rbdr_intr_handler(int irq, void *nicvf_irq)
+{
+	struct nicvf *nic = (struct nicvf *)nicvf_irq;
+	u8 qidx;
+
+
+	nicvf_dump_intr_status(nic);
+
+	/* Disable RBDR interrupt and schedule softirq */
+	for (qidx = 0; qidx < nic->qs->rbdr_cnt; qidx++) {
+		if (!nicvf_is_intr_enabled(nic, NICVF_INTR_RBDR, qidx))
+			continue;
+		nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx);
+		tasklet_hi_schedule(&nic->rbdr_task);
+		/* Clear interrupt */
+		nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t nicvf_qs_err_intr_handler(int irq, void *nicvf_irq)
+{
+	struct nicvf *nic = (struct nicvf *)nicvf_irq;
+
+	nicvf_dump_intr_status(nic);
+
+	/* Disable Qset err interrupt and schedule softirq */
+	nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0);
+	tasklet_hi_schedule(&nic->qs_err_task);
+	nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0);
+
+	return IRQ_HANDLED;
+}
+
+static int nicvf_enable_msix(struct nicvf *nic)
+{
+	int ret, vec;
+
+	nic->num_vec = NIC_VF_MSIX_VECTORS;
+
+	for (vec = 0; vec < nic->num_vec; vec++)
+		nic->msix_entries[vec].entry = vec;
+
+	ret = pci_enable_msix(nic->pdev, nic->msix_entries, nic->num_vec);
+	if (ret) {
+		netdev_err(nic->netdev,
+			   "Req for #%d msix vectors failed\n", nic->num_vec);
+		return 0;
+	}
+	nic->msix_enabled = 1;
+	return 1;
+}
+
+static void nicvf_disable_msix(struct nicvf *nic)
+{
+	if (nic->msix_enabled) {
+		pci_disable_msix(nic->pdev);
+		nic->msix_enabled = 0;
+		nic->num_vec = 0;
+	}
+}
+
+static int nicvf_register_interrupts(struct nicvf *nic)
+{
+	int irq, ret = 0;
+	int vector;
+
+	for_each_cq_irq(irq)
+		sprintf(nic->irq_name[irq], "NICVF%d CQ%d",
+			nic->vf_id, irq);
+
+	for_each_sq_irq(irq)
+		sprintf(nic->irq_name[irq], "NICVF%d SQ%d",
+			nic->vf_id, irq - NICVF_INTR_ID_SQ);
+
+	for_each_rbdr_irq(irq)
+		sprintf(nic->irq_name[irq], "NICVF%d RBDR%d",
+			nic->vf_id, irq - NICVF_INTR_ID_RBDR);
+
+	/* Register CQ interrupts */
+	for (irq = 0; irq < nic->qs->cq_cnt; irq++) {
+		vector = nic->msix_entries[irq].vector;
+		ret = request_irq(vector, nicvf_intr_handler,
+				  0, nic->irq_name[irq], nic->napi[irq]);
+		if (ret)
+			goto err;
+		nic->irq_allocated[irq] = true;
+	}
+
+	/* Register RBDR interrupt */
+	for (irq = NICVF_INTR_ID_RBDR;
+	     irq < (NICVF_INTR_ID_RBDR + nic->qs->rbdr_cnt); irq++) {
+		vector = nic->msix_entries[irq].vector;
+		ret = request_irq(vector, nicvf_rbdr_intr_handler,
+				  0, nic->irq_name[irq], nic);
+		if (ret)
+			goto err;
+		nic->irq_allocated[irq] = true;
+	}
+
+	/* Register QS error interrupt */
+	sprintf(nic->irq_name[NICVF_INTR_ID_QS_ERR],
+		"NICVF%d Qset error", nic->vf_id);
+	irq = NICVF_INTR_ID_QS_ERR;
+	ret = request_irq(nic->msix_entries[irq].vector,
+			  nicvf_qs_err_intr_handler,
+			  0, nic->irq_name[irq], nic);
+	if (!ret)
+		nic->irq_allocated[irq] = true;
+
+err:
+	if (ret)
+		netdev_err(nic->netdev, "request_irq failed, vector %d\n", irq);
+
+	return ret;
+}
+
+static void nicvf_unregister_interrupts(struct nicvf *nic)
+{
+	int irq;
+
+	/* Free registered interrupts */
+	for (irq = 0; irq < nic->num_vec; irq++) {
+		if (!nic->irq_allocated[irq])
+			continue;
+
+		if (irq < NICVF_INTR_ID_SQ)
+			free_irq(nic->msix_entries[irq].vector, nic->napi[irq]);
+		else
+			free_irq(nic->msix_entries[irq].vector, nic);
+
+		nic->irq_allocated[irq] = false;
+	}
+
+	/* Disable MSI-X */
+	nicvf_disable_msix(nic);
+}
+
+/* Initialize MSIX vectors and register MISC interrupt.
+ * Send READY message to PF to check if its alive
+ */
+static int nicvf_register_misc_interrupt(struct nicvf *nic)
+{
+	int ret = 0;
+	int irq = NICVF_INTR_ID_MISC;
+
+	/* Return if mailbox interrupt is already registered */
+	if (nic->msix_enabled)
+		return 0;
+
+	/* Enable MSI-X */
+	if (!nicvf_enable_msix(nic))
+		return 1;
+
+	sprintf(nic->irq_name[irq], "%s Mbox", "NICVF");
+	/* Register Misc interrupt */
+	ret = request_irq(nic->msix_entries[irq].vector,
+			  nicvf_misc_intr_handler, 0, nic->irq_name[irq], nic);
+
+	if (ret)
+		return ret;
+	nic->irq_allocated[irq] = true;
+
+	/* Enable mailbox interrupt */
+	nicvf_enable_intr(nic, NICVF_INTR_MBOX, 0);
+
+	/* Check if VF is able to communicate with PF */
+	if (!nicvf_check_pf_ready(nic)) {
+		nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
+		nicvf_unregister_interrupts(nic);
+		return 1;
+	}
+
+	return 0;
+}
+
+static netdev_tx_t nicvf_xmit(struct sk_buff *skb, struct net_device *netdev)
+{
+	struct nicvf *nic = netdev_priv(netdev);
+	int qid = skb_get_queue_mapping(skb);
+	struct netdev_queue *txq = netdev_get_tx_queue(netdev, qid);
+
+	/* Check for minimum packet length */
+	if (skb->len <= ETH_HLEN) {
+		dev_kfree_skb(skb);
+		return NETDEV_TX_OK;
+	}
+
+	if (!netif_tx_queue_stopped(txq) && !nicvf_sq_append_skb(nic, skb)) {
+		netif_tx_stop_queue(txq);
+		nic->drv_stats.txq_stop++;
+		if (netif_msg_tx_err(nic))
+			netdev_warn(netdev,
+				    "%s: Transmit ring full, stopping SQ%d\n",
+				    netdev->name, qid);
+		return NETDEV_TX_BUSY;
+	}
+
+	return NETDEV_TX_OK;
+}
+
+static inline void nicvf_free_cq_poll(struct nicvf *nic)
+{
+	struct nicvf_cq_poll *cq_poll = NULL;
+	int qidx;
+
+	for (qidx = 0; qidx < nic->qs->cq_cnt; qidx++) {
+		cq_poll = nic->napi[qidx];
+		if (!cq_poll)
+			continue;
+		nic->napi[qidx] = NULL;
+		kfree(cq_poll);
+	}
+}
+
+int nicvf_stop(struct net_device *netdev)
+{
+	int irq, qidx;
+	struct nicvf *nic = netdev_priv(netdev);
+	struct queue_set *qs = nic->qs;
+	struct nicvf_cq_poll *cq_poll = NULL;
+	union nic_mbx mbx = {};
+
+	mbx.msg.msg = NIC_MBOX_MSG_SHUTDOWN;
+	nicvf_send_msg_to_pf(nic, &mbx);
+
+	netif_carrier_off(netdev);
+	netif_tx_stop_all_queues(nic->netdev);
+
+#ifdef	VNIC_MULTI_QSET_SUPPORT
+	/* Teardown secondary qsets first */
+	if (!nic->sqs_mode) {
+		for (qidx = 0; qidx < nic->sqs_count; qidx++) {
+			if (!nic->snicvf[qidx])
+				continue;
+			nicvf_stop(nic->snicvf[qidx]->netdev);
+			nic->snicvf[qidx] = NULL;
+		}
+	}
+#endif
+
+	/* Disable RBDR & QS error interrupts */
+	for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) {
+		nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx);
+		nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx);
+	}
+	nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0);
+	nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0);
+
+	/* Wait for pending IRQ handlers to finish */
+	for (irq = 0; irq < nic->num_vec; irq++)
+		synchronize_irq(nic->msix_entries[irq].vector);
+
+	tasklet_kill(&nic->rbdr_task);
+	tasklet_kill(&nic->qs_err_task);
+	if (nic->rb_work_scheduled)
+		cancel_delayed_work_sync(&nic->rbdr_work);
+
+	for (qidx = 0; qidx < nic->qs->cq_cnt; qidx++) {
+		cq_poll = nic->napi[qidx];
+		if (!cq_poll)
+			continue;
+		napi_synchronize(&cq_poll->napi);
+		/* CQ intr is enabled while napi_complete,
+		 * so disable it now
+		 */
+		nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
+		nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx);
+		napi_disable(&cq_poll->napi);
+		netif_napi_del(&cq_poll->napi);
+	}
+
+	netif_tx_disable(netdev);
+
+	/* Free resources */
+	nicvf_config_data_transfer(nic, false);
+
+	/* Disable HW Qset */
+	nicvf_qset_config(nic, false);
+
+	/* disable mailbox interrupt */
+	nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
+
+	nicvf_unregister_interrupts(nic);
+
+	nicvf_free_cq_poll(nic);
+
+#ifdef	VNIC_MULTI_QSET_SUPPORT
+	/* Clear multiqset info */
+	nic->pnicvf = nic;
+	nic->sqs_count = 0;
+#endif
+
+	return 0;
+}
+
+int nicvf_open(struct net_device *netdev)
+{
+	int err, qidx;
+	struct nicvf *nic = netdev_priv(netdev);
+	struct queue_set *qs = nic->qs;
+	struct nicvf_cq_poll *cq_poll = NULL;
+
+	nic->mtu = netdev->mtu;
+
+	netif_carrier_off(netdev);
+
+	err = nicvf_register_misc_interrupt(nic);
+	if (err)
+		return err;
+
+	/* Register NAPI handler for processing CQEs */
+	for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
+		cq_poll = kzalloc(sizeof(*cq_poll), GFP_KERNEL);
+		if (!cq_poll) {
+			err = -ENOMEM;
+			goto napi_del;
+		}
+		cq_poll->cq_idx = qidx;
+		cq_poll->nicvf = nic;
+		netif_napi_add(netdev, &cq_poll->napi, nicvf_poll,
+			       NAPI_POLL_WEIGHT);
+		napi_enable(&cq_poll->napi);
+		nic->napi[qidx] = cq_poll;
+	}
+
+	/* Check if we got MAC address from PF or else generate a radom MAC */
+	if (is_zero_ether_addr(netdev->dev_addr)) {
+		eth_hw_addr_random(netdev);
+		nicvf_hw_set_mac_addr(nic, netdev);
+	}
+
+	/* Init tasklet for handling Qset err interrupt */
+	tasklet_init(&nic->qs_err_task, nicvf_handle_qs_err,
+		     (unsigned long)nic);
+
+	/* Init RBDR tasklet which will refill RBDR */
+	tasklet_init(&nic->rbdr_task, nicvf_rbdr_task,
+		     (unsigned long)nic);
+	INIT_DELAYED_WORK(&nic->rbdr_work, nicvf_rbdr_work);
+
+	/* Configure CPI alorithm */
+	nic->cpi_alg = cpi_alg;
+	if (!nic->sqs_mode)
+		nicvf_config_cpi(nic);
+
+#ifdef VNIC_MULTI_QSET_SUPPORT
+	nicvf_request_sqs(nic);
+	if (nic->sqs_mode)
+		nicvf_get_primary_vf_struct(nic);
+#endif
+
+#ifdef	VNIC_RSS_SUPPORT
+	/* Configure receive side scaling */
+	if (!nic->sqs_mode)
+		nicvf_rss_init(nic);
+#endif
+
+	err = nicvf_register_interrupts(nic);
+	if (err)
+		goto cleanup;
+
+	/* Initialize the queues */
+	err = nicvf_init_resources(nic);
+	if (err)
+		goto cleanup;
+
+	/* Make sure queue initialization is written */
+	wmb();
+
+	nicvf_reg_write(nic, NIC_VF_INT, -1);
+	/* Enable Qset err interrupt */
+	nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0);
+
+	/* Enable completion queue interrupt */
+	for (qidx = 0; qidx < qs->cq_cnt; qidx++)
+		nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx);
+
+	/* Enable RBDR threshold interrupt */
+	for (qidx = 0; qidx < qs->rbdr_cnt; qidx++)
+		nicvf_enable_intr(nic, NICVF_INTR_RBDR, qidx);
+
+	nic->drv_stats.txq_stop = 0;
+	nic->drv_stats.txq_wake = 0;
+
+	netif_carrier_on(netdev);
+	netif_tx_start_all_queues(netdev);
+
+	return 0;
+cleanup:
+	nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
+	nicvf_unregister_interrupts(nic);
+	tasklet_kill(&nic->qs_err_task);
+	tasklet_kill(&nic->rbdr_task);
+napi_del:
+	for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
+		cq_poll = nic->napi[qidx];
+		if (!cq_poll)
+			continue;
+		napi_disable(&cq_poll->napi);
+		netif_napi_del(&cq_poll->napi);
+	}
+	nicvf_free_cq_poll(nic);
+	return err;
+}
+
+static int nicvf_update_hw_max_frs(struct nicvf *nic, int mtu)
+{
+	union nic_mbx mbx = {};
+
+	mbx.frs.msg = NIC_MBOX_MSG_SET_MAX_FRS;
+	mbx.frs.max_frs = mtu;
+	mbx.frs.vf_id = nic->vf_id;
+
+	return nicvf_send_msg_to_pf(nic, &mbx);
+}
+
+static int nicvf_change_mtu(struct net_device *netdev, int new_mtu)
+{
+	struct nicvf *nic = netdev_priv(netdev);
+
+	if (new_mtu > NIC_HW_MAX_FRS)
+		return -EINVAL;
+
+	if (new_mtu < NIC_HW_MIN_FRS)
+		return -EINVAL;
+
+	if (nicvf_update_hw_max_frs(nic, new_mtu))
+		return -EINVAL;
+	netdev->mtu = new_mtu;
+	nic->mtu = new_mtu;
+
+	return 0;
+}
+
+static int nicvf_set_mac_address(struct net_device *netdev, void *p)
+{
+	struct sockaddr *addr = p;
+	struct nicvf *nic = netdev_priv(netdev);
+
+	if (!is_valid_ether_addr(addr->sa_data))
+		return -EADDRNOTAVAIL;
+
+	memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
+
+	if (nic->msix_enabled)
+		if (nicvf_hw_set_mac_addr(nic, netdev))
+			return -EBUSY;
+
+	return 0;
+}
+
+void nicvf_update_lmac_stats(struct nicvf *nic)
+{
+	int stat = 0;
+	union nic_mbx mbx = {};
+
+	if (!netif_running(nic->netdev))
+		return;
+
+	mbx.bgx_stats.msg = NIC_MBOX_MSG_BGX_STATS;
+	mbx.bgx_stats.vf_id = nic->vf_id;
+	/* Rx stats */
+	mbx.bgx_stats.rx = 1;
+	while (stat < BGX_RX_STATS_COUNT) {
+		mbx.bgx_stats.idx = stat;
+		if (nicvf_send_msg_to_pf(nic, &mbx))
+			return;
+		stat++;
+	}
+
+	stat = 0;
+
+	/* Tx stats */
+	mbx.bgx_stats.rx = 0;
+	while (stat < BGX_TX_STATS_COUNT) {
+		mbx.bgx_stats.idx = stat;
+		if (nicvf_send_msg_to_pf(nic, &mbx))
+			return;
+		stat++;
+	}
+}
+
+void nicvf_update_stats(struct nicvf *nic)
+{
+	int qidx;
+	struct nicvf_hw_stats *stats = &nic->hw_stats;
+	struct nicvf_drv_stats *drv_stats = &nic->drv_stats;
+	struct queue_set *qs = nic->qs;
+
+#define GET_RX_STATS(reg) \
+	nicvf_reg_read(nic, NIC_VNIC_RX_STAT_0_13 | (reg << 3))
+#define GET_TX_STATS(reg) \
+	nicvf_reg_read(nic, NIC_VNIC_TX_STAT_0_4 | (reg << 3))
+
+	stats->rx_bytes = GET_RX_STATS(RX_OCTS);
+	stats->rx_ucast_frames = GET_RX_STATS(RX_UCAST);
+	stats->rx_bcast_frames = GET_RX_STATS(RX_BCAST);
+	stats->rx_mcast_frames = GET_RX_STATS(RX_MCAST);
+	stats->rx_fcs_errors = GET_RX_STATS(RX_FCS);
+	stats->rx_l2_errors = GET_RX_STATS(RX_L2ERR);
+	stats->rx_drop_red = GET_RX_STATS(RX_RED);
+	stats->rx_drop_red_bytes = GET_RX_STATS(RX_RED_OCTS);
+	stats->rx_drop_overrun = GET_RX_STATS(RX_ORUN);
+	stats->rx_drop_overrun_bytes = GET_RX_STATS(RX_ORUN_OCTS);
+	stats->rx_drop_bcast = GET_RX_STATS(RX_DRP_BCAST);
+	stats->rx_drop_mcast = GET_RX_STATS(RX_DRP_MCAST);
+	stats->rx_drop_l3_bcast = GET_RX_STATS(RX_DRP_L3BCAST);
+	stats->rx_drop_l3_mcast = GET_RX_STATS(RX_DRP_L3MCAST);
+
+	stats->tx_bytes_ok = GET_TX_STATS(TX_OCTS);
+	stats->tx_ucast_frames_ok = GET_TX_STATS(TX_UCAST);
+	stats->tx_bcast_frames_ok = GET_TX_STATS(TX_BCAST);
+	stats->tx_mcast_frames_ok = GET_TX_STATS(TX_MCAST);
+	stats->tx_drops = GET_TX_STATS(TX_DROP);
+
+	drv_stats->tx_frames_ok = stats->tx_ucast_frames_ok +
+				  stats->tx_bcast_frames_ok +
+				  stats->tx_mcast_frames_ok;
+	drv_stats->rx_drops = stats->rx_drop_red +
+			      stats->rx_drop_overrun;
+	drv_stats->tx_drops = stats->tx_drops;
+
+	/* Update RQ and SQ stats */
+	for (qidx = 0; qidx < qs->rq_cnt; qidx++)
+		nicvf_update_rq_stats(nic, qidx);
+	for (qidx = 0; qidx < qs->sq_cnt; qidx++)
+		nicvf_update_sq_stats(nic, qidx);
+}
+
+static struct rtnl_link_stats64 *nicvf_get_stats64(struct net_device *netdev,
+					    struct rtnl_link_stats64 *stats)
+{
+	struct nicvf *nic = netdev_priv(netdev);
+	struct nicvf_hw_stats *hw_stats = &nic->hw_stats;
+	struct nicvf_drv_stats *drv_stats = &nic->drv_stats;
+
+	nicvf_update_stats(nic);
+
+	stats->rx_bytes = hw_stats->rx_bytes;
+	stats->rx_packets = drv_stats->rx_frames_ok;
+	stats->rx_dropped = drv_stats->rx_drops;
+	stats->multicast = hw_stats->rx_mcast_frames;
+
+	stats->tx_bytes = hw_stats->tx_bytes_ok;
+	stats->tx_packets = drv_stats->tx_frames_ok;
+	stats->tx_dropped = drv_stats->tx_drops;
+
+	return stats;
+}
+
+static void nicvf_tx_timeout(struct net_device *dev)
+{
+	struct nicvf *nic = netdev_priv(dev);
+
+	if (netif_msg_tx_err(nic))
+		netdev_warn(dev, "%s: Transmit timed out, resetting\n",
+			    dev->name);
+
+	schedule_work(&nic->reset_task);
+}
+
+static void nicvf_reset_task(struct work_struct *work)
+{
+	struct nicvf *nic;
+
+	nic = container_of(work, struct nicvf, reset_task);
+
+	if (!netif_running(nic->netdev))
+		return;
+
+	nicvf_stop(nic->netdev);
+	nicvf_open(nic->netdev);
+	nic->netdev->trans_start = jiffies;
+}
+
+static int nicvf_config_loopback(struct nicvf *nic,
+				 netdev_features_t features)
+{
+	union nic_mbx mbx = {};
+
+	mbx.lbk.msg = NIC_MBOX_MSG_LOOPBACK;
+	mbx.lbk.vf_id = nic->vf_id;
+	mbx.lbk.enable = (features & NETIF_F_LOOPBACK) != 0;
+
+	return nicvf_send_msg_to_pf(nic, &mbx);
+}
+
+static netdev_features_t nicvf_fix_features(struct net_device *netdev,
+					    netdev_features_t features)
+{
+	struct nicvf *nic = netdev_priv(netdev);
+
+	if ((features & NETIF_F_LOOPBACK) &&
+	    netif_running(netdev) && !nic->loopback_supported)
+		features &= ~NETIF_F_LOOPBACK;
+
+	return features;
+}
+
+static int nicvf_set_features(struct net_device *netdev,
+			      netdev_features_t features)
+{
+	struct nicvf *nic = netdev_priv(netdev);
+	netdev_features_t changed = features ^ netdev->features;
+
+	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
+		nicvf_config_vlan_stripping(nic, features);
+
+	if ((changed & NETIF_F_LOOPBACK) && netif_running(netdev))
+		return nicvf_config_loopback(nic, features);
+
+	return 0;
+}
+
+static const struct net_device_ops nicvf_netdev_ops = {
+	.ndo_open		= nicvf_open,
+	.ndo_stop		= nicvf_stop,
+	.ndo_start_xmit		= nicvf_xmit,
+	.ndo_change_mtu		= nicvf_change_mtu,
+	.ndo_set_mac_address	= nicvf_set_mac_address,
+	.ndo_get_stats64	= nicvf_get_stats64,
+	.ndo_tx_timeout         = nicvf_tx_timeout,
+	.ndo_fix_features       = nicvf_fix_features,
+	.ndo_set_features       = nicvf_set_features,
+};
+
+static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+	struct device *dev = &pdev->dev;
+	struct net_device *netdev;
+	struct nicvf *nic;
+	int    err, qcount;
+
+	err = pci_enable_device(pdev);
+	if (err) {
+		dev_err(dev, "Failed to enable PCI device\n");
+		return err;
+	}
+
+	err = pci_request_regions(pdev, DRV_NAME);
+	if (err) {
+		dev_err(dev, "PCI request regions failed 0x%x\n", err);
+		goto err_disable_device;
+	}
+
+	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
+	if (err) {
+		dev_err(dev, "Unable to get usable DMA configuration\n");
+		goto err_release_regions;
+	}
+
+	err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
+	if (err) {
+		dev_err(dev, "unable to get 48-bit DMA for consistent allocations\n");
+		goto err_release_regions;
+	}
+
+	qcount = MAX_CMP_QUEUES_PER_QS;
+
+#ifdef VNIC_MULTI_QSET_SUPPORT
+	/* Restrict multiqset support only for host bound VFs */
+	if (pdev->is_virtfn) {
+		/* Set max number of queues per VF */
+		qcount = roundup(num_online_cpus(), MAX_CMP_QUEUES_PER_QS);
+		qcount = min(qcount,
+			     (MAX_SQS_PER_VF + 1) * MAX_CMP_QUEUES_PER_QS);
+	}
+#endif
+
+	netdev = alloc_etherdev_mqs(sizeof(struct nicvf), qcount, qcount);
+	if (!netdev) {
+		err = -ENOMEM;
+		goto err_release_regions;
+	}
+
+	pci_set_drvdata(pdev, netdev);
+
+	SET_NETDEV_DEV(netdev, &pdev->dev);
+
+	nic = netdev_priv(netdev);
+	nic->netdev = netdev;
+	nic->pdev = pdev;
+	nic->pnicvf = nic;
+	nic->max_queues = qcount;
+
+	/* MAP VF's configuration registers */
+	nic->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0);
+	if (!nic->reg_base) {
+		dev_err(dev, "Cannot map config register space, aborting\n");
+		err = -ENOMEM;
+		goto err_free_netdev;
+	}
+
+	err = nicvf_set_qset_resources(nic);
+	if (err)
+		goto err_free_netdev;
+
+	/* Check if PF is alive and get MAC address for this VF */
+	err = nicvf_register_misc_interrupt(nic);
+	if (err)
+		goto err_free_netdev;
+
+#ifdef VNIC_MULTI_QSET_SUPPORT
+	nicvf_send_vf_struct(nic);
+
+	/* Check if this VF is in QS only mode */
+	if (nic->sqs_mode)
+		return 0;
+#endif
+
+	err = nicvf_set_real_num_queues(netdev, nic->tx_queues, nic->rx_queues);
+	if (err)
+		goto err_unregister_interrupts;
+
+	netdev->hw_features = (NETIF_F_RXCSUM | NETIF_F_IP_CSUM | NETIF_F_SG |
+			       NETIF_F_GRO |
+			       NETIF_F_HW_VLAN_CTAG_RX);
+#ifdef	VNIC_RSS_SUPPORT
+	netdev->hw_features |= NETIF_F_RXHASH;
+#endif
+
+	netdev->features |= netdev->hw_features;
+	netdev->hw_features |= NETIF_F_LOOPBACK;
+
+	netdev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM;
+
+	netdev->netdev_ops = &nicvf_netdev_ops;
+	netdev->watchdog_timeo = NICVF_TX_TIMEOUT;
+
+	INIT_WORK(&nic->reset_task, nicvf_reset_task);
+
+	err = register_netdev(netdev);
+	if (err) {
+		dev_err(dev, "Failed to register netdevice\n");
+		goto err_unregister_interrupts;
+	}
+
+	nic->msg_enable = debug;
+
+	nicvf_set_ethtool_ops(netdev);
+
+	return 0;
+
+err_unregister_interrupts:
+	nicvf_unregister_interrupts(nic);
+err_free_netdev:
+	pci_set_drvdata(pdev, NULL);
+	free_netdev(netdev);
+err_release_regions:
+	pci_release_regions(pdev);
+err_disable_device:
+	pci_disable_device(pdev);
+	return err;
+}
+
+static void nicvf_remove(struct pci_dev *pdev)
+{
+	struct net_device *netdev = pci_get_drvdata(pdev);
+	struct nicvf *nic = netdev_priv(netdev);
+	struct net_device *pnetdev = nic->pnicvf->netdev;
+
+	/* Check if this Qset is assigned to different VF.
+	 * If yes, clean primary and all secondary Qsets.
+	 */
+	if (pnetdev && (pnetdev->reg_state == NETREG_REGISTERED))
+		unregister_netdev(pnetdev);
+	nicvf_unregister_interrupts(nic);
+	pci_set_drvdata(pdev, NULL);
+	free_netdev(netdev);
+	pci_release_regions(pdev);
+	pci_disable_device(pdev);
+}
+
+static void nicvf_shutdown(struct pci_dev *pdev)
+{
+	nicvf_remove(pdev);
+}
+
+static struct pci_driver nicvf_driver = {
+	.name = DRV_NAME,
+	.id_table = nicvf_id_table,
+	.probe = nicvf_probe,
+	.remove = nicvf_remove,
+	.shutdown = nicvf_shutdown,
+};
+
+static int __init nicvf_init_module(void)
+{
+	pr_info("%s, ver %s\n", DRV_NAME, DRV_VERSION);
+
+	return pci_register_driver(&nicvf_driver);
+}
+
+static void __exit nicvf_cleanup_module(void)
+{
+	pci_unregister_driver(&nicvf_driver);
+}
+
+module_init(nicvf_init_module);
+module_exit(nicvf_cleanup_module);
diff --git a/sys/dev/vnic/nicvf_queues.c b/sys/dev/vnic/nicvf_queues.c
new file mode 100644
index 000000000000..9fa902403fc5
--- /dev/null
+++ b/sys/dev/vnic/nicvf_queues.c
@@ -0,0 +1,1467 @@
+/*
+ * Copyright (C) 2015 Cavium Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ *
+ */
+
+#include <linux/pci.h>
+#include <linux/netdevice.h>
+#include <linux/ip.h>
+#include <linux/etherdevice.h>
+#include <net/ip.h>
+#include <net/tso.h>
+
+#include "nic_reg.h"
+#include "nic.h"
+#include "q_struct.h"
+#include "nicvf_queues.h"
+
+struct rbuf_info {
+	struct page *page;
+	void	*data;
+	u64	offset;
+};
+
+#define GET_RBUF_INFO(x) ((struct rbuf_info *)(x - NICVF_RCV_BUF_ALIGN_BYTES))
+
+/* Poll a register for a specific value */
+static int nicvf_poll_reg(struct nicvf *nic, int qidx,
+			  u64 reg, int bit_pos, int bits, int val)
+{
+	u64 bit_mask;
+	u64 reg_val;
+	int timeout = 10;
+
+	bit_mask = (1ULL << bits) - 1;
+	bit_mask = (bit_mask << bit_pos);
+
+	while (timeout) {
+		reg_val = nicvf_queue_reg_read(nic, reg, qidx);
+		if (((reg_val & bit_mask) >> bit_pos) == val)
+			return 0;
+		usleep_range(1000, 2000);
+		timeout--;
+	}
+	netdev_err(nic->netdev, "Poll on reg 0x%llx failed\n", reg);
+	return 1;
+}
+
+/* Allocate memory for a queue's descriptors */
+static int nicvf_alloc_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem,
+				  int q_len, int desc_size, int align_bytes)
+{
+	dmem->q_len = q_len;
+	dmem->size = (desc_size * q_len) + align_bytes;
+	/* Save address, need it while freeing */
+	dmem->unalign_base = dma_zalloc_coherent(&nic->pdev->dev, dmem->size,
+						&dmem->dma, GFP_KERNEL);
+	if (!dmem->unalign_base)
+		return -ENOMEM;
+
+	/* Align memory address for 'align_bytes' */
+	dmem->phys_base = NICVF_ALIGNED_ADDR((u64)dmem->dma, align_bytes);
+	dmem->base = dmem->unalign_base + (dmem->phys_base - dmem->dma);
+	return 0;
+}
+
+/* Free queue's descriptor memory */
+static void nicvf_free_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem)
+{
+	if (!dmem)
+		return;
+
+	dma_free_coherent(&nic->pdev->dev, dmem->size,
+			  dmem->unalign_base, dmem->dma);
+	dmem->unalign_base = NULL;
+	dmem->base = NULL;
+}
+
+/* Allocate buffer for packet reception
+ * HW returns memory address where packet is DMA'ed but not a pointer
+ * into RBDR ring, so save buffer address at the start of fragment and
+ * align the start address to a cache aligned address
+ */
+static inline int nicvf_alloc_rcv_buffer(struct nicvf *nic, gfp_t gfp,
+					 u32 buf_len, u64 **rbuf)
+{
+	u64 data;
+	struct rbuf_info *rinfo;
+	int order = get_order(buf_len);
+
+	/* Check if request can be accomodated in previous allocated page */
+	if (nic->rb_page) {
+		if ((nic->rb_page_offset + buf_len + buf_len) >
+		    (PAGE_SIZE << order)) {
+			nic->rb_page = NULL;
+		} else {
+			nic->rb_page_offset += buf_len;
+			get_page(nic->rb_page);
+		}
+	}
+
+	/* Allocate a new page */
+	if (!nic->rb_page) {
+		nic->rb_page = alloc_pages(gfp | __GFP_COMP | __GFP_NOWARN,
+					   order);
+		if (!nic->rb_page) {
+			netdev_err(nic->netdev,
+				   "Failed to allocate new rcv buffer\n");
+			return -ENOMEM;
+		}
+		nic->rb_page_offset = 0;
+	}
+
+	data = (u64)page_address(nic->rb_page) + nic->rb_page_offset;
+
+	/* Align buffer addr to cache line i.e 128 bytes */
+	rinfo = (struct rbuf_info *)(data + NICVF_RCV_BUF_ALIGN_LEN(data));
+	/* Save page address for reference updation */
+	rinfo->page = nic->rb_page;
+	/* Store start address for later retrieval */
+	rinfo->data = (void *)data;
+	/* Store alignment offset */
+	rinfo->offset = NICVF_RCV_BUF_ALIGN_LEN(data);
+
+	data += rinfo->offset;
+
+	/* Give next aligned address to hw for DMA */
+	*rbuf = (u64 *)(data + NICVF_RCV_BUF_ALIGN_BYTES);
+	return 0;
+}
+
+/* Retrieve actual buffer start address and build skb for received packet */
+static struct sk_buff *nicvf_rb_ptr_to_skb(struct nicvf *nic,
+					   u64 rb_ptr, int len)
+{
+	struct sk_buff *skb;
+	struct rbuf_info *rinfo;
+
+	rb_ptr = (u64)phys_to_virt(rb_ptr);
+	/* Get buffer start address and alignment offset */
+	rinfo = GET_RBUF_INFO(rb_ptr);
+
+	/* Now build an skb to give to stack */
+	skb = build_skb(rinfo->data, RCV_FRAG_LEN);
+	if (!skb) {
+		put_page(rinfo->page);
+		return NULL;
+	}
+
+	/* Set correct skb->data */
+	skb_reserve(skb, rinfo->offset + NICVF_RCV_BUF_ALIGN_BYTES);
+
+	prefetch((void *)rb_ptr);
+	return skb;
+}
+
+/* Allocate RBDR ring and populate receive buffers */
+static int  nicvf_init_rbdr(struct nicvf *nic, struct rbdr *rbdr,
+			    int ring_len, int buf_size)
+{
+	int idx;
+	u64 *rbuf;
+	struct rbdr_entry_t *desc;
+	int err;
+
+	err = nicvf_alloc_q_desc_mem(nic, &rbdr->dmem, ring_len,
+				     sizeof(struct rbdr_entry_t),
+				     NICVF_RCV_BUF_ALIGN_BYTES);
+	if (err)
+		return err;
+
+	rbdr->desc = rbdr->dmem.base;
+	/* Buffer size has to be in multiples of 128 bytes */
+	rbdr->dma_size = buf_size;
+	rbdr->enable = true;
+	rbdr->thresh = RBDR_THRESH;
+
+	nic->rb_page = NULL;
+	for (idx = 0; idx < ring_len; idx++) {
+		err = nicvf_alloc_rcv_buffer(nic, GFP_KERNEL, RCV_FRAG_LEN,
+					     &rbuf);
+		if (err)
+			return err;
+
+		desc = GET_RBDR_DESC(rbdr, idx);
+		desc->buf_addr = virt_to_phys(rbuf) >> NICVF_RCV_BUF_ALIGN;
+	}
+	return 0;
+}
+
+/* Free RBDR ring and its receive buffers */
+static void nicvf_free_rbdr(struct nicvf *nic, struct rbdr *rbdr)
+{
+	int head, tail;
+	u64 buf_addr;
+	struct rbdr_entry_t *desc;
+	struct rbuf_info *rinfo;
+
+	if (!rbdr)
+		return;
+
+	rbdr->enable = false;
+	if (!rbdr->dmem.base)
+		return;
+
+	head = rbdr->head;
+	tail = rbdr->tail;
+
+	/* Free SKBs */
+	while (head != tail) {
+		desc = GET_RBDR_DESC(rbdr, head);
+		buf_addr = desc->buf_addr << NICVF_RCV_BUF_ALIGN;
+		rinfo = GET_RBUF_INFO((u64)phys_to_virt(buf_addr));
+		put_page(rinfo->page);
+		head++;
+		head &= (rbdr->dmem.q_len - 1);
+	}
+	/* Free SKB of tail desc */
+	desc = GET_RBDR_DESC(rbdr, tail);
+	buf_addr = desc->buf_addr << NICVF_RCV_BUF_ALIGN;
+	rinfo = GET_RBUF_INFO((u64)phys_to_virt(buf_addr));
+	put_page(rinfo->page);
+
+	/* Free RBDR ring */
+	nicvf_free_q_desc_mem(nic, &rbdr->dmem);
+}
+
+/* Refill receive buffer descriptors with new buffers.
+ */
+static void nicvf_refill_rbdr(struct nicvf *nic, gfp_t gfp)
+{
+	struct queue_set *qs = nic->qs;
+	int rbdr_idx = qs->rbdr_cnt;
+	int tail, qcount;
+	int refill_rb_cnt;
+	struct rbdr *rbdr;
+	struct rbdr_entry_t *desc;
+	u64 *rbuf;
+	int new_rb = 0;
+
+refill:
+	if (!rbdr_idx)
+		return;
+	rbdr_idx--;
+	rbdr = &qs->rbdr[rbdr_idx];
+	/* Check if it's enabled */
+	if (!rbdr->enable)
+		goto next_rbdr;
+
+	/* Get no of desc's to be refilled */
+	qcount = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_STATUS0, rbdr_idx);
+	qcount &= 0x7FFFF;
+	/* Doorbell can be ringed with a max of ring size minus 1 */
+	if (qcount >= (qs->rbdr_len - 1))
+		goto next_rbdr;
+	else
+		refill_rb_cnt = qs->rbdr_len - qcount - 1;
+
+	/* Start filling descs from tail */
+	tail = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_TAIL, rbdr_idx) >> 3;
+	while (refill_rb_cnt) {
+		tail++;
+		tail &= (rbdr->dmem.q_len - 1);
+
+		if (nicvf_alloc_rcv_buffer(nic, gfp, RCV_FRAG_LEN, &rbuf))
+			break;
+
+		desc = GET_RBDR_DESC(rbdr, tail);
+		desc->buf_addr = virt_to_phys(rbuf) >> NICVF_RCV_BUF_ALIGN;
+		refill_rb_cnt--;
+		new_rb++;
+	}
+
+	/* make sure all memory stores are done before ringing doorbell */
+	smp_wmb();
+
+	/* Check if buffer allocation failed */
+	if (refill_rb_cnt)
+		nic->rb_alloc_fail = true;
+	else
+		nic->rb_alloc_fail = false;
+
+	/* Notify HW */
+	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_DOOR,
+			      rbdr_idx, new_rb);
+next_rbdr:
+	/* Re-enable RBDR interrupts only if buffer allocation is success */
+	if (!nic->rb_alloc_fail && rbdr->enable)
+		nicvf_enable_intr(nic, NICVF_INTR_RBDR, rbdr_idx);
+
+	if (rbdr_idx)
+		goto refill;
+}
+
+/* Alloc rcv buffers in non-atomic mode for better success */
+void nicvf_rbdr_work(struct work_struct *work)
+{
+	struct nicvf *nic = container_of(work, struct nicvf, rbdr_work.work);
+
+	nicvf_refill_rbdr(nic, GFP_KERNEL);
+	if (nic->rb_alloc_fail)
+		schedule_delayed_work(&nic->rbdr_work, msecs_to_jiffies(10));
+	else
+		nic->rb_work_scheduled = false;
+}
+
+/* In Softirq context, alloc rcv buffers in atomic mode */
+void nicvf_rbdr_task(unsigned long data)
+{
+	struct nicvf *nic = (struct nicvf *)data;
+
+	nicvf_refill_rbdr(nic, GFP_ATOMIC);
+	if (nic->rb_alloc_fail) {
+		nic->rb_work_scheduled = true;
+		schedule_delayed_work(&nic->rbdr_work, msecs_to_jiffies(10));
+	}
+}
+
+/* Initialize completion queue */
+static int nicvf_init_cmp_queue(struct nicvf *nic,
+				struct cmp_queue *cq, int q_len)
+{
+	int err;
+
+	err = nicvf_alloc_q_desc_mem(nic, &cq->dmem, q_len, CMP_QUEUE_DESC_SIZE,
+				     NICVF_CQ_BASE_ALIGN_BYTES);
+	if (err)
+		return err;
+
+	cq->desc = cq->dmem.base;
+	cq->thresh = CMP_QUEUE_CQE_THRESH;
+	nic->cq_coalesce_usecs = (CMP_QUEUE_TIMER_THRESH * 0.05) - 1;
+
+	return 0;
+}
+
+static void nicvf_free_cmp_queue(struct nicvf *nic, struct cmp_queue *cq)
+{
+	if (!cq)
+		return;
+	if (!cq->dmem.base)
+		return;
+
+	nicvf_free_q_desc_mem(nic, &cq->dmem);
+}
+
+/* Initialize transmit queue */
+static int nicvf_init_snd_queue(struct nicvf *nic,
+				struct snd_queue *sq, int q_len)
+{
+	int err;
+
+	err = nicvf_alloc_q_desc_mem(nic, &sq->dmem, q_len, SND_QUEUE_DESC_SIZE,
+				     NICVF_SQ_BASE_ALIGN_BYTES);
+	if (err)
+		return err;
+
+	sq->desc = sq->dmem.base;
+	sq->skbuff = kcalloc(q_len, sizeof(u64), GFP_KERNEL);
+	if (!sq->skbuff)
+		return -ENOMEM;
+	sq->head = 0;
+	sq->tail = 0;
+	atomic_set(&sq->free_cnt, q_len - 1);
+	sq->thresh = SND_QUEUE_THRESH;
+
+	/* Preallocate memory for TSO segment's header */
+	sq->tso_hdrs = dma_alloc_coherent(&nic->pdev->dev,
+					  q_len * TSO_HEADER_SIZE,
+					  &sq->tso_hdrs_phys, GFP_KERNEL);
+	if (!sq->tso_hdrs)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void nicvf_free_snd_queue(struct nicvf *nic, struct snd_queue *sq)
+{
+	if (!sq)
+		return;
+	if (!sq->dmem.base)
+		return;
+
+	if (sq->tso_hdrs)
+		dma_free_coherent(&nic->pdev->dev,
+				  sq->dmem.q_len * TSO_HEADER_SIZE,
+				  sq->tso_hdrs, sq->tso_hdrs_phys);
+
+	kfree(sq->skbuff);
+	nicvf_free_q_desc_mem(nic, &sq->dmem);
+}
+
+static void nicvf_reclaim_snd_queue(struct nicvf *nic,
+				    struct queue_set *qs, int qidx)
+{
+	/* Disable send queue */
+	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, 0);
+	/* Check if SQ is stopped */
+	if (nicvf_poll_reg(nic, qidx, NIC_QSET_SQ_0_7_STATUS, 21, 1, 0x01))
+		return;
+	/* Reset send queue */
+	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, NICVF_SQ_RESET);
+}
+
+static void nicvf_reclaim_rcv_queue(struct nicvf *nic,
+				    struct queue_set *qs, int qidx)
+{
+	union nic_mbx mbx = {};
+
+	/* Make sure all packets in the pipeline are written back into mem */
+	mbx.msg.msg = NIC_MBOX_MSG_RQ_SW_SYNC;
+	nicvf_send_msg_to_pf(nic, &mbx);
+}
+
+static void nicvf_reclaim_cmp_queue(struct nicvf *nic,
+				    struct queue_set *qs, int qidx)
+{
+	/* Disable timer threshold (doesn't get reset upon CQ reset */
+	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG2, qidx, 0);
+	/* Disable completion queue */
+	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, 0);
+	/* Reset completion queue */
+	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, NICVF_CQ_RESET);
+}
+
+static void nicvf_reclaim_rbdr(struct nicvf *nic,
+			       struct rbdr *rbdr, int qidx)
+{
+	u64 tmp, fifo_state;
+	int timeout = 10;
+
+	/* Save head and tail pointers for feeing up buffers */
+	rbdr->head = nicvf_queue_reg_read(nic,
+					  NIC_QSET_RBDR_0_1_HEAD,
+					  qidx) >> 3;
+	rbdr->tail = nicvf_queue_reg_read(nic,
+					  NIC_QSET_RBDR_0_1_TAIL,
+					  qidx) >> 3;
+
+	/* If RBDR FIFO is in 'FAIL' state then do a reset first
+	 * before relaiming.
+	 */
+	fifo_state = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_STATUS0, qidx);
+	if (((fifo_state >> 62) & 0x03) == 0x3)
+		nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG,
+				      qidx, NICVF_RBDR_RESET);
+
+	/* Disable RBDR */
+	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx, 0);
+	if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x00))
+		return;
+	while (1) {
+		tmp = nicvf_queue_reg_read(nic,
+					   NIC_QSET_RBDR_0_1_PREFETCH_STATUS,
+					   qidx);
+		if ((tmp & 0xFFFFFFFF) == ((tmp >> 32) & 0xFFFFFFFF))
+			break;
+		usleep_range(1000, 2000);
+		timeout--;
+		if (!timeout) {
+			netdev_err(nic->netdev,
+				   "Failed polling on prefetch status\n");
+			return;
+		}
+	}
+	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG,
+			      qidx, NICVF_RBDR_RESET);
+
+	if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x02))
+		return;
+	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx, 0x00);
+	if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x00))
+		return;
+}
+
+void nicvf_config_vlan_stripping(struct nicvf *nic, netdev_features_t features)
+{
+	u64 rq_cfg;
+#ifdef VNIC_MULTI_QSET_SUPPORT
+	int sqs = 0;
+#endif
+
+	rq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_RQ_GEN_CFG, 0);
+
+	/* Enable first VLAN stripping */
+	if (features & NETIF_F_HW_VLAN_CTAG_RX)
+		rq_cfg |= (1ULL << 25);
+	else
+		rq_cfg &= ~(1ULL << 25);
+	nicvf_queue_reg_write(nic, NIC_QSET_RQ_GEN_CFG, 0, rq_cfg);
+
+#ifdef VNIC_MULTI_QSET_SUPPORT
+	/* Configure Secondary Qsets, if any */
+	for (sqs = 0; sqs < nic->sqs_count; sqs++)
+		if (nic->snicvf[sqs])
+			nicvf_queue_reg_write(nic->snicvf[sqs],
+					      NIC_QSET_RQ_GEN_CFG, 0, rq_cfg);
+#endif
+}
+
+/* Configures receive queue */
+static void nicvf_rcv_queue_config(struct nicvf *nic, struct queue_set *qs,
+				   int qidx, bool enable)
+{
+	union nic_mbx mbx = {};
+	struct rcv_queue *rq;
+	struct cmp_queue *cq;
+	struct rq_cfg rq_cfg;
+
+	rq = &qs->rq[qidx];
+	rq->enable = enable;
+
+	/* Disable receive queue */
+	nicvf_queue_reg_write(nic, NIC_QSET_RQ_0_7_CFG, qidx, 0);
+
+	if (!rq->enable) {
+		nicvf_reclaim_rcv_queue(nic, qs, qidx);
+		return;
+	}
+
+	rq->cq_qs = qs->vnic_id;
+	rq->cq_idx = qidx;
+	rq->start_rbdr_qs = qs->vnic_id;
+	rq->start_qs_rbdr_idx = qs->rbdr_cnt - 1;
+	rq->cont_rbdr_qs = qs->vnic_id;
+	rq->cont_qs_rbdr_idx = qs->rbdr_cnt - 1;
+	/* all writes of RBDR data to be loaded into L2 Cache as well*/
+	rq->caching = 1;
+
+	/* Send a mailbox msg to PF to config RQ */
+	mbx.rq.msg = NIC_MBOX_MSG_RQ_CFG;
+	mbx.rq.qs_num = qs->vnic_id;
+	mbx.rq.rq_num = qidx;
+	mbx.rq.cfg = (rq->caching << 26) | (rq->cq_qs << 19) |
+			  (rq->cq_idx << 16) | (rq->cont_rbdr_qs << 9) |
+			  (rq->cont_qs_rbdr_idx << 8) |
+			  (rq->start_rbdr_qs << 1) | (rq->start_qs_rbdr_idx);
+	nicvf_send_msg_to_pf(nic, &mbx);
+
+	mbx.rq.msg = NIC_MBOX_MSG_RQ_BP_CFG;
+	mbx.rq.cfg = (1ULL << 63) | (1ULL << 62) | (qs->vnic_id << 0);
+	nicvf_send_msg_to_pf(nic, &mbx);
+
+	/* RQ drop config
+	 * Enable CQ drop to reserve sufficient CQEs for all tx packets
+	 */
+	mbx.rq.msg = NIC_MBOX_MSG_RQ_DROP_CFG;
+	mbx.rq.cfg = (1ULL << 62) | (RQ_CQ_DROP << 8);
+	nicvf_send_msg_to_pf(nic, &mbx);
+
+	nicvf_queue_reg_write(nic, NIC_QSET_RQ_GEN_CFG, 0, 0x00);
+	if (!nic->sqs_mode)
+		nicvf_config_vlan_stripping(nic, nic->netdev->features);
+
+	/* Enable Receive queue */
+	rq_cfg.ena = 1;
+	rq_cfg.tcp_ena = 0;
+	nicvf_queue_reg_write(nic, NIC_QSET_RQ_0_7_CFG, qidx, *(u64 *)&rq_cfg);
+}
+
+/* Configures completion queue */
+void nicvf_cmp_queue_config(struct nicvf *nic, struct queue_set *qs,
+			    int qidx, bool enable)
+{
+	struct cmp_queue *cq;
+	struct cq_cfg cq_cfg;
+
+	cq = &qs->cq[qidx];
+	cq->enable = enable;
+
+	if (!cq->enable) {
+		nicvf_reclaim_cmp_queue(nic, qs, qidx);
+		return;
+	}
+
+	/* Reset completion queue */
+	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, NICVF_CQ_RESET);
+
+	if (!cq->enable)
+		return;
+
+	spin_lock_init(&cq->lock);
+	/* Set completion queue base address */
+	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_BASE,
+			      qidx, (u64)(cq->dmem.phys_base));
+
+	/* Enable Completion queue */
+	cq_cfg.ena = 1;
+	cq_cfg.reset = 0;
+	cq_cfg.caching = 0;
+	cq_cfg.qsize = CMP_QSIZE;
+	cq_cfg.avg_con = 0;
+	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, *(u64 *)&cq_cfg);
+
+	/* Set threshold value for interrupt generation */
+	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_THRESH, qidx, cq->thresh);
+	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG2,
+			      qidx, nic->cq_coalesce_usecs);
+}
+
+/* Configures transmit queue */
+static void nicvf_snd_queue_config(struct nicvf *nic, struct queue_set *qs,
+				   int qidx, bool enable)
+{
+	union nic_mbx mbx = {};
+	struct snd_queue *sq;
+	struct sq_cfg sq_cfg;
+
+	sq = &qs->sq[qidx];
+	sq->enable = enable;
+
+	if (!sq->enable) {
+		nicvf_reclaim_snd_queue(nic, qs, qidx);
+		return;
+	}
+
+	/* Reset send queue */
+	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, NICVF_SQ_RESET);
+
+	sq->cq_qs = qs->vnic_id;
+	sq->cq_idx = qidx;
+
+	/* Send a mailbox msg to PF to config SQ */
+	mbx.sq.msg = NIC_MBOX_MSG_SQ_CFG;
+	mbx.sq.qs_num = qs->vnic_id;
+	mbx.sq.sq_num = qidx;
+	mbx.sq.sqs_mode = nic->sqs_mode;
+	mbx.sq.cfg = (sq->cq_qs << 3) | sq->cq_idx;
+	nicvf_send_msg_to_pf(nic, &mbx);
+
+	/* Set queue base address */
+	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_BASE,
+			      qidx, (u64)(sq->dmem.phys_base));
+
+	/* Enable send queue  & set queue size */
+	sq_cfg.ena = 1;
+	sq_cfg.reset = 0;
+	sq_cfg.ldwb = 0;
+	sq_cfg.qsize = SND_QSIZE;
+	sq_cfg.tstmp_bgx_intf = 0;
+	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, *(u64 *)&sq_cfg);
+
+	/* Set threshold value for interrupt generation */
+	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_THRESH, qidx, sq->thresh);
+}
+
+/* Configures receive buffer descriptor ring */
+static void nicvf_rbdr_config(struct nicvf *nic, struct queue_set *qs,
+			      int qidx, bool enable)
+{
+	struct rbdr *rbdr;
+	struct rbdr_cfg rbdr_cfg;
+
+	rbdr = &qs->rbdr[qidx];
+	nicvf_reclaim_rbdr(nic, rbdr, qidx);
+	if (!enable)
+		return;
+
+	/* Set descriptor base address */
+	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_BASE,
+			      qidx, (u64)(rbdr->dmem.phys_base));
+
+	/* Enable RBDR  & set queue size */
+	/* Buffer size should be in multiples of 128 bytes */
+	rbdr_cfg.ena = 1;
+	rbdr_cfg.reset = 0;
+	rbdr_cfg.ldwb = 0;
+	rbdr_cfg.qsize = RBDR_SIZE;
+	rbdr_cfg.avg_con = 0;
+	rbdr_cfg.lines = rbdr->dma_size / 128;
+	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG,
+			      qidx, *(u64 *)&rbdr_cfg);
+
+	/* Notify HW */
+	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_DOOR,
+			      qidx, qs->rbdr_len - 1);
+
+	/* Set threshold value for interrupt generation */
+	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_THRESH,
+			      qidx, rbdr->thresh - 1);
+}
+
+/* Requests PF to assign and enable Qset */
+void nicvf_qset_config(struct nicvf *nic, bool enable)
+{
+	union nic_mbx mbx = {};
+	struct queue_set *qs = nic->qs;
+	struct qs_cfg *qs_cfg;
+
+	if (!qs) {
+		netdev_warn(nic->netdev,
+			    "Qset is still not allocated, don't init queues\n");
+		return;
+	}
+
+	qs->enable = enable;
+	qs->vnic_id = nic->vf_id;
+
+	/* Send a mailbox msg to PF to config Qset */
+	mbx.qs.msg = NIC_MBOX_MSG_QS_CFG;
+	mbx.qs.num = qs->vnic_id;
+#ifdef VNIC_MULTI_QSET_SUPPORT
+	mbx.qs.sqs_count = nic->sqs_count;
+#endif
+
+	mbx.qs.cfg = 0;
+	qs_cfg = (struct qs_cfg *)&mbx.qs.cfg;
+	if (qs->enable) {
+		qs_cfg->ena = 1;
+#ifdef __BIG_ENDIAN
+		qs_cfg->be = 1;
+#endif
+		qs_cfg->vnic = qs->vnic_id;
+	}
+	nicvf_send_msg_to_pf(nic, &mbx);
+}
+
+static void nicvf_free_resources(struct nicvf *nic)
+{
+	int qidx;
+	struct queue_set *qs = nic->qs;
+
+	/* Free receive buffer descriptor ring */
+	for (qidx = 0; qidx < qs->rbdr_cnt; qidx++)
+		nicvf_free_rbdr(nic, &qs->rbdr[qidx]);
+
+	/* Free completion queue */
+	for (qidx = 0; qidx < qs->cq_cnt; qidx++)
+		nicvf_free_cmp_queue(nic, &qs->cq[qidx]);
+
+	/* Free send queue */
+	for (qidx = 0; qidx < qs->sq_cnt; qidx++)
+		nicvf_free_snd_queue(nic, &qs->sq[qidx]);
+}
+
+static int nicvf_alloc_resources(struct nicvf *nic)
+{
+	int qidx;
+	struct queue_set *qs = nic->qs;
+
+	/* Alloc receive buffer descriptor ring */
+	for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) {
+		if (nicvf_init_rbdr(nic, &qs->rbdr[qidx], qs->rbdr_len,
+				    DMA_BUFFER_LEN))
+			goto alloc_fail;
+	}
+
+	/* Alloc send queue */
+	for (qidx = 0; qidx < qs->sq_cnt; qidx++) {
+		if (nicvf_init_snd_queue(nic, &qs->sq[qidx], qs->sq_len))
+			goto alloc_fail;
+	}
+
+	/* Alloc completion queue */
+	for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
+		if (nicvf_init_cmp_queue(nic, &qs->cq[qidx], qs->cq_len))
+			goto alloc_fail;
+	}
+
+	return 0;
+alloc_fail:
+	nicvf_free_resources(nic);
+	return -ENOMEM;
+}
+
+int nicvf_set_qset_resources(struct nicvf *nic)
+{
+	struct queue_set *qs;
+
+	qs = devm_kzalloc(&nic->pdev->dev, sizeof(*qs), GFP_KERNEL);
+	if (!qs)
+		return -ENOMEM;
+	nic->qs = qs;
+
+	/* Set count of each queue */
+	qs->rbdr_cnt = RBDR_CNT;
+#ifdef VNIC_RSS_SUPPORT
+	qs->rq_cnt = RCV_QUEUE_CNT;
+#else
+	qs->rq_cnt = 1;
+#endif
+	qs->sq_cnt = SND_QUEUE_CNT;
+	qs->cq_cnt = CMP_QUEUE_CNT;
+
+	/* Set queue lengths */
+	qs->rbdr_len = RCV_BUF_COUNT;
+	qs->sq_len = SND_QUEUE_LEN;
+	qs->cq_len = CMP_QUEUE_LEN;
+
+	nic->rx_queues = qs->rq_cnt;
+	nic->tx_queues = qs->sq_cnt;
+
+	return 0;
+}
+
+int nicvf_config_data_transfer(struct nicvf *nic, bool enable)
+{
+	bool disable = false;
+	struct queue_set *qs = nic->qs;
+	int qidx;
+
+	if (!qs)
+		return 0;
+
+	if (enable) {
+		if (nicvf_alloc_resources(nic))
+			return -ENOMEM;
+
+		for (qidx = 0; qidx < qs->sq_cnt; qidx++)
+			nicvf_snd_queue_config(nic, qs, qidx, enable);
+		for (qidx = 0; qidx < qs->cq_cnt; qidx++)
+			nicvf_cmp_queue_config(nic, qs, qidx, enable);
+		for (qidx = 0; qidx < qs->rbdr_cnt; qidx++)
+			nicvf_rbdr_config(nic, qs, qidx, enable);
+		for (qidx = 0; qidx < qs->rq_cnt; qidx++)
+			nicvf_rcv_queue_config(nic, qs, qidx, enable);
+	} else {
+		for (qidx = 0; qidx < qs->rq_cnt; qidx++)
+			nicvf_rcv_queue_config(nic, qs, qidx, disable);
+		for (qidx = 0; qidx < qs->rbdr_cnt; qidx++)
+			nicvf_rbdr_config(nic, qs, qidx, disable);
+		for (qidx = 0; qidx < qs->sq_cnt; qidx++)
+			nicvf_snd_queue_config(nic, qs, qidx, disable);
+		for (qidx = 0; qidx < qs->cq_cnt; qidx++)
+			nicvf_cmp_queue_config(nic, qs, qidx, disable);
+
+		nicvf_free_resources(nic);
+	}
+
+	return 0;
+}
+
+/* Get a free desc from SQ
+ * returns descriptor ponter & descriptor number
+ */
+static inline int nicvf_get_sq_desc(struct snd_queue *sq, int desc_cnt)
+{
+	int qentry;
+
+	qentry = sq->tail;
+	atomic_sub(desc_cnt, &sq->free_cnt);
+	sq->tail += desc_cnt;
+	sq->tail &= (sq->dmem.q_len - 1);
+
+	return qentry;
+}
+
+/* Free descriptor back to SQ for future use */
+void nicvf_put_sq_desc(struct snd_queue *sq, int desc_cnt)
+{
+	atomic_add(desc_cnt, &sq->free_cnt);
+	sq->head += desc_cnt;
+	sq->head &= (sq->dmem.q_len - 1);
+}
+
+static inline int nicvf_get_nxt_sqentry(struct snd_queue *sq, int qentry)
+{
+	qentry++;
+	qentry &= (sq->dmem.q_len - 1);
+	return qentry;
+}
+
+void nicvf_sq_enable(struct nicvf *nic, struct snd_queue *sq, int qidx)
+{
+	u64 sq_cfg;
+
+	sq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_CFG, qidx);
+	sq_cfg |= NICVF_SQ_EN;
+	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, sq_cfg);
+	/* Ring doorbell so that H/W restarts processing SQEs */
+	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR, qidx, 0);
+}
+
+void nicvf_sq_disable(struct nicvf *nic, int qidx)
+{
+	u64 sq_cfg;
+
+	sq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_CFG, qidx);
+	sq_cfg &= ~NICVF_SQ_EN;
+	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, sq_cfg);
+}
+
+void nicvf_sq_free_used_descs(struct net_device *netdev, struct snd_queue *sq,
+			      int qidx)
+{
+	u64 head, tail;
+	struct sk_buff *skb;
+	struct nicvf *nic = netdev_priv(netdev);
+	struct sq_hdr_subdesc *hdr;
+
+	head = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_HEAD, qidx) >> 4;
+	tail = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_TAIL, qidx) >> 4;
+	while (sq->head != head) {
+		hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, sq->head);
+		if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER) {
+			nicvf_put_sq_desc(sq, 1);
+			continue;
+		}
+		skb = (struct sk_buff *)sq->skbuff[sq->head];
+		if (skb)
+			dev_kfree_skb_any(skb);
+		atomic64_add(1, (atomic64_t *)&netdev->stats.tx_packets);
+		atomic64_add(hdr->tot_len,
+			     (atomic64_t *)&netdev->stats.tx_bytes);
+		nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1);
+	}
+}
+
+/* Get the number of SQ descriptors needed to xmit this skb */
+static int nicvf_sq_subdesc_required(struct nicvf *nic, struct sk_buff *skb)
+{
+	int subdesc_cnt = MIN_SQ_DESC_PER_PKT_XMIT;
+
+	if (skb_shinfo(skb)->gso_size) {
+		subdesc_cnt = nicvf_tso_count_subdescs(skb);
+		return subdesc_cnt;
+	}
+
+	if (skb_shinfo(skb)->nr_frags)
+		subdesc_cnt += skb_shinfo(skb)->nr_frags;
+
+	return subdesc_cnt;
+}
+
+/* Add SQ HEADER subdescriptor.
+ * First subdescriptor for every send descriptor.
+ */
+static inline void
+nicvf_sq_add_hdr_subdesc(struct snd_queue *sq, int qentry,
+			 int subdesc_cnt, struct sk_buff *skb, int len)
+{
+	int proto;
+	struct sq_hdr_subdesc *hdr;
+
+	hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, qentry);
+	sq->skbuff[qentry] = (u64)skb;
+
+	memset(hdr, 0, SND_QUEUE_DESC_SIZE);
+	hdr->subdesc_type = SQ_DESC_TYPE_HEADER;
+	/* Enable notification via CQE after processing SQE */
+	hdr->post_cqe = 1;
+	/* No of subdescriptors following this */
+	hdr->subdesc_cnt = subdesc_cnt;
+	hdr->tot_len = len;
+
+	/* Offload checksum calculation to HW */
+	if (skb->ip_summed == CHECKSUM_PARTIAL) {
+		hdr->csum_l3 = 1; /* Enable IP csum calculation */
+		hdr->l3_offset = skb_network_offset(skb);
+		hdr->l4_offset = skb_transport_offset(skb);
+
+		proto = ip_hdr(skb)->protocol;
+		switch (proto) {
+		case IPPROTO_TCP:
+			hdr->csum_l4 = SEND_L4_CSUM_TCP;
+			break;
+		case IPPROTO_UDP:
+			hdr->csum_l4 = SEND_L4_CSUM_UDP;
+			break;
+		case IPPROTO_SCTP:
+			hdr->csum_l4 = SEND_L4_CSUM_SCTP;
+			break;
+		}
+	}
+}
+
+/* SQ GATHER subdescriptor
+ * Must follow HDR descriptor
+ */
+static inline void nicvf_sq_add_gather_subdesc(struct snd_queue *sq, int qentry,
+					       int size, u64 data)
+{
+	struct sq_gather_subdesc *gather;
+
+	qentry &= (sq->dmem.q_len - 1);
+	gather = (struct sq_gather_subdesc *)GET_SQ_DESC(sq, qentry);
+
+	memset(gather, 0, SND_QUEUE_DESC_SIZE);
+	gather->subdesc_type = SQ_DESC_TYPE_GATHER;
+	gather->ld_type = NIC_SEND_LD_TYPE_E_LDD;
+	gather->size = size;
+	gather->addr = data;
+}
+
+/* Append an skb to a SQ for packet transfer. */
+int nicvf_sq_append_skb(struct nicvf *nic, struct sk_buff *skb)
+{
+	int i, size;
+	int subdesc_cnt;
+	int sq_num, qentry;
+	struct queue_set *qs;
+	struct snd_queue *sq;
+
+	sq_num = skb_get_queue_mapping(skb);
+#ifdef VNIC_MULTI_QSET_SUPPORT
+	if (sq_num >= MAX_SND_QUEUES_PER_QS) {
+		/* Get secondary Qset's SQ structure */
+		i = sq_num / MAX_SND_QUEUES_PER_QS;
+		if (!nic->snicvf[i - 1]) {
+			netdev_warn(nic->netdev,
+				    "Secondary Qset#%d's ptr not initialized\n",
+				    i - 1);
+			return 1;
+		}
+		nic = (struct nicvf *)nic->snicvf[i - 1];
+		sq_num = sq_num % MAX_SND_QUEUES_PER_QS;
+	}
+#endif
+
+	qs = nic->qs;
+	sq = &qs->sq[sq_num];
+
+	subdesc_cnt = nicvf_sq_subdesc_required(nic, skb);
+	if (subdesc_cnt > atomic_read(&sq->free_cnt))
+		goto append_fail;
+
+	qentry = nicvf_get_sq_desc(sq, subdesc_cnt);
+
+	/* Add SQ header subdesc */
+	nicvf_sq_add_hdr_subdesc(sq, qentry, subdesc_cnt - 1, skb, skb->len);
+
+	/* Add SQ gather subdescs */
+	qentry = nicvf_get_nxt_sqentry(sq, qentry);
+	size = skb_is_nonlinear(skb) ? skb_headlen(skb) : skb->len;
+	nicvf_sq_add_gather_subdesc(sq, qentry, size, virt_to_phys(skb->data));
+
+	/* Check for scattered buffer */
+	if (!skb_is_nonlinear(skb))
+		goto doorbell;
+
+	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+		const struct skb_frag_struct *frag;
+
+		frag = &skb_shinfo(skb)->frags[i];
+
+		qentry = nicvf_get_nxt_sqentry(sq, qentry);
+		size = skb_frag_size(frag);
+		nicvf_sq_add_gather_subdesc(sq, qentry, size,
+					    virt_to_phys(
+					    skb_frag_address(frag)));
+	}
+
+doorbell:
+	/* make sure all memory stores are done before ringing doorbell */
+	smp_wmb();
+
+	/* Inform HW to xmit new packet */
+	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR,
+			      sq_num, subdesc_cnt);
+	return 1;
+
+append_fail:
+	/* Use original PCI dev for debug log */
+	nic = nic->pnicvf;
+	netdev_dbg(nic->netdev, "Not enough SQ descriptors to xmit pkt\n");
+	return 0;
+}
+
+static inline unsigned frag_num(unsigned i)
+{
+#ifdef __BIG_ENDIAN
+	return (i & ~3) + 3 - (i & 3);
+#else
+	return i;
+#endif
+}
+
+/* Returns SKB for a received packet */
+struct sk_buff *nicvf_get_rcv_skb(struct nicvf *nic, struct cqe_rx_t *cqe_rx)
+{
+	int frag;
+	int payload_len = 0;
+	struct sk_buff *skb = NULL;
+	struct sk_buff *skb_frag = NULL;
+	struct sk_buff *prev_frag = NULL;
+	u16 *rb_lens = NULL;
+	u64 *rb_ptrs = NULL;
+
+	rb_lens = (void *)cqe_rx + (3 * sizeof(u64));
+	rb_ptrs = (void *)cqe_rx + (6 * sizeof(u64));
+
+	netdev_dbg(nic->netdev, "%s rb_cnt %d rb0_ptr %llx rb0_sz %d\n",
+		   __func__, cqe_rx->rb_cnt, cqe_rx->rb0_ptr, cqe_rx->rb0_sz);
+
+	for (frag = 0; frag < cqe_rx->rb_cnt; frag++) {
+		payload_len = rb_lens[frag_num(frag)];
+		if (!frag) {
+			/* First fragment */
+			skb = nicvf_rb_ptr_to_skb(nic,
+						  *rb_ptrs - cqe_rx->align_pad,
+						  payload_len);
+			if (!skb)
+				return NULL;
+			skb_reserve(skb, cqe_rx->align_pad);
+			skb_put(skb, payload_len);
+		} else {
+			/* Add fragments */
+			skb_frag = nicvf_rb_ptr_to_skb(nic, *rb_ptrs,
+						       payload_len);
+			if (!skb_frag) {
+				dev_kfree_skb(skb);
+				return NULL;
+			}
+
+			if (!skb_shinfo(skb)->frag_list)
+				skb_shinfo(skb)->frag_list = skb_frag;
+			else
+				prev_frag->next = skb_frag;
+
+			prev_frag = skb_frag;
+			skb->len += payload_len;
+			skb->data_len += payload_len;
+			skb_frag->len = payload_len;
+		}
+		/* Next buffer pointer */
+		rb_ptrs++;
+	}
+	return skb;
+}
+
+/* Enable interrupt */
+void nicvf_enable_intr(struct nicvf *nic, int int_type, int q_idx)
+{
+	u64 reg_val;
+
+	reg_val = nicvf_reg_read(nic, NIC_VF_ENA_W1S);
+
+	switch (int_type) {
+	case NICVF_INTR_CQ:
+		reg_val |= ((1ULL << q_idx) << NICVF_INTR_CQ_SHIFT);
+		break;
+	case NICVF_INTR_SQ:
+		reg_val |= ((1ULL << q_idx) << NICVF_INTR_SQ_SHIFT);
+		break;
+	case NICVF_INTR_RBDR:
+		reg_val |= ((1ULL << q_idx) << NICVF_INTR_RBDR_SHIFT);
+		break;
+	case NICVF_INTR_PKT_DROP:
+		reg_val |= (1ULL << NICVF_INTR_PKT_DROP_SHIFT);
+		break;
+	case NICVF_INTR_TCP_TIMER:
+		reg_val |= (1ULL << NICVF_INTR_TCP_TIMER_SHIFT);
+		break;
+	case NICVF_INTR_MBOX:
+		reg_val |= (1ULL << NICVF_INTR_MBOX_SHIFT);
+		break;
+	case NICVF_INTR_QS_ERR:
+		reg_val |= (1ULL << NICVF_INTR_QS_ERR_SHIFT);
+		break;
+	default:
+		netdev_err(nic->netdev,
+			   "Failed to enable interrupt: unknown type\n");
+		break;
+	}
+
+	nicvf_reg_write(nic, NIC_VF_ENA_W1S, reg_val);
+}
+
+/* Disable interrupt */
+void nicvf_disable_intr(struct nicvf *nic, int int_type, int q_idx)
+{
+	u64 reg_val = 0;
+
+	switch (int_type) {
+	case NICVF_INTR_CQ:
+		reg_val |= ((1ULL << q_idx) << NICVF_INTR_CQ_SHIFT);
+		break;
+	case NICVF_INTR_SQ:
+		reg_val |= ((1ULL << q_idx) << NICVF_INTR_SQ_SHIFT);
+		break;
+	case NICVF_INTR_RBDR:
+		reg_val |= ((1ULL << q_idx) << NICVF_INTR_RBDR_SHIFT);
+		break;
+	case NICVF_INTR_PKT_DROP:
+		reg_val |= (1ULL << NICVF_INTR_PKT_DROP_SHIFT);
+		break;
+	case NICVF_INTR_TCP_TIMER:
+		reg_val |= (1ULL << NICVF_INTR_TCP_TIMER_SHIFT);
+		break;
+	case NICVF_INTR_MBOX:
+		reg_val |= (1ULL << NICVF_INTR_MBOX_SHIFT);
+		break;
+	case NICVF_INTR_QS_ERR:
+		reg_val |= (1ULL << NICVF_INTR_QS_ERR_SHIFT);
+		break;
+	default:
+		netdev_err(nic->netdev,
+			   "Failed to disable interrupt: unknown type\n");
+		break;
+	}
+
+	nicvf_reg_write(nic, NIC_VF_ENA_W1C, reg_val);
+}
+
+/* Clear interrupt */
+void nicvf_clear_intr(struct nicvf *nic, int int_type, int q_idx)
+{
+	u64 reg_val = 0;
+
+	switch (int_type) {
+	case NICVF_INTR_CQ:
+		reg_val = ((1ULL << q_idx) << NICVF_INTR_CQ_SHIFT);
+		break;
+	case NICVF_INTR_SQ:
+		reg_val = ((1ULL << q_idx) << NICVF_INTR_SQ_SHIFT);
+		break;
+	case NICVF_INTR_RBDR:
+		reg_val = ((1ULL << q_idx) << NICVF_INTR_RBDR_SHIFT);
+		break;
+	case NICVF_INTR_PKT_DROP:
+		reg_val = (1ULL << NICVF_INTR_PKT_DROP_SHIFT);
+		break;
+	case NICVF_INTR_TCP_TIMER:
+		reg_val = (1ULL << NICVF_INTR_TCP_TIMER_SHIFT);
+		break;
+	case NICVF_INTR_MBOX:
+		reg_val = (1ULL << NICVF_INTR_MBOX_SHIFT);
+		break;
+	case NICVF_INTR_QS_ERR:
+		reg_val |= (1ULL << NICVF_INTR_QS_ERR_SHIFT);
+		break;
+	default:
+		netdev_err(nic->netdev,
+			   "Failed to clear interrupt: unknown type\n");
+		break;
+	}
+
+	nicvf_reg_write(nic, NIC_VF_INT, reg_val);
+}
+
+/* Check if interrupt is enabled */
+int nicvf_is_intr_enabled(struct nicvf *nic, int int_type, int q_idx)
+{
+	u64 reg_val;
+	u64 mask = 0xff;
+
+	reg_val = nicvf_reg_read(nic, NIC_VF_ENA_W1S);
+
+	switch (int_type) {
+	case NICVF_INTR_CQ:
+		mask = ((1ULL << q_idx) << NICVF_INTR_CQ_SHIFT);
+		break;
+	case NICVF_INTR_SQ:
+		mask = ((1ULL << q_idx) << NICVF_INTR_SQ_SHIFT);
+		break;
+	case NICVF_INTR_RBDR:
+		mask = ((1ULL << q_idx) << NICVF_INTR_RBDR_SHIFT);
+		break;
+	case NICVF_INTR_PKT_DROP:
+		mask = NICVF_INTR_PKT_DROP_MASK;
+		break;
+	case NICVF_INTR_TCP_TIMER:
+		mask = NICVF_INTR_TCP_TIMER_MASK;
+		break;
+	case NICVF_INTR_MBOX:
+		mask = NICVF_INTR_MBOX_MASK;
+		break;
+	case NICVF_INTR_QS_ERR:
+		mask = NICVF_INTR_QS_ERR_MASK;
+		break;
+	default:
+		netdev_err(nic->netdev,
+			   "Failed to check interrupt enable: unknown type\n");
+		break;
+	}
+
+	return (reg_val & mask);
+}
+
+void nicvf_update_rq_stats(struct nicvf *nic, int rq_idx)
+{
+	struct rcv_queue *rq;
+
+#define GET_RQ_STATS(reg) \
+	nicvf_reg_read(nic, NIC_QSET_RQ_0_7_STAT_0_1 |\
+			    (rq_idx << NIC_Q_NUM_SHIFT) | (reg << 3))
+
+	rq = &nic->qs->rq[rq_idx];
+	rq->stats.bytes = GET_RQ_STATS(RQ_SQ_STATS_OCTS);
+	rq->stats.pkts = GET_RQ_STATS(RQ_SQ_STATS_PKTS);
+}
+
+void nicvf_update_sq_stats(struct nicvf *nic, int sq_idx)
+{
+	struct snd_queue *sq;
+
+#define GET_SQ_STATS(reg) \
+	nicvf_reg_read(nic, NIC_QSET_SQ_0_7_STAT_0_1 |\
+			    (sq_idx << NIC_Q_NUM_SHIFT) | (reg << 3))
+
+	sq = &nic->qs->sq[sq_idx];
+	sq->stats.bytes = GET_SQ_STATS(RQ_SQ_STATS_OCTS);
+	sq->stats.pkts = GET_SQ_STATS(RQ_SQ_STATS_PKTS);
+}
+
+/* Check for errors in the receive cmp.queue entry */
+int nicvf_check_cqe_rx_errs(struct nicvf *nic,
+			    struct cmp_queue *cq, struct cqe_rx_t *cqe_rx)
+{
+	struct nicvf_hw_stats *stats = &nic->hw_stats;
+	struct nicvf_drv_stats *drv_stats = &nic->drv_stats;
+
+	if (!cqe_rx->err_level && !cqe_rx->err_opcode) {
+		drv_stats->rx_frames_ok++;
+		return 0;
+	}
+
+	if (netif_msg_rx_err(nic))
+		netdev_err(nic->netdev,
+			   "%s: RX error CQE err_level 0x%x err_opcode 0x%x\n",
+			   nic->netdev->name,
+			   cqe_rx->err_level, cqe_rx->err_opcode);
+
+	switch (cqe_rx->err_opcode) {
+	case CQ_RX_ERROP_RE_PARTIAL:
+		stats->rx_bgx_truncated_pkts++;
+		break;
+	case CQ_RX_ERROP_RE_JABBER:
+		stats->rx_jabber_errs++;
+		break;
+	case CQ_RX_ERROP_RE_FCS:
+		stats->rx_fcs_errs++;
+		break;
+	case CQ_RX_ERROP_RE_RX_CTL:
+		stats->rx_bgx_errs++;
+		break;
+	case CQ_RX_ERROP_PREL2_ERR:
+		stats->rx_prel2_errs++;
+		break;
+	case CQ_RX_ERROP_L2_MAL:
+		stats->rx_l2_hdr_malformed++;
+		break;
+	case CQ_RX_ERROP_L2_OVERSIZE:
+		stats->rx_oversize++;
+		break;
+	case CQ_RX_ERROP_L2_UNDERSIZE:
+		stats->rx_undersize++;
+		break;
+	case CQ_RX_ERROP_L2_LENMISM:
+		stats->rx_l2_len_mismatch++;
+		break;
+	case CQ_RX_ERROP_L2_PCLP:
+		stats->rx_l2_pclp++;
+		break;
+	case CQ_RX_ERROP_IP_NOT:
+		stats->rx_ip_ver_errs++;
+		break;
+	case CQ_RX_ERROP_IP_CSUM_ERR:
+		stats->rx_ip_csum_errs++;
+		break;
+	case CQ_RX_ERROP_IP_MAL:
+		stats->rx_ip_hdr_malformed++;
+		break;
+	case CQ_RX_ERROP_IP_MALD:
+		stats->rx_ip_payload_malformed++;
+		break;
+	case CQ_RX_ERROP_IP_HOP:
+		stats->rx_ip_ttl_errs++;
+		break;
+	case CQ_RX_ERROP_L3_PCLP:
+		stats->rx_l3_pclp++;
+		break;
+	case CQ_RX_ERROP_L4_MAL:
+		stats->rx_l4_malformed++;
+		break;
+	case CQ_RX_ERROP_L4_CHK:
+		stats->rx_l4_csum_errs++;
+		break;
+	case CQ_RX_ERROP_UDP_LEN:
+		stats->rx_udp_len_errs++;
+		break;
+	case CQ_RX_ERROP_L4_PORT:
+		stats->rx_l4_port_errs++;
+		break;
+	case CQ_RX_ERROP_TCP_FLAG:
+		stats->rx_tcp_flag_errs++;
+		break;
+	case CQ_RX_ERROP_TCP_OFFSET:
+		stats->rx_tcp_offset_errs++;
+		break;
+	case CQ_RX_ERROP_L4_PCLP:
+		stats->rx_l4_pclp++;
+		break;
+	case CQ_RX_ERROP_RBDR_TRUNC:
+		stats->rx_truncated_pkts++;
+		break;
+	}
+
+	return 1;
+}
+
+/* Check for errors in the send cmp.queue entry */
+int nicvf_check_cqe_tx_errs(struct nicvf *nic,
+			    struct cmp_queue *cq, struct cqe_send_t *cqe_tx)
+{
+	struct cmp_queue_stats *stats = &cq->stats;
+
+	switch (cqe_tx->send_status) {
+	case CQ_TX_ERROP_GOOD:
+		stats->tx.good++;
+		return 0;
+	case CQ_TX_ERROP_DESC_FAULT:
+		stats->tx.desc_fault++;
+		break;
+	case CQ_TX_ERROP_HDR_CONS_ERR:
+		stats->tx.hdr_cons_err++;
+		break;
+	case CQ_TX_ERROP_SUBDC_ERR:
+		stats->tx.subdesc_err++;
+		break;
+	case CQ_TX_ERROP_IMM_SIZE_OFLOW:
+		stats->tx.imm_size_oflow++;
+		break;
+	case CQ_TX_ERROP_DATA_SEQUENCE_ERR:
+		stats->tx.data_seq_err++;
+		break;
+	case CQ_TX_ERROP_MEM_SEQUENCE_ERR:
+		stats->tx.mem_seq_err++;
+		break;
+	case CQ_TX_ERROP_LOCK_VIOL:
+		stats->tx.lock_viol++;
+		break;
+	case CQ_TX_ERROP_DATA_FAULT:
+		stats->tx.data_fault++;
+		break;
+	case CQ_TX_ERROP_TSTMP_CONFLICT:
+		stats->tx.tstmp_conflict++;
+		break;
+	case CQ_TX_ERROP_TSTMP_TIMEOUT:
+		stats->tx.tstmp_timeout++;
+		break;
+	case CQ_TX_ERROP_MEM_FAULT:
+		stats->tx.mem_fault++;
+		break;
+	case CQ_TX_ERROP_CK_OVERLAP:
+		stats->tx.csum_overlap++;
+		break;
+	case CQ_TX_ERROP_CK_OFLOW:
+		stats->tx.csum_overflow++;
+		break;
+	}
+
+	return 1;
+}
diff --git a/sys/dev/vnic/nicvf_queues.h b/sys/dev/vnic/nicvf_queues.h
new file mode 100644
index 000000000000..d5842e928453
--- /dev/null
+++ b/sys/dev/vnic/nicvf_queues.h
@@ -0,0 +1,366 @@
+/*
+ * Copyright (C) 2015 Cavium Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ *
+ */
+
+#ifndef NICVF_QUEUES_H
+#define NICVF_QUEUES_H
+
+#include <linux/netdevice.h>
+#include "q_struct.h"
+
+#define MAX_QUEUE_SET			128
+#define MAX_RCV_QUEUES_PER_QS		8
+#define MAX_RCV_BUF_DESC_RINGS_PER_QS	2
+#define MAX_SND_QUEUES_PER_QS		8
+#define MAX_CMP_QUEUES_PER_QS		8
+
+/* VF's queue interrupt ranges */
+#define	NICVF_INTR_ID_CQ		0
+#define	NICVF_INTR_ID_SQ		8
+#define	NICVF_INTR_ID_RBDR		16
+#define	NICVF_INTR_ID_MISC		18
+#define	NICVF_INTR_ID_QS_ERR		19
+
+#define	for_each_cq_irq(irq)	\
+	for (irq = NICVF_INTR_ID_CQ; irq < NICVF_INTR_ID_SQ; irq++)
+#define	for_each_sq_irq(irq)	\
+	for (irq = NICVF_INTR_ID_SQ; irq < NICVF_INTR_ID_RBDR; irq++)
+#define	for_each_rbdr_irq(irq)	\
+	for (irq = NICVF_INTR_ID_RBDR; irq < NICVF_INTR_ID_MISC; irq++)
+
+#define RBDR_SIZE0		0ULL /* 8K entries */
+#define RBDR_SIZE1		1ULL /* 16K entries */
+#define RBDR_SIZE2		2ULL /* 32K entries */
+#define RBDR_SIZE3		3ULL /* 64K entries */
+#define RBDR_SIZE4		4ULL /* 126K entries */
+#define RBDR_SIZE5		5ULL /* 256K entries */
+#define RBDR_SIZE6		6ULL /* 512K entries */
+
+#define SND_QUEUE_SIZE0		0ULL /* 1K entries */
+#define SND_QUEUE_SIZE1		1ULL /* 2K entries */
+#define SND_QUEUE_SIZE2		2ULL /* 4K entries */
+#define SND_QUEUE_SIZE3		3ULL /* 8K entries */
+#define SND_QUEUE_SIZE4		4ULL /* 16K entries */
+#define SND_QUEUE_SIZE5		5ULL /* 32K entries */
+#define SND_QUEUE_SIZE6		6ULL /* 64K entries */
+
+#define CMP_QUEUE_SIZE0		0ULL /* 1K entries */
+#define CMP_QUEUE_SIZE1		1ULL /* 2K entries */
+#define CMP_QUEUE_SIZE2		2ULL /* 4K entries */
+#define CMP_QUEUE_SIZE3		3ULL /* 8K entries */
+#define CMP_QUEUE_SIZE4		4ULL /* 16K entries */
+#define CMP_QUEUE_SIZE5		5ULL /* 32K entries */
+#define CMP_QUEUE_SIZE6		6ULL /* 64K entries */
+
+/* Default queue count per QS, its lengths and threshold values */
+#define RBDR_CNT		1
+#define RCV_QUEUE_CNT		8
+#define SND_QUEUE_CNT		8
+#define CMP_QUEUE_CNT		8 /* Max of RCV and SND qcount */
+
+#define SND_QSIZE		SND_QUEUE_SIZE2
+#define SND_QUEUE_LEN		(1ULL << (SND_QSIZE + 10))
+#define MAX_SND_QUEUE_LEN	(1ULL << (SND_QUEUE_SIZE6 + 10))
+#define SND_QUEUE_THRESH	2ULL
+#define MIN_SQ_DESC_PER_PKT_XMIT	2
+/* Since timestamp not enabled, otherwise 2 */
+#define MAX_CQE_PER_PKT_XMIT		1
+
+/* Keep CQ and SQ sizes same, if timestamping
+ * is enabled this equation will change.
+ */
+#define CMP_QSIZE		CMP_QUEUE_SIZE2
+#define CMP_QUEUE_LEN		(1ULL << (CMP_QSIZE + 10))
+#define CMP_QUEUE_CQE_THRESH	0
+#define CMP_QUEUE_TIMER_THRESH	220 /* 10usec */
+
+#define RBDR_SIZE		RBDR_SIZE0
+#define RCV_BUF_COUNT		(1ULL << (RBDR_SIZE + 13))
+#define MAX_RCV_BUF_COUNT	(1ULL << (RBDR_SIZE6 + 13))
+#define RBDR_THRESH		(RCV_BUF_COUNT / 2)
+#define DMA_BUFFER_LEN		2048 /* In multiples of 128bytes */
+#define RCV_FRAG_LEN	(SKB_DATA_ALIGN(DMA_BUFFER_LEN + NET_SKB_PAD) + \
+			 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + \
+			 (NICVF_RCV_BUF_ALIGN_BYTES * 2))
+#define RCV_DATA_OFFSET		NICVF_RCV_BUF_ALIGN_BYTES
+
+#define MAX_CQES_FOR_TX		((SND_QUEUE_LEN / MIN_SQ_DESC_PER_PKT_XMIT) * \
+				 MAX_CQE_PER_PKT_XMIT)
+/* Calculate number of CQEs to reserve for all SQEs.
+ * Its 1/256th level of CQ size.
+ * '+ 1' to account for pipelining
+ */
+#define RQ_CQ_DROP		((256 / (CMP_QUEUE_LEN / \
+				 (CMP_QUEUE_LEN - MAX_CQES_FOR_TX))) + 1)
+
+/* Descriptor size in bytes */
+#define SND_QUEUE_DESC_SIZE	16
+#define CMP_QUEUE_DESC_SIZE	512
+
+/* Buffer / descriptor alignments */
+#define NICVF_RCV_BUF_ALIGN		7
+#define NICVF_RCV_BUF_ALIGN_BYTES	(1ULL << NICVF_RCV_BUF_ALIGN)
+#define NICVF_CQ_BASE_ALIGN_BYTES	512  /* 9 bits */
+#define NICVF_SQ_BASE_ALIGN_BYTES	128  /* 7 bits */
+
+#define NICVF_ALIGNED_ADDR(ADDR, ALIGN_BYTES)	ALIGN(ADDR, ALIGN_BYTES)
+#define NICVF_ADDR_ALIGN_LEN(ADDR, BYTES)\
+	(NICVF_ALIGNED_ADDR(ADDR, BYTES) - BYTES)
+#define NICVF_RCV_BUF_ALIGN_LEN(X)\
+	(NICVF_ALIGNED_ADDR(X, NICVF_RCV_BUF_ALIGN_BYTES) - X)
+
+/* Queue enable/disable */
+#define NICVF_SQ_EN		BIT_ULL(19)
+
+/* Queue reset */
+#define NICVF_CQ_RESET		BIT_ULL(41)
+#define NICVF_SQ_RESET		BIT_ULL(17)
+#define NICVF_RBDR_RESET	BIT_ULL(43)
+
+enum CQ_RX_ERRLVL_E {
+	CQ_ERRLVL_MAC,
+	CQ_ERRLVL_L2,
+	CQ_ERRLVL_L3,
+	CQ_ERRLVL_L4,
+};
+
+enum CQ_RX_ERROP_E {
+	CQ_RX_ERROP_RE_NONE = 0x0,
+	CQ_RX_ERROP_RE_PARTIAL = 0x1,
+	CQ_RX_ERROP_RE_JABBER = 0x2,
+	CQ_RX_ERROP_RE_FCS = 0x7,
+	CQ_RX_ERROP_RE_TERMINATE = 0x9,
+	CQ_RX_ERROP_RE_RX_CTL = 0xb,
+	CQ_RX_ERROP_PREL2_ERR = 0x1f,
+	CQ_RX_ERROP_L2_FRAGMENT = 0x20,
+	CQ_RX_ERROP_L2_OVERRUN = 0x21,
+	CQ_RX_ERROP_L2_PFCS = 0x22,
+	CQ_RX_ERROP_L2_PUNY = 0x23,
+	CQ_RX_ERROP_L2_MAL = 0x24,
+	CQ_RX_ERROP_L2_OVERSIZE = 0x25,
+	CQ_RX_ERROP_L2_UNDERSIZE = 0x26,
+	CQ_RX_ERROP_L2_LENMISM = 0x27,
+	CQ_RX_ERROP_L2_PCLP = 0x28,
+	CQ_RX_ERROP_IP_NOT = 0x41,
+	CQ_RX_ERROP_IP_CSUM_ERR = 0x42,
+	CQ_RX_ERROP_IP_MAL = 0x43,
+	CQ_RX_ERROP_IP_MALD = 0x44,
+	CQ_RX_ERROP_IP_HOP = 0x45,
+	CQ_RX_ERROP_L3_ICRC = 0x46,
+	CQ_RX_ERROP_L3_PCLP = 0x47,
+	CQ_RX_ERROP_L4_MAL = 0x61,
+	CQ_RX_ERROP_L4_CHK = 0x62,
+	CQ_RX_ERROP_UDP_LEN = 0x63,
+	CQ_RX_ERROP_L4_PORT = 0x64,
+	CQ_RX_ERROP_TCP_FLAG = 0x65,
+	CQ_RX_ERROP_TCP_OFFSET = 0x66,
+	CQ_RX_ERROP_L4_PCLP = 0x67,
+	CQ_RX_ERROP_RBDR_TRUNC = 0x70,
+};
+
+enum CQ_TX_ERROP_E {
+	CQ_TX_ERROP_GOOD = 0x0,
+	CQ_TX_ERROP_DESC_FAULT = 0x10,
+	CQ_TX_ERROP_HDR_CONS_ERR = 0x11,
+	CQ_TX_ERROP_SUBDC_ERR = 0x12,
+	CQ_TX_ERROP_IMM_SIZE_OFLOW = 0x80,
+	CQ_TX_ERROP_DATA_SEQUENCE_ERR = 0x81,
+	CQ_TX_ERROP_MEM_SEQUENCE_ERR = 0x82,
+	CQ_TX_ERROP_LOCK_VIOL = 0x83,
+	CQ_TX_ERROP_DATA_FAULT = 0x84,
+	CQ_TX_ERROP_TSTMP_CONFLICT = 0x85,
+	CQ_TX_ERROP_TSTMP_TIMEOUT = 0x86,
+	CQ_TX_ERROP_MEM_FAULT = 0x87,
+	CQ_TX_ERROP_CK_OVERLAP = 0x88,
+	CQ_TX_ERROP_CK_OFLOW = 0x89,
+	CQ_TX_ERROP_ENUM_LAST = 0x8a,
+};
+
+struct cmp_queue_stats {
+	struct tx_stats {
+		u64 good;
+		u64 desc_fault;
+		u64 hdr_cons_err;
+		u64 subdesc_err;
+		u64 imm_size_oflow;
+		u64 data_seq_err;
+		u64 mem_seq_err;
+		u64 lock_viol;
+		u64 data_fault;
+		u64 tstmp_conflict;
+		u64 tstmp_timeout;
+		u64 mem_fault;
+		u64 csum_overlap;
+		u64 csum_overflow;
+	} tx;
+} ____cacheline_aligned_in_smp;
+
+enum RQ_SQ_STATS {
+	RQ_SQ_STATS_OCTS,
+	RQ_SQ_STATS_PKTS,
+};
+
+struct rx_tx_queue_stats {
+	u64	bytes;
+	u64	pkts;
+} ____cacheline_aligned_in_smp;
+
+struct q_desc_mem {
+	dma_addr_t	dma;
+	u64		size;
+	u16		q_len;
+	dma_addr_t	phys_base;
+	void		*base;
+	void		*unalign_base;
+};
+
+struct rbdr {
+	bool		enable;
+	u32		dma_size;
+	u32		frag_len;
+	u32		thresh;		/* Threshold level for interrupt */
+	void		*desc;
+	u32		head;
+	u32		tail;
+	struct q_desc_mem   dmem;
+} ____cacheline_aligned_in_smp;
+
+struct rcv_queue {
+	bool		enable;
+	struct	rbdr	*rbdr_start;
+	struct	rbdr	*rbdr_cont;
+	bool		en_tcp_reassembly;
+	u8		cq_qs;  /* CQ's QS to which this RQ is assigned */
+	u8		cq_idx; /* CQ index (0 to 7) in the QS */
+	u8		cont_rbdr_qs;      /* Continue buffer ptrs - QS num */
+	u8		cont_qs_rbdr_idx;  /* RBDR idx in the cont QS */
+	u8		start_rbdr_qs;     /* First buffer ptrs - QS num */
+	u8		start_qs_rbdr_idx; /* RBDR idx in the above QS */
+	u8		caching;
+	struct		rx_tx_queue_stats stats;
+} ____cacheline_aligned_in_smp;
+
+struct cmp_queue {
+	bool		enable;
+	u16		thresh;
+	spinlock_t	lock;  /* lock to serialize processing CQEs */
+	void		*desc;
+	struct q_desc_mem   dmem;
+	struct cmp_queue_stats	stats;
+	int		irq;
+} ____cacheline_aligned_in_smp;
+
+struct snd_queue {
+	bool		enable;
+	u8		cq_qs;  /* CQ's QS to which this SQ is pointing */
+	u8		cq_idx; /* CQ index (0 to 7) in the above QS */
+	u16		thresh;
+	atomic_t	free_cnt;
+	u32		head;
+	u32		tail;
+	u64		*skbuff;
+	void		*desc;
+
+	struct q_desc_mem   dmem;
+	struct rx_tx_queue_stats stats;
+} ____cacheline_aligned_in_smp;
+
+struct queue_set {
+	bool		enable;
+	bool		be_en;
+	u8		vnic_id;
+	u8		rq_cnt;
+	u8		cq_cnt;
+	u64		cq_len;
+	u8		sq_cnt;
+	u64		sq_len;
+	u8		rbdr_cnt;
+	u64		rbdr_len;
+	struct	rcv_queue	rq[MAX_RCV_QUEUES_PER_QS];
+	struct	cmp_queue	cq[MAX_CMP_QUEUES_PER_QS];
+	struct	snd_queue	sq[MAX_SND_QUEUES_PER_QS];
+	struct	rbdr		rbdr[MAX_RCV_BUF_DESC_RINGS_PER_QS];
+} ____cacheline_aligned_in_smp;
+
+#define GET_RBDR_DESC(RING, idx)\
+		(&(((struct rbdr_entry_t *)((RING)->desc))[idx]))
+#define GET_SQ_DESC(RING, idx)\
+		(&(((struct sq_hdr_subdesc *)((RING)->desc))[idx]))
+#define GET_CQ_DESC(RING, idx)\
+		(&(((union cq_desc_t *)((RING)->desc))[idx]))
+
+/* CQ status bits */
+#define	CQ_WR_FULL	BIT(26)
+#define	CQ_WR_DISABLE	BIT(25)
+#define	CQ_WR_FAULT	BIT(24)
+#define	CQ_CQE_COUNT	(0xFFFF << 0)
+
+#define	CQ_ERR_MASK	(CQ_WR_FULL | CQ_WR_DISABLE | CQ_WR_FAULT)
+
+void nicvf_config_vlan_stripping(struct nicvf *nic,
+				 netdev_features_t features);
+int nicvf_set_qset_resources(struct nicvf *nic);
+int nicvf_config_data_transfer(struct nicvf *nic, bool enable);
+void nicvf_qset_config(struct nicvf *nic, bool enable);
+void nicvf_cmp_queue_config(struct nicvf *nic, struct queue_set *qs,
+			    int qidx, bool enable);
+
+void nicvf_sq_enable(struct nicvf *nic, struct snd_queue *sq, int qidx);
+void nicvf_sq_disable(struct nicvf *nic, int qidx);
+void nicvf_put_sq_desc(struct snd_queue *sq, int desc_cnt);
+void nicvf_sq_free_used_descs(struct net_device *netdev,
+			      struct snd_queue *sq, int qidx);
+int nicvf_sq_append_skb(struct nicvf *nic, struct sk_buff *skb);
+
+struct sk_buff *nicvf_get_rcv_skb(struct nicvf *nic, struct cqe_rx_t *cqe_rx);
+void nicvf_rbdr_task(unsigned long data);
+void nicvf_rbdr_work(struct work_struct *work);
+
+void nicvf_enable_intr(struct nicvf *nic, int int_type, int q_idx);
+void nicvf_disable_intr(struct nicvf *nic, int int_type, int q_idx);
+void nicvf_clear_intr(struct nicvf *nic, int int_type, int q_idx);
+int nicvf_is_intr_enabled(struct nicvf *nic, int int_type, int q_idx);
+
+/* Register access APIs */
+void nicvf_reg_write(struct nicvf *nic, u64 offset, u64 val);
+u64  nicvf_reg_read(struct nicvf *nic, u64 offset);
+void nicvf_qset_reg_write(struct nicvf *nic, u64 offset, u64 val);
+u64 nicvf_qset_reg_read(struct nicvf *nic, u64 offset);
+void nicvf_queue_reg_write(struct nicvf *nic, u64 offset,
+			   u64 qidx, u64 val);
+u64  nicvf_queue_reg_read(struct nicvf *nic,
+			  u64 offset, u64 qidx);
+
+/* Stats */
+void nicvf_update_rq_stats(struct nicvf *nic, int rq_idx);
+void nicvf_update_sq_stats(struct nicvf *nic, int sq_idx);
+int nicvf_check_cqe_rx_errs(struct nicvf *nic,
+			    struct cmp_queue *cq, struct cqe_rx_t *cqe_rx);
+int nicvf_check_cqe_tx_errs(struct nicvf *nic,
+			    struct cmp_queue *cq, struct cqe_send_t *cqe_tx);
+#endif /* NICVF_QUEUES_H */
diff --git a/sys/dev/vnic/q_struct.h b/sys/dev/vnic/q_struct.h
new file mode 100644
index 000000000000..6d33c7b097e2
--- /dev/null
+++ b/sys/dev/vnic/q_struct.h
@@ -0,0 +1,719 @@
+/*
+ * Copyright (C) 2015 Cavium Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ *
+ */
+
+#ifndef Q_STRUCT_H
+#define Q_STRUCT_H
+
+/* Load transaction types for reading segment bytes specified by
+ * NIC_SEND_GATHER_S[LD_TYPE].
+ */
+enum nic_send_ld_type_e {
+	NIC_SEND_LD_TYPE_E_LDD = 0x0,
+	NIC_SEND_LD_TYPE_E_LDT = 0x1,
+	NIC_SEND_LD_TYPE_E_LDWB = 0x2,
+	NIC_SEND_LD_TYPE_E_ENUM_LAST = 0x3,
+};
+
+enum ether_type_algorithm {
+	ETYPE_ALG_NONE = 0x0,
+	ETYPE_ALG_SKIP = 0x1,
+	ETYPE_ALG_ENDPARSE = 0x2,
+	ETYPE_ALG_VLAN = 0x3,
+	ETYPE_ALG_VLAN_STRIP = 0x4,
+};
+
+enum layer3_type {
+	L3TYPE_NONE = 0x00,
+	L3TYPE_GRH = 0x01,
+	L3TYPE_IPV4 = 0x04,
+	L3TYPE_IPV4_OPTIONS = 0x05,
+	L3TYPE_IPV6 = 0x06,
+	L3TYPE_IPV6_OPTIONS = 0x07,
+	L3TYPE_ET_STOP = 0x0D,
+	L3TYPE_OTHER = 0x0E,
+};
+
+enum layer4_type {
+	L4TYPE_NONE = 0x00,
+	L4TYPE_IPSEC_ESP = 0x01,
+	L4TYPE_IPFRAG = 0x02,
+	L4TYPE_IPCOMP = 0x03,
+	L4TYPE_TCP = 0x04,
+	L4TYPE_UDP = 0x05,
+	L4TYPE_SCTP = 0x06,
+	L4TYPE_GRE = 0x07,
+	L4TYPE_ROCE_BTH = 0x08,
+	L4TYPE_OTHER = 0x0E,
+};
+
+/* CPI and RSSI configuration */
+enum cpi_algorithm_type {
+	CPI_ALG_NONE = 0x0,
+	CPI_ALG_VLAN = 0x1,
+	CPI_ALG_VLAN16 = 0x2,
+	CPI_ALG_DIFF = 0x3,
+};
+
+enum rss_algorithm_type {
+	RSS_ALG_NONE = 0x00,
+	RSS_ALG_PORT = 0x01,
+	RSS_ALG_IP = 0x02,
+	RSS_ALG_TCP_IP = 0x03,
+	RSS_ALG_UDP_IP = 0x04,
+	RSS_ALG_SCTP_IP = 0x05,
+	RSS_ALG_GRE_IP = 0x06,
+	RSS_ALG_ROCE = 0x07,
+};
+
+enum rss_hash_cfg {
+	RSS_HASH_L2ETC = 0x00,
+	RSS_HASH_IP = 0x01,
+	RSS_HASH_TCP = 0x02,
+	RSS_HASH_TCP_SYN_DIS = 0x03,
+	RSS_HASH_UDP = 0x04,
+	RSS_HASH_L4ETC = 0x05,
+	RSS_HASH_ROCE = 0x06,
+	RSS_L3_BIDI = 0x07,
+	RSS_L4_BIDI = 0x08,
+};
+
+/* Completion queue entry types */
+enum cqe_type {
+	CQE_TYPE_INVALID = 0x0,
+	CQE_TYPE_RX = 0x2,
+	CQE_TYPE_RX_SPLIT = 0x3,
+	CQE_TYPE_RX_TCP = 0x4,
+	CQE_TYPE_SEND = 0x8,
+	CQE_TYPE_SEND_PTP = 0x9,
+};
+
+enum cqe_rx_tcp_status {
+	CQE_RX_STATUS_VALID_TCP_CNXT = 0x00,
+	CQE_RX_STATUS_INVALID_TCP_CNXT = 0x0F,
+};
+
+enum cqe_send_status {
+	CQE_SEND_STATUS_GOOD = 0x00,
+	CQE_SEND_STATUS_DESC_FAULT = 0x01,
+	CQE_SEND_STATUS_HDR_CONS_ERR = 0x11,
+	CQE_SEND_STATUS_SUBDESC_ERR = 0x12,
+	CQE_SEND_STATUS_IMM_SIZE_OFLOW = 0x80,
+	CQE_SEND_STATUS_CRC_SEQ_ERR = 0x81,
+	CQE_SEND_STATUS_DATA_SEQ_ERR = 0x82,
+	CQE_SEND_STATUS_MEM_SEQ_ERR = 0x83,
+	CQE_SEND_STATUS_LOCK_VIOL = 0x84,
+	CQE_SEND_STATUS_LOCK_UFLOW = 0x85,
+	CQE_SEND_STATUS_DATA_FAULT = 0x86,
+	CQE_SEND_STATUS_TSTMP_CONFLICT = 0x87,
+	CQE_SEND_STATUS_TSTMP_TIMEOUT = 0x88,
+	CQE_SEND_STATUS_MEM_FAULT = 0x89,
+	CQE_SEND_STATUS_CSUM_OVERLAP = 0x8A,
+	CQE_SEND_STATUS_CSUM_OVERFLOW = 0x8B,
+};
+
+enum cqe_rx_tcp_end_reason {
+	CQE_RX_TCP_END_FIN_FLAG_DET = 0,
+	CQE_RX_TCP_END_INVALID_FLAG = 1,
+	CQE_RX_TCP_END_TIMEOUT = 2,
+	CQE_RX_TCP_END_OUT_OF_SEQ = 3,
+	CQE_RX_TCP_END_PKT_ERR = 4,
+	CQE_RX_TCP_END_QS_DISABLED = 0x0F,
+};
+
+/* Packet protocol level error enumeration */
+enum cqe_rx_err_level {
+	CQE_RX_ERRLVL_RE = 0x0,
+	CQE_RX_ERRLVL_L2 = 0x1,
+	CQE_RX_ERRLVL_L3 = 0x2,
+	CQE_RX_ERRLVL_L4 = 0x3,
+};
+
+/* Packet protocol level error type enumeration */
+enum cqe_rx_err_opcode {
+	CQE_RX_ERR_RE_NONE = 0x0,
+	CQE_RX_ERR_RE_PARTIAL = 0x1,
+	CQE_RX_ERR_RE_JABBER = 0x2,
+	CQE_RX_ERR_RE_FCS = 0x7,
+	CQE_RX_ERR_RE_TERMINATE = 0x9,
+	CQE_RX_ERR_RE_RX_CTL = 0xb,
+	CQE_RX_ERR_PREL2_ERR = 0x1f,
+	CQE_RX_ERR_L2_FRAGMENT = 0x20,
+	CQE_RX_ERR_L2_OVERRUN = 0x21,
+	CQE_RX_ERR_L2_PFCS = 0x22,
+	CQE_RX_ERR_L2_PUNY = 0x23,
+	CQE_RX_ERR_L2_MAL = 0x24,
+	CQE_RX_ERR_L2_OVERSIZE = 0x25,
+	CQE_RX_ERR_L2_UNDERSIZE = 0x26,
+	CQE_RX_ERR_L2_LENMISM = 0x27,
+	CQE_RX_ERR_L2_PCLP = 0x28,
+	CQE_RX_ERR_IP_NOT = 0x41,
+	CQE_RX_ERR_IP_CHK = 0x42,
+	CQE_RX_ERR_IP_MAL = 0x43,
+	CQE_RX_ERR_IP_MALD = 0x44,
+	CQE_RX_ERR_IP_HOP = 0x45,
+	CQE_RX_ERR_L3_ICRC = 0x46,
+	CQE_RX_ERR_L3_PCLP = 0x47,
+	CQE_RX_ERR_L4_MAL = 0x61,
+	CQE_RX_ERR_L4_CHK = 0x62,
+	CQE_RX_ERR_UDP_LEN = 0x63,
+	CQE_RX_ERR_L4_PORT = 0x64,
+	CQE_RX_ERR_TCP_FLAG = 0x65,
+	CQE_RX_ERR_TCP_OFFSET = 0x66,
+	CQE_RX_ERR_L4_PCLP = 0x67,
+	CQE_RX_ERR_RBDR_TRUNC = 0x70,
+};
+
+struct cqe_rx_t {
+#if defined(__BIG_ENDIAN_BITFIELD)
+	u64   cqe_type:4; /* W0 */
+	u64   stdn_fault:1;
+	u64   rsvd0:1;
+	u64   rq_qs:7;
+	u64   rq_idx:3;
+	u64   rsvd1:12;
+	u64   rss_alg:4;
+	u64   rsvd2:4;
+	u64   rb_cnt:4;
+	u64   vlan_found:1;
+	u64   vlan_stripped:1;
+	u64   vlan2_found:1;
+	u64   vlan2_stripped:1;
+	u64   l4_type:4;
+	u64   l3_type:4;
+	u64   l2_present:1;
+	u64   err_level:3;
+	u64   err_opcode:8;
+
+	u64   pkt_len:16; /* W1 */
+	u64   l2_ptr:8;
+	u64   l3_ptr:8;
+	u64   l4_ptr:8;
+	u64   cq_pkt_len:8;
+	u64   align_pad:3;
+	u64   rsvd3:1;
+	u64   chan:12;
+
+	u64   rss_tag:32; /* W2 */
+	u64   vlan_tci:16;
+	u64   vlan_ptr:8;
+	u64   vlan2_ptr:8;
+
+	u64   rb3_sz:16; /* W3 */
+	u64   rb2_sz:16;
+	u64   rb1_sz:16;
+	u64   rb0_sz:16;
+
+	u64   rb7_sz:16; /* W4 */
+	u64   rb6_sz:16;
+	u64   rb5_sz:16;
+	u64   rb4_sz:16;
+
+	u64   rb11_sz:16; /* W5 */
+	u64   rb10_sz:16;
+	u64   rb9_sz:16;
+	u64   rb8_sz:16;
+#elif defined(__LITTLE_ENDIAN_BITFIELD)
+	u64   err_opcode:8;
+	u64   err_level:3;
+	u64   l2_present:1;
+	u64   l3_type:4;
+	u64   l4_type:4;
+	u64   vlan2_stripped:1;
+	u64   vlan2_found:1;
+	u64   vlan_stripped:1;
+	u64   vlan_found:1;
+	u64   rb_cnt:4;
+	u64   rsvd2:4;
+	u64   rss_alg:4;
+	u64   rsvd1:12;
+	u64   rq_idx:3;
+	u64   rq_qs:7;
+	u64   rsvd0:1;
+	u64   stdn_fault:1;
+	u64   cqe_type:4; /* W0 */
+	u64   chan:12;
+	u64   rsvd3:1;
+	u64   align_pad:3;
+	u64   cq_pkt_len:8;
+	u64   l4_ptr:8;
+	u64   l3_ptr:8;
+	u64   l2_ptr:8;
+	u64   pkt_len:16; /* W1 */
+	u64   vlan2_ptr:8;
+	u64   vlan_ptr:8;
+	u64   vlan_tci:16;
+	u64   rss_tag:32; /* W2 */
+	u64   rb0_sz:16;
+	u64   rb1_sz:16;
+	u64   rb2_sz:16;
+	u64   rb3_sz:16; /* W3 */
+	u64   rb4_sz:16;
+	u64   rb5_sz:16;
+	u64   rb6_sz:16;
+	u64   rb7_sz:16; /* W4 */
+	u64   rb8_sz:16;
+	u64   rb9_sz:16;
+	u64   rb10_sz:16;
+	u64   rb11_sz:16; /* W5 */
+#endif
+	u64   rb0_ptr:64;
+	u64   rb1_ptr:64;
+	u64   rb2_ptr:64;
+	u64   rb3_ptr:64;
+	u64   rb4_ptr:64;
+	u64   rb5_ptr:64;
+	u64   rb6_ptr:64;
+	u64   rb7_ptr:64;
+	u64   rb8_ptr:64;
+	u64   rb9_ptr:64;
+	u64   rb10_ptr:64;
+	u64   rb11_ptr:64;
+};
+
+struct cqe_rx_tcp_err_t {
+#if defined(__BIG_ENDIAN_BITFIELD)
+	u64   cqe_type:4; /* W0 */
+	u64   rsvd0:60;
+
+	u64   rsvd1:4; /* W1 */
+	u64   partial_first:1;
+	u64   rsvd2:27;
+	u64   rbdr_bytes:8;
+	u64   rsvd3:24;
+#elif defined(__LITTLE_ENDIAN_BITFIELD)
+	u64   rsvd0:60;
+	u64   cqe_type:4;
+
+	u64   rsvd3:24;
+	u64   rbdr_bytes:8;
+	u64   rsvd2:27;
+	u64   partial_first:1;
+	u64   rsvd1:4;
+#endif
+};
+
+struct cqe_rx_tcp_t {
+#if defined(__BIG_ENDIAN_BITFIELD)
+	u64   cqe_type:4; /* W0 */
+	u64   rsvd0:52;
+	u64   cq_tcp_status:8;
+
+	u64   rsvd1:32; /* W1 */
+	u64   tcp_cntx_bytes:8;
+	u64   rsvd2:8;
+	u64   tcp_err_bytes:16;
+#elif defined(__LITTLE_ENDIAN_BITFIELD)
+	u64   cq_tcp_status:8;
+	u64   rsvd0:52;
+	u64   cqe_type:4; /* W0 */
+
+	u64   tcp_err_bytes:16;
+	u64   rsvd2:8;
+	u64   tcp_cntx_bytes:8;
+	u64   rsvd1:32; /* W1 */
+#endif
+};
+
+struct cqe_send_t {
+#if defined(__BIG_ENDIAN_BITFIELD)
+	u64   cqe_type:4; /* W0 */
+	u64   rsvd0:4;
+	u64   sqe_ptr:16;
+	u64   rsvd1:4;
+	u64   rsvd2:10;
+	u64   sq_qs:7;
+	u64   sq_idx:3;
+	u64   rsvd3:8;
+	u64   send_status:8;
+
+	u64   ptp_timestamp:64; /* W1 */
+#elif defined(__LITTLE_ENDIAN_BITFIELD)
+	u64   send_status:8;
+	u64   rsvd3:8;
+	u64   sq_idx:3;
+	u64   sq_qs:7;
+	u64   rsvd2:10;
+	u64   rsvd1:4;
+	u64   sqe_ptr:16;
+	u64   rsvd0:4;
+	u64   cqe_type:4; /* W0 */
+
+	u64   ptp_timestamp:64; /* W1 */
+#endif
+};
+
+union cq_desc_t {
+	u64    u[64];
+	struct cqe_send_t snd_hdr;
+	struct cqe_rx_t rx_hdr;
+	struct cqe_rx_tcp_t rx_tcp_hdr;
+	struct cqe_rx_tcp_err_t rx_tcp_err_hdr;
+};
+
+struct rbdr_entry_t {
+#if defined(__BIG_ENDIAN_BITFIELD)
+	u64   rsvd0:15;
+	u64   buf_addr:42;
+	u64   cache_align:7;
+#elif defined(__LITTLE_ENDIAN_BITFIELD)
+	u64   cache_align:7;
+	u64   buf_addr:42;
+	u64   rsvd0:15;
+#endif
+};
+
+/* TCP reassembly context */
+struct rbe_tcp_cnxt_t {
+#if defined(__BIG_ENDIAN_BITFIELD)
+	u64   tcp_pkt_cnt:12;
+	u64   rsvd1:4;
+	u64   align_hdr_bytes:4;
+	u64   align_ptr_bytes:4;
+	u64   ptr_bytes:16;
+	u64   rsvd2:24;
+	u64   cqe_type:4;
+	u64   rsvd0:54;
+	u64   tcp_end_reason:2;
+	u64   tcp_status:4;
+#elif defined(__LITTLE_ENDIAN_BITFIELD)
+	u64   tcp_status:4;
+	u64   tcp_end_reason:2;
+	u64   rsvd0:54;
+	u64   cqe_type:4;
+	u64   rsvd2:24;
+	u64   ptr_bytes:16;
+	u64   align_ptr_bytes:4;
+	u64   align_hdr_bytes:4;
+	u64   rsvd1:4;
+	u64   tcp_pkt_cnt:12;
+#endif
+};
+
+/* Always Big endian */
+struct rx_hdr_t {
+	u64   opaque:32;
+	u64   rss_flow:8;
+	u64   skip_length:6;
+	u64   disable_rss:1;
+	u64   disable_tcp_reassembly:1;
+	u64   nodrop:1;
+	u64   dest_alg:2;
+	u64   rsvd0:2;
+	u64   dest_rq:11;
+};
+
+enum send_l4_csum_type {
+	SEND_L4_CSUM_DISABLE = 0x00,
+	SEND_L4_CSUM_UDP = 0x01,
+	SEND_L4_CSUM_TCP = 0x02,
+	SEND_L4_CSUM_SCTP = 0x03,
+};
+
+enum send_crc_alg {
+	SEND_CRCALG_CRC32 = 0x00,
+	SEND_CRCALG_CRC32C = 0x01,
+	SEND_CRCALG_ICRC = 0x02,
+};
+
+enum send_load_type {
+	SEND_LD_TYPE_LDD = 0x00,
+	SEND_LD_TYPE_LDT = 0x01,
+	SEND_LD_TYPE_LDWB = 0x02,
+};
+
+enum send_mem_alg_type {
+	SEND_MEMALG_SET = 0x00,
+	SEND_MEMALG_ADD = 0x08,
+	SEND_MEMALG_SUB = 0x09,
+	SEND_MEMALG_ADDLEN = 0x0A,
+	SEND_MEMALG_SUBLEN = 0x0B,
+};
+
+enum send_mem_dsz_type {
+	SEND_MEMDSZ_B64 = 0x00,
+	SEND_MEMDSZ_B32 = 0x01,
+	SEND_MEMDSZ_B8 = 0x03,
+};
+
+enum sq_subdesc_type {
+	SQ_DESC_TYPE_INVALID = 0x00,
+	SQ_DESC_TYPE_HEADER = 0x01,
+	SQ_DESC_TYPE_CRC = 0x02,
+	SQ_DESC_TYPE_IMMEDIATE = 0x03,
+	SQ_DESC_TYPE_GATHER = 0x04,
+	SQ_DESC_TYPE_MEMORY = 0x05,
+};
+
+struct sq_crc_subdesc {
+#if defined(__BIG_ENDIAN_BITFIELD)
+	u64    rsvd1:32;
+	u64    crc_ival:32;
+	u64    subdesc_type:4;
+	u64    crc_alg:2;
+	u64    rsvd0:10;
+	u64    crc_insert_pos:16;
+	u64    hdr_start:16;
+	u64    crc_len:16;
+#elif defined(__LITTLE_ENDIAN_BITFIELD)
+	u64    crc_len:16;
+	u64    hdr_start:16;
+	u64    crc_insert_pos:16;
+	u64    rsvd0:10;
+	u64    crc_alg:2;
+	u64    subdesc_type:4;
+	u64    crc_ival:32;
+	u64    rsvd1:32;
+#endif
+};
+
+struct sq_gather_subdesc {
+#if defined(__BIG_ENDIAN_BITFIELD)
+	u64    subdesc_type:4; /* W0 */
+	u64    ld_type:2;
+	u64    rsvd0:42;
+	u64    size:16;
+
+	u64    rsvd1:15; /* W1 */
+	u64    addr:49;
+#elif defined(__LITTLE_ENDIAN_BITFIELD)
+	u64    size:16;
+	u64    rsvd0:42;
+	u64    ld_type:2;
+	u64    subdesc_type:4; /* W0 */
+
+	u64    addr:49;
+	u64    rsvd1:15; /* W1 */
+#endif
+};
+
+/* SQ immediate subdescriptor */
+struct sq_imm_subdesc {
+#if defined(__BIG_ENDIAN_BITFIELD)
+	u64    subdesc_type:4; /* W0 */
+	u64    rsvd0:46;
+	u64    len:14;
+
+	u64    data:64; /* W1 */
+#elif defined(__LITTLE_ENDIAN_BITFIELD)
+	u64    len:14;
+	u64    rsvd0:46;
+	u64    subdesc_type:4; /* W0 */
+
+	u64    data:64; /* W1 */
+#endif
+};
+
+struct sq_mem_subdesc {
+#if defined(__BIG_ENDIAN_BITFIELD)
+	u64    subdesc_type:4; /* W0 */
+	u64    mem_alg:4;
+	u64    mem_dsz:2;
+	u64    wmem:1;
+	u64    rsvd0:21;
+	u64    offset:32;
+
+	u64    rsvd1:15; /* W1 */
+	u64    addr:49;
+#elif defined(__LITTLE_ENDIAN_BITFIELD)
+	u64    offset:32;
+	u64    rsvd0:21;
+	u64    wmem:1;
+	u64    mem_dsz:2;
+	u64    mem_alg:4;
+	u64    subdesc_type:4; /* W0 */
+
+	u64    addr:49;
+	u64    rsvd1:15; /* W1 */
+#endif
+};
+
+struct sq_hdr_subdesc {
+#if defined(__BIG_ENDIAN_BITFIELD)
+	u64    subdesc_type:4;
+	u64    tso:1;
+	u64    post_cqe:1; /* Post CQE on no error also */
+	u64    dont_send:1;
+	u64    tstmp:1;
+	u64    subdesc_cnt:8;
+	u64    csum_l4:2;
+	u64    csum_l3:1;
+	u64    rsvd0:5;
+	u64    l4_offset:8;
+	u64    l3_offset:8;
+	u64    rsvd1:4;
+	u64    tot_len:20; /* W0 */
+
+	u64    tso_sdc_cont:8;
+	u64    tso_sdc_first:8;
+	u64    tso_l4_offset:8;
+	u64    tso_flags_last:12;
+	u64    tso_flags_first:12;
+	u64    rsvd2:2;
+	u64    tso_max_paysize:14; /* W1 */
+#elif defined(__LITTLE_ENDIAN_BITFIELD)
+	u64    tot_len:20;
+	u64    rsvd1:4;
+	u64    l3_offset:8;
+	u64    l4_offset:8;
+	u64    rsvd0:5;
+	u64    csum_l3:1;
+	u64    csum_l4:2;
+	u64    subdesc_cnt:8;
+	u64    tstmp:1;
+	u64    dont_send:1;
+	u64    post_cqe:1; /* Post CQE on no error also */
+	u64    tso:1;
+	u64    subdesc_type:4; /* W0 */
+
+	u64    tso_max_paysize:14;
+	u64    rsvd2:2;
+	u64    tso_flags_first:12;
+	u64    tso_flags_last:12;
+	u64    tso_l4_offset:8;
+	u64    tso_sdc_first:8;
+	u64    tso_sdc_cont:8; /* W1 */
+#endif
+};
+
+/* Queue config register formats */
+struct rq_cfg {
+#if defined(__BIG_ENDIAN_BITFIELD)
+	u64 reserved_2_63:62;
+	u64 ena:1;
+	u64 tcp_ena:1;
+#elif defined(__LITTLE_ENDIAN_BITFIELD)
+	u64 tcp_ena:1;
+	u64 ena:1;
+	u64 reserved_2_63:62;
+#endif
+};
+
+struct cq_cfg {
+#if defined(__BIG_ENDIAN_BITFIELD)
+	u64 reserved_43_63:21;
+	u64 ena:1;
+	u64 reset:1;
+	u64 caching:1;
+	u64 reserved_35_39:5;
+	u64 qsize:3;
+	u64 reserved_25_31:7;
+	u64 avg_con:9;
+	u64 reserved_0_15:16;
+#elif defined(__LITTLE_ENDIAN_BITFIELD)
+	u64 reserved_0_15:16;
+	u64 avg_con:9;
+	u64 reserved_25_31:7;
+	u64 qsize:3;
+	u64 reserved_35_39:5;
+	u64 caching:1;
+	u64 reset:1;
+	u64 ena:1;
+	u64 reserved_43_63:21;
+#endif
+};
+
+struct sq_cfg {
+#if defined(__BIG_ENDIAN_BITFIELD)
+	u64 reserved_20_63:44;
+	u64 ena:1;
+	u64 reserved_18_18:1;
+	u64 reset:1;
+	u64 ldwb:1;
+	u64 reserved_11_15:5;
+	u64 qsize:3;
+	u64 reserved_3_7:5;
+	u64 tstmp_bgx_intf:3;
+#elif defined(__LITTLE_ENDIAN_BITFIELD)
+	u64 tstmp_bgx_intf:3;
+	u64 reserved_3_7:5;
+	u64 qsize:3;
+	u64 reserved_11_15:5;
+	u64 ldwb:1;
+	u64 reset:1;
+	u64 reserved_18_18:1;
+	u64 ena:1;
+	u64 reserved_20_63:44;
+#endif
+};
+
+struct rbdr_cfg {
+#if defined(__BIG_ENDIAN_BITFIELD)
+	u64 reserved_45_63:19;
+	u64 ena:1;
+	u64 reset:1;
+	u64 ldwb:1;
+	u64 reserved_36_41:6;
+	u64 qsize:4;
+	u64 reserved_25_31:7;
+	u64 avg_con:9;
+	u64 reserved_12_15:4;
+	u64 lines:12;
+#elif defined(__LITTLE_ENDIAN_BITFIELD)
+	u64 lines:12;
+	u64 reserved_12_15:4;
+	u64 avg_con:9;
+	u64 reserved_25_31:7;
+	u64 qsize:4;
+	u64 reserved_36_41:6;
+	u64 ldwb:1;
+	u64 reset:1;
+	u64 ena: 1;
+	u64 reserved_45_63:19;
+#endif
+};
+
+struct qs_cfg {
+#if defined(__BIG_ENDIAN_BITFIELD)
+	u64 reserved_32_63:32;
+	u64 ena:1;
+	u64 reserved_27_30:4;
+	u64 sq_ins_ena:1;
+	u64 sq_ins_pos:6;
+	u64 lock_ena:1;
+	u64 lock_viol_cqe_ena:1;
+	u64 send_tstmp_ena:1;
+	u64 be:1;
+	u64 reserved_7_15:9;
+	u64 vnic:7;
+#elif defined(__LITTLE_ENDIAN_BITFIELD)
+	u64 vnic:7;
+	u64 reserved_7_15:9;
+	u64 be:1;
+	u64 send_tstmp_ena:1;
+	u64 lock_viol_cqe_ena:1;
+	u64 lock_ena:1;
+	u64 sq_ins_pos:6;
+	u64 sq_ins_ena:1;
+	u64 reserved_27_30:4;
+	u64 ena:1;
+	u64 reserved_32_63:32;
+#endif
+};
+
+#endif /* Q_STRUCT_H */
diff --git a/sys/dev/vnic/thunder_bgx.c b/sys/dev/vnic/thunder_bgx.c
new file mode 100644
index 000000000000..3dabd59668c8
--- /dev/null
+++ b/sys/dev/vnic/thunder_bgx.c
@@ -0,0 +1,1212 @@
+/*
+ * Copyright (C) 2015 Cavium Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ *
+ */
+
+#include <linux/acpi.h>
+#include <linux/module.h>
+#include <linux/interrupt.h>
+#include <linux/pci.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/phy.h>
+#include <linux/of.h>
+#include <linux/of_mdio.h>
+#include <linux/of_net.h>
+
+#include "nic_reg.h"
+#include "nic.h"
+#include "thunder_bgx.h"
+
+#define DRV_NAME	"thunder-BGX"
+#define DRV_VERSION	"1.0"
+
+struct lmac {
+	struct bgx		*bgx;
+	int			dmac;
+	u8			mac[ETH_ALEN];
+	bool			link_up;
+	int			lmacid; /* ID within BGX */
+	int			lmacid_bd; /* ID on board */
+	struct net_device       netdev;
+	struct phy_device       *phydev;
+	unsigned int            last_duplex;
+	unsigned int            last_link;
+	unsigned int            last_speed;
+	bool			is_sgmii;
+	struct delayed_work	dwork;
+	struct workqueue_struct *check_link;
+};
+
+struct bgx {
+	u8			bgx_id;
+	u8			qlm_mode;
+	struct	lmac		lmac[MAX_LMAC_PER_BGX];
+	int			lmac_count;
+	int                     lmac_type;
+	int                     lane_to_sds;
+	int			use_training;
+	void __iomem		*reg_base;
+	struct pci_dev		*pdev;
+};
+
+static struct bgx *bgx_vnic[MAX_BGX_THUNDER];
+static int lmac_count; /* Total no of LMACs in system */
+
+static int bgx_xaui_check_link(struct lmac *lmac);
+
+/* Supported devices */
+static const struct pci_device_id bgx_id_table[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_BGX) },
+	{ 0, }  /* end of table */
+};
+
+MODULE_AUTHOR("Cavium Inc");
+MODULE_DESCRIPTION("Cavium Thunder BGX/MAC Driver");
+MODULE_VERSION(DRV_VERSION);
+MODULE_DEVICE_TABLE(pci, bgx_id_table);
+
+/* The Cavium ThunderX network controller can *only* be found in SoCs
+ * containing the ThunderX ARM64 CPU implementation.  All accesses to the device
+ * registers on this platform are implicitly strongly ordered with respect
+ * to memory accesses. So writeq_relaxed() and readq_relaxed() are safe to use
+ * with no memory barriers in this driver.  The readq()/writeq() functions add
+ * explicit ordering operation which in this case are redundant, and only
+ * add overhead.
+ */
+
+/* Register read/write APIs */
+static u64 bgx_reg_read(struct bgx *bgx, u8 lmac, u64 offset)
+{
+	void __iomem *addr = bgx->reg_base + ((u32)lmac << 20) + offset;
+
+	return readq_relaxed(addr);
+}
+
+static void bgx_reg_write(struct bgx *bgx, u8 lmac, u64 offset, u64 val)
+{
+	void __iomem *addr = bgx->reg_base + ((u32)lmac << 20) + offset;
+
+	writeq_relaxed(val, addr);
+}
+
+static void bgx_reg_modify(struct bgx *bgx, u8 lmac, u64 offset, u64 val)
+{
+	void __iomem *addr = bgx->reg_base + ((u32)lmac << 20) + offset;
+
+	writeq_relaxed(val | readq_relaxed(addr), addr);
+}
+
+static int bgx_poll_reg(struct bgx *bgx, u8 lmac, u64 reg, u64 mask, bool zero)
+{
+	int timeout = 100;
+	u64 reg_val;
+
+	while (timeout) {
+		reg_val = bgx_reg_read(bgx, lmac, reg);
+		if (zero && !(reg_val & mask))
+			return 0;
+		if (!zero && (reg_val & mask))
+			return 0;
+		usleep_range(1000, 2000);
+		timeout--;
+	}
+	return 1;
+}
+
+/* Return number of BGX present in HW */
+unsigned bgx_get_map(int node)
+{
+	int i;
+	unsigned map = 0;
+
+	for (i = 0; i < MAX_BGX_PER_CN88XX; i++) {
+		if (bgx_vnic[(node * MAX_BGX_PER_CN88XX) + i])
+			map |= (1 << i);
+	}
+
+	return map;
+}
+EXPORT_SYMBOL(bgx_get_map);
+
+/* Return number of LMAC configured for this BGX */
+int bgx_get_lmac_count(int node, int bgx_idx)
+{
+	struct bgx *bgx;
+
+	bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx];
+	if (bgx)
+		return bgx->lmac_count;
+
+	return 0;
+}
+EXPORT_SYMBOL(bgx_get_lmac_count);
+
+/* Returns the current link status of LMAC */
+void bgx_get_lmac_link_state(int node, int bgx_idx, int lmacid, void *status)
+{
+	struct bgx_link_status *link = (struct bgx_link_status *)status;
+	struct bgx *bgx;
+	struct lmac *lmac;
+
+	bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx];
+	if (!bgx)
+		return;
+
+	lmac = &bgx->lmac[lmacid];
+	link->link_up = lmac->link_up;
+	link->duplex = lmac->last_duplex;
+	link->speed = lmac->last_speed;
+}
+EXPORT_SYMBOL(bgx_get_lmac_link_state);
+
+const u8 *bgx_get_lmac_mac(int node, int bgx_idx, int lmacid)
+{
+	struct bgx *bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx];
+
+	if (bgx)
+		return bgx->lmac[lmacid].mac;
+
+	return NULL;
+}
+EXPORT_SYMBOL(bgx_get_lmac_mac);
+
+void bgx_set_lmac_mac(int node, int bgx_idx, int lmacid, const u8 *mac)
+{
+	struct bgx *bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx];
+
+	if (!bgx)
+		return;
+
+	ether_addr_copy(bgx->lmac[lmacid].mac, mac);
+}
+EXPORT_SYMBOL(bgx_set_lmac_mac);
+
+static void bgx_sgmii_change_link_state(struct lmac *lmac)
+{
+	struct bgx *bgx = lmac->bgx;
+	u64 cmr_cfg;
+	u64 port_cfg = 0;
+	u64 misc_ctl = 0;
+
+	cmr_cfg = bgx_reg_read(bgx, lmac->lmacid, BGX_CMRX_CFG);
+	cmr_cfg &= ~CMR_EN;
+	bgx_reg_write(bgx, lmac->lmacid, BGX_CMRX_CFG, cmr_cfg);
+
+	port_cfg = bgx_reg_read(bgx, lmac->lmacid, BGX_GMP_GMI_PRTX_CFG);
+	misc_ctl = bgx_reg_read(bgx, lmac->lmacid, BGX_GMP_PCS_MISCX_CTL);
+
+	if (lmac->link_up) {
+		misc_ctl &= ~PCS_MISC_CTL_GMX_ENO;
+		port_cfg &= ~GMI_PORT_CFG_DUPLEX;
+		port_cfg |=  (lmac->last_duplex << 2);
+	} else {
+		misc_ctl |= PCS_MISC_CTL_GMX_ENO;
+	}
+
+	switch (lmac->last_speed) {
+	case 10:
+		port_cfg &= ~GMI_PORT_CFG_SPEED; /* speed 0 */
+		port_cfg |= GMI_PORT_CFG_SPEED_MSB;  /* speed_msb 1 */
+		port_cfg &= ~GMI_PORT_CFG_SLOT_TIME; /* slottime 0 */
+		misc_ctl &= ~PCS_MISC_CTL_SAMP_PT_MASK;
+		misc_ctl |= 50; /* samp_pt */
+		bgx_reg_write(bgx, lmac->lmacid, BGX_GMP_GMI_TXX_SLOT, 64);
+		bgx_reg_write(bgx, lmac->lmacid, BGX_GMP_GMI_TXX_BURST, 0);
+		break;
+	case 100:
+		port_cfg &= ~GMI_PORT_CFG_SPEED; /* speed 0 */
+		port_cfg &= ~GMI_PORT_CFG_SPEED_MSB; /* speed_msb 0 */
+		port_cfg &= ~GMI_PORT_CFG_SLOT_TIME; /* slottime 0 */
+		misc_ctl &= ~PCS_MISC_CTL_SAMP_PT_MASK;
+		misc_ctl |= 5; /* samp_pt */
+		bgx_reg_write(bgx, lmac->lmacid, BGX_GMP_GMI_TXX_SLOT, 64);
+		bgx_reg_write(bgx, lmac->lmacid, BGX_GMP_GMI_TXX_BURST, 0);
+		break;
+	case 1000:
+		port_cfg |= GMI_PORT_CFG_SPEED; /* speed 1 */
+		port_cfg &= ~GMI_PORT_CFG_SPEED_MSB; /* speed_msb 0 */
+		port_cfg |= GMI_PORT_CFG_SLOT_TIME; /* slottime 1 */
+		misc_ctl &= ~PCS_MISC_CTL_SAMP_PT_MASK;
+		misc_ctl |= 1; /* samp_pt */
+		bgx_reg_write(bgx, lmac->lmacid, BGX_GMP_GMI_TXX_SLOT, 512);
+		if (lmac->last_duplex)
+			bgx_reg_write(bgx, lmac->lmacid,
+				      BGX_GMP_GMI_TXX_BURST, 0);
+		else
+			bgx_reg_write(bgx, lmac->lmacid,
+				      BGX_GMP_GMI_TXX_BURST, 8192);
+		break;
+	default:
+		break;
+	}
+	bgx_reg_write(bgx, lmac->lmacid, BGX_GMP_PCS_MISCX_CTL, misc_ctl);
+	bgx_reg_write(bgx, lmac->lmacid, BGX_GMP_GMI_PRTX_CFG, port_cfg);
+
+	port_cfg = bgx_reg_read(bgx, lmac->lmacid, BGX_GMP_GMI_PRTX_CFG);
+
+	/* renable lmac */
+	cmr_cfg |= CMR_EN;
+	bgx_reg_write(bgx, lmac->lmacid, BGX_CMRX_CFG, cmr_cfg);
+}
+
+static void bgx_lmac_handler(struct net_device *netdev)
+{
+	struct lmac *lmac = container_of(netdev, struct lmac, netdev);
+	struct phy_device *phydev = lmac->phydev;
+	int link_changed = 0;
+
+	if (!lmac)
+		return;
+
+	if (!phydev->link && lmac->last_link)
+		link_changed = -1;
+
+	if (phydev->link &&
+	    (lmac->last_duplex != phydev->duplex ||
+	     lmac->last_link != phydev->link ||
+	     lmac->last_speed != phydev->speed)) {
+			link_changed = 1;
+	}
+
+	lmac->last_link = phydev->link;
+	lmac->last_speed = phydev->speed;
+	lmac->last_duplex = phydev->duplex;
+
+	if (!link_changed)
+		return;
+
+	if (link_changed > 0)
+		lmac->link_up = true;
+	else
+		lmac->link_up = false;
+
+	if (lmac->is_sgmii)
+		bgx_sgmii_change_link_state(lmac);
+	else
+		bgx_xaui_check_link(lmac);
+}
+
+u64 bgx_get_rx_stats(int node, int bgx_idx, int lmac, int idx)
+{
+	struct bgx *bgx;
+
+	bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx];
+	if (!bgx)
+		return 0;
+
+	if (idx > 8)
+		lmac = 0;
+	return bgx_reg_read(bgx, lmac, BGX_CMRX_RX_STAT0 + (idx * 8));
+}
+EXPORT_SYMBOL(bgx_get_rx_stats);
+
+u64 bgx_get_tx_stats(int node, int bgx_idx, int lmac, int idx)
+{
+	struct bgx *bgx;
+
+	bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx];
+	if (!bgx)
+		return 0;
+
+	return bgx_reg_read(bgx, lmac, BGX_CMRX_TX_STAT0 + (idx * 8));
+}
+EXPORT_SYMBOL(bgx_get_tx_stats);
+
+static void bgx_flush_dmac_addrs(struct bgx *bgx, int lmac)
+{
+	u64 offset;
+
+	while (bgx->lmac[lmac].dmac > 0) {
+		offset = ((bgx->lmac[lmac].dmac - 1) * sizeof(u64)) +
+			(lmac * MAX_DMAC_PER_LMAC * sizeof(u64));
+		bgx_reg_write(bgx, 0, BGX_CMR_RX_DMACX_CAM + offset, 0);
+		bgx->lmac[lmac].dmac--;
+	}
+}
+
+void bgx_add_dmac_addr(u64 dmac, int node, int bgx_idx, int lmac)
+{
+	u64 offset;
+	struct bgx *bgx;
+
+#ifdef BGX_IN_PROMISCUOUS_MODE
+	return;
+#endif
+
+	bgx_idx += node * MAX_BGX_PER_CN88XX;
+	bgx = bgx_vnic[bgx_idx];
+
+	if (!bgx) {
+		dev_err(&bgx->pdev->dev,
+			"BGX%d not yet initialized, ignoring DMAC addition\n",
+			bgx_idx);
+		return;
+	}
+
+	dmac = dmac | (1ULL << 48) | ((u64)lmac << 49); /* Enable DMAC */
+	if (bgx->lmac[lmac].dmac == MAX_DMAC_PER_LMAC) {
+		dev_err(&bgx->pdev->dev,
+			"Max DMAC filters for LMAC%d reached, ignoring\n",
+			lmac);
+		return;
+	}
+
+	if (bgx->lmac[lmac].dmac == MAX_DMAC_PER_LMAC_TNS_BYPASS_MODE)
+		bgx->lmac[lmac].dmac = 1;
+
+	offset = (bgx->lmac[lmac].dmac * sizeof(u64)) +
+		(lmac * MAX_DMAC_PER_LMAC * sizeof(u64));
+	bgx_reg_write(bgx, 0, BGX_CMR_RX_DMACX_CAM + offset, dmac);
+	bgx->lmac[lmac].dmac++;
+
+	bgx_reg_write(bgx, lmac, BGX_CMRX_RX_DMAC_CTL,
+		      (CAM_ACCEPT << 3) | (MCAST_MODE_CAM_FILTER << 1)
+		      | (BCAST_ACCEPT << 0));
+}
+EXPORT_SYMBOL(bgx_add_dmac_addr);
+
+/* Configure BGX LMAC in internal loopback mode */
+void bgx_lmac_internal_loopback(int node, int bgx_idx,
+				int lmac_idx, bool enable)
+{
+	struct bgx *bgx;
+	struct lmac *lmac;
+	u64    cfg;
+
+	bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx];
+	if (!bgx)
+		return;
+
+	lmac = &bgx->lmac[lmac_idx];
+	if (lmac->is_sgmii) {
+		cfg = bgx_reg_read(bgx, lmac_idx, BGX_GMP_PCS_MRX_CTL);
+		if (enable)
+			cfg |= PCS_MRX_CTL_LOOPBACK1;
+		else
+			cfg &= ~PCS_MRX_CTL_LOOPBACK1;
+		bgx_reg_write(bgx, lmac_idx, BGX_GMP_PCS_MRX_CTL, cfg);
+	} else {
+		cfg = bgx_reg_read(bgx, lmac_idx, BGX_SPUX_CONTROL1);
+		if (enable)
+			cfg |= SPU_CTL_LOOPBACK;
+		else
+			cfg &= ~SPU_CTL_LOOPBACK;
+		bgx_reg_write(bgx, lmac_idx, BGX_SPUX_CONTROL1, cfg);
+	}
+}
+EXPORT_SYMBOL(bgx_lmac_internal_loopback);
+
+static int bgx_lmac_sgmii_init(struct bgx *bgx, int lmacid)
+{
+	u64 cfg;
+
+	bgx_reg_modify(bgx, lmacid, BGX_GMP_GMI_TXX_THRESH, 0x30);
+	/* max packet size */
+	bgx_reg_modify(bgx, lmacid, BGX_GMP_GMI_RXX_JABBER, MAX_FRAME_SIZE);
+
+	/* Disable frame alignment if using preamble */
+	cfg = bgx_reg_read(bgx, lmacid, BGX_GMP_GMI_TXX_APPEND);
+	if (cfg & 1)
+		bgx_reg_write(bgx, lmacid, BGX_GMP_GMI_TXX_SGMII_CTL, 0);
+
+	/* Enable lmac */
+	bgx_reg_modify(bgx, lmacid, BGX_CMRX_CFG, CMR_EN);
+
+	/* PCS reset */
+	bgx_reg_modify(bgx, lmacid, BGX_GMP_PCS_MRX_CTL, PCS_MRX_CTL_RESET);
+	if (bgx_poll_reg(bgx, lmacid, BGX_GMP_PCS_MRX_CTL,
+			 PCS_MRX_CTL_RESET, true)) {
+		dev_err(&bgx->pdev->dev, "BGX PCS reset not completed\n");
+		return -1;
+	}
+
+	/* power down, reset autoneg, autoneg enable */
+	cfg = bgx_reg_read(bgx, lmacid, BGX_GMP_PCS_MRX_CTL);
+	cfg &= ~PCS_MRX_CTL_PWR_DN;
+	cfg |= (PCS_MRX_CTL_RST_AN | PCS_MRX_CTL_AN_EN);
+	bgx_reg_write(bgx, lmacid, BGX_GMP_PCS_MRX_CTL, cfg);
+
+	if (bgx_poll_reg(bgx, lmacid, BGX_GMP_PCS_MRX_STATUS,
+			 PCS_MRX_STATUS_AN_CPT, false)) {
+		dev_err(&bgx->pdev->dev, "BGX AN_CPT not completed\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+static int bgx_lmac_xaui_init(struct bgx *bgx, int lmacid, int lmac_type)
+{
+	u64 cfg;
+
+	/* Reset SPU */
+	bgx_reg_modify(bgx, lmacid, BGX_SPUX_CONTROL1, SPU_CTL_RESET);
+	if (bgx_poll_reg(bgx, lmacid, BGX_SPUX_CONTROL1, SPU_CTL_RESET, true)) {
+		dev_err(&bgx->pdev->dev, "BGX SPU reset not completed\n");
+		return -1;
+	}
+
+	/* Disable LMAC */
+	cfg = bgx_reg_read(bgx, lmacid, BGX_CMRX_CFG);
+	cfg &= ~CMR_EN;
+	bgx_reg_write(bgx, lmacid, BGX_CMRX_CFG, cfg);
+
+	bgx_reg_modify(bgx, lmacid, BGX_SPUX_CONTROL1, SPU_CTL_LOW_POWER);
+	/* Set interleaved running disparity for RXAUI */
+	if (bgx->lmac_type != BGX_MODE_RXAUI)
+		bgx_reg_modify(bgx, lmacid,
+			       BGX_SPUX_MISC_CONTROL, SPU_MISC_CTL_RX_DIS);
+	else
+		bgx_reg_modify(bgx, lmacid, BGX_SPUX_MISC_CONTROL,
+			       SPU_MISC_CTL_RX_DIS | SPU_MISC_CTL_INTLV_RDISP);
+
+	/* clear all interrupts */
+	cfg = bgx_reg_read(bgx, lmacid, BGX_SMUX_RX_INT);
+	bgx_reg_write(bgx, lmacid, BGX_SMUX_RX_INT, cfg);
+	cfg = bgx_reg_read(bgx, lmacid, BGX_SMUX_TX_INT);
+	bgx_reg_write(bgx, lmacid, BGX_SMUX_TX_INT, cfg);
+	cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_INT);
+	bgx_reg_write(bgx, lmacid, BGX_SPUX_INT, cfg);
+
+	if (bgx->use_training) {
+		bgx_reg_write(bgx, lmacid, BGX_SPUX_BR_PMD_LP_CUP, 0x00);
+		bgx_reg_write(bgx, lmacid, BGX_SPUX_BR_PMD_LD_CUP, 0x00);
+		bgx_reg_write(bgx, lmacid, BGX_SPUX_BR_PMD_LD_REP, 0x00);
+		/* training enable */
+		bgx_reg_modify(bgx, lmacid,
+			       BGX_SPUX_BR_PMD_CRTL, SPU_PMD_CRTL_TRAIN_EN);
+	}
+
+	/* Append FCS to each packet */
+	bgx_reg_modify(bgx, lmacid, BGX_SMUX_TX_APPEND, SMU_TX_APPEND_FCS_D);
+
+	/* Disable forward error correction */
+	cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_FEC_CONTROL);
+	cfg &= ~SPU_FEC_CTL_FEC_EN;
+	bgx_reg_write(bgx, lmacid, BGX_SPUX_FEC_CONTROL, cfg);
+
+	/* Disable autoneg */
+	cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_AN_CONTROL);
+	cfg = cfg & ~(SPU_AN_CTL_AN_EN | SPU_AN_CTL_XNP_EN);
+	bgx_reg_write(bgx, lmacid, BGX_SPUX_AN_CONTROL, cfg);
+
+	cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_AN_ADV);
+	if (bgx->lmac_type == BGX_MODE_10G_KR)
+		cfg |= (1 << 23);
+	else if (bgx->lmac_type == BGX_MODE_40G_KR)
+		cfg |= (1 << 24);
+	else
+		cfg &= ~((1 << 23) | (1 << 24));
+	cfg = cfg & (~((1ULL << 25) | (1ULL << 22) | (1ULL << 12)));
+	bgx_reg_write(bgx, lmacid, BGX_SPUX_AN_ADV, cfg);
+
+	cfg = bgx_reg_read(bgx, 0, BGX_SPU_DBG_CONTROL);
+	cfg &= ~SPU_DBG_CTL_AN_ARB_LINK_CHK_EN;
+	bgx_reg_write(bgx, 0, BGX_SPU_DBG_CONTROL, cfg);
+
+	/* Enable lmac */
+	bgx_reg_modify(bgx, lmacid, BGX_CMRX_CFG, CMR_EN);
+
+	cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_CONTROL1);
+	cfg &= ~SPU_CTL_LOW_POWER;
+	bgx_reg_write(bgx, lmacid, BGX_SPUX_CONTROL1, cfg);
+
+	cfg = bgx_reg_read(bgx, lmacid, BGX_SMUX_TX_CTL);
+	cfg &= ~SMU_TX_CTL_UNI_EN;
+	cfg |= SMU_TX_CTL_DIC_EN;
+	bgx_reg_write(bgx, lmacid, BGX_SMUX_TX_CTL, cfg);
+
+	/* take lmac_count into account */
+	bgx_reg_modify(bgx, lmacid, BGX_SMUX_TX_THRESH, (0x100 - 1));
+	/* max packet size */
+	bgx_reg_modify(bgx, lmacid, BGX_SMUX_RX_JABBER, MAX_FRAME_SIZE);
+
+	return 0;
+}
+
+static int bgx_xaui_check_link(struct lmac *lmac)
+{
+	struct bgx *bgx = lmac->bgx;
+	int lmacid = lmac->lmacid;
+	int lmac_type = bgx->lmac_type;
+	u64 cfg;
+
+	bgx_reg_modify(bgx, lmacid, BGX_SPUX_MISC_CONTROL, SPU_MISC_CTL_RX_DIS);
+	if (bgx->use_training) {
+		cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_INT);
+		if (!(cfg & (1ull << 13))) {
+			cfg = (1ull << 13) | (1ull << 14);
+			bgx_reg_write(bgx, lmacid, BGX_SPUX_INT, cfg);
+			cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_BR_PMD_CRTL);
+			cfg |= (1ull << 0);
+			bgx_reg_write(bgx, lmacid, BGX_SPUX_BR_PMD_CRTL, cfg);
+			return -1;
+		}
+	}
+
+	/* wait for PCS to come out of reset */
+	if (bgx_poll_reg(bgx, lmacid, BGX_SPUX_CONTROL1, SPU_CTL_RESET, true)) {
+		dev_err(&bgx->pdev->dev, "BGX SPU reset not completed\n");
+		return -1;
+	}
+
+	if ((lmac_type == BGX_MODE_10G_KR) || (lmac_type == BGX_MODE_XFI) ||
+	    (lmac_type == BGX_MODE_40G_KR) || (lmac_type == BGX_MODE_XLAUI)) {
+		if (bgx_poll_reg(bgx, lmacid, BGX_SPUX_BR_STATUS1,
+				 SPU_BR_STATUS_BLK_LOCK, false)) {
+			dev_err(&bgx->pdev->dev,
+				"SPU_BR_STATUS_BLK_LOCK not completed\n");
+			return -1;
+		}
+	} else {
+		if (bgx_poll_reg(bgx, lmacid, BGX_SPUX_BX_STATUS,
+				 SPU_BX_STATUS_RX_ALIGN, false)) {
+			dev_err(&bgx->pdev->dev,
+				"SPU_BX_STATUS_RX_ALIGN not completed\n");
+			return -1;
+		}
+	}
+
+	/* Clear rcvflt bit (latching high) and read it back */
+	bgx_reg_modify(bgx, lmacid, BGX_SPUX_STATUS2, SPU_STATUS2_RCVFLT);
+	if (bgx_reg_read(bgx, lmacid, BGX_SPUX_STATUS2) & SPU_STATUS2_RCVFLT) {
+		dev_err(&bgx->pdev->dev, "Receive fault, retry training\n");
+		if (bgx->use_training) {
+			cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_INT);
+			if (!(cfg & (1ull << 13))) {
+				cfg = (1ull << 13) | (1ull << 14);
+				bgx_reg_write(bgx, lmacid, BGX_SPUX_INT, cfg);
+				cfg = bgx_reg_read(bgx, lmacid,
+						   BGX_SPUX_BR_PMD_CRTL);
+				cfg |= (1ull << 0);
+				bgx_reg_write(bgx, lmacid,
+					      BGX_SPUX_BR_PMD_CRTL, cfg);
+				return -1;
+			}
+		}
+		return -1;
+	}
+
+	/* Wait for MAC RX to be ready */
+	if (bgx_poll_reg(bgx, lmacid, BGX_SMUX_RX_CTL,
+			 SMU_RX_CTL_STATUS, true)) {
+		dev_err(&bgx->pdev->dev, "SMU RX link not okay\n");
+		return -1;
+	}
+
+	/* Wait for BGX RX to be idle */
+	if (bgx_poll_reg(bgx, lmacid, BGX_SMUX_CTL, SMU_CTL_RX_IDLE, false)) {
+		dev_err(&bgx->pdev->dev, "SMU RX not idle\n");
+		return -1;
+	}
+
+	/* Wait for BGX TX to be idle */
+	if (bgx_poll_reg(bgx, lmacid, BGX_SMUX_CTL, SMU_CTL_TX_IDLE, false)) {
+		dev_err(&bgx->pdev->dev, "SMU TX not idle\n");
+		return -1;
+	}
+
+	if (bgx_reg_read(bgx, lmacid, BGX_SPUX_STATUS2) & SPU_STATUS2_RCVFLT) {
+		dev_err(&bgx->pdev->dev, "Receive fault\n");
+		return -1;
+	}
+
+	/* Receive link is latching low. Force it high and verify it */
+	bgx_reg_modify(bgx, lmacid, BGX_SPUX_STATUS1, SPU_STATUS1_RCV_LNK);
+	if (bgx_poll_reg(bgx, lmacid, BGX_SPUX_STATUS1,
+			 SPU_STATUS1_RCV_LNK, false)) {
+		dev_err(&bgx->pdev->dev, "SPU receive link down\n");
+		return -1;
+	}
+
+	cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_MISC_CONTROL);
+	cfg &= ~SPU_MISC_CTL_RX_DIS;
+	bgx_reg_write(bgx, lmacid, BGX_SPUX_MISC_CONTROL, cfg);
+	return 0;
+}
+
+static void bgx_poll_for_link(struct work_struct *work)
+{
+	struct lmac *lmac;
+	u64 link;
+
+	lmac = container_of(work, struct lmac, dwork.work);
+
+	/* Receive link is latching low. Force it high and verify it */
+	bgx_reg_modify(lmac->bgx, lmac->lmacid,
+		       BGX_SPUX_STATUS1, SPU_STATUS1_RCV_LNK);
+	bgx_poll_reg(lmac->bgx, lmac->lmacid, BGX_SPUX_STATUS1,
+		     SPU_STATUS1_RCV_LNK, false);
+
+	link = bgx_reg_read(lmac->bgx, lmac->lmacid, BGX_SPUX_STATUS1);
+	if (link & SPU_STATUS1_RCV_LNK) {
+		lmac->link_up = 1;
+		if (lmac->bgx->lmac_type == BGX_MODE_XLAUI)
+			lmac->last_speed = 40000;
+		else
+			lmac->last_speed = 10000;
+		lmac->last_duplex = 1;
+	} else {
+		lmac->link_up = 0;
+	}
+
+	if (lmac->last_link != lmac->link_up) {
+		lmac->last_link = lmac->link_up;
+		if (lmac->link_up)
+			bgx_xaui_check_link(lmac);
+	}
+
+	queue_delayed_work(lmac->check_link, &lmac->dwork, HZ * 2);
+}
+
+static int bgx_lmac_enable(struct bgx *bgx, u8 lmacid)
+{
+	u64 dmac_bcast = (1ULL << 48) - 1;
+	struct lmac *lmac;
+	u64 cfg;
+
+	lmac = &bgx->lmac[lmacid];
+	lmac->bgx = bgx;
+
+	if (bgx->lmac_type == BGX_MODE_SGMII) {
+		lmac->is_sgmii = 1;
+		if (bgx_lmac_sgmii_init(bgx, lmacid))
+			return -1;
+	} else {
+		lmac->is_sgmii = 0;
+		if (bgx_lmac_xaui_init(bgx, lmacid, bgx->lmac_type))
+			return -1;
+	}
+
+	if (lmac->is_sgmii) {
+		cfg = bgx_reg_read(bgx, lmacid, BGX_GMP_GMI_TXX_APPEND);
+		cfg |= ((1ull << 2) | (1ull << 1)); /* FCS and PAD */
+		bgx_reg_modify(bgx, lmacid, BGX_GMP_GMI_TXX_APPEND, cfg);
+		bgx_reg_write(bgx, lmacid, BGX_GMP_GMI_TXX_MIN_PKT, 60 - 1);
+	} else {
+		cfg = bgx_reg_read(bgx, lmacid, BGX_SMUX_TX_APPEND);
+		cfg |= ((1ull << 2) | (1ull << 1)); /* FCS and PAD */
+		bgx_reg_modify(bgx, lmacid, BGX_SMUX_TX_APPEND, cfg);
+		bgx_reg_write(bgx, lmacid, BGX_SMUX_TX_MIN_PKT, 60 + 4);
+	}
+
+	/* Enable lmac */
+	bgx_reg_modify(bgx, lmacid, BGX_CMRX_CFG,
+		       CMR_EN | CMR_PKT_RX_EN | CMR_PKT_TX_EN);
+
+	/* Restore default cfg, incase low level firmware changed it */
+	bgx_reg_write(bgx, lmacid, BGX_CMRX_RX_DMAC_CTL, 0x03);
+
+	/* Add broadcast MAC into all LMAC's DMAC filters */
+	bgx_add_dmac_addr(dmac_bcast, 0, bgx->bgx_id, lmacid);
+
+	if ((bgx->lmac_type != BGX_MODE_XFI) &&
+	    (bgx->lmac_type != BGX_MODE_XLAUI) &&
+	    (bgx->lmac_type != BGX_MODE_40G_KR) &&
+	    (bgx->lmac_type != BGX_MODE_10G_KR)) {
+		if (!lmac->phydev)
+			return -ENODEV;
+
+		lmac->phydev->dev_flags = 0;
+
+		if (phy_connect_direct(&lmac->netdev, lmac->phydev,
+				       bgx_lmac_handler,
+				       PHY_INTERFACE_MODE_SGMII))
+			return -ENODEV;
+
+		phy_start_aneg(lmac->phydev);
+	} else {
+		lmac->check_link = alloc_workqueue("check_link", WQ_UNBOUND |
+						   WQ_MEM_RECLAIM, 1);
+		if (!lmac->check_link)
+			return -ENOMEM;
+		INIT_DELAYED_WORK(&lmac->dwork, bgx_poll_for_link);
+		queue_delayed_work(lmac->check_link, &lmac->dwork, 0);
+	}
+
+	return 0;
+}
+
+static void bgx_lmac_disable(struct bgx *bgx, u8 lmacid)
+{
+	struct lmac *lmac;
+	u64 cmrx_cfg;
+
+	lmac = &bgx->lmac[lmacid];
+	if (lmac->check_link) {
+		/* Destroy work queue */
+		cancel_delayed_work(&lmac->dwork);
+		flush_workqueue(lmac->check_link);
+		destroy_workqueue(lmac->check_link);
+	}
+
+	cmrx_cfg = bgx_reg_read(bgx, lmacid, BGX_CMRX_CFG);
+	cmrx_cfg &= ~(1 << 15);
+	bgx_reg_write(bgx, lmacid, BGX_CMRX_CFG, cmrx_cfg);
+	bgx_flush_dmac_addrs(bgx, lmacid);
+
+	if ((bgx->lmac_type != BGX_MODE_XFI) &&
+	    (bgx->lmac_type != BGX_MODE_XLAUI) &&
+	    (bgx->lmac_type != BGX_MODE_40G_KR) &&
+	    (bgx->lmac_type != BGX_MODE_10G_KR)) {
+		if (lmac->phydev)
+			phy_disconnect(lmac->phydev);
+	}
+	lmac->phydev = NULL;
+}
+
+static void bgx_set_num_ports(struct bgx *bgx)
+{
+	u64 lmac_count;
+
+	switch (bgx->qlm_mode) {
+	case QLM_MODE_SGMII:
+		bgx->lmac_count = 4;
+		bgx->lmac_type = BGX_MODE_SGMII;
+		bgx->lane_to_sds = 0;
+		break;
+	case QLM_MODE_XAUI_1X4:
+		bgx->lmac_count = 1;
+		bgx->lmac_type = BGX_MODE_XAUI;
+		bgx->lane_to_sds = 0xE4;
+			break;
+	case QLM_MODE_RXAUI_2X2:
+		bgx->lmac_count = 2;
+		bgx->lmac_type = BGX_MODE_RXAUI;
+		bgx->lane_to_sds = 0xE4;
+			break;
+	case QLM_MODE_XFI_4X1:
+		bgx->lmac_count = 4;
+		bgx->lmac_type = BGX_MODE_XFI;
+		bgx->lane_to_sds = 0;
+		break;
+	case QLM_MODE_XLAUI_1X4:
+		bgx->lmac_count = 1;
+		bgx->lmac_type = BGX_MODE_XLAUI;
+		bgx->lane_to_sds = 0xE4;
+		break;
+	case QLM_MODE_10G_KR_4X1:
+		bgx->lmac_count = 4;
+		bgx->lmac_type = BGX_MODE_10G_KR;
+		bgx->lane_to_sds = 0;
+		bgx->use_training = 1;
+		break;
+	case QLM_MODE_40G_KR4_1X4:
+		bgx->lmac_count = 1;
+		bgx->lmac_type = BGX_MODE_40G_KR;
+		bgx->lane_to_sds = 0xE4;
+		bgx->use_training = 1;
+		break;
+	default:
+		bgx->lmac_count = 0;
+		break;
+	}
+
+	/* Check if low level firmware has programmed LMAC count
+	 * based on board type, if yes consider that otherwise
+	 * the default static values
+	 */
+	lmac_count = bgx_reg_read(bgx, 0, BGX_CMR_RX_LMACS) & 0x7;
+	if (lmac_count != 4)
+		bgx->lmac_count = lmac_count;
+}
+
+static void bgx_init_hw(struct bgx *bgx)
+{
+	int i;
+
+	bgx_set_num_ports(bgx);
+
+	bgx_reg_modify(bgx, 0, BGX_CMR_GLOBAL_CFG, CMR_GLOBAL_CFG_FCS_STRIP);
+	if (bgx_reg_read(bgx, 0, BGX_CMR_BIST_STATUS))
+		dev_err(&bgx->pdev->dev, "BGX%d BIST failed\n", bgx->bgx_id);
+
+	/* Set lmac type and lane2serdes mapping */
+	for (i = 0; i < bgx->lmac_count; i++) {
+		if (bgx->lmac_type == BGX_MODE_RXAUI) {
+			if (i)
+				bgx->lane_to_sds = 0x0e;
+			else
+				bgx->lane_to_sds = 0x04;
+			bgx_reg_write(bgx, i, BGX_CMRX_CFG,
+				      (bgx->lmac_type << 8) | bgx->lane_to_sds);
+			continue;
+		}
+		bgx_reg_write(bgx, i, BGX_CMRX_CFG,
+			      (bgx->lmac_type << 8) | (bgx->lane_to_sds + i));
+		bgx->lmac[i].lmacid_bd = lmac_count;
+		lmac_count++;
+	}
+
+	bgx_reg_write(bgx, 0, BGX_CMR_TX_LMACS, bgx->lmac_count);
+	bgx_reg_write(bgx, 0, BGX_CMR_RX_LMACS, bgx->lmac_count);
+
+	/* Set the backpressure AND mask */
+	for (i = 0; i < bgx->lmac_count; i++)
+		bgx_reg_modify(bgx, 0, BGX_CMR_CHAN_MSK_AND,
+			       ((1ULL << MAX_BGX_CHANS_PER_LMAC) - 1) <<
+			       (i * MAX_BGX_CHANS_PER_LMAC));
+
+	/* Disable all MAC filtering */
+	for (i = 0; i < RX_DMAC_COUNT; i++)
+		bgx_reg_write(bgx, 0, BGX_CMR_RX_DMACX_CAM + (i * 8), 0x00);
+
+	/* Disable MAC steering (NCSI traffic) */
+	for (i = 0; i < RX_TRAFFIC_STEER_RULE_COUNT; i++)
+		bgx_reg_write(bgx, 0, BGX_CMR_RX_STREERING + (i * 8), 0x00);
+}
+
+static void bgx_get_qlm_mode(struct bgx *bgx)
+{
+	struct device *dev = &bgx->pdev->dev;
+	int lmac_type;
+	int train_en;
+
+	/* Read LMAC0 type to figure out QLM mode
+	 * This is configured by low level firmware
+	 */
+	lmac_type = bgx_reg_read(bgx, 0, BGX_CMRX_CFG);
+	lmac_type = (lmac_type >> 8) & 0x07;
+
+	train_en = bgx_reg_read(bgx, 0, BGX_SPUX_BR_PMD_CRTL) &
+				SPU_PMD_CRTL_TRAIN_EN;
+
+	switch (lmac_type) {
+	case BGX_MODE_SGMII:
+		bgx->qlm_mode = QLM_MODE_SGMII;
+		dev_info(dev, "BGX%d QLM mode: SGMII\n", bgx->bgx_id);
+		break;
+	case BGX_MODE_XAUI:
+		bgx->qlm_mode = QLM_MODE_XAUI_1X4;
+		dev_info(dev, "BGX%d QLM mode: XAUI\n", bgx->bgx_id);
+		break;
+	case BGX_MODE_RXAUI:
+		bgx->qlm_mode = QLM_MODE_RXAUI_2X2;
+		dev_info(dev, "BGX%d QLM mode: RXAUI\n", bgx->bgx_id);
+		break;
+	case BGX_MODE_XFI:
+		if (!train_en) {
+			bgx->qlm_mode = QLM_MODE_XFI_4X1;
+			dev_info(dev, "BGX%d QLM mode: XFI\n", bgx->bgx_id);
+		} else {
+			bgx->qlm_mode = QLM_MODE_10G_KR_4X1;
+			dev_info(dev, "BGX%d QLM mode: 10G_KR\n", bgx->bgx_id);
+		}
+		break;
+	case BGX_MODE_XLAUI:
+		if (!train_en) {
+			bgx->qlm_mode = QLM_MODE_XLAUI_1X4;
+			dev_info(dev, "BGX%d QLM mode: XLAUI\n", bgx->bgx_id);
+		} else {
+			bgx->qlm_mode = QLM_MODE_40G_KR4_1X4;
+			dev_info(dev, "BGX%d QLM mode: 40G_KR4\n", bgx->bgx_id);
+		}
+		break;
+	default:
+		bgx->qlm_mode = QLM_MODE_SGMII;
+		dev_info(dev, "BGX%d QLM default mode: SGMII\n", bgx->bgx_id);
+	}
+}
+
+#ifdef CONFIG_ACPI
+
+static int bgx_match_phy_id(struct device *dev, void *data)
+{
+	struct phy_device *phydev = to_phy_device(dev);
+	u32 *phy_id = data;
+
+	if (phydev->addr == *phy_id)
+		return 1;
+
+	return 0;
+}
+
+static const char *addr_propnames[] = {
+	"mac-address",
+	"local-mac-address",
+	"address",
+};
+
+static int acpi_get_mac_address(struct acpi_device *adev, u8 *dst)
+{
+	u64 mac;
+	int i;
+	int ret;
+
+	for (i = 0; i < ARRAY_SIZE(addr_propnames); i++) {
+		ret = acpi_dev_prop_read_single(adev, addr_propnames[i],
+						DEV_PROP_U64, &mac);
+		if (ret)
+			continue;
+
+		if (mac & (~0ULL << 48))
+			continue;	/* more than 6 bytes */
+
+		if (!is_valid_ether_addr((u8 *)&mac))
+			continue;
+
+		ether_addr_copy(dst, (u8 *)&mac);
+
+		return 0;
+	}
+
+	return ret ? ret : -EINVAL;
+}
+
+static acpi_status bgx_acpi_register_phy(acpi_handle handle,
+					u32 lvl, void *context, void **rv)
+{
+	struct acpi_reference_args args;
+	struct bgx *bgx = context;
+	struct acpi_device *adev;
+	struct device *phy_dev;
+	u32 phy_id;
+
+	if (acpi_bus_get_device(handle, &adev))
+		return AE_OK;
+
+	if (acpi_dev_get_property_reference(adev, "phy-handle", 0, &args))
+		return AE_OK;
+
+	if (acpi_dev_prop_read_single(args.adev, "phy-channel", DEV_PROP_U32,
+					&phy_id))
+		return AE_OK;
+
+	phy_dev = bus_find_device(&mdio_bus_type, NULL, (void *)&phy_id,
+				  bgx_match_phy_id);
+	if (!phy_dev)
+		return AE_OK;
+
+	SET_NETDEV_DEV(&bgx->lmac[bgx->lmac_count].netdev, &bgx->pdev->dev);
+	bgx->lmac[bgx->lmac_count].phydev = to_phy_device(phy_dev);
+
+	acpi_get_mac_address(adev, bgx->lmac[bgx->lmac_count].mac);
+
+	bgx->lmac[bgx->lmac_count].lmacid = bgx->lmac_count;
+	bgx->lmac_count++;
+
+	return AE_OK;
+}
+
+static acpi_status bgx_acpi_match_id(acpi_handle handle, u32 lvl,
+				void *context, void **ret_val)
+{
+	struct acpi_buffer string = { ACPI_ALLOCATE_BUFFER, NULL };
+	struct bgx *bgx = context;
+	char bgx_sel[5];
+
+	snprintf(bgx_sel, 5, "BGX%d", bgx->bgx_id);
+	if (ACPI_FAILURE(acpi_get_name(handle, ACPI_SINGLE_NAME, &string))) {
+		pr_warn("Invalid link device\n");
+		return AE_OK;
+	}
+
+	if (strncmp(string.pointer, bgx_sel, 4))
+		return AE_OK;
+
+	acpi_walk_namespace(ACPI_TYPE_DEVICE, handle, 1,
+			    bgx_acpi_register_phy, NULL, bgx, NULL);
+
+	kfree(string.pointer);
+	return AE_CTRL_TERMINATE;
+}
+
+static int bgx_init_acpi_phy(struct bgx *bgx)
+{
+	acpi_get_devices(NULL, bgx_acpi_match_id, bgx, (void **)NULL);
+	return 0;
+}
+
+#else
+
+static int bgx_init_acpi_phy(struct bgx *bgx)
+{
+	return -ENODEV;
+}
+
+#endif /* CONFIG_ACPI */
+
+#if IS_ENABLED(CONFIG_OF_MDIO)
+
+static int bgx_init_of_phy(struct bgx *bgx)
+{
+	struct device_node *np;
+	struct device_node *np_child;
+	u8 lmac = 0;
+	char bgx_sel[5];
+	const char *mac;
+
+	/* Get BGX node from DT */
+	snprintf(bgx_sel, 5, "bgx%d", bgx->bgx_id);
+	np = of_find_node_by_name(NULL, bgx_sel);
+	if (!np)
+		return -ENODEV;
+
+	for_each_child_of_node(np, np_child) {
+		struct device_node *phy_np = of_parse_phandle(np_child,
+							      "phy-handle", 0);
+		if (!phy_np)
+			continue;
+		bgx->lmac[lmac].phydev = of_phy_find_device(phy_np);
+
+		mac = of_get_mac_address(np_child);
+		if (mac)
+			ether_addr_copy(bgx->lmac[lmac].mac, mac);
+
+		SET_NETDEV_DEV(&bgx->lmac[lmac].netdev, &bgx->pdev->dev);
+		bgx->lmac[lmac].lmacid = lmac;
+		lmac++;
+		if (lmac == MAX_LMAC_PER_BGX)
+			break;
+	}
+	return 0;
+}
+
+#else
+
+static int bgx_init_of_phy(struct bgx *bgx)
+{
+	return -ENODEV;
+}
+
+#endif /* CONFIG_OF_MDIO */
+
+static int bgx_init_phy(struct bgx *bgx)
+{
+	int err = bgx_init_of_phy(bgx);
+
+	if (err != -ENODEV)
+		return err;
+
+	return bgx_init_acpi_phy(bgx);
+}
+
+static int bgx_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+	int err;
+	struct device *dev = &pdev->dev;
+	struct bgx *bgx = NULL;
+	u8 lmac;
+
+	bgx = devm_kzalloc(dev, sizeof(*bgx), GFP_KERNEL);
+	if (!bgx)
+		return -ENOMEM;
+	bgx->pdev = pdev;
+
+	pci_set_drvdata(pdev, bgx);
+
+	err = pci_enable_device(pdev);
+	if (err) {
+		dev_err(dev, "Failed to enable PCI device\n");
+		pci_set_drvdata(pdev, NULL);
+		return err;
+	}
+
+	err = pci_request_regions(pdev, DRV_NAME);
+	if (err) {
+		dev_err(dev, "PCI request regions failed 0x%x\n", err);
+		goto err_disable_device;
+	}
+
+	/* MAP configuration registers */
+	bgx->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0);
+	if (!bgx->reg_base) {
+		dev_err(dev, "BGX: Cannot map CSR memory space, aborting\n");
+		err = -ENOMEM;
+		goto err_release_regions;
+	}
+	bgx->bgx_id = (pci_resource_start(pdev, PCI_CFG_REG_BAR_NUM) >> 24) & 1;
+	bgx->bgx_id += nic_get_node_id(pdev) * MAX_BGX_PER_CN88XX;
+
+	bgx_vnic[bgx->bgx_id] = bgx;
+	bgx_get_qlm_mode(bgx);
+
+	err = bgx_init_phy(bgx);
+	if (err)
+		goto err_enable;
+
+	bgx_init_hw(bgx);
+
+	/* Enable all LMACs */
+	for (lmac = 0; lmac < bgx->lmac_count; lmac++) {
+		err = bgx_lmac_enable(bgx, lmac);
+		if (err) {
+			dev_err(dev, "BGX%d failed to enable lmac%d\n",
+				bgx->bgx_id, lmac);
+			goto err_enable;
+		}
+	}
+
+	return 0;
+
+err_enable:
+	bgx_vnic[bgx->bgx_id] = NULL;
+err_release_regions:
+	pci_release_regions(pdev);
+err_disable_device:
+	pci_disable_device(pdev);
+	pci_set_drvdata(pdev, NULL);
+	return err;
+}
+
+static void bgx_remove(struct pci_dev *pdev)
+{
+	struct bgx *bgx = pci_get_drvdata(pdev);
+	u8 lmac;
+
+	/* Disable all LMACs */
+	for (lmac = 0; lmac < bgx->lmac_count; lmac++)
+		bgx_lmac_disable(bgx, lmac);
+
+	bgx_vnic[bgx->bgx_id] = NULL;
+	pci_release_regions(pdev);
+	pci_disable_device(pdev);
+	pci_set_drvdata(pdev, NULL);
+}
+
+static struct pci_driver bgx_driver = {
+	.name = DRV_NAME,
+	.id_table = bgx_id_table,
+	.probe = bgx_probe,
+	.remove = bgx_remove,
+};
+
+static int __init bgx_init_module(void)
+{
+	pr_info("%s, ver %s\n", DRV_NAME, DRV_VERSION);
+
+	return pci_register_driver(&bgx_driver);
+}
+
+static void __exit bgx_cleanup_module(void)
+{
+	pci_unregister_driver(&bgx_driver);
+}
+
+module_init(bgx_init_module);
+module_exit(bgx_cleanup_module);
diff --git a/sys/dev/vnic/thunder_bgx.h b/sys/dev/vnic/thunder_bgx.h
new file mode 100644
index 000000000000..b1c761866b55
--- /dev/null
+++ b/sys/dev/vnic/thunder_bgx.h
@@ -0,0 +1,248 @@
+/*
+ * Copyright (C) 2015 Cavium Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ *
+ */
+
+#ifndef THUNDER_BGX_H
+#define THUNDER_BGX_H
+
+#define    MAX_BGX_THUNDER			8 /* Max 4 nodes, 2 per node */
+#define    MAX_BGX_PER_CN88XX			2
+#define    MAX_LMAC_PER_BGX			4
+#define    MAX_BGX_CHANS_PER_LMAC		16
+#define    MAX_DMAC_PER_LMAC			8
+#define    MAX_FRAME_SIZE			9216
+
+#define    MAX_DMAC_PER_LMAC_TNS_BYPASS_MODE	2
+
+#define    MAX_LMAC	(MAX_BGX_PER_CN88XX * MAX_LMAC_PER_BGX)
+
+/* Registers */
+#define BGX_CMRX_CFG			0x00
+#define  CMR_PKT_TX_EN				BIT_ULL(13)
+#define  CMR_PKT_RX_EN				BIT_ULL(14)
+#define  CMR_EN					BIT_ULL(15)
+#define BGX_CMR_GLOBAL_CFG		0x08
+#define  CMR_GLOBAL_CFG_FCS_STRIP		BIT_ULL(6)
+#define BGX_CMRX_RX_ID_MAP		0x60
+#define BGX_CMRX_RX_STAT0		0x70
+#define BGX_CMRX_RX_STAT1		0x78
+#define BGX_CMRX_RX_STAT2		0x80
+#define BGX_CMRX_RX_STAT3		0x88
+#define BGX_CMRX_RX_STAT4		0x90
+#define BGX_CMRX_RX_STAT5		0x98
+#define BGX_CMRX_RX_STAT6		0xA0
+#define BGX_CMRX_RX_STAT7		0xA8
+#define BGX_CMRX_RX_STAT8		0xB0
+#define BGX_CMRX_RX_STAT9		0xB8
+#define BGX_CMRX_RX_STAT10		0xC0
+#define BGX_CMRX_RX_BP_DROP		0xC8
+#define BGX_CMRX_RX_DMAC_CTL		0x0E8
+#define BGX_CMR_RX_DMACX_CAM		0x200
+#define  RX_DMACX_CAM_EN			BIT_ULL(48)
+#define  RX_DMACX_CAM_LMACID(x)			(x << 49)
+#define  RX_DMAC_COUNT				32
+#define BGX_CMR_RX_STREERING		0x300
+#define  RX_TRAFFIC_STEER_RULE_COUNT		8
+#define BGX_CMR_CHAN_MSK_AND		0x450
+#define BGX_CMR_BIST_STATUS		0x460
+#define BGX_CMR_RX_LMACS		0x468
+#define BGX_CMRX_TX_STAT0		0x600
+#define BGX_CMRX_TX_STAT1		0x608
+#define BGX_CMRX_TX_STAT2		0x610
+#define BGX_CMRX_TX_STAT3		0x618
+#define BGX_CMRX_TX_STAT4		0x620
+#define BGX_CMRX_TX_STAT5		0x628
+#define BGX_CMRX_TX_STAT6		0x630
+#define BGX_CMRX_TX_STAT7		0x638
+#define BGX_CMRX_TX_STAT8		0x640
+#define BGX_CMRX_TX_STAT9		0x648
+#define BGX_CMRX_TX_STAT10		0x650
+#define BGX_CMRX_TX_STAT11		0x658
+#define BGX_CMRX_TX_STAT12		0x660
+#define BGX_CMRX_TX_STAT13		0x668
+#define BGX_CMRX_TX_STAT14		0x670
+#define BGX_CMRX_TX_STAT15		0x678
+#define BGX_CMRX_TX_STAT16		0x680
+#define BGX_CMRX_TX_STAT17		0x688
+#define BGX_CMR_TX_LMACS		0x1000
+
+#define BGX_SPUX_CONTROL1		0x10000
+#define  SPU_CTL_LOW_POWER			BIT_ULL(11)
+#define  SPU_CTL_LOOPBACK			BIT_ULL(14)
+#define  SPU_CTL_RESET				BIT_ULL(15)
+#define BGX_SPUX_STATUS1		0x10008
+#define  SPU_STATUS1_RCV_LNK			BIT_ULL(2)
+#define BGX_SPUX_STATUS2		0x10020
+#define  SPU_STATUS2_RCVFLT			BIT_ULL(10)
+#define BGX_SPUX_BX_STATUS		0x10028
+#define  SPU_BX_STATUS_RX_ALIGN			BIT_ULL(12)
+#define BGX_SPUX_BR_STATUS1		0x10030
+#define  SPU_BR_STATUS_BLK_LOCK			BIT_ULL(0)
+#define  SPU_BR_STATUS_RCV_LNK			BIT_ULL(12)
+#define BGX_SPUX_BR_PMD_CRTL		0x10068
+#define  SPU_PMD_CRTL_TRAIN_EN			BIT_ULL(1)
+#define BGX_SPUX_BR_PMD_LP_CUP		0x10078
+#define BGX_SPUX_BR_PMD_LD_CUP		0x10088
+#define BGX_SPUX_BR_PMD_LD_REP		0x10090
+#define BGX_SPUX_FEC_CONTROL		0x100A0
+#define  SPU_FEC_CTL_FEC_EN			BIT_ULL(0)
+#define  SPU_FEC_CTL_ERR_EN			BIT_ULL(1)
+#define BGX_SPUX_AN_CONTROL		0x100C8
+#define  SPU_AN_CTL_AN_EN			BIT_ULL(12)
+#define  SPU_AN_CTL_XNP_EN			BIT_ULL(13)
+#define BGX_SPUX_AN_ADV			0x100D8
+#define BGX_SPUX_MISC_CONTROL		0x10218
+#define  SPU_MISC_CTL_INTLV_RDISP		BIT_ULL(10)
+#define  SPU_MISC_CTL_RX_DIS			BIT_ULL(12)
+#define BGX_SPUX_INT			0x10220	/* +(0..3) << 20 */
+#define BGX_SPUX_INT_W1S		0x10228
+#define BGX_SPUX_INT_ENA_W1C		0x10230
+#define BGX_SPUX_INT_ENA_W1S		0x10238
+#define BGX_SPU_DBG_CONTROL		0x10300
+#define  SPU_DBG_CTL_AN_ARB_LINK_CHK_EN		BIT_ULL(18)
+#define  SPU_DBG_CTL_AN_NONCE_MCT_DIS		BIT_ULL(29)
+
+#define BGX_SMUX_RX_INT			0x20000
+#define BGX_SMUX_RX_JABBER		0x20030
+#define BGX_SMUX_RX_CTL			0x20048
+#define  SMU_RX_CTL_STATUS			(3ull << 0)
+#define BGX_SMUX_TX_APPEND		0x20100
+#define  SMU_TX_APPEND_FCS_D			BIT_ULL(2)
+#define BGX_SMUX_TX_MIN_PKT		0x20118
+#define BGX_SMUX_TX_INT			0x20140
+#define BGX_SMUX_TX_CTL			0x20178
+#define  SMU_TX_CTL_DIC_EN			BIT_ULL(0)
+#define  SMU_TX_CTL_UNI_EN			BIT_ULL(1)
+#define  SMU_TX_CTL_LNK_STATUS			(3ull << 4)
+#define BGX_SMUX_TX_THRESH		0x20180
+#define BGX_SMUX_CTL			0x20200
+#define  SMU_CTL_RX_IDLE			BIT_ULL(0)
+#define  SMU_CTL_TX_IDLE			BIT_ULL(1)
+
+#define BGX_GMP_PCS_MRX_CTL		0x30000
+#define	 PCS_MRX_CTL_RST_AN			BIT_ULL(9)
+#define	 PCS_MRX_CTL_PWR_DN			BIT_ULL(11)
+#define	 PCS_MRX_CTL_AN_EN			BIT_ULL(12)
+#define	 PCS_MRX_CTL_LOOPBACK1			BIT_ULL(14)
+#define	 PCS_MRX_CTL_RESET			BIT_ULL(15)
+#define BGX_GMP_PCS_MRX_STATUS		0x30008
+#define	 PCS_MRX_STATUS_AN_CPT			BIT_ULL(5)
+#define BGX_GMP_PCS_ANX_AN_RESULTS	0x30020
+#define BGX_GMP_PCS_SGM_AN_ADV		0x30068
+#define BGX_GMP_PCS_MISCX_CTL		0x30078
+#define  PCS_MISC_CTL_GMX_ENO			BIT_ULL(11)
+#define  PCS_MISC_CTL_SAMP_PT_MASK	0x7Full
+#define BGX_GMP_GMI_PRTX_CFG		0x38020
+#define  GMI_PORT_CFG_SPEED			BIT_ULL(1)
+#define  GMI_PORT_CFG_DUPLEX			BIT_ULL(2)
+#define  GMI_PORT_CFG_SLOT_TIME			BIT_ULL(3)
+#define  GMI_PORT_CFG_SPEED_MSB			BIT_ULL(8)
+#define BGX_GMP_GMI_RXX_JABBER		0x38038
+#define BGX_GMP_GMI_TXX_THRESH		0x38210
+#define BGX_GMP_GMI_TXX_APPEND		0x38218
+#define BGX_GMP_GMI_TXX_SLOT		0x38220
+#define BGX_GMP_GMI_TXX_BURST		0x38228
+#define BGX_GMP_GMI_TXX_MIN_PKT		0x38240
+#define BGX_GMP_GMI_TXX_SGMII_CTL	0x38300
+
+#define BGX_MSIX_VEC_0_29_ADDR		0x400000 /* +(0..29) << 4 */
+#define BGX_MSIX_VEC_0_29_CTL		0x400008
+#define BGX_MSIX_PBA_0			0x4F0000
+
+/* MSI-X interrupts */
+#define BGX_MSIX_VECTORS	30
+#define BGX_LMAC_VEC_OFFSET	7
+#define BGX_MSIX_VEC_SHIFT	4
+
+#define CMRX_INT		0
+#define SPUX_INT		1
+#define SMUX_RX_INT		2
+#define SMUX_TX_INT		3
+#define GMPX_PCS_INT		4
+#define GMPX_GMI_RX_INT		5
+#define GMPX_GMI_TX_INT		6
+#define CMR_MEM_INT		28
+#define SPU_MEM_INT		29
+
+#define LMAC_INTR_LINK_UP	BIT(0)
+#define LMAC_INTR_LINK_DOWN	BIT(1)
+
+/*  RX_DMAC_CTL configuration*/
+enum MCAST_MODE {
+		MCAST_MODE_REJECT,
+		MCAST_MODE_ACCEPT,
+		MCAST_MODE_CAM_FILTER,
+		RSVD
+};
+
+#define BCAST_ACCEPT	1
+#define CAM_ACCEPT	1
+
+void octeon_mdiobus_force_mod_depencency(void);
+void bgx_add_dmac_addr(u64 dmac, int node, int bgx_idx, int lmac);
+unsigned bgx_get_map(int node);
+int bgx_get_lmac_count(int node, int bgx);
+const u8 *bgx_get_lmac_mac(int node, int bgx_idx, int lmacid);
+void bgx_set_lmac_mac(int node, int bgx_idx, int lmacid, const u8 *mac);
+void bgx_get_lmac_link_state(int node, int bgx_idx, int lmacid, void *status);
+void bgx_lmac_internal_loopback(int node, int bgx_idx,
+				int lmac_idx, bool enable);
+u64 bgx_get_rx_stats(int node, int bgx_idx, int lmac, int idx);
+u64 bgx_get_tx_stats(int node, int bgx_idx, int lmac, int idx);
+#define BGX_RX_STATS_COUNT 11
+#define BGX_TX_STATS_COUNT 18
+
+struct bgx_stats {
+	u64 rx_stats[BGX_RX_STATS_COUNT];
+	u64 tx_stats[BGX_TX_STATS_COUNT];
+};
+
+#define BGX_IN_PROMISCUOUS_MODE 1
+
+enum LMAC_TYPE {
+	BGX_MODE_SGMII = 0, /* 1 lane, 1.250 Gbaud */
+	BGX_MODE_XAUI = 1,  /* 4 lanes, 3.125 Gbaud */
+	BGX_MODE_DXAUI = 1, /* 4 lanes, 6.250 Gbaud */
+	BGX_MODE_RXAUI = 2, /* 2 lanes, 6.250 Gbaud */
+	BGX_MODE_XFI = 3,   /* 1 lane, 10.3125 Gbaud */
+	BGX_MODE_XLAUI = 4, /* 4 lanes, 10.3125 Gbaud */
+	BGX_MODE_10G_KR = 3,/* 1 lane, 10.3125 Gbaud */
+	BGX_MODE_40G_KR = 4,/* 4 lanes, 10.3125 Gbaud */
+};
+
+enum qlm_mode {
+	QLM_MODE_SGMII,         /* SGMII, each lane independent */
+	QLM_MODE_XAUI_1X4,      /* 1 XAUI or DXAUI, 4 lanes */
+	QLM_MODE_RXAUI_2X2,     /* 2 RXAUI, 2 lanes each */
+	QLM_MODE_XFI_4X1,       /* 4 XFI, 1 lane each */
+	QLM_MODE_XLAUI_1X4,     /* 1 XLAUI, 4 lanes each */
+	QLM_MODE_10G_KR_4X1,    /* 4 10GBASE-KR, 1 lane each */
+	QLM_MODE_40G_KR4_1X4,   /* 1 40GBASE-KR4, 4 lanes each */
+};
+
+#endif /* THUNDER_BGX_H */

From 710dcf20352bbf19a7f847496a8355d5a0750ee2 Mon Sep 17 00:00:00 2001
From: zbb <zbb@FreeBSD.org>
Date: Sun, 18 Oct 2015 22:02:58 +0000
Subject: [PATCH 364/380] Introduce initial support for Cavium's ThunderX
 networking interface

- The driver consists of three main componens: PF, VF, BGX
- Requires appropriate entries in DTS and MDIO driver
- Supports only FDT configuration
- Multiple Tx queues and single Rx queue supported
- No RSS, HW checksum and TSO support
- No more than 8 queues per-IF (only one Queue Set per IF)
- HW statistics enabled
- Works in all available MAC modes (1,10,20,40G)
- Style converted to BSD according to style(9)
- The code brings lmac_if interface used by the BGX driver to
  update its logical MACs state.

Obtained from: Semihalf
Sponsored by:  The FreeBSD Foundation
---
 sys/conf/files.arm64           |    6 +
 sys/dev/vnic/lmac_if.m         |  102 ++
 sys/dev/vnic/nic.h             |  447 +++---
 sys/dev/vnic/nic_main.c        | 1227 +++++++-------
 sys/dev/vnic/nic_reg.h         |  356 ++--
 sys/dev/vnic/nicvf_main.c      | 2770 +++++++++++++++-----------------
 sys/dev/vnic/nicvf_queues.c    | 1780 +++++++++++++-------
 sys/dev/vnic/nicvf_queues.h    |  412 ++---
 sys/dev/vnic/q_struct.h        |  714 ++++----
 sys/dev/vnic/thunder_bgx.c     | 1001 ++++++------
 sys/dev/vnic/thunder_bgx.h     |  320 ++--
 sys/dev/vnic/thunder_bgx_fdt.c |  207 +++
 sys/dev/vnic/thunder_bgx_var.h |   67 +
 13 files changed, 5046 insertions(+), 4363 deletions(-)
 create mode 100644 sys/dev/vnic/lmac_if.m
 create mode 100644 sys/dev/vnic/thunder_bgx_fdt.c
 create mode 100644 sys/dev/vnic/thunder_bgx_var.h

diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64
index 399a35ec0663..698c053166f1 100644
--- a/sys/conf/files.arm64
+++ b/sys/conf/files.arm64
@@ -68,6 +68,12 @@ dev/psci/psci_arm64.S		optional	psci
 dev/uart/uart_cpu_fdt.c		optional	uart fdt
 dev/uart/uart_dev_pl011.c	optional	uart pl011
 dev/usb/controller/dwc_otg_hisi.c optional	dwcotg soc_hisi_hi6220
+dev/vnic/nic_main.c		optional	vnic pci
+dev/vnic/nicvf_main.c		optional	vnic pci pci_iov
+dev/vnic/nicvf_queues.c		optional	vnic pci pci_iov
+dev/vnic/thunder_bgx_fdt.c	optional	vnic fdt
+dev/vnic/thunder_bgx.c		optional	vnic pci
+dev/vnic/lmac_if.m		optional	vnic
 kern/kern_clocksource.c		standard
 kern/subr_dummy_vdso_tc.c	standard
 libkern/bcmp.c			standard
diff --git a/sys/dev/vnic/lmac_if.m b/sys/dev/vnic/lmac_if.m
new file mode 100644
index 000000000000..e8770f22ce11
--- /dev/null
+++ b/sys/dev/vnic/lmac_if.m
@@ -0,0 +1,102 @@
+#-
+# Copyright (c) 2015 The FreeBSD Foundation
+# All rights reserved.
+#
+# This software was developed by Semihalf under
+# the sponsorship of the FreeBSD Foundation.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+
+# LMAC (BGX controller) interface description
+#
+
+INTERFACE lmac;
+
+CODE {
+	static int null_lmac_media_status(device_t dev, int lmacid, int *link,
+	    int *duplex, int *speed)
+	{
+		return (ENXIO);
+	}
+
+	static int null_lmac_media_change(device_t dev, int lmacid, int link,
+	    int duplex, int speed)
+	{
+		return (ENXIO);
+	}
+
+	static int null_lmac_phy_connect(device_t dev, int lmacid, int phy)
+	{
+		return (ENXIO);
+	}
+
+	static int null_lmac_phy_disconnect(device_t dev, int lmacid, int phy)
+	{
+		return (ENXIO);
+	}
+};
+
+# Get link status
+#
+# 0 : Success
+#
+METHOD int media_status {
+	device_t		dev;
+	int			lmacid;
+	int *			link;
+	int *			duplex;
+	int *			speed;
+} DEFAULT null_lmac_media_status;
+
+# Change link status
+#
+# 0 : Success
+#
+METHOD int media_change {
+	device_t		dev;
+	int			lmacid;
+	int			link;
+	int			duplex;
+	int			speed;
+} DEFAULT null_lmac_media_change;
+
+# Connect PHY
+#
+# 0 : Success
+#
+METHOD int phy_connect {
+	device_t		dev;
+	int			lmacid;
+	int			phy;
+} DEFAULT null_lmac_phy_connect;
+
+# Disconnect PHY
+#
+# 0 : Success
+#
+METHOD int phy_disconnect {
+	device_t		dev;
+	int			lmacid;
+	int			phy;
+} DEFAULT null_lmac_phy_disconnect;
diff --git a/sys/dev/vnic/nic.h b/sys/dev/vnic/nic.h
index 3023deabd05c..f2a997aa88e1 100644
--- a/sys/dev/vnic/nic.h
+++ b/sys/dev/vnic/nic.h
@@ -30,11 +30,8 @@
 #ifndef NIC_H
 #define	NIC_H
 
-#include <linux/netdevice.h>
-#include <linux/interrupt.h>
-#include <linux/pci.h>
-#include "thunder_bgx.h"
-
+/* PCI vendor ID */
+#define PCI_VENDOR_ID_CAVIUM			0x177D
 /* PCI device IDs */
 #define	PCI_DEVICE_ID_THUNDER_NIC_PF		0xA01E
 #define	PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF	0x0011
@@ -53,12 +50,15 @@
 #define	NIC_TNS_MODE			1
 
 /* NIC priv flags */
-#define	NIC_SRIOV_ENABLED		BIT(0)
-#define	NIC_TNS_ENABLED			BIT(1)
+#define	NIC_SRIOV_ENABLED		(1 << 0)
+#define	NIC_TNS_ENABLED			(1 << 1)
 
+/* ARM64TODO */
+#if 0
 /* VNIC HW optimiation features */
 #define VNIC_RSS_SUPPORT
 #define VNIC_MULTI_QSET_SUPPORT
+#endif
 
 /* Min/Max packet size */
 #define	NIC_HW_MIN_FRS			64
@@ -67,7 +67,8 @@
 /* Max pkinds */
 #define	NIC_MAX_PKIND			16
 
-/* Rx Channels */
+/*
+ * Rx Channels */
 /* Receive channel configuration in TNS bypass mode
  * Below is configuration in TNS bypass mode
  * BGX0-LMAC0-CHAN0 - VNIC CHAN0
@@ -83,7 +84,7 @@
 #define	NIC_CPI_COUNT			2048 /* No of channel parse indices */
 
 /* TNS bypass mode: 1-1 mapping between VNIC and BGX:LMAC */
-#define NIC_MAX_BGX			MAX_BGX_PER_CN88XX
+#define	NIC_MAX_BGX			MAX_BGX_PER_CN88XX
 #define	NIC_CPI_PER_BGX			(NIC_CPI_COUNT / NIC_MAX_BGX)
 #define	NIC_MAX_CPI_PER_LMAC		64 /* Max when CPI_ALG is IP diffserv */
 #define	NIC_RSSI_PER_BGX		(NIC_RSSI_COUNT / NIC_MAX_BGX)
@@ -122,27 +123,33 @@
 #define	NICVF_INTR_CQ_MASK		(0xFF << NICVF_INTR_CQ_SHIFT)
 #define	NICVF_INTR_SQ_MASK		(0xFF << NICVF_INTR_SQ_SHIFT)
 #define	NICVF_INTR_RBDR_MASK		(0x03 << NICVF_INTR_RBDR_SHIFT)
-#define	NICVF_INTR_PKT_DROP_MASK	BIT(NICVF_INTR_PKT_DROP_SHIFT)
-#define	NICVF_INTR_TCP_TIMER_MASK	BIT(NICVF_INTR_TCP_TIMER_SHIFT)
-#define	NICVF_INTR_MBOX_MASK		BIT(NICVF_INTR_MBOX_SHIFT)
-#define	NICVF_INTR_QS_ERR_MASK		BIT(NICVF_INTR_QS_ERR_SHIFT)
+#define	NICVF_INTR_PKT_DROP_MASK	(1 << NICVF_INTR_PKT_DROP_SHIFT)
+#define	NICVF_INTR_TCP_TIMER_MASK	(1 << NICVF_INTR_TCP_TIMER_SHIFT)
+#define	NICVF_INTR_MBOX_MASK		(1 << NICVF_INTR_MBOX_SHIFT)
+#define	NICVF_INTR_QS_ERR_MASK		(1 << NICVF_INTR_QS_ERR_SHIFT)
 
 /* MSI-X interrupts */
 #define	NIC_PF_MSIX_VECTORS		10
 #define	NIC_VF_MSIX_VECTORS		20
 
-#define NIC_PF_INTR_ID_ECC0_SBE		0
-#define NIC_PF_INTR_ID_ECC0_DBE		1
-#define NIC_PF_INTR_ID_ECC1_SBE		2
-#define NIC_PF_INTR_ID_ECC1_DBE		3
-#define NIC_PF_INTR_ID_ECC2_SBE		4
-#define NIC_PF_INTR_ID_ECC2_DBE		5
-#define NIC_PF_INTR_ID_ECC3_SBE		6
-#define NIC_PF_INTR_ID_ECC3_DBE		7
-#define NIC_PF_INTR_ID_MBOX0		8
-#define NIC_PF_INTR_ID_MBOX1		9
+#define	NIC_PF_INTR_ID_ECC0_SBE		0
+#define	NIC_PF_INTR_ID_ECC0_DBE		1
+#define	NIC_PF_INTR_ID_ECC1_SBE		2
+#define	NIC_PF_INTR_ID_ECC1_DBE		3
+#define	NIC_PF_INTR_ID_ECC2_SBE		4
+#define	NIC_PF_INTR_ID_ECC2_DBE		5
+#define	NIC_PF_INTR_ID_ECC3_SBE		6
+#define	NIC_PF_INTR_ID_ECC3_DBE		7
+#define	NIC_PF_INTR_ID_MBOX0		8
+#define	NIC_PF_INTR_ID_MBOX1		9
 
-/* Global timer for CQ timer thresh interrupts
+struct msix_entry {
+	struct resource *	irq_res;
+	void *			handle;
+};
+
+/*
+ * Global timer for CQ timer thresh interrupts
  * Calculated for SCLK of 700Mhz
  * value written should be a 1/16th of what is expected
  *
@@ -151,7 +158,8 @@
  */
 #define NICPF_CLK_PER_INT_TICK		2
 
-/* Time to wait before we decide that a SQ is stuck.
+/*
+ * Time to wait before we decide that a SQ is stuck.
  *
  * Since both pkt rx and tx notifications are done with same CQ,
  * when packets are being received at very high rate (eg: L2 forwarding)
@@ -160,36 +168,10 @@
  */
 #define	NICVF_TX_TIMEOUT		(50 * HZ)
 
-struct nicvf_cq_poll {
-	struct  nicvf *nicvf;
-	u8	cq_idx;		/* Completion queue index */
-	struct	napi_struct napi;
-};
-
 #define	NIC_RSSI_COUNT			4096 /* Total no of RSS indices */
-#define NIC_MAX_RSS_HASH_BITS		8
-#define NIC_MAX_RSS_IDR_TBL_SIZE	(1 << NIC_MAX_RSS_HASH_BITS)
-#define RSS_HASH_KEY_SIZE		5 /* 320 bit key */
-
-#ifdef VNIC_RSS_SUPPORT
-struct nicvf_rss_info {
-	bool enable;
-#define	RSS_L2_EXTENDED_HASH_ENA	BIT(0)
-#define	RSS_IP_HASH_ENA			BIT(1)
-#define	RSS_TCP_HASH_ENA		BIT(2)
-#define	RSS_TCP_SYN_DIS			BIT(3)
-#define	RSS_UDP_HASH_ENA		BIT(4)
-#define RSS_L4_EXTENDED_HASH_ENA	BIT(5)
-#define	RSS_ROCE_ENA			BIT(6)
-#define	RSS_L3_BI_DIRECTION_ENA		BIT(7)
-#define	RSS_L4_BI_DIRECTION_ENA		BIT(8)
-	u64 cfg;
-	u8  hash_bits;
-	u16 rss_size;
-	u8  ind_tbl[NIC_MAX_RSS_IDR_TBL_SIZE];
-	u64 key[RSS_HASH_KEY_SIZE];
-} ____cacheline_aligned_in_smp;
-#endif
+#define	NIC_MAX_RSS_HASH_BITS		8
+#define	NIC_MAX_RSS_IDR_TBL_SIZE	(1 << NIC_MAX_RSS_HASH_BITS)
+#define	RSS_HASH_KEY_SIZE		5 /* 320 bit key */
 
 enum rx_stats_reg_offset {
 	RX_OCTS = 0x0,
@@ -219,132 +201,124 @@ enum tx_stats_reg_offset {
 };
 
 struct nicvf_hw_stats {
-	u64 rx_bytes;
-	u64 rx_ucast_frames;
-	u64 rx_bcast_frames;
-	u64 rx_mcast_frames;
-	u64 rx_fcs_errors;
-	u64 rx_l2_errors;
-	u64 rx_drop_red;
-	u64 rx_drop_red_bytes;
-	u64 rx_drop_overrun;
-	u64 rx_drop_overrun_bytes;
-	u64 rx_drop_bcast;
-	u64 rx_drop_mcast;
-	u64 rx_drop_l3_bcast;
-	u64 rx_drop_l3_mcast;
-	u64 rx_bgx_truncated_pkts;
-	u64 rx_jabber_errs;
-	u64 rx_fcs_errs;
-	u64 rx_bgx_errs;
-	u64 rx_prel2_errs;
-	u64 rx_l2_hdr_malformed;
-	u64 rx_oversize;
-	u64 rx_undersize;
-	u64 rx_l2_len_mismatch;
-	u64 rx_l2_pclp;
-	u64 rx_ip_ver_errs;
-	u64 rx_ip_csum_errs;
-	u64 rx_ip_hdr_malformed;
-	u64 rx_ip_payload_malformed;
-	u64 rx_ip_ttl_errs;
-	u64 rx_l3_pclp;
-	u64 rx_l4_malformed;
-	u64 rx_l4_csum_errs;
-	u64 rx_udp_len_errs;
-	u64 rx_l4_port_errs;
-	u64 rx_tcp_flag_errs;
-	u64 rx_tcp_offset_errs;
-	u64 rx_l4_pclp;
-	u64 rx_truncated_pkts;
+	uint64_t rx_bytes;
+	uint64_t rx_ucast_frames;
+	uint64_t rx_bcast_frames;
+	uint64_t rx_mcast_frames;
+	uint64_t rx_fcs_errors;
+	uint64_t rx_l2_errors;
+	uint64_t rx_drop_red;
+	uint64_t rx_drop_red_bytes;
+	uint64_t rx_drop_overrun;
+	uint64_t rx_drop_overrun_bytes;
+	uint64_t rx_drop_bcast;
+	uint64_t rx_drop_mcast;
+	uint64_t rx_drop_l3_bcast;
+	uint64_t rx_drop_l3_mcast;
+	uint64_t rx_bgx_truncated_pkts;
+	uint64_t rx_jabber_errs;
+	uint64_t rx_fcs_errs;
+	uint64_t rx_bgx_errs;
+	uint64_t rx_prel2_errs;
+	uint64_t rx_l2_hdr_malformed;
+	uint64_t rx_oversize;
+	uint64_t rx_undersize;
+	uint64_t rx_l2_len_mismatch;
+	uint64_t rx_l2_pclp;
+	uint64_t rx_ip_ver_errs;
+	uint64_t rx_ip_csum_errs;
+	uint64_t rx_ip_hdr_malformed;
+	uint64_t rx_ip_payload_malformed;
+	uint64_t rx_ip_ttl_errs;
+	uint64_t rx_l3_pclp;
+	uint64_t rx_l4_malformed;
+	uint64_t rx_l4_csum_errs;
+	uint64_t rx_udp_len_errs;
+	uint64_t rx_l4_port_errs;
+	uint64_t rx_tcp_flag_errs;
+	uint64_t rx_tcp_offset_errs;
+	uint64_t rx_l4_pclp;
+	uint64_t rx_truncated_pkts;
 
-	u64 tx_bytes_ok;
-	u64 tx_ucast_frames_ok;
-	u64 tx_bcast_frames_ok;
-	u64 tx_mcast_frames_ok;
-	u64 tx_drops;
+	uint64_t tx_bytes_ok;
+	uint64_t tx_ucast_frames_ok;
+	uint64_t tx_bcast_frames_ok;
+	uint64_t tx_mcast_frames_ok;
+	uint64_t tx_drops;
 };
 
 struct nicvf_drv_stats {
 	/* Rx */
-	u64 rx_frames_ok;
-	u64 rx_frames_64;
-	u64 rx_frames_127;
-	u64 rx_frames_255;
-	u64 rx_frames_511;
-	u64 rx_frames_1023;
-	u64 rx_frames_1518;
-	u64 rx_frames_jumbo;
-	u64 rx_drops;
+	uint64_t rx_frames_ok;
+	uint64_t rx_frames_64;
+	uint64_t rx_frames_127;
+	uint64_t rx_frames_255;
+	uint64_t rx_frames_511;
+	uint64_t rx_frames_1023;
+	uint64_t rx_frames_1518;
+	uint64_t rx_frames_jumbo;
+	uint64_t rx_drops;
 
 	/* Tx */
-	u64 tx_frames_ok;
-	u64 tx_drops;
-	u64 tx_tso;
-	u64 txq_stop;
-	u64 txq_wake;
+	uint64_t tx_frames_ok;
+	uint64_t tx_drops;
+	uint64_t tx_tso;
+	uint64_t txq_stop;
+	uint64_t txq_wake;
 };
 
 struct nicvf {
 	struct nicvf		*pnicvf;
-	struct net_device	*netdev;
-	struct pci_dev		*pdev;
-	u8			vf_id;
-	u8			node;
-	bool			tns_mode:1;
-	bool                    sqs_mode:1;
-	bool			loopback_supported:1;
-	u16			mtu;
-	struct queue_set	*qs;
-#ifdef VNIC_MULTI_QSET_SUPPORT
-#define	MAX_SQS_PER_VF_SINGLE_NODE		5
-#define	MAX_SQS_PER_VF				11
-	u8			sqs_id;
-	u8			sqs_count; /* Secondary Qset count */
-	struct nicvf		*snicvf[MAX_SQS_PER_VF];
-#endif
-	u8			rx_queues;
-	u8			tx_queues;
-	u8			max_queues;
-	void __iomem		*reg_base;
-	bool			link_up;
-	u8			duplex;
-	u32			speed;
-	struct page		*rb_page;
-	u32			rb_page_offset;
-	bool			rb_alloc_fail;
-	bool			rb_work_scheduled;
-	struct delayed_work	rbdr_work;
-	struct tasklet_struct	rbdr_task;
-	struct tasklet_struct	qs_err_task;
-	struct tasklet_struct	cq_task;
-	struct nicvf_cq_poll	*napi[8];
-#ifdef VNIC_RSS_SUPPORT
-	struct nicvf_rss_info	rss_info;
-#endif
-	u8			cpi_alg;
-	/* Interrupt coalescing settings */
-	u32			cq_coalesce_usecs;
+	device_t		dev;
 
-	u32			msg_enable;
-	struct nicvf_hw_stats   hw_stats;
-	struct nicvf_drv_stats  drv_stats;
+	struct ifnet *		ifp;
+	struct sx		core_sx;
+	struct ifmedia		if_media;
+	uint32_t		if_flags;
+
+	uint8_t			hwaddr[ETHER_ADDR_LEN];
+	uint8_t			vf_id;
+	uint8_t			node;
+	boolean_t		tns_mode:1;
+	boolean_t		sqs_mode:1;
+	bool			loopback_supported:1;
+	uint16_t		mtu;
+	struct queue_set	*qs;
+	uint8_t			rx_queues;
+	uint8_t			tx_queues;
+	uint8_t			max_queues;
+	struct resource		*reg_base;
+	boolean_t		link_up;
+	uint8_t			duplex;
+	uint32_t		speed;
+	uint8_t			cpi_alg;
+	/* Interrupt coalescing settings */
+	uint32_t		cq_coalesce_usecs;
+
+	uint32_t		msg_enable;
+	struct nicvf_hw_stats	hw_stats;
+	struct nicvf_drv_stats	drv_stats;
 	struct bgx_stats	bgx_stats;
-	struct work_struct	reset_task;
+
+	/* Interface statistics */
+	struct callout		stats_callout;
+	struct mtx		stats_mtx;
 
 	/* MSI-X  */
-	bool			msix_enabled;
-	u8			num_vec;
+	boolean_t		msix_enabled;
+	uint8_t			num_vec;
 	struct msix_entry	msix_entries[NIC_VF_MSIX_VECTORS];
+	struct resource *	msix_table_res;
 	char			irq_name[NIC_VF_MSIX_VECTORS][20];
-	bool			irq_allocated[NIC_VF_MSIX_VECTORS];
+	boolean_t		irq_allocated[NIC_VF_MSIX_VECTORS];
 
 	/* VF <-> PF mailbox communication */
-	bool			pf_acked;
-	bool			pf_nacked;
-} ____cacheline_aligned_in_smp;
+	boolean_t		pf_acked;
+	boolean_t		pf_nacked;
+} __aligned(CACHE_LINE_SIZE);
 
-/* PF <--> VF Mailbox communication
+/*
+ * PF <--> VF Mailbox communication
  * Eight 64bit registers are shared between PF and VF.
  * Separate set for each VF.
  * Writing '1' into last register mbx7 means end of message.
@@ -381,123 +355,108 @@ struct nicvf {
 #define	NIC_MBOX_MSG_SHUTDOWN		0xF1	/* VF is being shutdown */
 
 struct nic_cfg_msg {
-	u8    msg;
-	u8    vf_id;
-	u8    node_id;
-	bool  tns_mode:1;
-	bool  sqs_mode:1;
-	bool  loopback_supported:1;
-	u8    mac_addr[ETH_ALEN];
+	uint8_t		msg;
+	uint8_t		vf_id;
+	uint8_t		node_id;
+	boolean_t	tns_mode:1;
+	boolean_t	sqs_mode:1;
+	boolean_t	loopback_supported:1;
+	uint8_t	mac_addr[ETHER_ADDR_LEN];
 };
 
 /* Qset configuration */
 struct qs_cfg_msg {
-	u8    msg;
-	u8    num;
-	u8    sqs_count;
-	u64   cfg;
+	uint8_t		msg;
+	uint8_t		num;
+	uint8_t		sqs_count;
+	uint64_t	cfg;
 };
 
 /* Receive queue configuration */
 struct rq_cfg_msg {
-	u8    msg;
-	u8    qs_num;
-	u8    rq_num;
-	u64   cfg;
+	uint8_t		msg;
+	uint8_t		qs_num;
+	uint8_t		rq_num;
+	uint64_t	cfg;
 };
 
 /* Send queue configuration */
 struct sq_cfg_msg {
-	u8    msg;
-	u8    qs_num;
-	u8    sq_num;
-	bool  sqs_mode;
-	u64   cfg;
+	uint8_t		msg;
+	uint8_t		qs_num;
+	uint8_t		sq_num;
+	boolean_t	sqs_mode;
+	uint64_t	cfg;
 };
 
 /* Set VF's MAC address */
 struct set_mac_msg {
-	u8    msg;
-	u8    vf_id;
-	u8    mac_addr[ETH_ALEN];
+	uint8_t		msg;
+	uint8_t		vf_id;
+	uint8_t		mac_addr[ETHER_ADDR_LEN];
 };
 
 /* Set Maximum frame size */
 struct set_frs_msg {
-	u8    msg;
-	u8    vf_id;
-	u16   max_frs;
+	uint8_t		msg;
+	uint8_t		vf_id;
+	uint16_t	max_frs;
 };
 
 /* Set CPI algorithm type */
 struct cpi_cfg_msg {
-	u8    msg;
-	u8    vf_id;
-	u8    rq_cnt;
-	u8    cpi_alg;
+	uint8_t		msg;
+	uint8_t		vf_id;
+	uint8_t		rq_cnt;
+	uint8_t		cpi_alg;
 };
 
 /* Get RSS table size */
 struct rss_sz_msg {
-	u8    msg;
-	u8    vf_id;
-	u16   ind_tbl_size;
+	uint8_t		msg;
+	uint8_t		vf_id;
+	uint16_t	ind_tbl_size;
 };
 
 /* Set RSS configuration */
 struct rss_cfg_msg {
-	u8    msg;
-	u8    vf_id;
-	u8    hash_bits;
-	u8    tbl_len;
-	u8    tbl_offset;
-#define RSS_IND_TBL_LEN_PER_MBX_MSG	8
-	u8    ind_tbl[RSS_IND_TBL_LEN_PER_MBX_MSG];
+	uint8_t		msg;
+	uint8_t		vf_id;
+	uint8_t		hash_bits;
+	uint8_t		tbl_len;
+	uint8_t		tbl_offset;
+#define	RSS_IND_TBL_LEN_PER_MBX_MSG	8
+	uint8_t		ind_tbl[RSS_IND_TBL_LEN_PER_MBX_MSG];
 };
 
 struct bgx_stats_msg {
-	u8    msg;
-	u8    vf_id;
-	u8    rx;
-	u8    idx;
-	u64   stats;
+	uint8_t		msg;
+	uint8_t		vf_id;
+	uint8_t		rx;
+	uint8_t		idx;
+	uint64_t	stats;
 };
 
 /* Physical interface link status */
 struct bgx_link_status {
-	u8    msg;
-	u8    link_up;
-	u8    duplex;
-	u32   speed;
+	uint8_t		msg;
+	uint8_t		link_up;
+	uint8_t		duplex;
+	uint32_t	speed;
 };
 
-#ifdef VNIC_MULTI_QSET_SUPPORT
-/* Get Extra Qset IDs */
-struct sqs_alloc {
-	u8    msg;
-	u8    vf_id;
-	u8    qs_count;
-};
-
-struct nicvf_ptr {
-	u8    msg;
-	u8    vf_id;
-	bool  sqs_mode;
-	u8    sqs_id;
-	u64   nicvf;
-};
-#endif
-
 /* Set interface in loopback mode */
 struct set_loopback {
-	u8    msg;
-	u8    vf_id;
-	bool  enable;
+	uint8_t		msg;
+	uint8_t		vf_id;
+	boolean_t	enable;
 };
 
 /* 128 bit shared memory between PF and each VF */
 union nic_mbx {
-	struct { u8 msg; }	msg;
+	struct {
+		uint8_t msg;
+	} msg;
 	struct nic_cfg_msg	nic_cfg;
 	struct qs_cfg_msg	qs;
 	struct rq_cfg_msg	rq;
@@ -507,33 +466,23 @@ union nic_mbx {
 	struct cpi_cfg_msg	cpi_cfg;
 	struct rss_sz_msg	rss_size;
 	struct rss_cfg_msg	rss_cfg;
-	struct bgx_stats_msg    bgx_stats;
-	struct bgx_link_status  link_status;
-#ifdef VNIC_MULTI_QSET_SUPPORT
-	struct sqs_alloc        sqs_alloc;
-	struct nicvf_ptr	nicvf;
-#endif
+	struct bgx_stats_msg	bgx_stats;
+	struct bgx_link_status	link_status;
 	struct set_loopback	lbk;
 };
 
-#define NIC_NODE_ID_MASK	0x03
-#define NIC_NODE_ID_SHIFT	44
+#define	NIC_NODE_ID_MASK	0x03
+#define	NIC_NODE_ID_SHIFT	44
 
-static inline int nic_get_node_id(struct pci_dev *pdev)
+static __inline int
+nic_get_node_id(struct resource *res)
 {
-	u64 addr = pci_resource_start(pdev, PCI_CFG_REG_BAR_NUM);
+	pci_addr_t addr;
+
+	addr = rman_get_start(res);
 	return ((addr >> NIC_NODE_ID_SHIFT) & NIC_NODE_ID_MASK);
 }
 
-int nicvf_set_real_num_queues(struct net_device *netdev,
-			      int tx_queues, int rx_queues);
-int nicvf_open(struct net_device *netdev);
-int nicvf_stop(struct net_device *netdev);
 int nicvf_send_msg_to_pf(struct nicvf *vf, union nic_mbx *mbx);
-void nicvf_config_rss(struct nicvf *nic);
-void nicvf_set_rss_key(struct nicvf *nic);
-void nicvf_set_ethtool_ops(struct net_device *netdev);
-void nicvf_update_stats(struct nicvf *nic);
-void nicvf_update_lmac_stats(struct nicvf *nic);
 
 #endif /* NIC_H */
diff --git a/sys/dev/vnic/nic_main.c b/sys/dev/vnic/nic_main.c
index de2f22ce0daa..1c350589af62 100644
--- a/sys/dev/vnic/nic_main.c
+++ b/sys/dev/vnic/nic_main.c
@@ -27,135 +27,411 @@
  *
  */
 
-#include <linux/module.h>
-#include <linux/interrupt.h>
-#include <linux/pci.h>
-#include <linux/etherdevice.h>
-#include <linux/of.h>
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
 
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bitset.h>
+#include <sys/bitstring.h>
+#include <sys/bus.h>
+#include <sys/endian.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/rman.h>
+#include <sys/pciio.h>
+#include <sys/pcpu.h>
+#include <sys/proc.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
+#include <sys/cpuset.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+
+#include <net/ethernet.h>
+#include <net/if.h>
+#include <net/if_media.h>
+
+#include <machine/bus.h>
+#include <machine/_inttypes.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#include <sys/dnv.h>
+#include <sys/nv.h>
+#ifdef PCI_IOV
+#include <sys/iov_schema.h>
+#include <dev/pci/pci_iov.h>
+#endif
+
+#include "thunder_bgx.h"
 #include "nic_reg.h"
 #include "nic.h"
 #include "q_struct.h"
-#include "thunder_bgx.h"
 
-#define DRV_NAME	"thunder-nic"
-#define DRV_VERSION	"1.0"
+#define	VNIC_PF_DEVSTR		"Cavium Thunder NIC Physical Function Driver"
+
+#define	VNIC_PF_REG_RID		PCIR_BAR(PCI_CFG_REG_BAR_NUM)
+
+#define	NIC_SET_VF_LMAC_MAP(bgx, lmac)		((((bgx) & 0xF) << 4) | ((lmac) & 0xF))
+#define	NIC_GET_BGX_FROM_VF_LMAC_MAP(map)	(((map) >> 4) & 0xF)
+#define	NIC_GET_LMAC_FROM_VF_LMAC_MAP(map)	((map) & 0xF)
+
+/* Structure to be used by the SR-IOV for VF configuration schemas */
+struct nicvf_info {
+	boolean_t		vf_enabled;
+	int			vf_flags;
+};
 
 struct nicpf {
-	struct pci_dev		*pdev;
-	u8			rev_id;
-	u8			node;
-	unsigned int		flags;
-	u8			num_vf_en;      /* No of VF enabled */
-	bool			vf_enabled[MAX_NUM_VFS_SUPPORTED];
-	void __iomem		*reg_base;       /* Register start address */
-#ifdef VNIC_MULTI_QSET_SUPPORT
-	u8			num_sqs_en;	/* Secondary qsets enabled */
-	u64			nicvf[MAX_NUM_VFS_SUPPORTED];
-	u8			vf_sqs[MAX_NUM_VFS_SUPPORTED][MAX_SQS_PER_VF];
-	u8			pqs_vf[MAX_NUM_VFS_SUPPORTED];
-	bool			sqs_used[MAX_NUM_VFS_SUPPORTED];
-#endif
+	device_t		dev;
+	uint8_t			rev_id;
+	uint8_t			node;
+	u_int			flags;
+	uint8_t			num_vf_en;      /* No of VF enabled */
+	struct nicvf_info	vf_info[MAX_NUM_VFS_SUPPORTED];
+	struct resource *	reg_base;       /* Register start address */
 	struct pkind_cfg	pkind;
-#define	NIC_SET_VF_LMAC_MAP(bgx, lmac)	(((bgx & 0xF) << 4) | (lmac & 0xF))
-#define	NIC_GET_BGX_FROM_VF_LMAC_MAP(map)	((map >> 4) & 0xF)
-#define	NIC_GET_LMAC_FROM_VF_LMAC_MAP(map)	(map & 0xF)
-	u8			vf_lmac_map[MAX_LMAC];
-	struct delayed_work     dwork;
-	struct workqueue_struct *check_link;
-	u8			link[MAX_LMAC];
-	u8			duplex[MAX_LMAC];
-	u32			speed[MAX_LMAC];
-	u16			cpi_base[MAX_NUM_VFS_SUPPORTED];
-	u16			rss_ind_tbl_size;
-	bool			mbx_lock[MAX_NUM_VFS_SUPPORTED];
+	uint8_t			vf_lmac_map[MAX_LMAC];
+	boolean_t		mbx_lock[MAX_NUM_VFS_SUPPORTED];
+
+	struct callout		check_link;
+	struct mtx		check_link_mtx;
+
+	uint8_t			link[MAX_LMAC];
+	uint8_t			duplex[MAX_LMAC];
+	uint32_t		speed[MAX_LMAC];
+	uint16_t		cpi_base[MAX_NUM_VFS_SUPPORTED];
+	uint16_t		rss_ind_tbl_size;
 
 	/* MSI-X */
-	bool			msix_enabled;
-	u8			num_vec;
+	boolean_t		msix_enabled;
+	uint8_t			num_vec;
 	struct msix_entry	msix_entries[NIC_PF_MSIX_VECTORS];
-	bool			irq_allocated[NIC_PF_MSIX_VECTORS];
+	struct resource *	msix_table_res;
 };
 
-/* Supported devices */
-static const struct pci_device_id nic_id_table[] = {
-	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_NIC_PF) },
-	{ 0, }  /* end of table */
+static int nicpf_probe(device_t);
+static int nicpf_attach(device_t);
+static int nicpf_detach(device_t);
+
+#ifdef PCI_IOV
+static int nicpf_iov_init(device_t, uint16_t, const nvlist_t *);
+static void nicpf_iov_uninit(device_t);
+static int nicpf_iov_addr_vf(device_t, uint16_t, const nvlist_t *);
+#endif
+
+static device_method_t nicpf_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		nicpf_probe),
+	DEVMETHOD(device_attach,	nicpf_attach),
+	DEVMETHOD(device_detach,	nicpf_detach),
+	/* PCI SR-IOV interface */
+#ifdef PCI_IOV
+	DEVMETHOD(pci_iov_init,		nicpf_iov_init),
+	DEVMETHOD(pci_iov_uninit,	nicpf_iov_uninit),
+	DEVMETHOD(pci_iov_add_vf,	nicpf_iov_addr_vf),
+#endif
+	DEVMETHOD_END,
 };
 
-MODULE_AUTHOR("Sunil Goutham");
-MODULE_DESCRIPTION("Cavium Thunder NIC Physical Function Driver");
-MODULE_VERSION(DRV_VERSION);
-MODULE_DEVICE_TABLE(pci, nic_id_table);
+static driver_t nicpf_driver = {
+	"vnicpf",
+	nicpf_methods,
+	sizeof(struct nicpf),
+};
 
-/* The Cavium ThunderX network controller can *only* be found in SoCs
- * containing the ThunderX ARM64 CPU implementation.  All accesses to the device
- * registers on this platform are implicitly strongly ordered with respect
- * to memory accesses. So writeq_relaxed() and readq_relaxed() are safe to use
- * with no memory barriers in this driver.  The readq()/writeq() functions add
- * explicit ordering operation which in this case are redundant, and only
- * add overhead.
+static devclass_t nicpf_devclass;
+
+DRIVER_MODULE(nicpf, pci, nicpf_driver, nicpf_devclass, 0, 0);
+MODULE_DEPEND(nicpf, pci, 1, 1, 1);
+MODULE_DEPEND(nicpf, ether, 1, 1, 1);
+MODULE_DEPEND(nicpf, thunder_bgx, 1, 1, 1);
+
+static int nicpf_alloc_res(struct nicpf *);
+static void nicpf_free_res(struct nicpf *);
+static void nic_set_lmac_vf_mapping(struct nicpf *);
+static void nic_init_hw(struct nicpf *);
+static int nic_sriov_init(device_t, struct nicpf *);
+static void nic_poll_for_link(void *);
+static int nic_register_interrupts(struct nicpf *);
+static void nic_unregister_interrupts(struct nicpf *);
+
+/*
+ * Device interface
  */
-
-/* Register read/write APIs */
-static void nic_reg_write(struct nicpf *nic, u64 offset, u64 val)
+static int
+nicpf_probe(device_t dev)
 {
-	writeq_relaxed(val, nic->reg_base + offset);
+	uint16_t vendor_id;
+	uint16_t device_id;
+
+	vendor_id = pci_get_vendor(dev);
+	device_id = pci_get_device(dev);
+
+	if (vendor_id == PCI_VENDOR_ID_CAVIUM &&
+	    device_id == PCI_DEVICE_ID_THUNDER_NIC_PF) {
+		device_set_desc(dev, VNIC_PF_DEVSTR);
+		return (BUS_PROBE_DEFAULT);
+	}
+
+	return (ENXIO);
 }
 
-static u64 nic_reg_read(struct nicpf *nic, u64 offset)
+static int
+nicpf_attach(device_t dev)
 {
-	return readq_relaxed(nic->reg_base + offset);
+	struct nicpf *nic;
+	int err;
+
+	nic = device_get_softc(dev);
+	nic->dev = dev;
+
+	/* Enable bus mastering */
+	pci_enable_busmaster(dev);
+
+	/* Allocate PCI resources */
+	err = nicpf_alloc_res(nic);
+	if (err != 0) {
+		device_printf(dev, "Could not allocate PCI resources\n");
+		return (err);
+	}
+
+	nic->node = nic_get_node_id(nic->reg_base);
+	nic->rev_id = pci_read_config(dev, PCIR_REVID, 1);
+
+	/* Enable Traffic Network Switch (TNS) bypass mode by default */
+	nic->flags &= ~NIC_TNS_ENABLED;
+	nic_set_lmac_vf_mapping(nic);
+
+	/* Initialize hardware */
+	nic_init_hw(nic);
+
+	/* Set RSS TBL size for each VF */
+	nic->rss_ind_tbl_size = NIC_MAX_RSS_IDR_TBL_SIZE;
+
+	/* Setup interrupts */
+	err = nic_register_interrupts(nic);
+	if (err != 0)
+		goto err_free_res;
+
+	/* Configure SRIOV */
+	err = nic_sriov_init(dev, nic);
+	if (err != 0)
+		goto err_free_intr;
+
+	if (nic->flags & NIC_TNS_ENABLED)
+		return (0);
+
+	mtx_init(&nic->check_link_mtx, "VNIC PF link poll", NULL, MTX_DEF);
+	/* Register physical link status poll callout */
+	callout_init_mtx(&nic->check_link, &nic->check_link_mtx, 0);
+	mtx_lock(&nic->check_link_mtx);
+	nic_poll_for_link(nic);
+	mtx_unlock(&nic->check_link_mtx);
+
+	return (0);
+
+err_free_intr:
+	nic_unregister_interrupts(nic);
+err_free_res:
+	nicpf_free_res(nic);
+	pci_disable_busmaster(dev);
+
+	return (err);
+}
+
+static int
+nicpf_detach(device_t dev)
+{
+	struct nicpf *nic;
+
+	nic = device_get_softc(dev);
+
+	callout_drain(&nic->check_link);
+	mtx_destroy(&nic->check_link_mtx);
+
+	nic_unregister_interrupts(nic);
+	nicpf_free_res(nic);
+	pci_disable_busmaster(dev);
+
+	return (0);
+}
+
+/*
+ * SR-IOV interface
+ */
+#ifdef PCI_IOV
+static int
+nicpf_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params)
+{
+	struct nicpf *nic;
+
+	nic = device_get_softc(dev);
+
+	nic->num_vf_en = 0;
+	if (num_vfs == 0)
+		return (ENXIO);
+	if (num_vfs > MAX_NUM_VFS_SUPPORTED)
+		return (EINVAL);
+
+	/*
+	 * Just set variables here.
+	 * The number of VFs will be written to configuration
+	 * space later in PCI_ADD_VF().
+	 */
+	nic->num_vf_en = num_vfs;
+	nic->flags |= NIC_SRIOV_ENABLED;
+
+	return (0);
+}
+
+static void
+nicpf_iov_uninit(device_t dev)
+{
+
+	/* ARM64TODO: Implement this function */
+}
+
+static int
+nicpf_iov_addr_vf(device_t dev, uint16_t vfnum, const nvlist_t *params)
+{
+	const void *mac;
+	struct nicpf *nic;
+	size_t size;
+	int bgx, lmac;
+
+	nic = device_get_softc(dev);
+
+	if ((nic->flags & NIC_SRIOV_ENABLED) == 0)
+		return (ENXIO);
+
+	if (nvlist_exists_binary(params, "mac-addr") != 0) {
+		mac = nvlist_get_binary(params, "mac-addr", &size);
+		bgx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vfnum]);
+		lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vfnum]);
+		bgx_set_lmac_mac(nic->node, bgx, lmac, mac);
+	}
+
+	return (0);
+}
+#endif
+
+/*
+ * Helper routines
+ */
+static int
+nicpf_alloc_res(struct nicpf *nic)
+{
+	device_t dev;
+	int rid;
+
+	dev = nic->dev;
+
+	rid = VNIC_PF_REG_RID;
+	nic->reg_base = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
+	    RF_ACTIVE);
+	if (nic->reg_base == NULL) {
+		/* For verbose output print some more details */
+		if (bootverbose) {
+			device_printf(dev,
+			    "Could not allocate registers memory\n");
+		}
+		return (ENXIO);
+	}
+
+	return (0);
+}
+
+static void
+nicpf_free_res(struct nicpf *nic)
+{
+	device_t dev;
+
+	dev = nic->dev;
+
+	if (nic->reg_base != NULL) {
+		bus_release_resource(dev, SYS_RES_MEMORY,
+		    rman_get_rid(nic->reg_base), nic->reg_base);
+	}
+}
+
+/* Register read/write APIs */
+static __inline void
+nic_reg_write(struct nicpf *nic, bus_space_handle_t offset,
+    uint64_t val)
+{
+
+	bus_write_8(nic->reg_base, offset, val);
+}
+
+static __inline uint64_t
+nic_reg_read(struct nicpf *nic, uint64_t offset)
+{
+	uint64_t val;
+
+	val = bus_read_8(nic->reg_base, offset);
+	return (val);
 }
 
 /* PF -> VF mailbox communication APIs */
-static void nic_enable_mbx_intr(struct nicpf *nic)
+static void
+nic_enable_mbx_intr(struct nicpf *nic)
 {
+
 	/* Enable mailbox interrupt for all 128 VFs */
-	nic_reg_write(nic, NIC_PF_MAILBOX_ENA_W1S, ~0ull);
-	nic_reg_write(nic, NIC_PF_MAILBOX_ENA_W1S + sizeof(u64), ~0ull);
+	nic_reg_write(nic, NIC_PF_MAILBOX_ENA_W1S, ~0UL);
+	nic_reg_write(nic, NIC_PF_MAILBOX_ENA_W1S + sizeof(uint64_t), ~0UL);
 }
 
-static void nic_clear_mbx_intr(struct nicpf *nic, int vf, int mbx_reg)
+static void
+nic_clear_mbx_intr(struct nicpf *nic, int vf, int mbx_reg)
 {
-	nic_reg_write(nic, NIC_PF_MAILBOX_INT + (mbx_reg << 3), BIT_ULL(vf));
+
+	nic_reg_write(nic, NIC_PF_MAILBOX_INT + (mbx_reg << 3), (1UL << vf));
 }
 
-static u64 nic_get_mbx_addr(int vf)
+static uint64_t
+nic_get_mbx_addr(int vf)
 {
-	return NIC_PF_VF_0_127_MAILBOX_0_1 + (vf << NIC_VF_NUM_SHIFT);
+
+	return (NIC_PF_VF_0_127_MAILBOX_0_1 + (vf << NIC_VF_NUM_SHIFT));
 }
 
-/* Send a mailbox message to VF
+/*
+ * Send a mailbox message to VF
  * @vf: vf to which this message to be sent
  * @mbx: Message to be sent
  */
-static void nic_send_msg_to_vf(struct nicpf *nic, int vf, union nic_mbx *mbx)
+static void
+nic_send_msg_to_vf(struct nicpf *nic, int vf, union nic_mbx *mbx)
 {
-	void __iomem *mbx_addr = nic->reg_base + nic_get_mbx_addr(vf);
-	u64 *msg = (u64 *)mbx;
+	bus_space_handle_t mbx_addr = nic_get_mbx_addr(vf);
+	uint64_t *msg = (uint64_t *)mbx;
 
-	/* In first revision HW, mbox interrupt is triggerred
+	/*
+	 * In first revision HW, mbox interrupt is triggerred
 	 * when PF writes to MBOX(1), in next revisions when
 	 * PF writes to MBOX(0)
 	 */
 	if (nic->rev_id == 0) {
-		/* see the comment for nic_reg_write()/nic_reg_read()
-		 * functions above
-		 */
-		writeq_relaxed(msg[0], mbx_addr);
-		writeq_relaxed(msg[1], mbx_addr + 8);
+		nic_reg_write(nic, mbx_addr + 0, msg[0]);
+		nic_reg_write(nic, mbx_addr + 8, msg[1]);
 	} else {
-		writeq_relaxed(msg[1], mbx_addr + 8);
-		writeq_relaxed(msg[0], mbx_addr);
+		nic_reg_write(nic, mbx_addr + 8, msg[1]);
+		nic_reg_write(nic, mbx_addr + 0, msg[0]);
 	}
 }
 
-/* Responds to VF's READY message with VF's
+/*
+ * Responds to VF's READY message with VF's
  * ID, node, MAC address e.t.c
  * @vf: VF which sent READY message
  */
-static void nic_mbx_send_ready(struct nicpf *nic, int vf)
+static void
+nic_mbx_send_ready(struct nicpf *nic, int vf)
 {
 	union nic_mbx mbx = {};
 	int bgx_idx, lmac;
@@ -174,12 +450,11 @@ static void nic_mbx_send_ready(struct nicpf *nic, int vf)
 		lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]);
 
 		mac = bgx_get_lmac_mac(nic->node, bgx_idx, lmac);
-		if (mac)
-			ether_addr_copy((u8 *)&mbx.nic_cfg.mac_addr, mac);
+		if (mac) {
+			memcpy((uint8_t *)&mbx.nic_cfg.mac_addr, mac,
+			    ETHER_ADDR_LEN);
+		}
 	}
-#ifdef VNIC_MULTI_QSET_SUPPORT
-	mbx.nic_cfg.sqs_mode = (vf >= nic->num_vf_en) ? true : false;
-#endif
 	mbx.nic_cfg.node_id = nic->node;
 
 	mbx.nic_cfg.loopback_supported = vf < MAX_LMAC;
@@ -187,10 +462,12 @@ static void nic_mbx_send_ready(struct nicpf *nic, int vf)
 	nic_send_msg_to_vf(nic, vf, &mbx);
 }
 
-/* ACKs VF's mailbox message
+/*
+ * ACKs VF's mailbox message
  * @vf: VF to which ACK to be sent
  */
-static void nic_mbx_send_ack(struct nicpf *nic, int vf)
+static void
+nic_mbx_send_ack(struct nicpf *nic, int vf)
 {
 	union nic_mbx mbx = {};
 
@@ -198,11 +475,13 @@ static void nic_mbx_send_ack(struct nicpf *nic, int vf)
 	nic_send_msg_to_vf(nic, vf, &mbx);
 }
 
-/* NACKs VF's mailbox message that PF is not able to
+/*
+ * NACKs VF's mailbox message that PF is not able to
  * complete the action
  * @vf: VF to which ACK to be sent
  */
-static void nic_mbx_send_nack(struct nicpf *nic, int vf)
+static void
+nic_mbx_send_nack(struct nicpf *nic, int vf)
 {
 	union nic_mbx mbx = {};
 
@@ -210,12 +489,14 @@ static void nic_mbx_send_nack(struct nicpf *nic, int vf)
 	nic_send_msg_to_vf(nic, vf, &mbx);
 }
 
-/* Flush all in flight receive packets to memory and
+/*
+ * Flush all in flight receive packets to memory and
  * bring down an active RQ
  */
-static int nic_rcv_queue_sw_sync(struct nicpf *nic)
+static int
+nic_rcv_queue_sw_sync(struct nicpf *nic)
 {
-	u16 timeout = ~0x00;
+	uint16_t timeout = ~0x00;
 
 	nic_reg_write(nic, NIC_PF_SW_SYNC_RX, 0x01);
 	/* Wait till sync cycle is finished */
@@ -226,14 +507,15 @@ static int nic_rcv_queue_sw_sync(struct nicpf *nic)
 	}
 	nic_reg_write(nic, NIC_PF_SW_SYNC_RX, 0x00);
 	if (!timeout) {
-		dev_err(&nic->pdev->dev, "Receive queue software sync failed");
-		return 1;
+		device_printf(nic->dev, "Receive queue software sync failed\n");
+		return (ETIMEDOUT);
 	}
-	return 0;
+	return (0);
 }
 
 /* Get BGX Rx/Tx stats and respond to VF's request */
-static void nic_get_bgx_stats(struct nicpf *nic, struct bgx_stats_msg *bgx)
+static void
+nic_get_bgx_stats(struct nicpf *nic, struct bgx_stats_msg *bgx)
 {
 	int bgx_idx, lmac;
 	union nic_mbx mbx = {};
@@ -245,38 +527,43 @@ static void nic_get_bgx_stats(struct nicpf *nic, struct bgx_stats_msg *bgx)
 	mbx.bgx_stats.vf_id = bgx->vf_id;
 	mbx.bgx_stats.rx = bgx->rx;
 	mbx.bgx_stats.idx = bgx->idx;
-	if (bgx->rx)
-		mbx.bgx_stats.stats = bgx_get_rx_stats(nic->node, bgx_idx,
-							    lmac, bgx->idx);
-	else
-		mbx.bgx_stats.stats = bgx_get_tx_stats(nic->node, bgx_idx,
-							    lmac, bgx->idx);
+	if (bgx->rx != 0) {
+		mbx.bgx_stats.stats =
+		    bgx_get_rx_stats(nic->node, bgx_idx, lmac, bgx->idx);
+	} else {
+		mbx.bgx_stats.stats =
+		    bgx_get_tx_stats(nic->node, bgx_idx, lmac, bgx->idx);
+	}
 	nic_send_msg_to_vf(nic, bgx->vf_id, &mbx);
 }
 
 /* Update hardware min/max frame size */
-static int nic_update_hw_frs(struct nicpf *nic, int new_frs, int vf)
+static int
+nic_update_hw_frs(struct nicpf *nic, int new_frs, int vf)
 {
+
 	if ((new_frs > NIC_HW_MAX_FRS) || (new_frs < NIC_HW_MIN_FRS)) {
-		dev_err(&nic->pdev->dev,
-			"Invalid MTU setting from VF%d rejected, should be between %d and %d\n",
-			   vf, NIC_HW_MIN_FRS, NIC_HW_MAX_FRS);
-		return 1;
+		device_printf(nic->dev,
+		    "Invalid MTU setting from VF%d rejected, "
+		    "should be between %d and %d\n",
+		    vf, NIC_HW_MIN_FRS, NIC_HW_MAX_FRS);
+		return (EINVAL);
 	}
-	new_frs += ETH_HLEN;
+	new_frs += ETHER_HDR_LEN;
 	if (new_frs <= nic->pkind.maxlen)
-		return 0;
+		return (0);
 
 	nic->pkind.maxlen = new_frs;
-	nic_reg_write(nic, NIC_PF_PKIND_0_15_CFG, *(u64 *)&nic->pkind);
-	return 0;
+	nic_reg_write(nic, NIC_PF_PKIND_0_15_CFG, *(uint64_t *)&nic->pkind);
+	return (0);
 }
 
 /* Set minimum transmit packet size */
-static void nic_set_tx_pkt_pad(struct nicpf *nic, int size)
+static void
+nic_set_tx_pkt_pad(struct nicpf *nic, int size)
 {
 	int lmac;
-	u64 lmac_cfg;
+	uint64_t lmac_cfg;
 
 	/* Max value that can be set is 60 */
 	if (size > 60)
@@ -290,15 +577,17 @@ static void nic_set_tx_pkt_pad(struct nicpf *nic, int size)
 	}
 }
 
-/* Function to check number of LMACs present and set VF::LMAC mapping.
+/*
+ * Function to check number of LMACs present and set VF::LMAC mapping.
  * Mapping will be used while initializing channels.
  */
-static void nic_set_lmac_vf_mapping(struct nicpf *nic)
+static void
+nic_set_lmac_vf_mapping(struct nicpf *nic)
 {
 	unsigned bgx_map = bgx_get_map(nic->node);
 	int bgx, next_bgx_lmac = 0;
 	int lmac, lmac_cnt = 0;
-	u64 lmac_credit;
+	uint64_t lmac_credit;
 
 	nic->num_vf_en = 0;
 	if (nic->flags & NIC_TNS_ENABLED) {
@@ -307,7 +596,7 @@ static void nic_set_lmac_vf_mapping(struct nicpf *nic)
 	}
 
 	for (bgx = 0; bgx < NIC_MAX_BGX; bgx++) {
-		if (!(bgx_map & (1 << bgx)))
+		if ((bgx_map & (1 << bgx)) == 0)
 			continue;
 		lmac_cnt = bgx_get_lmac_count(nic->node, bgx);
 		for (lmac = 0; lmac < lmac_cnt; lmac++)
@@ -316,16 +605,16 @@ static void nic_set_lmac_vf_mapping(struct nicpf *nic)
 		nic->num_vf_en += lmac_cnt;
 
 		/* Program LMAC credits */
-		lmac_credit = (1ull << 1); /* channel credit enable */
+		lmac_credit = (1UL << 1); /* channel credit enable */
 		lmac_credit |= (0x1ff << 2); /* Max outstanding pkt count */
 		/* 48KB BGX Tx buffer size, each unit is of size 16bytes */
 		lmac_credit |= (((((48 * 1024) / lmac_cnt) -
-				NIC_HW_MAX_FRS) / 16) << 12);
+		    NIC_HW_MAX_FRS) / 16) << 12);
 		lmac = bgx * MAX_LMAC_PER_BGX;
-		for (; lmac < lmac_cnt + (bgx * MAX_LMAC_PER_BGX); lmac++)
-			nic_reg_write(nic,
-				      NIC_PF_LMAC_0_7_CREDIT + (lmac * 8),
-				      lmac_credit);
+		for (; lmac < lmac_cnt + (bgx * MAX_LMAC_PER_BGX); lmac++) {
+			nic_reg_write(nic, NIC_PF_LMAC_0_7_CREDIT + (lmac * 8),
+			    lmac_credit);
+		}
 	}
 }
 
@@ -334,7 +623,8 @@ static void nic_set_lmac_vf_mapping(struct nicpf *nic)
 #define BGX0_BLOCK 8
 #define BGX1_BLOCK 9
 
-static void nic_init_hw(struct nicpf *nic)
+static void
+nic_init_hw(struct nicpf *nic)
 {
 	int i;
 
@@ -345,40 +635,41 @@ static void nic_init_hw(struct nicpf *nic)
 	nic_reg_write(nic, NIC_PF_CFG, 0x3);
 
 	/* Enable backpressure */
-	nic_reg_write(nic, NIC_PF_BP_CFG, (1ULL << 6) | 0x03);
+	nic_reg_write(nic, NIC_PF_BP_CFG, (1UL << 6) | 0x03);
 
 	if (nic->flags & NIC_TNS_ENABLED) {
 		nic_reg_write(nic, NIC_PF_INTF_0_1_SEND_CFG,
-			      (NIC_TNS_MODE << 7) | TNS_PORT0_BLOCK);
+		    (NIC_TNS_MODE << 7) | TNS_PORT0_BLOCK);
 		nic_reg_write(nic, NIC_PF_INTF_0_1_SEND_CFG | (1 << 8),
-			      (NIC_TNS_MODE << 7) | TNS_PORT1_BLOCK);
+		    (NIC_TNS_MODE << 7) | TNS_PORT1_BLOCK);
 		nic_reg_write(nic, NIC_PF_INTF_0_1_BP_CFG,
-			      (1ULL << 63) | TNS_PORT0_BLOCK);
+		    (1UL << 63) | TNS_PORT0_BLOCK);
 		nic_reg_write(nic, NIC_PF_INTF_0_1_BP_CFG + (1 << 8),
-			      (1ULL << 63) | TNS_PORT1_BLOCK);
+		    (1UL << 63) | TNS_PORT1_BLOCK);
 
 	} else {
 		/* Disable TNS mode on both interfaces */
 		nic_reg_write(nic, NIC_PF_INTF_0_1_SEND_CFG,
-			      (NIC_TNS_BYPASS_MODE << 7) | BGX0_BLOCK);
+		    (NIC_TNS_BYPASS_MODE << 7) | BGX0_BLOCK);
 		nic_reg_write(nic, NIC_PF_INTF_0_1_SEND_CFG | (1 << 8),
-			      (NIC_TNS_BYPASS_MODE << 7) | BGX1_BLOCK);
+		    (NIC_TNS_BYPASS_MODE << 7) | BGX1_BLOCK);
 		nic_reg_write(nic, NIC_PF_INTF_0_1_BP_CFG,
-			      (1ULL << 63) | BGX0_BLOCK);
+		    (1UL << 63) | BGX0_BLOCK);
 		nic_reg_write(nic, NIC_PF_INTF_0_1_BP_CFG + (1 << 8),
-			      (1ULL << 63) | BGX1_BLOCK);
+		    (1UL << 63) | BGX1_BLOCK);
 	}
 
 	/* PKIND configuration */
 	nic->pkind.minlen = 0;
-	nic->pkind.maxlen = NIC_HW_MAX_FRS + ETH_HLEN;
+	nic->pkind.maxlen = NIC_HW_MAX_FRS + ETHER_HDR_LEN;
 	nic->pkind.lenerr_en = 1;
 	nic->pkind.rx_hdr = 0;
 	nic->pkind.hdr_sl = 0;
 
-	for (i = 0; i < NIC_MAX_PKIND; i++)
+	for (i = 0; i < NIC_MAX_PKIND; i++) {
 		nic_reg_write(nic, NIC_PF_PKIND_0_15_CFG | (i << 3),
-			      *(u64 *)&nic->pkind);
+		    *(uint64_t *)&nic->pkind);
+	}
 
 	nic_set_tx_pkt_pad(nic, NIC_HW_MIN_FRS);
 
@@ -387,16 +678,17 @@ static void nic_init_hw(struct nicpf *nic)
 
 	/* Enable VLAN ethertype matching and stripping */
 	nic_reg_write(nic, NIC_PF_RX_ETYPE_0_7,
-		      (2 << 19) | (ETYPE_ALG_VLAN_STRIP << 16) | ETH_P_8021Q);
+	    (2 << 19) | (ETYPE_ALG_VLAN_STRIP << 16) | ETHERTYPE_VLAN);
 }
 
 /* Channel parse index configuration */
-static void nic_config_cpi(struct nicpf *nic, struct cpi_cfg_msg *cfg)
+static void
+nic_config_cpi(struct nicpf *nic, struct cpi_cfg_msg *cfg)
 {
-	u32 vnic, bgx, lmac, chan;
-	u32 padd, cpi_count = 0;
-	u64 cpi_base, cpi, rssi_base, rssi;
-	u8  qset, rq_idx = 0;
+	uint32_t vnic, bgx, lmac, chan;
+	uint32_t padd, cpi_count = 0;
+	uint64_t cpi_base, cpi, rssi_base, rssi;
+	uint8_t qset, rq_idx = 0;
 
 	vnic = cfg->vf_id;
 	bgx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vnic]);
@@ -408,9 +700,9 @@ static void nic_config_cpi(struct nicpf *nic, struct cpi_cfg_msg *cfg)
 
 	/* Rx channel configuration */
 	nic_reg_write(nic, NIC_PF_CHAN_0_255_RX_BP_CFG | (chan << 3),
-		      (1ull << 63) | (vnic << 0));
+	    (1UL << 63) | (vnic << 0));
 	nic_reg_write(nic, NIC_PF_CHAN_0_255_RX_CFG | (chan << 3),
-		      ((u64)cfg->cpi_alg << 62) | (cpi_base << 48));
+	    ((uint64_t)cfg->cpi_alg << 62) | (cpi_base << 48));
 
 	if (cfg->cpi_alg == CPI_ALG_NONE)
 		cpi_count = 1;
@@ -426,7 +718,7 @@ static void nic_config_cpi(struct nicpf *nic, struct cpi_cfg_msg *cfg)
 	rssi = rssi_base;
 	for (; rssi < (rssi_base + cfg->rq_cnt); rssi++) {
 		nic_reg_write(nic, NIC_PF_RSSI_0_4097_RQ | (rssi << 3),
-			      (qset << 3) | rq_idx);
+		    (qset << 3) | rq_idx);
 		rq_idx++;
 	}
 
@@ -441,7 +733,7 @@ static void nic_config_cpi(struct nicpf *nic, struct cpi_cfg_msg *cfg)
 
 		/* Leave RSS_SIZE as '0' to disable RSS */
 		nic_reg_write(nic, NIC_PF_CPI_0_2047_CFG | (cpi << 3),
-			      (vnic << 24) | (padd << 16) | (rssi_base + rssi));
+		    (vnic << 24) | (padd << 16) | (rssi_base + rssi));
 
 		if ((rssi + 1) >= cfg->rq_cnt)
 			continue;
@@ -456,59 +748,8 @@ static void nic_config_cpi(struct nicpf *nic, struct cpi_cfg_msg *cfg)
 	nic->cpi_base[cfg->vf_id] = cpi_base;
 }
 
-#ifdef VNIC_RSS_SUPPORT
-/* Responsds to VF with its RSS indirection table size */
-static void nic_send_rss_size(struct nicpf *nic, int vf)
-{
-	union nic_mbx mbx = {};
-	u64  *msg;
-
-	msg = (u64 *)&mbx;
-
-	mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE;
-	mbx.rss_size.ind_tbl_size = nic->rss_ind_tbl_size;
-	nic_send_msg_to_vf(nic, vf, &mbx);
-}
-
-/* Receive side scaling configuration
- * configure:
- * - RSS index
- * - indir table i.e hash::RQ mapping
- * - no of hash bits to consider
- */
-static void nic_config_rss(struct nicpf *nic, struct rss_cfg_msg *cfg)
-{
-	u8  qset, idx = 0;
-	u64 cpi_cfg, cpi_base, rssi_base, rssi;
-
-	cpi_base = nic->cpi_base[cfg->vf_id];
-	cpi_cfg = nic_reg_read(nic, NIC_PF_CPI_0_2047_CFG | (cpi_base << 3));
-	rssi_base = (cpi_cfg & 0x0FFF) + cfg->tbl_offset;
-
-	rssi = rssi_base;
-	qset = cfg->vf_id;
-
-	for (; rssi < (rssi_base + cfg->tbl_len); rssi++) {
-#ifdef VNIC_MULTI_QSET_SUPPORT
-		u8 svf = cfg->ind_tbl[idx] >> 3;
-
-		if (svf)
-			qset = nic->vf_sqs[cfg->vf_id][svf - 1];
-		else
-			qset = cfg->vf_id;
-#endif
-		nic_reg_write(nic, NIC_PF_RSSI_0_4097_RQ | (rssi << 3),
-			      (qset << 3) | (cfg->ind_tbl[idx] & 0x7));
-		idx++;
-	}
-
-	cpi_cfg &= ~(0xFULL << 20);
-	cpi_cfg |= (cfg->hash_bits << 20);
-	nic_reg_write(nic, NIC_PF_CPI_0_2047_CFG | (cpi_base << 3), cpi_cfg);
-}
-#endif
-
-/* 4 level transmit side scheduler configutation
+/*
+ * 4 level transmit side scheduler configutation
  * for TNS bypass mode
  *
  * Sample configuration for SQ0
@@ -521,21 +762,16 @@ static void nic_config_rss(struct nicpf *nic, struct rss_cfg_msg *cfg)
  * VNIC6-SQ0 -> TL4(528) -> TL3[132] -> TL2[33] -> TL1[1] -> BGX1
  * VNIC7-SQ0 -> TL4(536) -> TL3[134] -> TL2[33] -> TL1[1] -> BGX1
  */
-static void nic_tx_channel_cfg(struct nicpf *nic, u8 vnic,
-			       struct sq_cfg_msg *sq)
+static void
+nic_tx_channel_cfg(struct nicpf *nic, uint8_t vnic, struct sq_cfg_msg *sq)
 {
-	u32 bgx, lmac, chan;
-	u32 tl2, tl3, tl4;
-	u32 rr_quantum;
-	u8 sq_idx = sq->sq_num;
-	u8 pqs_vnic;
+	uint32_t bgx, lmac, chan;
+	uint32_t tl2, tl3, tl4;
+	uint32_t rr_quantum;
+	uint8_t sq_idx = sq->sq_num;
+	uint8_t pqs_vnic;
 
-#ifdef VNIC_MULTI_QSET_SUPPORT
-	if (sq->sqs_mode)
-		pqs_vnic = nic->pqs_vf[vnic];
-	else
-#endif
-		pqs_vnic = vnic;
+	pqs_vnic = vnic;
 
 	bgx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[pqs_vnic]);
 	lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[pqs_vnic]);
@@ -545,17 +781,13 @@ static void nic_tx_channel_cfg(struct nicpf *nic, u8 vnic,
 
 	tl4 = (lmac * NIC_TL4_PER_LMAC) + (bgx * NIC_TL4_PER_BGX);
 	tl4 += sq_idx;
-#ifdef VNIC_MULTI_QSET_SUPPORT
-	if (sq->sqs_mode)
-		tl4 += vnic * 8;
-#endif
 
 	tl3 = tl4 / (NIC_MAX_TL4 / NIC_MAX_TL3);
 	nic_reg_write(nic, NIC_PF_QSET_0_127_SQ_0_7_CFG2 |
-		      ((u64)vnic << NIC_QS_ID_SHIFT) |
-		      ((u32)sq_idx << NIC_Q_NUM_SHIFT), tl4);
+	    ((uint64_t)vnic << NIC_QS_ID_SHIFT) |
+	    ((uint32_t)sq_idx << NIC_Q_NUM_SHIFT), tl4);
 	nic_reg_write(nic, NIC_PF_TL4_0_1023_CFG | (tl4 << 3),
-		      ((u64)vnic << 27) | ((u32)sq_idx << 24) | rr_quantum);
+	    ((uint64_t)vnic << 27) | ((uint32_t)sq_idx << 24) | rr_quantum);
 
 	nic_reg_write(nic, NIC_PF_TL3_0_255_CFG | (tl3 << 3), rr_quantum);
 	chan = (lmac * MAX_BGX_CHANS_PER_LMAC) + (bgx * NIC_CHANS_PER_INF);
@@ -570,113 +802,46 @@ static void nic_tx_channel_cfg(struct nicpf *nic, u8 vnic,
 	nic_reg_write(nic, NIC_PF_TL2_0_63_PRI | (tl2 << 3), 0x00);
 }
 
-#ifdef VNIC_MULTI_QSET_SUPPORT
-/* Send primary nicvf pointer to secondary QS's VF */
-static void nic_send_pnicvf(struct nicpf *nic, int sqs)
-{
-	union nic_mbx mbx = {};
-
-	mbx.nicvf.msg = NIC_MBOX_MSG_PNICVF_PTR;
-	mbx.nicvf.nicvf = nic->nicvf[nic->pqs_vf[sqs]];
-	nic_send_msg_to_vf(nic, sqs, &mbx);
-}
-
-/* Send SQS's nicvf pointer to primary QS's VF */
-static void nic_send_snicvf(struct nicpf *nic, struct nicvf_ptr *nicvf)
-{
-	union nic_mbx mbx = {};
-	int sqs_id = nic->vf_sqs[nicvf->vf_id][nicvf->sqs_id];
-
-	mbx.nicvf.msg = NIC_MBOX_MSG_SNICVF_PTR;
-	mbx.nicvf.sqs_id = nicvf->sqs_id;
-	mbx.nicvf.nicvf = nic->nicvf[sqs_id];
-	nic_send_msg_to_vf(nic, nicvf->vf_id, &mbx);
-}
-
-/* Find next available Qset that can be assigned as a
- * secondary Qset to a VF.
- */
-static int nic_nxt_avail_sqs(struct nicpf *nic)
-{
-	int sqs;
-
-	for (sqs = 0; sqs < nic->num_sqs_en; sqs++) {
-		if (!nic->sqs_used[sqs])
-			nic->sqs_used[sqs] = true;
-		else
-			continue;
-		return sqs + nic->num_vf_en;
-	}
-	return -1;
-}
-
-/* Allocate additional Qsets for requested VF */
-static void nic_alloc_sqs(struct nicpf *nic, struct sqs_alloc *sqs)
-{
-	union nic_mbx mbx = {};
-	int idx, alloc_qs = 0;
-	int sqs_id;
-
-	if (!nic->num_sqs_en)
-		goto send_mbox;
-
-	for (idx = 0; idx < sqs->qs_count; idx++) {
-		sqs_id = nic_nxt_avail_sqs(nic);
-		if (sqs_id < 0)
-			break;
-		nic->vf_sqs[sqs->vf_id][idx] = sqs_id;
-		nic->pqs_vf[sqs_id] = sqs->vf_id;
-		alloc_qs++;
-	}
-
-send_mbox:
-	mbx.sqs_alloc.msg = NIC_MBOX_MSG_ALLOC_SQS;
-	mbx.sqs_alloc.vf_id = sqs->vf_id;
-	mbx.sqs_alloc.qs_count = alloc_qs;
-	nic_send_msg_to_vf(nic, sqs->vf_id, &mbx);
-}
-#endif
-
-static int nic_config_loopback(struct nicpf *nic, struct set_loopback *lbk)
+static int
+nic_config_loopback(struct nicpf *nic, struct set_loopback *lbk)
 {
 	int bgx_idx, lmac_idx;
 
 	if (lbk->vf_id > MAX_LMAC)
-		return -1;
+		return (ENXIO);
 
 	bgx_idx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[lbk->vf_id]);
 	lmac_idx = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[lbk->vf_id]);
 
 	bgx_lmac_internal_loopback(nic->node, bgx_idx, lmac_idx, lbk->enable);
 
-	return 0;
+	return (0);
 }
 
 /* Interrupt handler to handle mailbox messages from VFs */
-static void nic_handle_mbx_intr(struct nicpf *nic, int vf)
+static void
+nic_handle_mbx_intr(struct nicpf *nic, int vf)
 {
 	union nic_mbx mbx = {};
-	u64 *mbx_data;
-	u64 mbx_addr;
-	u64 reg_addr;
-	u64 cfg;
+	uint64_t *mbx_data;
+	uint64_t mbx_addr;
+	uint64_t reg_addr;
+	uint64_t cfg;
 	int bgx, lmac;
 	int i;
 	int ret = 0;
 
-	nic->mbx_lock[vf] = true;
+	nic->mbx_lock[vf] = TRUE;
 
 	mbx_addr = nic_get_mbx_addr(vf);
-	mbx_data = (u64 *)&mbx;
+	mbx_data = (uint64_t *)&mbx;
 
 	for (i = 0; i < NIC_PF_VF_MAILBOX_SIZE; i++) {
 		*mbx_data = nic_reg_read(nic, mbx_addr);
 		mbx_data++;
-		mbx_addr += sizeof(u64);
+		mbx_addr += sizeof(uint64_t);
 	}
 
-	dev_dbg(&nic->pdev->dev, "%s: Mailbox msg %d from VF%d\n",
-		__func__, mbx.msg.msg, vf);
 	switch (mbx.msg.msg) {
 	case NIC_MBOX_MSG_READY:
 		nic_mbx_send_ready(nic, vf);
@@ -689,28 +854,20 @@ static void nic_handle_mbx_intr(struct nicpf *nic, int vf)
 		break;
 	case NIC_MBOX_MSG_QS_CFG:
 		reg_addr = NIC_PF_QSET_0_127_CFG |
-			   (mbx.qs.num << NIC_QS_ID_SHIFT);
+		    (mbx.qs.num << NIC_QS_ID_SHIFT);
 		cfg = mbx.qs.cfg;
-#ifdef VNIC_MULTI_QSET_SUPPORT
-		/* Check if its a secondary Qset */
-		if (vf >= nic->num_vf_en) {
-			cfg = cfg & (~0x7FULL);
-			/* Assign this Qset to primary Qset's VF */
-			cfg |= nic->pqs_vf[vf];
-		}
-#endif
 		nic_reg_write(nic, reg_addr, cfg);
 		break;
 	case NIC_MBOX_MSG_RQ_CFG:
 		reg_addr = NIC_PF_QSET_0_127_RQ_0_7_CFG |
-			   (mbx.rq.qs_num << NIC_QS_ID_SHIFT) |
-			   (mbx.rq.rq_num << NIC_Q_NUM_SHIFT);
+		    (mbx.rq.qs_num << NIC_QS_ID_SHIFT) |
+		    (mbx.rq.rq_num << NIC_Q_NUM_SHIFT);
 		nic_reg_write(nic, reg_addr, mbx.rq.cfg);
 		break;
 	case NIC_MBOX_MSG_RQ_BP_CFG:
 		reg_addr = NIC_PF_QSET_0_127_RQ_0_7_BP_CFG |
-			   (mbx.rq.qs_num << NIC_QS_ID_SHIFT) |
-			   (mbx.rq.rq_num << NIC_Q_NUM_SHIFT);
+		    (mbx.rq.qs_num << NIC_QS_ID_SHIFT) |
+		    (mbx.rq.rq_num << NIC_Q_NUM_SHIFT);
 		nic_reg_write(nic, reg_addr, mbx.rq.cfg);
 		break;
 	case NIC_MBOX_MSG_RQ_SW_SYNC:
@@ -718,70 +875,37 @@ static void nic_handle_mbx_intr(struct nicpf *nic, int vf)
 		break;
 	case NIC_MBOX_MSG_RQ_DROP_CFG:
 		reg_addr = NIC_PF_QSET_0_127_RQ_0_7_DROP_CFG |
-			   (mbx.rq.qs_num << NIC_QS_ID_SHIFT) |
-			   (mbx.rq.rq_num << NIC_Q_NUM_SHIFT);
+		    (mbx.rq.qs_num << NIC_QS_ID_SHIFT) |
+		    (mbx.rq.rq_num << NIC_Q_NUM_SHIFT);
 		nic_reg_write(nic, reg_addr, mbx.rq.cfg);
 		break;
 	case NIC_MBOX_MSG_SQ_CFG:
 		reg_addr = NIC_PF_QSET_0_127_SQ_0_7_CFG |
-			   (mbx.sq.qs_num << NIC_QS_ID_SHIFT) |
-			   (mbx.sq.sq_num << NIC_Q_NUM_SHIFT);
+		    (mbx.sq.qs_num << NIC_QS_ID_SHIFT) |
+		    (mbx.sq.sq_num << NIC_Q_NUM_SHIFT);
 		nic_reg_write(nic, reg_addr, mbx.sq.cfg);
 		nic_tx_channel_cfg(nic, mbx.qs.num, &mbx.sq);
 		break;
 	case NIC_MBOX_MSG_SET_MAC:
-#ifdef VNIC_MULTI_QSET_SUPPORT
-		if (vf >= nic->num_vf_en)
-			break;
-#endif
 		lmac = mbx.mac.vf_id;
 		bgx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[lmac]);
 		lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[lmac]);
 		bgx_set_lmac_mac(nic->node, bgx, lmac, mbx.mac.mac_addr);
 		break;
 	case NIC_MBOX_MSG_SET_MAX_FRS:
-		ret = nic_update_hw_frs(nic, mbx.frs.max_frs,
-					mbx.frs.vf_id);
+		ret = nic_update_hw_frs(nic, mbx.frs.max_frs, mbx.frs.vf_id);
 		break;
 	case NIC_MBOX_MSG_CPI_CFG:
 		nic_config_cpi(nic, &mbx.cpi_cfg);
 		break;
-#ifdef VNIC_RSS_SUPPORT
-	case NIC_MBOX_MSG_RSS_SIZE:
-		nic_send_rss_size(nic, vf);
-		goto unlock;
-	case NIC_MBOX_MSG_RSS_CFG:
-	case NIC_MBOX_MSG_RSS_CFG_CONT:
-		nic_config_rss(nic, &mbx.rss_cfg);
-		break;
-#endif
 	case NIC_MBOX_MSG_CFG_DONE:
 		/* Last message of VF config msg sequence */
-		nic->vf_enabled[vf] = true;
+		nic->vf_info[vf].vf_enabled = TRUE;
 		goto unlock;
 	case NIC_MBOX_MSG_SHUTDOWN:
 		/* First msg in VF teardown sequence */
-		nic->vf_enabled[vf] = false;
-#ifdef VNIC_MULTI_QSET_SUPPORT
-		if (vf >= nic->num_vf_en)
-			nic->sqs_used[vf - nic->num_vf_en] = false;
-		nic->pqs_vf[vf] = 0;
-#endif
+		nic->vf_info[vf].vf_enabled = FALSE;
 		break;
-#ifdef VNIC_MULTI_QSET_SUPPORT
-	case NIC_MBOX_MSG_ALLOC_SQS:
-		nic_alloc_sqs(nic, &mbx.sqs_alloc);
-		goto unlock;
-	case NIC_MBOX_MSG_NICVF_PTR:
-		nic->nicvf[vf] = mbx.nicvf.nicvf;
-		break;
-	case NIC_MBOX_MSG_PNICVF_PTR:
-		nic_send_pnicvf(nic, vf);
-		goto unlock;
-	case NIC_MBOX_MSG_SNICVF_PTR:
-		nic_send_snicvf(nic, &mbx.nicvf);
-		goto unlock;
-#endif
 	case NIC_MBOX_MSG_BGX_STATS:
 		nic_get_bgx_stats(nic, &mbx.bgx_stats);
 		goto unlock;
@@ -789,241 +913,232 @@ static void nic_handle_mbx_intr(struct nicpf *nic, int vf)
 		ret = nic_config_loopback(nic, &mbx.lbk);
 		break;
 	default:
-		dev_err(&nic->pdev->dev,
-			"Invalid msg from VF%d, msg 0x%x\n", vf, mbx.msg.msg);
+		device_printf(nic->dev,
+		    "Invalid msg from VF%d, msg 0x%x\n", vf, mbx.msg.msg);
 		break;
 	}
 
-	if (!ret)
+	if (ret == 0)
 		nic_mbx_send_ack(nic, vf);
 	else if (mbx.msg.msg != NIC_MBOX_MSG_READY)
 		nic_mbx_send_nack(nic, vf);
 unlock:
-	nic->mbx_lock[vf] = false;
+	nic->mbx_lock[vf] = FALSE;
 }
 
-static void nic_mbx_intr_handler (struct nicpf *nic, int mbx)
+static void
+nic_mbx_intr_handler(struct nicpf *nic, int mbx)
 {
-	u64 intr;
-	u8  vf, vf_per_mbx_reg = 64;
+	uint64_t intr;
+	uint8_t  vf, vf_per_mbx_reg = 64;
 
 	intr = nic_reg_read(nic, NIC_PF_MAILBOX_INT + (mbx << 3));
-	dev_dbg(&nic->pdev->dev, "PF interrupt Mbox%d 0x%llx\n", mbx, intr);
 	for (vf = 0; vf < vf_per_mbx_reg; vf++) {
-		if (intr & (1ULL << vf)) {
-			dev_dbg(&nic->pdev->dev, "Intr from VF %d\n",
-				vf + (mbx * vf_per_mbx_reg));
-
+		if (intr & (1UL << vf)) {
 			nic_handle_mbx_intr(nic, vf + (mbx * vf_per_mbx_reg));
 			nic_clear_mbx_intr(nic, vf, mbx);
 		}
 	}
 }
 
-static irqreturn_t nic_mbx0_intr_handler (int irq, void *nic_irq)
+static void
+nic_mbx0_intr_handler (void *arg)
 {
-	struct nicpf *nic = (struct nicpf *)nic_irq;
+	struct nicpf *nic = (struct nicpf *)arg;
 
 	nic_mbx_intr_handler(nic, 0);
-
-	return IRQ_HANDLED;
 }
 
-static irqreturn_t nic_mbx1_intr_handler (int irq, void *nic_irq)
+static void
+nic_mbx1_intr_handler (void *arg)
 {
-	struct nicpf *nic = (struct nicpf *)nic_irq;
+	struct nicpf *nic = (struct nicpf *)arg;
 
 	nic_mbx_intr_handler(nic, 1);
-
-	return IRQ_HANDLED;
 }
 
-static int nic_enable_msix(struct nicpf *nic)
+static int
+nic_enable_msix(struct nicpf *nic)
 {
-	int i, ret;
+	struct pci_devinfo *dinfo;
+	int rid, count;
+	int ret;
 
-	nic->num_vec = NIC_PF_MSIX_VECTORS;
+	dinfo = device_get_ivars(nic->dev);
+	rid = dinfo->cfg.msix.msix_table_bar;
+	nic->msix_table_res =
+	    bus_alloc_resource_any(nic->dev, SYS_RES_MEMORY, &rid, RF_ACTIVE);
+	if (nic->msix_table_res == NULL) {
+		device_printf(nic->dev,
+		    "Could not allocate memory for MSI-X table\n");
+		return (ENXIO);
+	}
 
-	for (i = 0; i < nic->num_vec; i++)
-		nic->msix_entries[i].entry = i;
+	count = nic->num_vec = NIC_PF_MSIX_VECTORS;
 
-	ret = pci_enable_msix(nic->pdev, nic->msix_entries, nic->num_vec);
-	if (ret) {
-		dev_err(&nic->pdev->dev,
-			"Request for #%d msix vectors failed\n",
-			   nic->num_vec);
-		return ret;
+	ret = pci_alloc_msix(nic->dev, &count);
+	if ((ret != 0) || (count != nic->num_vec)) {
+		device_printf(nic->dev,
+		    "Request for #%d msix vectors failed, error: %d\n",
+		    nic->num_vec, ret);
+		return (ret);
 	}
 
 	nic->msix_enabled = 1;
-	return 0;
+	return (0);
 }
 
-static void nic_disable_msix(struct nicpf *nic)
+static void
+nic_disable_msix(struct nicpf *nic)
 {
 	if (nic->msix_enabled) {
-		pci_disable_msix(nic->pdev);
+		pci_release_msi(nic->dev);
 		nic->msix_enabled = 0;
 		nic->num_vec = 0;
 	}
 }
 
-static void nic_free_all_interrupts(struct nicpf *nic)
+static void
+nic_free_all_interrupts(struct nicpf *nic)
 {
 	int irq;
 
 	for (irq = 0; irq < nic->num_vec; irq++) {
-		if (nic->irq_allocated[irq])
-			free_irq(nic->msix_entries[irq].vector, nic);
-		nic->irq_allocated[irq] = false;
+		if (nic->msix_entries[irq].irq_res == NULL)
+			continue;
+		if (nic->msix_entries[irq].handle != NULL) {
+			bus_teardown_intr(nic->dev,
+			    nic->msix_entries[irq].irq_res,
+			    nic->msix_entries[irq].handle);
+		}
+
+		bus_release_resource(nic->dev, SYS_RES_IRQ, irq,
+		    nic->msix_entries[irq].irq_res);
 	}
 }
 
-static int nic_register_interrupts(struct nicpf *nic)
+static int
+nic_register_interrupts(struct nicpf *nic)
 {
+	int irq, rid;
 	int ret;
 
 	/* Enable MSI-X */
 	ret = nic_enable_msix(nic);
-	if (ret)
-		return ret;
+	if (ret != 0)
+		return (ret);
 
 	/* Register mailbox interrupt handlers */
-	ret = request_irq(nic->msix_entries[NIC_PF_INTR_ID_MBOX0].vector,
-			  nic_mbx0_intr_handler, 0, "NIC Mbox0", nic);
-	if (ret)
+	irq = NIC_PF_INTR_ID_MBOX0;
+	rid = irq + 1;
+	nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev,
+	    SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE));
+	if (nic->msix_entries[irq].irq_res == NULL) {
+		ret = ENXIO;
+		goto fail;
+	}
+	ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res,
+	    (INTR_MPSAFE | INTR_TYPE_MISC), NULL, nic_mbx0_intr_handler, nic,
+	    &nic->msix_entries[irq].handle);
+	if (ret != 0)
 		goto fail;
 
-	nic->irq_allocated[NIC_PF_INTR_ID_MBOX0] = true;
-
-	ret = request_irq(nic->msix_entries[NIC_PF_INTR_ID_MBOX1].vector,
-			  nic_mbx1_intr_handler, 0, "NIC Mbox1", nic);
-	if (ret)
+	irq = NIC_PF_INTR_ID_MBOX1;
+	rid = irq + 1;
+	nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev,
+	    SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE));
+	if (nic->msix_entries[irq].irq_res == NULL) {
+		ret = ENXIO;
+		goto fail;
+	}
+	ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res,
+	    (INTR_MPSAFE | INTR_TYPE_MISC), NULL, nic_mbx1_intr_handler, nic,
+	    &nic->msix_entries[irq].handle);
+	if (ret != 0)
 		goto fail;
-
-	nic->irq_allocated[NIC_PF_INTR_ID_MBOX1] = true;
 
 	/* Enable mailbox interrupt */
 	nic_enable_mbx_intr(nic);
-	return 0;
+	return (0);
 
 fail:
-	dev_err(&nic->pdev->dev, "Request irq failed\n");
 	nic_free_all_interrupts(nic);
-	return ret;
+	return (ret);
 }
 
-static void nic_unregister_interrupts(struct nicpf *nic)
+static void
+nic_unregister_interrupts(struct nicpf *nic)
 {
+
 	nic_free_all_interrupts(nic);
 	nic_disable_msix(nic);
 }
 
-#ifdef VNIC_MULTI_QSET_SUPPORT
-int nic_num_sqs_en(struct nicpf *nic, int vf_en)
+static int nic_sriov_init(device_t dev, struct nicpf *nic)
 {
-	int pos = 0, sqs_per_vf = MAX_SQS_PER_VF_SINGLE_NODE;
-	u16 total_vf;
-
-	/* Check if its a multi-node environment */
-	if (nr_node_ids > 1)
-		sqs_per_vf = MAX_SQS_PER_VF;
-
-	pos = pci_find_ext_capability(nic->pdev, PCI_EXT_CAP_ID_SRIOV);
-	pci_read_config_word(nic->pdev, (pos + PCI_SRIOV_TOTAL_VF), &total_vf);
-	return min(total_vf - vf_en, vf_en * sqs_per_vf);
-}
-#endif
-
-int nic_sriov_configure(struct pci_dev *pdev, int num_vfs_requested)
-{
-	struct nicpf *nic = pci_get_drvdata(pdev);
-	int vf_en;
+#ifdef PCI_IOV
+	nvlist_t *pf_schema, *vf_schema;
+	int iov_pos;
 	int err;
+	uint16_t total_vf_cnt;
 
-	if (nic->num_vf_en == num_vfs_requested)
-		return num_vfs_requested;
-
-	if (nic->flags & NIC_SRIOV_ENABLED) {
-		nic->flags &= ~NIC_SRIOV_ENABLED;
+	err = pci_find_extcap(dev, PCIZ_SRIOV, &iov_pos);
+	if (err != 0) {
+		device_printf(dev,
+		    "SR-IOV capability is not found in PCIe config space\n");
+		return (err);
 	}
-
-	nic->num_vf_en = 0;
-	if (num_vfs_requested > MAX_NUM_VFS_SUPPORTED ||
-	    num_vfs_requested < 0)
-		return -EINVAL;
-
-	if (num_vfs_requested) {
-		vf_en = num_vfs_requested;
-#ifdef VNIC_MULTI_QSET_SUPPORT
-		nic->num_sqs_en = nic_num_sqs_en(nic, num_vfs_requested);
-		vf_en += nic->num_sqs_en;
-#endif
-		nic->num_vf_en = num_vfs_requested;
-		nic->flags |= NIC_SRIOV_ENABLED;
-	}
-
-	return num_vfs_requested;
-}
-
-static int nic_sriov_init(struct pci_dev *pdev, struct nicpf *nic)
-{
-	int pos = 0;
-	int vf_en;
-	int err;
-	u16 total_vf_cnt;
-
-	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
-	if (!pos) {
-		dev_err(&pdev->dev, "SRIOV capability is not found in PCIe config space\n");
-		return -ENODEV;
-	}
-
-	pci_read_config_word(pdev, (pos + PCI_SRIOV_TOTAL_VF), &total_vf_cnt);
+	/* Fix-up the number of enabled VFs */
+	total_vf_cnt = pci_read_config(dev, iov_pos + PCIR_SRIOV_TOTAL_VFS, 2);
 	if (total_vf_cnt < nic->num_vf_en)
 		nic->num_vf_en = total_vf_cnt;
 
-	if (!total_vf_cnt)
-		return 0;
+	if (total_vf_cnt == 0)
+		return (0);
 
-	vf_en = nic->num_vf_en;
-#ifdef VNIC_MULTI_QSET_SUPPORT
-	nic->num_sqs_en = nic_num_sqs_en(nic, nic->num_vf_en);
-	vf_en += nic->num_sqs_en;
-#endif
+	/* Attach SR-IOV */
+	pf_schema = pci_iov_schema_alloc_node();
+	vf_schema = pci_iov_schema_alloc_node();
+	pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL);
+	/*
+	 * All VFs can change their MACs.
+	 * This flag will be ignored but we set it just for the record.
+	 */
+	pci_iov_schema_add_bool(vf_schema, "allow-set-mac",
+	    IOV_SCHEMA_HASDEFAULT, TRUE);
 
-	if (err) {
-		dev_err(&pdev->dev, "SRIOV enable failed, num VF is %d\n",
-			vf_en);
+	err = pci_iov_attach(dev, pf_schema, vf_schema);
+	if (err != 0) {
+		device_printf(dev,
+		    "Failed to initialize SR-IOV (error=%d)\n",
+		    err);
 		nic->num_vf_en = 0;
-		return err;
+		return (err);
 	}
-
-	dev_info(&pdev->dev, "SRIOV enabled, number of VF available %d\n",
-		 vf_en);
-
-	nic->flags |= NIC_SRIOV_ENABLED;
-	return 0;
+#endif
+	return (0);
 }
 
-/* Poll for BGX LMAC link status and update corresponding VF
+/*
+ * Poll for BGX LMAC link status and update corresponding VF
  * if there is a change, valid only if internal L2 switch
  * is not present otherwise VF link is always treated as up
  */
-static void nic_poll_for_link(struct work_struct *work)
+static void
+nic_poll_for_link(void *arg)
 {
 	union nic_mbx mbx = {};
 	struct nicpf *nic;
 	struct bgx_link_status link;
-	u8 vf, bgx, lmac;
+	uint8_t vf, bgx, lmac;
 
-	nic = container_of(work, struct nicpf, dwork.work);
+	nic = (struct nicpf *)arg;
 
 	mbx.link_status.msg = NIC_MBOX_MSG_BGX_LINK_CHANGE;
 
 	for (vf = 0; vf < nic->num_vf_en; vf++) {
 		/* Poll only if VF is UP */
-		if (!nic->vf_enabled[vf])
+		if (!nic->vf_info[vf].vf_enabled)
 			continue;
 
 		/* Get BGX, LMAC indices for the VF */
@@ -1048,145 +1163,5 @@ static void nic_poll_for_link(struct work_struct *work)
 			nic_send_msg_to_vf(nic, vf, &mbx);
 		}
 	}
-	queue_delayed_work(nic->check_link, &nic->dwork, HZ * 2);
+	callout_reset(&nic->check_link, hz * 2, nic_poll_for_link, nic);
 }
-
-static int nic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
-{
-	struct device *dev = &pdev->dev;
-	struct nicpf *nic;
-	int    err;
-
-	BUILD_BUG_ON(sizeof(union nic_mbx) > 16);
-
-	nic = devm_kzalloc(dev, sizeof(*nic), GFP_KERNEL);
-	if (!nic)
-		return -ENOMEM;
-
-	pci_set_drvdata(pdev, nic);
-
-	nic->pdev = pdev;
-
-	err = pci_enable_device(pdev);
-	if (err) {
-		dev_err(dev, "Failed to enable PCI device\n");
-		pci_set_drvdata(pdev, NULL);
-		return err;
-	}
-
-	err = pci_request_regions(pdev, DRV_NAME);
-	if (err) {
-		dev_err(dev, "PCI request regions failed 0x%x\n", err);
-		goto err_disable_device;
-	}
-
-	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
-	if (err) {
-		dev_err(dev, "Unable to get usable DMA configuration\n");
-		goto err_release_regions;
-	}
-
-	err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
-	if (err) {
-		dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n");
-		goto err_release_regions;
-	}
-
-	/* MAP PF's configuration registers */
-	nic->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0);
-	if (!nic->reg_base) {
-		dev_err(dev, "Cannot map config register space, aborting\n");
-		err = -ENOMEM;
-		goto err_release_regions;
-	}
-
-	pci_read_config_byte(pdev, PCI_REVISION_ID, &nic->rev_id);
-
-	nic->node = nic_get_node_id(pdev);
-
-	/* By default set NIC in TNS bypass mode */
-	nic->flags &= ~NIC_TNS_ENABLED;
-	nic_set_lmac_vf_mapping(nic);
-
-	/* Initialize hardware */
-	nic_init_hw(nic);
-
-	/* Set RSS TBL size for each VF */
-	nic->rss_ind_tbl_size = NIC_MAX_RSS_IDR_TBL_SIZE;
-
-	/* Register interrupts */
-	err = nic_register_interrupts(nic);
-	if (err)
-		goto err_release_regions;
-
-	/* Configure SRIOV */
-	err = nic_sriov_init(pdev, nic);
-	if (err)
-		goto err_unregister_interrupts;
-
-	if (nic->flags & NIC_TNS_ENABLED)
-		return 0;
-
-	/* Register a physical link status poll fn() */
-	nic->check_link = alloc_workqueue("check_link_status",
-					  WQ_UNBOUND | WQ_MEM_RECLAIM, 1);
-	if (!nic->check_link) {
-		err = -ENOMEM;
-		goto err_disable_sriov;
-	}
-
-	INIT_DELAYED_WORK(&nic->dwork, nic_poll_for_link);
-	queue_delayed_work(nic->check_link, &nic->dwork, 0);
-
-	return 0;
-
-err_disable_sriov:
-err_unregister_interrupts:
-	nic_unregister_interrupts(nic);
-err_release_regions:
-	pci_release_regions(pdev);
-err_disable_device:
-	pci_disable_device(pdev);
-	pci_set_drvdata(pdev, NULL);
-	return err;
-}
-
-static void nic_remove(struct pci_dev *pdev)
-{
-	struct nicpf *nic = pci_get_drvdata(pdev);
-
-	if (nic->check_link) {
-		/* Destroy work Queue */
-		cancel_delayed_work(&nic->dwork);
-		flush_workqueue(nic->check_link);
-		destroy_workqueue(nic->check_link);
-	}
-
-	nic_unregister_interrupts(nic);
-	pci_release_regions(pdev);
-	pci_disable_device(pdev);
-	pci_set_drvdata(pdev, NULL);
-}
-
-static struct pci_driver nic_driver = {
-	.name = DRV_NAME,
-	.id_table = nic_id_table,
-	.probe = nic_probe,
-	.remove = nic_remove,
-	.sriov_configure = nic_sriov_configure,
-};
-
-static int __init nic_init_module(void)
-{
-	pr_info("%s, ver %s\n", DRV_NAME, DRV_VERSION);
-
-	return pci_register_driver(&nic_driver);
-}
-
-static void __exit nic_cleanup_module(void)
-{
-	pci_unregister_driver(&nic_driver);
-}
-
-module_init(nic_init_module);
-module_exit(nic_cleanup_module);
diff --git a/sys/dev/vnic/nic_reg.h b/sys/dev/vnic/nic_reg.h
index b66f7e38633f..8d508c747a7e 100644
--- a/sys/dev/vnic/nic_reg.h
+++ b/sys/dev/vnic/nic_reg.h
@@ -30,205 +30,195 @@
 #ifndef NIC_REG_H
 #define NIC_REG_H
 
-#define   NIC_PF_REG_COUNT			29573
-#define   NIC_VF_REG_COUNT			249
+#define	NIC_PF_REG_COUNT			29573
+#define	NIC_VF_REG_COUNT			249
 
 /* Physical function register offsets */
-#define   NIC_PF_CFG				(0x0000)
-#define   NIC_PF_STATUS				(0x0010)
-#define   NIC_PF_INTR_TIMER_CFG			(0x0030)
-#define   NIC_PF_BIST_STATUS			(0x0040)
-#define   NIC_PF_SOFT_RESET			(0x0050)
-#define   NIC_PF_TCP_TIMER			(0x0060)
-#define   NIC_PF_BP_CFG				(0x0080)
-#define   NIC_PF_RRM_CFG			(0x0088)
-#define   NIC_PF_CQM_CF				(0x00A0)
-#define   NIC_PF_CNM_CF				(0x00A8)
-#define   NIC_PF_CNM_STATUS			(0x00B0)
-#define   NIC_PF_CQ_AVG_CFG			(0x00C0)
-#define   NIC_PF_RRM_AVG_CFG			(0x00C8)
-#define   NIC_PF_INTF_0_1_SEND_CFG		(0x0200)
-#define   NIC_PF_INTF_0_1_BP_CFG		(0x0208)
-#define   NIC_PF_INTF_0_1_BP_DIS_0_1		(0x0210)
-#define   NIC_PF_INTF_0_1_BP_SW_0_1		(0x0220)
-#define   NIC_PF_RBDR_BP_STATE_0_3		(0x0240)
-#define   NIC_PF_MAILBOX_INT			(0x0410)
-#define   NIC_PF_MAILBOX_INT_W1S		(0x0430)
-#define   NIC_PF_MAILBOX_ENA_W1C		(0x0450)
-#define   NIC_PF_MAILBOX_ENA_W1S		(0x0470)
-#define   NIC_PF_RX_ETYPE_0_7			(0x0500)
-#define   NIC_PF_PKIND_0_15_CFG			(0x0600)
-#define   NIC_PF_ECC0_FLIP0			(0x1000)
-#define   NIC_PF_ECC1_FLIP0			(0x1008)
-#define   NIC_PF_ECC2_FLIP0			(0x1010)
-#define   NIC_PF_ECC3_FLIP0			(0x1018)
-#define   NIC_PF_ECC0_FLIP1			(0x1080)
-#define   NIC_PF_ECC1_FLIP1			(0x1088)
-#define   NIC_PF_ECC2_FLIP1			(0x1090)
-#define   NIC_PF_ECC3_FLIP1			(0x1098)
-#define   NIC_PF_ECC0_CDIS			(0x1100)
-#define   NIC_PF_ECC1_CDIS			(0x1108)
-#define   NIC_PF_ECC2_CDIS			(0x1110)
-#define   NIC_PF_ECC3_CDIS			(0x1118)
-#define   NIC_PF_BIST0_STATUS			(0x1280)
-#define   NIC_PF_BIST1_STATUS			(0x1288)
-#define   NIC_PF_BIST2_STATUS			(0x1290)
-#define   NIC_PF_BIST3_STATUS			(0x1298)
-#define   NIC_PF_ECC0_SBE_INT			(0x2000)
-#define   NIC_PF_ECC0_SBE_INT_W1S		(0x2008)
-#define   NIC_PF_ECC0_SBE_ENA_W1C		(0x2010)
-#define   NIC_PF_ECC0_SBE_ENA_W1S		(0x2018)
-#define   NIC_PF_ECC0_DBE_INT			(0x2100)
-#define   NIC_PF_ECC0_DBE_INT_W1S		(0x2108)
-#define   NIC_PF_ECC0_DBE_ENA_W1C		(0x2110)
-#define   NIC_PF_ECC0_DBE_ENA_W1S		(0x2118)
-#define   NIC_PF_ECC1_SBE_INT			(0x2200)
-#define   NIC_PF_ECC1_SBE_INT_W1S		(0x2208)
-#define   NIC_PF_ECC1_SBE_ENA_W1C		(0x2210)
-#define   NIC_PF_ECC1_SBE_ENA_W1S		(0x2218)
-#define   NIC_PF_ECC1_DBE_INT			(0x2300)
-#define   NIC_PF_ECC1_DBE_INT_W1S		(0x2308)
-#define   NIC_PF_ECC1_DBE_ENA_W1C		(0x2310)
-#define   NIC_PF_ECC1_DBE_ENA_W1S		(0x2318)
-#define   NIC_PF_ECC2_SBE_INT			(0x2400)
-#define   NIC_PF_ECC2_SBE_INT_W1S		(0x2408)
-#define   NIC_PF_ECC2_SBE_ENA_W1C		(0x2410)
-#define   NIC_PF_ECC2_SBE_ENA_W1S		(0x2418)
-#define   NIC_PF_ECC2_DBE_INT			(0x2500)
-#define   NIC_PF_ECC2_DBE_INT_W1S		(0x2508)
-#define   NIC_PF_ECC2_DBE_ENA_W1C		(0x2510)
-#define   NIC_PF_ECC2_DBE_ENA_W1S		(0x2518)
-#define   NIC_PF_ECC3_SBE_INT			(0x2600)
-#define   NIC_PF_ECC3_SBE_INT_W1S		(0x2608)
-#define   NIC_PF_ECC3_SBE_ENA_W1C		(0x2610)
-#define   NIC_PF_ECC3_SBE_ENA_W1S		(0x2618)
-#define   NIC_PF_ECC3_DBE_INT			(0x2700)
-#define   NIC_PF_ECC3_DBE_INT_W1S		(0x2708)
-#define   NIC_PF_ECC3_DBE_ENA_W1C		(0x2710)
-#define   NIC_PF_ECC3_DBE_ENA_W1S		(0x2718)
-#define   NIC_PF_CPI_0_2047_CFG			(0x200000)
-#define   NIC_PF_RSSI_0_4097_RQ			(0x220000)
-#define   NIC_PF_LMAC_0_7_CFG			(0x240000)
-#define   NIC_PF_LMAC_0_7_SW_XOFF		(0x242000)
-#define   NIC_PF_LMAC_0_7_CREDIT		(0x244000)
-#define   NIC_PF_CHAN_0_255_TX_CFG		(0x400000)
-#define   NIC_PF_CHAN_0_255_RX_CFG		(0x420000)
-#define   NIC_PF_CHAN_0_255_SW_XOFF		(0x440000)
-#define   NIC_PF_CHAN_0_255_CREDIT		(0x460000)
-#define   NIC_PF_CHAN_0_255_RX_BP_CFG		(0x480000)
-#define   NIC_PF_SW_SYNC_RX			(0x490000)
-#define   NIC_PF_SW_SYNC_RX_DONE		(0x490008)
-#define   NIC_PF_TL2_0_63_CFG			(0x500000)
-#define   NIC_PF_TL2_0_63_PRI			(0x520000)
-#define   NIC_PF_TL2_0_63_SH_STATUS		(0x580000)
-#define   NIC_PF_TL3A_0_63_CFG			(0x5F0000)
-#define   NIC_PF_TL3_0_255_CFG			(0x600000)
-#define   NIC_PF_TL3_0_255_CHAN			(0x620000)
-#define   NIC_PF_TL3_0_255_PIR			(0x640000)
-#define   NIC_PF_TL3_0_255_SW_XOFF		(0x660000)
-#define   NIC_PF_TL3_0_255_CNM_RATE		(0x680000)
-#define   NIC_PF_TL3_0_255_SH_STATUS		(0x6A0000)
-#define   NIC_PF_TL4A_0_255_CFG			(0x6F0000)
-#define   NIC_PF_TL4_0_1023_CFG			(0x800000)
-#define   NIC_PF_TL4_0_1023_SW_XOFF		(0x820000)
-#define   NIC_PF_TL4_0_1023_SH_STATUS		(0x840000)
-#define   NIC_PF_TL4A_0_1023_CNM_RATE		(0x880000)
-#define   NIC_PF_TL4A_0_1023_CNM_STATUS		(0x8A0000)
-#define   NIC_PF_VF_0_127_MAILBOX_0_1		(0x20002030)
-#define   NIC_PF_VNIC_0_127_TX_STAT_0_4		(0x20004000)
-#define   NIC_PF_VNIC_0_127_RX_STAT_0_13	(0x20004100)
-#define   NIC_PF_QSET_0_127_LOCK_0_15		(0x20006000)
-#define   NIC_PF_QSET_0_127_CFG			(0x20010000)
-#define   NIC_PF_QSET_0_127_RQ_0_7_CFG		(0x20010400)
-#define   NIC_PF_QSET_0_127_RQ_0_7_DROP_CFG	(0x20010420)
-#define   NIC_PF_QSET_0_127_RQ_0_7_BP_CFG	(0x20010500)
-#define   NIC_PF_QSET_0_127_RQ_0_7_STAT_0_1	(0x20010600)
-#define   NIC_PF_QSET_0_127_SQ_0_7_CFG		(0x20010C00)
-#define   NIC_PF_QSET_0_127_SQ_0_7_CFG2		(0x20010C08)
-#define   NIC_PF_QSET_0_127_SQ_0_7_STAT_0_1	(0x20010D00)
+#define	NIC_PF_CFG				(0x0000)
+#define	NIC_PF_STATUS				(0x0010)
+#define	NIC_PF_INTR_TIMER_CFG			(0x0030)
+#define	NIC_PF_BIST_STATUS			(0x0040)
+#define	NIC_PF_SOFT_RESET			(0x0050)
+#define	NIC_PF_TCP_TIMER			(0x0060)
+#define	NIC_PF_BP_CFG				(0x0080)
+#define	NIC_PF_RRM_CFG				(0x0088)
+#define	NIC_PF_CQM_CF				(0x00A0)
+#define	NIC_PF_CNM_CF				(0x00A8)
+#define	NIC_PF_CNM_STATUS			(0x00B0)
+#define	NIC_PF_CQ_AVG_CFG			(0x00C0)
+#define	NIC_PF_RRM_AVG_CFG			(0x00C8)
+#define	NIC_PF_INTF_0_1_SEND_CFG		(0x0200)
+#define	NIC_PF_INTF_0_1_BP_CFG			(0x0208)
+#define	NIC_PF_INTF_0_1_BP_DIS_0_1		(0x0210)
+#define	NIC_PF_INTF_0_1_BP_SW_0_1		(0x0220)
+#define	NIC_PF_RBDR_BP_STATE_0_3		(0x0240)
+#define	NIC_PF_MAILBOX_INT			(0x0410)
+#define	NIC_PF_MAILBOX_INT_W1S			(0x0430)
+#define	NIC_PF_MAILBOX_ENA_W1C			(0x0450)
+#define	NIC_PF_MAILBOX_ENA_W1S			(0x0470)
+#define	NIC_PF_RX_ETYPE_0_7			(0x0500)
+#define	NIC_PF_PKIND_0_15_CFG			(0x0600)
+#define	NIC_PF_ECC0_FLIP0			(0x1000)
+#define	NIC_PF_ECC1_FLIP0			(0x1008)
+#define	NIC_PF_ECC2_FLIP0			(0x1010)
+#define	NIC_PF_ECC3_FLIP0			(0x1018)
+#define	NIC_PF_ECC0_FLIP1			(0x1080)
+#define	NIC_PF_ECC1_FLIP1			(0x1088)
+#define	NIC_PF_ECC2_FLIP1			(0x1090)
+#define	NIC_PF_ECC3_FLIP1			(0x1098)
+#define	NIC_PF_ECC0_CDIS			(0x1100)
+#define	NIC_PF_ECC1_CDIS			(0x1108)
+#define	NIC_PF_ECC2_CDIS			(0x1110)
+#define	NIC_PF_ECC3_CDIS			(0x1118)
+#define	NIC_PF_BIST0_STATUS			(0x1280)
+#define	NIC_PF_BIST1_STATUS			(0x1288)
+#define	NIC_PF_BIST2_STATUS			(0x1290)
+#define	NIC_PF_BIST3_STATUS			(0x1298)
+#define	NIC_PF_ECC0_SBE_INT			(0x2000)
+#define	NIC_PF_ECC0_SBE_INT_W1S			(0x2008)
+#define	NIC_PF_ECC0_SBE_ENA_W1C			(0x2010)
+#define	NIC_PF_ECC0_SBE_ENA_W1S			(0x2018)
+#define	NIC_PF_ECC0_DBE_INT			(0x2100)
+#define	NIC_PF_ECC0_DBE_INT_W1S			(0x2108)
+#define	NIC_PF_ECC0_DBE_ENA_W1C			(0x2110)
+#define	NIC_PF_ECC0_DBE_ENA_W1S			(0x2118)
+#define	NIC_PF_ECC1_SBE_INT			(0x2200)
+#define	NIC_PF_ECC1_SBE_INT_W1S			(0x2208)
+#define	NIC_PF_ECC1_SBE_ENA_W1C			(0x2210)
+#define	NIC_PF_ECC1_SBE_ENA_W1S			(0x2218)
+#define	NIC_PF_ECC1_DBE_INT			(0x2300)
+#define	NIC_PF_ECC1_DBE_INT_W1S			(0x2308)
+#define	NIC_PF_ECC1_DBE_ENA_W1C			(0x2310)
+#define	NIC_PF_ECC1_DBE_ENA_W1S			(0x2318)
+#define	NIC_PF_ECC2_SBE_INT			(0x2400)
+#define	NIC_PF_ECC2_SBE_INT_W1S			(0x2408)
+#define	NIC_PF_ECC2_SBE_ENA_W1C			(0x2410)
+#define	NIC_PF_ECC2_SBE_ENA_W1S			(0x2418)
+#define	NIC_PF_ECC2_DBE_INT			(0x2500)
+#define	NIC_PF_ECC2_DBE_INT_W1S			(0x2508)
+#define	NIC_PF_ECC2_DBE_ENA_W1C			(0x2510)
+#define	NIC_PF_ECC2_DBE_ENA_W1S			(0x2518)
+#define	NIC_PF_ECC3_SBE_INT			(0x2600)
+#define	NIC_PF_ECC3_SBE_INT_W1S			(0x2608)
+#define	NIC_PF_ECC3_SBE_ENA_W1C			(0x2610)
+#define	NIC_PF_ECC3_SBE_ENA_W1S			(0x2618)
+#define	NIC_PF_ECC3_DBE_INT			(0x2700)
+#define	NIC_PF_ECC3_DBE_INT_W1S			(0x2708)
+#define	NIC_PF_ECC3_DBE_ENA_W1C			(0x2710)
+#define	NIC_PF_ECC3_DBE_ENA_W1S			(0x2718)
+#define	NIC_PF_CPI_0_2047_CFG			(0x200000)
+#define	NIC_PF_RSSI_0_4097_RQ			(0x220000)
+#define	NIC_PF_LMAC_0_7_CFG			(0x240000)
+#define	NIC_PF_LMAC_0_7_SW_XOFF			(0x242000)
+#define	NIC_PF_LMAC_0_7_CREDIT			(0x244000)
+#define	NIC_PF_CHAN_0_255_TX_CFG		(0x400000)
+#define	NIC_PF_CHAN_0_255_RX_CFG		(0x420000)
+#define	NIC_PF_CHAN_0_255_SW_XOFF		(0x440000)
+#define	NIC_PF_CHAN_0_255_CREDIT		(0x460000)
+#define	NIC_PF_CHAN_0_255_RX_BP_CFG		(0x480000)
+#define	NIC_PF_SW_SYNC_RX			(0x490000)
+#define	NIC_PF_SW_SYNC_RX_DONE			(0x490008)
+#define	NIC_PF_TL2_0_63_CFG			(0x500000)
+#define	NIC_PF_TL2_0_63_PRI			(0x520000)
+#define	NIC_PF_TL2_0_63_SH_STATUS		(0x580000)
+#define	NIC_PF_TL3A_0_63_CFG			(0x5F0000)
+#define	NIC_PF_TL3_0_255_CFG			(0x600000)
+#define	NIC_PF_TL3_0_255_CHAN			(0x620000)
+#define	NIC_PF_TL3_0_255_PIR			(0x640000)
+#define	NIC_PF_TL3_0_255_SW_XOFF		(0x660000)
+#define	NIC_PF_TL3_0_255_CNM_RATE		(0x680000)
+#define	NIC_PF_TL3_0_255_SH_STATUS		(0x6A0000)
+#define	NIC_PF_TL4A_0_255_CFG			(0x6F0000)
+#define	NIC_PF_TL4_0_1023_CFG			(0x800000)
+#define	NIC_PF_TL4_0_1023_SW_XOFF		(0x820000)
+#define	NIC_PF_TL4_0_1023_SH_STATUS		(0x840000)
+#define	NIC_PF_TL4A_0_1023_CNM_RATE		(0x880000)
+#define	NIC_PF_TL4A_0_1023_CNM_STATUS		(0x8A0000)
+#define	NIC_PF_VF_0_127_MAILBOX_0_1		(0x20002030)
+#define	NIC_PF_VNIC_0_127_TX_STAT_0_4		(0x20004000)
+#define	NIC_PF_VNIC_0_127_RX_STAT_0_13		(0x20004100)
+#define	NIC_PF_QSET_0_127_LOCK_0_15		(0x20006000)
+#define	NIC_PF_QSET_0_127_CFG			(0x20010000)
+#define	NIC_PF_QSET_0_127_RQ_0_7_CFG		(0x20010400)
+#define	NIC_PF_QSET_0_127_RQ_0_7_DROP_CFG	(0x20010420)
+#define	NIC_PF_QSET_0_127_RQ_0_7_BP_CFG		(0x20010500)
+#define	NIC_PF_QSET_0_127_RQ_0_7_STAT_0_1	(0x20010600)
+#define	NIC_PF_QSET_0_127_SQ_0_7_CFG		(0x20010C00)
+#define	NIC_PF_QSET_0_127_SQ_0_7_CFG2		(0x20010C08)
+#define	NIC_PF_QSET_0_127_SQ_0_7_STAT_0_1	(0x20010D00)
 
-#define   NIC_PF_MSIX_VEC_0_18_ADDR		(0x000000)
-#define   NIC_PF_MSIX_VEC_0_CTL			(0x000008)
-#define   NIC_PF_MSIX_PBA_0			(0x0F0000)
+#define	NIC_PF_MSIX_VEC_0_18_ADDR		(0x000000)
+#define	NIC_PF_MSIX_VEC_0_CTL			(0x000008)
+#define	NIC_PF_MSIX_PBA_0			(0x0F0000)
 
 /* Virtual function register offsets */
-#define   NIC_VNIC_CFG				(0x000020)
-#define   NIC_VF_PF_MAILBOX_0_1			(0x000130)
-#define   NIC_VF_INT				(0x000200)
-#define   NIC_VF_INT_W1S			(0x000220)
-#define   NIC_VF_ENA_W1C			(0x000240)
-#define   NIC_VF_ENA_W1S			(0x000260)
+#define	NIC_VNIC_CFG				(0x000020)
+#define	NIC_VF_PF_MAILBOX_0_1			(0x000130)
+#define	NIC_VF_INT				(0x000200)
+#define	NIC_VF_INT_W1S				(0x000220)
+#define	NIC_VF_ENA_W1C				(0x000240)
+#define	NIC_VF_ENA_W1S				(0x000260)
 
-#define   NIC_VNIC_RSS_CFG			(0x0020E0)
-#define   NIC_VNIC_RSS_KEY_0_4			(0x002200)
-#define   NIC_VNIC_TX_STAT_0_4			(0x004000)
-#define   NIC_VNIC_RX_STAT_0_13			(0x004100)
-#define   NIC_QSET_RQ_GEN_CFG			(0x010010)
+#define	NIC_VNIC_RSS_CFG			(0x0020E0)
+#define	NIC_VNIC_RSS_KEY_0_4			(0x002200)
+#define	NIC_VNIC_TX_STAT_0_4			(0x004000)
+#define	NIC_VNIC_RX_STAT_0_13			(0x004100)
+#define	NIC_QSET_RQ_GEN_CFG			(0x010010)
 
-#define   NIC_QSET_CQ_0_7_CFG			(0x010400)
-#define   NIC_QSET_CQ_0_7_CFG2			(0x010408)
-#define   NIC_QSET_CQ_0_7_THRESH		(0x010410)
-#define   NIC_QSET_CQ_0_7_BASE			(0x010420)
-#define   NIC_QSET_CQ_0_7_HEAD			(0x010428)
-#define   NIC_QSET_CQ_0_7_TAIL			(0x010430)
-#define   NIC_QSET_CQ_0_7_DOOR			(0x010438)
-#define   NIC_QSET_CQ_0_7_STATUS		(0x010440)
-#define   NIC_QSET_CQ_0_7_STATUS2		(0x010448)
-#define   NIC_QSET_CQ_0_7_DEBUG			(0x010450)
+#define	NIC_QSET_CQ_0_7_CFG			(0x010400)
+#define	NIC_QSET_CQ_0_7_CFG2			(0x010408)
+#define	NIC_QSET_CQ_0_7_THRESH			(0x010410)
+#define	NIC_QSET_CQ_0_7_BASE			(0x010420)
+#define	NIC_QSET_CQ_0_7_HEAD			(0x010428)
+#define	NIC_QSET_CQ_0_7_TAIL			(0x010430)
+#define	NIC_QSET_CQ_0_7_DOOR			(0x010438)
+#define	NIC_QSET_CQ_0_7_STATUS			(0x010440)
+#define	NIC_QSET_CQ_0_7_STATUS2			(0x010448)
+#define	NIC_QSET_CQ_0_7_DEBUG			(0x010450)
 
-#define   NIC_QSET_RQ_0_7_CFG			(0x010600)
-#define   NIC_QSET_RQ_0_7_STAT_0_1		(0x010700)
+#define	NIC_QSET_RQ_0_7_CFG			(0x010600)
+#define	NIC_QSET_RQ_0_7_STAT_0_1		(0x010700)
 
-#define   NIC_QSET_SQ_0_7_CFG			(0x010800)
-#define   NIC_QSET_SQ_0_7_THRESH		(0x010810)
-#define   NIC_QSET_SQ_0_7_BASE			(0x010820)
-#define   NIC_QSET_SQ_0_7_HEAD			(0x010828)
-#define   NIC_QSET_SQ_0_7_TAIL			(0x010830)
-#define   NIC_QSET_SQ_0_7_DOOR			(0x010838)
-#define   NIC_QSET_SQ_0_7_STATUS		(0x010840)
-#define   NIC_QSET_SQ_0_7_DEBUG			(0x010848)
-#define   NIC_QSET_SQ_0_7_CNM_CHG		(0x010860)
-#define   NIC_QSET_SQ_0_7_STAT_0_1		(0x010900)
+#define	NIC_QSET_SQ_0_7_CFG			(0x010800)
+#define	NIC_QSET_SQ_0_7_THRESH			(0x010810)
+#define	NIC_QSET_SQ_0_7_BASE			(0x010820)
+#define	NIC_QSET_SQ_0_7_HEAD			(0x010828)
+#define	NIC_QSET_SQ_0_7_TAIL			(0x010830)
+#define	NIC_QSET_SQ_0_7_DOOR			(0x010838)
+#define	NIC_QSET_SQ_0_7_STATUS			(0x010840)
+#define	NIC_QSET_SQ_0_7_DEBUG			(0x010848)
+#define	NIC_QSET_SQ_0_7_CNM_CHG			(0x010860)
+#define	NIC_QSET_SQ_0_7_STAT_0_1		(0x010900)
 
-#define   NIC_QSET_RBDR_0_1_CFG			(0x010C00)
-#define   NIC_QSET_RBDR_0_1_THRESH		(0x010C10)
-#define   NIC_QSET_RBDR_0_1_BASE		(0x010C20)
-#define   NIC_QSET_RBDR_0_1_HEAD		(0x010C28)
-#define   NIC_QSET_RBDR_0_1_TAIL		(0x010C30)
-#define   NIC_QSET_RBDR_0_1_DOOR		(0x010C38)
-#define   NIC_QSET_RBDR_0_1_STATUS0		(0x010C40)
-#define   NIC_QSET_RBDR_0_1_STATUS1		(0x010C48)
-#define   NIC_QSET_RBDR_0_1_PREFETCH_STATUS	(0x010C50)
+#define	NIC_QSET_RBDR_0_1_CFG			(0x010C00)
+#define	NIC_QSET_RBDR_0_1_THRESH		(0x010C10)
+#define	NIC_QSET_RBDR_0_1_BASE			(0x010C20)
+#define	NIC_QSET_RBDR_0_1_HEAD			(0x010C28)
+#define	NIC_QSET_RBDR_0_1_TAIL			(0x010C30)
+#define	NIC_QSET_RBDR_0_1_DOOR			(0x010C38)
+#define	NIC_QSET_RBDR_0_1_STATUS0		(0x010C40)
+#define	NIC_QSET_RBDR_0_1_STATUS1		(0x010C48)
+#define	NIC_QSET_RBDR_0_1_PREFETCH_STATUS	(0x010C50)
 
-#define   NIC_VF_MSIX_VECTOR_0_19_ADDR		(0x000000)
-#define   NIC_VF_MSIX_VECTOR_0_19_CTL		(0x000008)
-#define   NIC_VF_MSIX_PBA			(0x0F0000)
+#define	NIC_VF_MSIX_VECTOR_0_19_ADDR		(0x000000)
+#define	NIC_VF_MSIX_VECTOR_0_19_CTL		(0x000008)
+#define	NIC_VF_MSIX_PBA				(0x0F0000)
 
 /* Offsets within registers */
-#define   NIC_MSIX_VEC_SHIFT			4
-#define   NIC_Q_NUM_SHIFT			18
-#define   NIC_QS_ID_SHIFT			21
-#define   NIC_VF_NUM_SHIFT			21
+#define	NIC_MSIX_VEC_SHIFT			4
+#define	NIC_Q_NUM_SHIFT				18
+#define	NIC_QS_ID_SHIFT				21
+#define	NIC_VF_NUM_SHIFT			21
 
 /* Port kind configuration register */
 struct pkind_cfg {
-#if defined(__BIG_ENDIAN_BITFIELD)
-	u64 reserved_42_63:22;
-	u64 hdr_sl:5;	/* Header skip length */
-	u64 rx_hdr:3;	/* TNS Receive header present */
-	u64 lenerr_en:1;/* L2 length error check enable */
-	u64 reserved_32_32:1;
-	u64 maxlen:16;	/* Max frame size */
-	u64 minlen:16;	/* Min frame size */
-#elif defined(__LITTLE_ENDIAN_BITFIELD)
-	u64 minlen:16;
-	u64 maxlen:16;
-	u64 reserved_32_32:1;
-	u64 lenerr_en:1;
-	u64 rx_hdr:3;
-	u64 hdr_sl:5;
-	u64 reserved_42_63:22;
-#endif
+	uint64_t minlen:16;
+	uint64_t maxlen:16;
+	uint64_t reserved_32_32:1;
+	uint64_t lenerr_en:1;
+	uint64_t rx_hdr:3;
+	uint64_t hdr_sl:5;
+	uint64_t reserved_42_63:22;
 };
 
 #endif /* NIC_REG_H */
diff --git a/sys/dev/vnic/nicvf_main.c b/sys/dev/vnic/nicvf_main.c
index 9e9e0d53a8f0..42a1d37dc3ad 100644
--- a/sys/dev/vnic/nicvf_main.c
+++ b/sys/dev/vnic/nicvf_main.c
@@ -26,1215 +26,528 @@
  * $FreeBSD$
  *
  */
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
 
-#include <linux/module.h>
-#include <linux/interrupt.h>
-#include <linux/pci.h>
-#include <linux/netdevice.h>
-#include <linux/if_vlan.h>
-#include <linux/etherdevice.h>
-#include <linux/ethtool.h>
-#include <linux/log2.h>
-#include <linux/prefetch.h>
-#include <linux/irq.h>
+#include "opt_inet.h"
+#include "opt_inet6.h"
 
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bitset.h>
+#include <sys/bitstring.h>
+#include <sys/bus.h>
+#include <sys/endian.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/mbuf.h>
+#include <sys/module.h>
+#include <sys/rman.h>
+#include <sys/pciio.h>
+#include <sys/pcpu.h>
+#include <sys/proc.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
+#include <sys/stdatomic.h>
+#include <sys/cpuset.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/smp.h>
+#include <sys/taskqueue.h>
+
+#include <net/bpf.h>
+#include <net/ethernet.h>
+#include <net/if.h>
+#include <net/if_var.h>
+#include <net/if_arp.h>
+#include <net/if_dl.h>
+#include <net/if_media.h>
+#include <net/if_types.h>
+#include <net/if_vlan_var.h>
+
+#include <netinet/in.h>
+#include <netinet/if_ether.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#include <sys/dnv.h>
+#include <sys/nv.h>
+#include <sys/iov_schema.h>
+
+#include <machine/bus.h>
+
+#include "thunder_bgx.h"
 #include "nic_reg.h"
 #include "nic.h"
 #include "nicvf_queues.h"
-#include "thunder_bgx.h"
 
-#define DRV_NAME	"thunder-nicvf"
-#define DRV_VERSION	"1.0"
+#define	VNIC_VF_DEVSTR		"Cavium Thunder NIC Virtual Function Driver"
 
-/* Supported devices */
-static const struct pci_device_id nicvf_id_table[] = {
-	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM,
-			 PCI_DEVICE_ID_THUNDER_NIC_VF,
-			 PCI_VENDOR_ID_CAVIUM, 0xA11E) },
-	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM,
-			 PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF,
-			 PCI_VENDOR_ID_CAVIUM, 0xA11E) },
-	{ 0, }  /* end of table */
+#define	VNIC_VF_REG_RID		PCIR_BAR(PCI_CFG_REG_BAR_NUM)
+
+/* Lock for core interface settings */
+#define	NICVF_CORE_LOCK_INIT(nic)				\
+    sx_init(&(nic)->core_sx, device_get_nameunit((nic)->dev))
+
+#define	NICVF_CORE_LOCK_DESTROY(nic)				\
+    sx_destroy(&(nic)->core_sx)
+
+#define	NICVF_CORE_LOCK(nic)		sx_xlock(&(nic)->core_sx)
+#define	NICVF_CORE_UNLOCK(nic)		sx_xunlock(&(nic)->core_sx)
+
+#define	NICVF_CORE_LOCK_ASSERT(nic)	sx_assert(&(nic)->core_sx, SA_XLOCKED)
+
+#define	SPEED_10	10
+#define	SPEED_100	100
+#define	SPEED_1000	1000
+#define	SPEED_10000	10000
+#define	SPEED_40000	40000
+
+MALLOC_DEFINE(M_NICVF, "nicvf", "ThunderX VNIC VF dynamic memory");
+
+static int nicvf_probe(device_t);
+static int nicvf_attach(device_t);
+static int nicvf_detach(device_t);
+
+static device_method_t nicvf_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		nicvf_probe),
+	DEVMETHOD(device_attach,	nicvf_attach),
+	DEVMETHOD(device_detach,	nicvf_detach),
+
+	DEVMETHOD_END,
 };
 
-MODULE_AUTHOR("Sunil Goutham");
-MODULE_DESCRIPTION("Cavium Thunder NIC Virtual Function Driver");
-MODULE_VERSION(DRV_VERSION);
-MODULE_DEVICE_TABLE(pci, nicvf_id_table);
+static driver_t nicvf_driver = {
+	"vnic",
+	nicvf_methods,
+	sizeof(struct nicvf),
+};
 
-static int debug = 0x00;
-module_param(debug, int, 0644);
-MODULE_PARM_DESC(debug, "Debug message level bitmap");
+static devclass_t nicvf_devclass;
 
-static int cpi_alg = CPI_ALG_NONE;
-module_param(cpi_alg, int, S_IRUGO);
-MODULE_PARM_DESC(cpi_alg,
-		 "PFC algorithm (0=none, 1=VLAN, 2=VLAN16, 3=IP Diffserv)");
+DRIVER_MODULE(nicvf, pci, nicvf_driver, nicvf_devclass, 0, 0);
+MODULE_DEPEND(nicvf, pci, 1, 1, 1);
+MODULE_DEPEND(nicvf, ether, 1, 1, 1);
+MODULE_DEPEND(nicvf, vnic_pf, 1, 1, 1);
 
-static inline u8 nicvf_netdev_qidx(struct nicvf *nic, u8 qidx)
+static int nicvf_allocate_misc_interrupt(struct nicvf *);
+static int nicvf_enable_misc_interrupt(struct nicvf *);
+static int nicvf_allocate_net_interrupts(struct nicvf *);
+static void nicvf_release_all_interrupts(struct nicvf *);
+static int nicvf_hw_set_mac_addr(struct nicvf *, uint8_t *);
+static void nicvf_config_cpi(struct nicvf *);
+static int nicvf_init_resources(struct nicvf *);
+
+static int nicvf_setup_ifnet(struct nicvf *);
+static int nicvf_setup_ifmedia(struct nicvf *);
+static void nicvf_hw_addr_random(uint8_t *);
+
+static int nicvf_if_ioctl(struct ifnet *, u_long, caddr_t);
+static void nicvf_if_init(void *);
+static void nicvf_if_init_locked(struct nicvf *);
+static int nicvf_if_transmit(struct ifnet *, struct mbuf *);
+static void nicvf_if_qflush(struct ifnet *);
+static uint64_t nicvf_if_getcounter(struct ifnet *, ift_counter);
+
+static int nicvf_stop_locked(struct nicvf *);
+
+static void nicvf_media_status(struct ifnet *, struct ifmediareq *);
+static int nicvf_media_change(struct ifnet *);
+
+static void nicvf_tick_stats(void *);
+
+static int
+nicvf_probe(device_t dev)
 {
-#ifdef VNIC_MULTI_QSET_SUPPORT
-	if (nic->sqs_mode)
-		return qidx + ((nic->sqs_id + 1) * MAX_CMP_QUEUES_PER_QS);
-	else
-#endif
-		return qidx;
-}
+	uint16_t vendor_id;
+	uint16_t device_id;
 
-static inline void nicvf_set_rx_frame_cnt(struct nicvf *nic,
-					  struct sk_buff *skb)
-{
-	if (skb->len <= 64)
-		nic->drv_stats.rx_frames_64++;
-	else if (skb->len <= 127)
-		nic->drv_stats.rx_frames_127++;
-	else if (skb->len <= 255)
-		nic->drv_stats.rx_frames_255++;
-	else if (skb->len <= 511)
-		nic->drv_stats.rx_frames_511++;
-	else if (skb->len <= 1023)
-		nic->drv_stats.rx_frames_1023++;
-	else if (skb->len <= 1518)
-		nic->drv_stats.rx_frames_1518++;
-	else
-		nic->drv_stats.rx_frames_jumbo++;
-}
+	vendor_id = pci_get_vendor(dev);
+	device_id = pci_get_device(dev);
 
-/* The Cavium ThunderX network controller can *only* be found in SoCs
- * containing the ThunderX ARM64 CPU implementation.  All accesses to the device
- * registers on this platform are implicitly strongly ordered with respect
- * to memory accesses. So writeq_relaxed() and readq_relaxed() are safe to use
- * with no memory barriers in this driver.  The readq()/writeq() functions add
- * explicit ordering operation which in this case are redundant, and only
- * add overhead.
- */
+	if (vendor_id != PCI_VENDOR_ID_CAVIUM)
+		return (ENXIO);
 
-/* Register read/write APIs */
-void nicvf_reg_write(struct nicvf *nic, u64 offset, u64 val)
-{
-	writeq_relaxed(val, nic->reg_base + offset);
-}
-
-u64 nicvf_reg_read(struct nicvf *nic, u64 offset)
-{
-	return readq_relaxed(nic->reg_base + offset);
-}
-
-void nicvf_queue_reg_write(struct nicvf *nic, u64 offset,
-			   u64 qidx, u64 val)
-{
-	void __iomem *addr = nic->reg_base + offset;
-
-	writeq_relaxed(val, addr + (qidx << NIC_Q_NUM_SHIFT));
-}
-
-u64 nicvf_queue_reg_read(struct nicvf *nic, u64 offset, u64 qidx)
-{
-	void __iomem *addr = nic->reg_base + offset;
-
-	return readq_relaxed(addr + (qidx << NIC_Q_NUM_SHIFT));
-}
-
-/* VF -> PF mailbox communication */
-static void nicvf_write_to_mbx(struct nicvf *nic, union nic_mbx *mbx)
-{
-	u64 *msg = (u64 *)mbx;
-
-	nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 0, msg[0]);
-	nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 8, msg[1]);
-}
-
-int nicvf_send_msg_to_pf(struct nicvf *nic, union nic_mbx *mbx)
-{
-	int timeout = NIC_MBOX_MSG_TIMEOUT;
-	int sleep = 10;
-
-	nic->pf_acked = false;
-	nic->pf_nacked = false;
-
-	nicvf_write_to_mbx(nic, mbx);
-
-	/* Wait for previous message to be acked, timeout 2sec */
-	while (!nic->pf_acked) {
-		if (nic->pf_nacked)
-			return -EINVAL;
-		msleep(sleep);
-		if (nic->pf_acked)
-			break;
-		timeout -= sleep;
-		if (!timeout) {
-			netdev_err(nic->netdev,
-				   "PF didn't ack to mbox msg %d from VF%d\n",
-				   (mbx->msg.msg & 0xFF), nic->vf_id);
-			return -EBUSY;
-		}
-	}
-	return 0;
-}
-
-/* Checks if VF is able to comminicate with PF
-* and also gets the VNIC number this VF is associated to.
-*/
-static int nicvf_check_pf_ready(struct nicvf *nic)
-{
-	union nic_mbx mbx = {};
-
-	mbx.msg.msg = NIC_MBOX_MSG_READY;
-	if (nicvf_send_msg_to_pf(nic, &mbx)) {
-		netdev_err(nic->netdev,
-			   "PF didn't respond to READY msg\n");
-		return 0;
+	if (device_id == PCI_DEVICE_ID_THUNDER_NIC_VF ||
+	    device_id == PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF) {
+		device_set_desc(dev, VNIC_VF_DEVSTR);
+		return (BUS_PROBE_DEFAULT);
 	}
 
-	return 1;
+	return (ENXIO);
 }
 
-static void nicvf_read_bgx_stats(struct nicvf *nic, struct bgx_stats_msg *bgx)
-{
-	if (bgx->rx)
-		nic->bgx_stats.rx_stats[bgx->idx] = bgx->stats;
-	else
-		nic->bgx_stats.tx_stats[bgx->idx] = bgx->stats;
-}
-
-static void  nicvf_handle_mbx_intr(struct nicvf *nic)
-{
-	union nic_mbx mbx = {};
-	u64 *mbx_data;
-	u64 mbx_addr;
-	int i;
-
-	mbx_addr = NIC_VF_PF_MAILBOX_0_1;
-	mbx_data = (u64 *)&mbx;
-
-	for (i = 0; i < NIC_PF_VF_MAILBOX_SIZE; i++) {
-		*mbx_data = nicvf_reg_read(nic, mbx_addr);
-		mbx_data++;
-		mbx_addr += sizeof(u64);
-	}
-
-	netdev_dbg(nic->netdev, "Mbox message: msg: 0x%x\n", mbx.msg.msg);
-	switch (mbx.msg.msg) {
-	case NIC_MBOX_MSG_READY:
-		nic->pf_acked = true;
-		nic->vf_id = mbx.nic_cfg.vf_id & 0x7F;
-		nic->tns_mode = mbx.nic_cfg.tns_mode & 0x7F;
-		nic->node = mbx.nic_cfg.node_id;
-		ether_addr_copy(nic->netdev->dev_addr, mbx.nic_cfg.mac_addr);
-#ifdef VNIC_MULTI_QSET_SUPPORT
-		nic->sqs_mode = mbx.nic_cfg.sqs_mode;
-#endif
-		nic->loopback_supported = mbx.nic_cfg.loopback_supported;
-		nic->link_up = false;
-		nic->duplex = 0;
-		nic->speed = 0;
-		break;
-	case NIC_MBOX_MSG_ACK:
-		nic->pf_acked = true;
-		break;
-	case NIC_MBOX_MSG_NACK:
-		nic->pf_nacked = true;
-		break;
-#ifdef VNIC_RSS_SUPPORT
-	case NIC_MBOX_MSG_RSS_SIZE:
-		nic->rss_info.rss_size = mbx.rss_size.ind_tbl_size;
-		nic->pf_acked = true;
-		break;
-#endif
-	case NIC_MBOX_MSG_BGX_STATS:
-		nicvf_read_bgx_stats(nic, &mbx.bgx_stats);
-		nic->pf_acked = true;
-		break;
-	case NIC_MBOX_MSG_BGX_LINK_CHANGE:
-		nic->pf_acked = true;
-		nic->link_up = mbx.link_status.link_up;
-		nic->duplex = mbx.link_status.duplex;
-		nic->speed = mbx.link_status.speed;
-		if (nic->link_up) {
-			netdev_info(nic->netdev, "%s: Link is Up %d Mbps %s\n",
-				    nic->netdev->name, nic->speed,
-				    nic->duplex == DUPLEX_FULL ?
-				"Full duplex" : "Half duplex");
-			netif_carrier_on(nic->netdev);
-			netif_tx_start_all_queues(nic->netdev);
-		} else {
-			netdev_info(nic->netdev, "%s: Link is Down\n",
-				    nic->netdev->name);
-			netif_carrier_off(nic->netdev);
-			netif_tx_stop_all_queues(nic->netdev);
-		}
-		break;
-#ifdef VNIC_MULTI_QSET_SUPPORT
-	case NIC_MBOX_MSG_ALLOC_SQS:
-		nic->sqs_count = mbx.sqs_alloc.qs_count;
-		nic->pf_acked = true;
-		break;
-	case NIC_MBOX_MSG_SNICVF_PTR:
-		/* Primary VF: make note of secondary VF's pointer
-		 * to be used while packet transmission.
-		 */
-		nic->snicvf[mbx.nicvf.sqs_id] =
-			(struct nicvf *)mbx.nicvf.nicvf;
-		nic->pf_acked = true;
-		break;
-	case NIC_MBOX_MSG_PNICVF_PTR:
-		/* Secondary VF/Qset: make note of primary VF's pointer
-		 * to be used while packet reception, to handover packet
-		 * to primary VF's netdev.
-		 */
-		nic->pnicvf = (struct nicvf *)mbx.nicvf.nicvf;
-		nic->pf_acked = true;
-		break;
-#endif
-	default:
-		netdev_err(nic->netdev,
-			   "Invalid message from PF, msg 0x%x\n", mbx.msg.msg);
-		break;
-	}
-	nicvf_clear_intr(nic, NICVF_INTR_MBOX, 0);
-}
-
-static int nicvf_hw_set_mac_addr(struct nicvf *nic, struct net_device *netdev)
-{
-	union nic_mbx mbx = {};
-
-	mbx.mac.msg = NIC_MBOX_MSG_SET_MAC;
-	mbx.mac.vf_id = nic->vf_id;
-	ether_addr_copy(mbx.mac.mac_addr, netdev->dev_addr);
-
-	return nicvf_send_msg_to_pf(nic, &mbx);
-}
-
-static void nicvf_config_cpi(struct nicvf *nic)
-{
-	union nic_mbx mbx = {};
-
-	mbx.cpi_cfg.msg = NIC_MBOX_MSG_CPI_CFG;
-	mbx.cpi_cfg.vf_id = nic->vf_id;
-	mbx.cpi_cfg.cpi_alg = nic->cpi_alg;
-	mbx.cpi_cfg.rq_cnt = nic->qs->rq_cnt;
-
-	nicvf_send_msg_to_pf(nic, &mbx);
-}
-
-#ifdef	VNIC_RSS_SUPPORT
-static void nicvf_get_rss_size(struct nicvf *nic)
-{
-	union nic_mbx mbx = {};
-
-	mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE;
-	mbx.rss_size.vf_id = nic->vf_id;
-	nicvf_send_msg_to_pf(nic, &mbx);
-}
-
-void nicvf_config_rss(struct nicvf *nic)
-{
-	union nic_mbx mbx = {};
-	struct nicvf_rss_info *rss = &nic->rss_info;
-	int ind_tbl_len = rss->rss_size;
-	int i, nextq = 0;
-
-	mbx.rss_cfg.vf_id = nic->vf_id;
-	mbx.rss_cfg.hash_bits = rss->hash_bits;
-	while (ind_tbl_len) {
-		mbx.rss_cfg.tbl_offset = nextq;
-		mbx.rss_cfg.tbl_len = min(ind_tbl_len,
-					       RSS_IND_TBL_LEN_PER_MBX_MSG);
-		mbx.rss_cfg.msg = mbx.rss_cfg.tbl_offset ?
-			  NIC_MBOX_MSG_RSS_CFG_CONT : NIC_MBOX_MSG_RSS_CFG;
-
-		for (i = 0; i < mbx.rss_cfg.tbl_len; i++)
-			mbx.rss_cfg.ind_tbl[i] = rss->ind_tbl[nextq++];
-
-		nicvf_send_msg_to_pf(nic, &mbx);
-
-		ind_tbl_len -= mbx.rss_cfg.tbl_len;
-	}
-}
-
-void nicvf_set_rss_key(struct nicvf *nic)
-{
-	struct nicvf_rss_info *rss = &nic->rss_info;
-	u64 key_addr = NIC_VNIC_RSS_KEY_0_4;
-	int idx;
-
-	for (idx = 0; idx < RSS_HASH_KEY_SIZE; idx++) {
-		nicvf_reg_write(nic, key_addr, rss->key[idx]);
-		key_addr += sizeof(u64);
-	}
-}
-
-static int nicvf_rss_init(struct nicvf *nic)
-{
-	struct nicvf_rss_info *rss = &nic->rss_info;
-	int idx;
-
-	nicvf_get_rss_size(nic);
-
-	if (cpi_alg != CPI_ALG_NONE) {
-		rss->enable = false;
-		rss->hash_bits = 0;
-		return 0;
-	}
-
-	rss->enable = true;
-
-	/* Using the HW reset value for now */
-	rss->key[0] = 0xFEED0BADFEED0BADULL;
-	rss->key[1] = 0xFEED0BADFEED0BADULL;
-	rss->key[2] = 0xFEED0BADFEED0BADULL;
-	rss->key[3] = 0xFEED0BADFEED0BADULL;
-	rss->key[4] = 0xFEED0BADFEED0BADULL;
-
-	nicvf_set_rss_key(nic);
-
-	rss->cfg = RSS_IP_HASH_ENA | RSS_TCP_HASH_ENA | RSS_UDP_HASH_ENA;
-	nicvf_reg_write(nic, NIC_VNIC_RSS_CFG, rss->cfg);
-
-	rss->hash_bits =  ilog2(rounddown_pow_of_two(rss->rss_size));
-
-	for (idx = 0; idx < rss->rss_size; idx++)
-		rss->ind_tbl[idx] = ethtool_rxfh_indir_default(idx,
-							       nic->rx_queues);
-	nicvf_config_rss(nic);
-	return 1;
-}
-#endif
-
-#ifdef VNIC_MULTI_QSET_SUPPORT
-/* Request PF to allocate additional Qsets */
-static void nicvf_request_sqs(struct nicvf *nic)
-{
-	union nic_mbx mbx = {};
-	int sqs;
-	int sqs_count = nic->sqs_count;
-	int rx_queues = 0, tx_queues = 0;
-
-	/* Only primary VF should request */
-	if (nic->sqs_mode ||  !nic->sqs_count)
-		return;
-
-	mbx.sqs_alloc.msg = NIC_MBOX_MSG_ALLOC_SQS;
-	mbx.sqs_alloc.vf_id = nic->vf_id;
-	mbx.sqs_alloc.qs_count = nic->sqs_count;
-	if (nicvf_send_msg_to_pf(nic, &mbx)) {
-		/* No response from PF */
-		nic->sqs_count = 0;
-		return;
-	}
-
-	/* Return if no Secondary Qsets available */
-	if (!nic->sqs_count)
-		return;
-
-	if (nic->rx_queues > MAX_RCV_QUEUES_PER_QS)
-		rx_queues = nic->rx_queues - MAX_RCV_QUEUES_PER_QS;
-	if (nic->tx_queues > MAX_SND_QUEUES_PER_QS)
-		tx_queues = nic->tx_queues - MAX_SND_QUEUES_PER_QS;
-
-	/* Set no of Rx/Tx queues in each of the SQsets */
-	for (sqs = 0; sqs < nic->sqs_count; sqs++) {
-		mbx.nicvf.msg = NIC_MBOX_MSG_SNICVF_PTR;
-		mbx.nicvf.vf_id = nic->vf_id;
-		mbx.nicvf.sqs_id = sqs;
-		nicvf_send_msg_to_pf(nic, &mbx);
-
-		nic->snicvf[sqs]->sqs_id = sqs;
-		if (rx_queues > MAX_RCV_QUEUES_PER_QS) {
-			nic->snicvf[sqs]->qs->rq_cnt = MAX_RCV_QUEUES_PER_QS;
-			rx_queues -= MAX_RCV_QUEUES_PER_QS;
-		} else {
-			nic->snicvf[sqs]->qs->rq_cnt = rx_queues;
-			rx_queues = 0;
-		}
-
-		if (tx_queues > MAX_SND_QUEUES_PER_QS) {
-			nic->snicvf[sqs]->qs->sq_cnt = MAX_SND_QUEUES_PER_QS;
-			tx_queues -= MAX_SND_QUEUES_PER_QS;
-		} else {
-			nic->snicvf[sqs]->qs->sq_cnt = tx_queues;
-			tx_queues = 0;
-		}
-
-		nic->snicvf[sqs]->qs->cq_cnt =
-		max(nic->snicvf[sqs]->qs->rq_cnt, nic->snicvf[sqs]->qs->sq_cnt);
-
-		/* Initialize secondary Qset's queues and its interrupts */
-		nicvf_open(nic->snicvf[sqs]->netdev);
-	}
-
-	/* Update stack with actual Rx/Tx queue count allocated */
-	if (sqs_count != nic->sqs_count)
-		nicvf_set_real_num_queues(nic->netdev,
-					  nic->tx_queues, nic->rx_queues);
-}
-
-/* Send this Qset's nicvf pointer to PF.
- * PF inturn sends primary VF's nicvf struct to secondary Qsets/VFs
- * so that packets received by these Qsets can use primary VF's netdev
- */
-static void nicvf_send_vf_struct(struct nicvf *nic)
-{
-	union nic_mbx mbx = {};
-
-	mbx.nicvf.msg = NIC_MBOX_MSG_NICVF_PTR;
-	mbx.nicvf.sqs_mode = nic->sqs_mode;
-	mbx.nicvf.nicvf = (u64)nic;
-	nicvf_send_msg_to_pf(nic, &mbx);
-}
-
-static void nicvf_get_primary_vf_struct(struct nicvf *nic)
-{
-	union nic_mbx mbx = {};
-
-	mbx.nicvf.msg = NIC_MBOX_MSG_PNICVF_PTR;
-	nicvf_send_msg_to_pf(nic, &mbx);
-}
-#endif
-
-int nicvf_set_real_num_queues(struct net_device *netdev,
-			      int tx_queues, int rx_queues)
+static int
+nicvf_attach(device_t dev)
 {
+	struct nicvf *nic;
+	int rid, qcount;
 	int err = 0;
+	uint8_t hwaddr[ETHER_ADDR_LEN];
+	uint8_t zeromac[] = {[0 ... (ETHER_ADDR_LEN - 1)] = 0};
 
-	err = netif_set_real_num_tx_queues(netdev, tx_queues);
-	if (err) {
-		netdev_err(netdev,
-			   "Failed to set no of Tx queues: %d\n", tx_queues);
-		return err;
-	}
-
-	err = netif_set_real_num_rx_queues(netdev, rx_queues);
-	if (err)
-		netdev_err(netdev,
-			   "Failed to set no of Rx queues: %d\n", rx_queues);
-	return err;
-}
-
-static int nicvf_init_resources(struct nicvf *nic)
-{
-	int err;
-	union nic_mbx mbx = {};
-
-	mbx.msg.msg = NIC_MBOX_MSG_CFG_DONE;
-
-	/* Enable Qset */
-	nicvf_qset_config(nic, true);
-
-	/* Initialize queues and HW for data transfer */
-	err = nicvf_config_data_transfer(nic, true);
-	if (err) {
-		netdev_err(nic->netdev,
-			   "Failed to alloc/config VF's QSet resources\n");
-		return err;
-	}
-
-	/* Send VF config done msg to PF */
-	nicvf_write_to_mbx(nic, &mbx);
-
-	return 0;
-}
-
-static void nicvf_snd_pkt_handler(struct net_device *netdev,
-				  struct cmp_queue *cq,
-				  struct cqe_send_t *cqe_tx, int cqe_type)
-{
-	struct sk_buff *skb = NULL;
-	struct nicvf *nic = netdev_priv(netdev);
-	struct snd_queue *sq;
-	struct sq_hdr_subdesc *hdr;
-
-	sq = &nic->qs->sq[cqe_tx->sq_idx];
-
-	hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, cqe_tx->sqe_ptr);
-	if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER)
-		return;
-
-	netdev_dbg(nic->netdev,
-		   "%s Qset #%d SQ #%d SQ ptr #%d subdesc count %d\n",
-		   __func__, cqe_tx->sq_qs, cqe_tx->sq_idx,
-		   cqe_tx->sqe_ptr, hdr->subdesc_cnt);
-
-	nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1);
-	nicvf_check_cqe_tx_errs(nic, cq, cqe_tx);
-	skb = (struct sk_buff *)sq->skbuff[cqe_tx->sqe_ptr];
-	/* For TSO offloaded packets only one head SKB needs to be freed */
-	if (skb) {
-		prefetch(skb);
-		dev_consume_skb_any(skb);
-		sq->skbuff[cqe_tx->sqe_ptr] = (u64)NULL;
-	}
-}
-
-static inline void nicvf_set_rxhash(struct net_device *netdev,
-				    struct cqe_rx_t *cqe_rx,
-				    struct sk_buff *skb)
-{
-	u8 hash_type;
-	u32 hash;
-
-	if (!(netdev->features & NETIF_F_RXHASH))
-		return;
-
-	switch (cqe_rx->rss_alg) {
-	case RSS_ALG_TCP_IP:
-	case RSS_ALG_UDP_IP:
-		hash_type = PKT_HASH_TYPE_L4;
-		hash = cqe_rx->rss_tag;
-		break;
-	case RSS_ALG_IP:
-		hash_type = PKT_HASH_TYPE_L3;
-		hash = cqe_rx->rss_tag;
-		break;
-	default:
-		hash_type = PKT_HASH_TYPE_NONE;
-		hash = 0;
-	}
-
-	skb_set_hash(skb, hash, hash_type);
-}
-
-static void nicvf_rcv_pkt_handler(struct net_device *netdev,
-				  struct napi_struct *napi,
-				  struct cmp_queue *cq,
-				  struct cqe_rx_t *cqe_rx, int cqe_type)
-{
-	struct sk_buff *skb;
-	struct nicvf *nic = netdev_priv(netdev);
-	int err = 0;
-	int rq_idx;
-
-	rq_idx = nicvf_netdev_qidx(nic, cqe_rx->rq_idx);
-
-#ifdef VNIC_MULTI_QSET_SUPPORT
-	if (nic->sqs_mode) {
-		/* Use primary VF's 'nicvf' struct */
-		nic = nic->pnicvf;
-		netdev = nic->netdev;
-	}
-#endif
-	/* Check for errors */
-	err = nicvf_check_cqe_rx_errs(nic, cq, cqe_rx);
-	if (err && !cqe_rx->rb_cnt)
-		return;
-
-	skb = nicvf_get_rcv_skb(nic, cqe_rx);
-	if (!skb) {
-		netdev_dbg(nic->netdev, "Packet not received\n");
-		return;
-	}
-
-	if (netif_msg_pktdata(nic)) {
-		netdev_info(nic->netdev, "%s: skb 0x%p, len=%d\n", netdev->name,
-			    skb, skb->len);
-		print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 1,
-			       skb->data, skb->len, true);
-	}
-
-	/* If error packet, drop it here */
-	if (err) {
-		dev_kfree_skb_any(skb);
-		return;
-	}
-
-	nicvf_set_rx_frame_cnt(nic, skb);
-
-	nicvf_set_rxhash(netdev, cqe_rx, skb);
-
-	skb_record_rx_queue(skb, rq_idx);
-	if (netdev->hw_features & NETIF_F_RXCSUM) {
-		/* HW by default verifies TCP/UDP/SCTP checksums */
-		skb->ip_summed = CHECKSUM_UNNECESSARY;
-	} else {
-		skb_checksum_none_assert(skb);
-	}
-
-	skb->protocol = eth_type_trans(skb, netdev);
-
-	/* Check for stripped VLAN */
-	if (cqe_rx->vlan_found && cqe_rx->vlan_stripped)
-		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
-				       ntohs(cqe_rx->vlan_tci));
-
-	if (napi && (netdev->features & NETIF_F_GRO))
-		napi_gro_receive(napi, skb);
-	else
-		netif_receive_skb(skb);
-}
-
-static int nicvf_cq_intr_handler(struct net_device *netdev, u8 cq_idx,
-				 struct napi_struct *napi, int budget)
-{
-	int processed_cqe, work_done = 0, tx_done = 0;
-	int cqe_count, cqe_head;
-	struct nicvf *nic = netdev_priv(netdev);
-	struct queue_set *qs = nic->qs;
-	struct cmp_queue *cq = &qs->cq[cq_idx];
-	struct cqe_rx_t *cq_desc;
-	struct netdev_queue *txq;
-
-	spin_lock_bh(&cq->lock);
-loop:
-	processed_cqe = 0;
-	/* Get no of valid CQ entries to process */
-	cqe_count = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS, cq_idx);
-	cqe_count &= CQ_CQE_COUNT;
-	if (!cqe_count)
-		goto done;
-
-	/* Get head of the valid CQ entries */
-	cqe_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD, cq_idx) >> 9;
-	cqe_head &= 0xFFFF;
-
-	netdev_dbg(nic->netdev, "%s CQ%d cqe_count %d cqe_head %d\n",
-		   __func__, cq_idx, cqe_count, cqe_head);
-	while (processed_cqe < cqe_count) {
-		/* Get the CQ descriptor */
-		cq_desc = (struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head);
-		cqe_head++;
-		cqe_head &= (cq->dmem.q_len - 1);
-		/* Initiate prefetch for next descriptor */
-		prefetch((struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head));
-
-		if ((work_done >= budget) && napi &&
-		    (cq_desc->cqe_type != CQE_TYPE_SEND)) {
-			break;
-		}
-
-		netdev_dbg(nic->netdev, "CQ%d cq_desc->cqe_type %d\n",
-			   cq_idx, cq_desc->cqe_type);
-		switch (cq_desc->cqe_type) {
-		case CQE_TYPE_RX:
-			nicvf_rcv_pkt_handler(netdev, napi, cq,
-					      cq_desc, CQE_TYPE_RX);
-			work_done++;
-		break;
-		case CQE_TYPE_SEND:
-			nicvf_snd_pkt_handler(netdev, cq,
-					      (void *)cq_desc, CQE_TYPE_SEND);
-			tx_done++;
-		break;
-		case CQE_TYPE_INVALID:
-		case CQE_TYPE_RX_SPLIT:
-		case CQE_TYPE_RX_TCP:
-		case CQE_TYPE_SEND_PTP:
-			/* Ignore for now */
-		break;
-		}
-		processed_cqe++;
-	}
-	netdev_dbg(nic->netdev,
-		   "%s CQ%d processed_cqe %d work_done %d budget %d\n",
-		   __func__, cq_idx, processed_cqe, work_done, budget);
-
-	/* Ring doorbell to inform H/W to reuse processed CQEs */
-	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_DOOR,
-			      cq_idx, processed_cqe);
-
-	if ((work_done < budget) && napi)
-		goto loop;
-
-done:
-	/* Wakeup TXQ if its stopped earlier due to SQ full */
-	if (tx_done) {
-#ifdef VNIC_MULTI_QSET_SUPPORT
-		netdev = nic->pnicvf->netdev;
-#endif
-		txq = netdev_get_tx_queue(netdev,
-					  nicvf_netdev_qidx(nic, cq_idx));
-#ifdef VNIC_MULTI_QSET_SUPPORT
-		nic = nic->pnicvf;
-#endif
-		if (netif_tx_queue_stopped(txq) && netif_carrier_ok(netdev)) {
-			netif_tx_start_queue(txq);
-			nic->drv_stats.txq_wake++;
-			if (netif_msg_tx_err(nic))
-				netdev_warn(netdev,
-					    "%s: Transmit queue wakeup SQ%d\n",
-					    netdev->name, cq_idx);
-		}
-	}
-
-	spin_unlock_bh(&cq->lock);
-	return work_done;
-}
-
-static int nicvf_poll(struct napi_struct *napi, int budget)
-{
-	u64  cq_head;
-	int  work_done = 0;
-	struct net_device *netdev = napi->dev;
-	struct nicvf *nic = netdev_priv(netdev);
-	struct nicvf_cq_poll *cq;
-
-	cq = container_of(napi, struct nicvf_cq_poll, napi);
-	work_done = nicvf_cq_intr_handler(netdev, cq->cq_idx, napi, budget);
-
-	if (work_done < budget) {
-		/* Slow packet rate, exit polling */
-		napi_complete(napi);
-		/* Re-enable interrupts */
-		cq_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD,
-					       cq->cq_idx);
-		nicvf_clear_intr(nic, NICVF_INTR_CQ, cq->cq_idx);
-		nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_HEAD,
-				      cq->cq_idx, cq_head);
-		nicvf_enable_intr(nic, NICVF_INTR_CQ, cq->cq_idx);
-	}
-	return work_done;
-}
-
-/* Qset error interrupt handler
- *
- * As of now only CQ errors are handled
- */
-static void nicvf_handle_qs_err(unsigned long data)
-{
-	struct nicvf *nic = (struct nicvf *)data;
-	struct queue_set *qs = nic->qs;
-	int qidx;
-	u64 status;
-
-	netif_tx_disable(nic->netdev);
-
-	/* Check if it is CQ err */
-	for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
-		status = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS,
-					      qidx);
-		if (!(status & CQ_ERR_MASK))
-			continue;
-		/* Process already queued CQEs and reconfig CQ */
-		nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
-		nicvf_sq_disable(nic, qidx);
-		nicvf_cq_intr_handler(nic->netdev, qidx, NULL, 0);
-		nicvf_cmp_queue_config(nic, qs, qidx, true);
-		nicvf_sq_free_used_descs(nic->netdev, &qs->sq[qidx], qidx);
-		nicvf_sq_enable(nic, &qs->sq[qidx], qidx);
-
-		nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx);
-	}
-
-	netif_tx_start_all_queues(nic->netdev);
-	/* Re-enable Qset error interrupt */
-	nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0);
-}
-
-static inline void nicvf_dump_intr_status(struct nicvf *nic)
-{
-	if (netif_msg_intr(nic))
-		netdev_info(nic->netdev, "%s: interrupt status 0x%llx\n",
-			    nic->netdev->name, nicvf_reg_read(nic, NIC_VF_INT));
-}
-
-static irqreturn_t nicvf_misc_intr_handler(int irq, void *nicvf_irq)
-{
-	struct nicvf *nic = (struct nicvf *)nicvf_irq;
-	u64 intr;
-
-	nicvf_dump_intr_status(nic);
-
-	intr = nicvf_reg_read(nic, NIC_VF_INT);
-	/* Check for spurious interrupt */
-	if (!(intr & NICVF_INTR_MBOX_MASK))
-		return IRQ_HANDLED;
-
-	nicvf_handle_mbx_intr(nic);
-
-	return IRQ_HANDLED;
-}
-
-static irqreturn_t nicvf_intr_handler(int irq, void *cq_irq)
-{
-	struct nicvf_cq_poll *cq_poll = (struct nicvf_cq_poll *)cq_irq;
-	struct nicvf *nic = cq_poll->nicvf;
-	int qidx = cq_poll->cq_idx;
-
-	nicvf_dump_intr_status(nic);
-
-	/* Disable interrupts */
-	nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
-
-	/* Schedule NAPI */
-	napi_schedule(&cq_poll->napi);
-
-	/* Clear interrupt */
-	nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx);
-
-	return IRQ_HANDLED;
-}
-
-static irqreturn_t nicvf_rbdr_intr_handler(int irq, void *nicvf_irq)
-{
-	struct nicvf *nic = (struct nicvf *)nicvf_irq;
-	u8 qidx;
-
-
-	nicvf_dump_intr_status(nic);
-
-	/* Disable RBDR interrupt and schedule softirq */
-	for (qidx = 0; qidx < nic->qs->rbdr_cnt; qidx++) {
-		if (!nicvf_is_intr_enabled(nic, NICVF_INTR_RBDR, qidx))
-			continue;
-		nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx);
-		tasklet_hi_schedule(&nic->rbdr_task);
-		/* Clear interrupt */
-		nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx);
-	}
-
-	return IRQ_HANDLED;
-}
-
-static irqreturn_t nicvf_qs_err_intr_handler(int irq, void *nicvf_irq)
-{
-	struct nicvf *nic = (struct nicvf *)nicvf_irq;
-
-	nicvf_dump_intr_status(nic);
-
-	/* Disable Qset err interrupt and schedule softirq */
-	nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0);
-	tasklet_hi_schedule(&nic->qs_err_task);
-	nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0);
-
-	return IRQ_HANDLED;
-}
-
-static int nicvf_enable_msix(struct nicvf *nic)
-{
-	int ret, vec;
-
-	nic->num_vec = NIC_VF_MSIX_VECTORS;
-
-	for (vec = 0; vec < nic->num_vec; vec++)
-		nic->msix_entries[vec].entry = vec;
-
-	ret = pci_enable_msix(nic->pdev, nic->msix_entries, nic->num_vec);
-	if (ret) {
-		netdev_err(nic->netdev,
-			   "Req for #%d msix vectors failed\n", nic->num_vec);
-		return 0;
-	}
-	nic->msix_enabled = 1;
-	return 1;
-}
-
-static void nicvf_disable_msix(struct nicvf *nic)
-{
-	if (nic->msix_enabled) {
-		pci_disable_msix(nic->pdev);
-		nic->msix_enabled = 0;
-		nic->num_vec = 0;
-	}
-}
-
-static int nicvf_register_interrupts(struct nicvf *nic)
-{
-	int irq, ret = 0;
-	int vector;
-
-	for_each_cq_irq(irq)
-		sprintf(nic->irq_name[irq], "NICVF%d CQ%d",
-			nic->vf_id, irq);
-
-	for_each_sq_irq(irq)
-		sprintf(nic->irq_name[irq], "NICVF%d SQ%d",
-			nic->vf_id, irq - NICVF_INTR_ID_SQ);
-
-	for_each_rbdr_irq(irq)
-		sprintf(nic->irq_name[irq], "NICVF%d RBDR%d",
-			nic->vf_id, irq - NICVF_INTR_ID_RBDR);
-
-	/* Register CQ interrupts */
-	for (irq = 0; irq < nic->qs->cq_cnt; irq++) {
-		vector = nic->msix_entries[irq].vector;
-		ret = request_irq(vector, nicvf_intr_handler,
-				  0, nic->irq_name[irq], nic->napi[irq]);
-		if (ret)
-			goto err;
-		nic->irq_allocated[irq] = true;
-	}
-
-	/* Register RBDR interrupt */
-	for (irq = NICVF_INTR_ID_RBDR;
-	     irq < (NICVF_INTR_ID_RBDR + nic->qs->rbdr_cnt); irq++) {
-		vector = nic->msix_entries[irq].vector;
-		ret = request_irq(vector, nicvf_rbdr_intr_handler,
-				  0, nic->irq_name[irq], nic);
-		if (ret)
-			goto err;
-		nic->irq_allocated[irq] = true;
-	}
-
-	/* Register QS error interrupt */
-	sprintf(nic->irq_name[NICVF_INTR_ID_QS_ERR],
-		"NICVF%d Qset error", nic->vf_id);
-	irq = NICVF_INTR_ID_QS_ERR;
-	ret = request_irq(nic->msix_entries[irq].vector,
-			  nicvf_qs_err_intr_handler,
-			  0, nic->irq_name[irq], nic);
-	if (!ret)
-		nic->irq_allocated[irq] = true;
-
-err:
-	if (ret)
-		netdev_err(nic->netdev, "request_irq failed, vector %d\n", irq);
-
-	return ret;
-}
-
-static void nicvf_unregister_interrupts(struct nicvf *nic)
-{
-	int irq;
-
-	/* Free registered interrupts */
-	for (irq = 0; irq < nic->num_vec; irq++) {
-		if (!nic->irq_allocated[irq])
-			continue;
-
-		if (irq < NICVF_INTR_ID_SQ)
-			free_irq(nic->msix_entries[irq].vector, nic->napi[irq]);
-		else
-			free_irq(nic->msix_entries[irq].vector, nic);
-
-		nic->irq_allocated[irq] = false;
-	}
-
-	/* Disable MSI-X */
-	nicvf_disable_msix(nic);
-}
-
-/* Initialize MSIX vectors and register MISC interrupt.
- * Send READY message to PF to check if its alive
- */
-static int nicvf_register_misc_interrupt(struct nicvf *nic)
-{
-	int ret = 0;
-	int irq = NICVF_INTR_ID_MISC;
-
-	/* Return if mailbox interrupt is already registered */
-	if (nic->msix_enabled)
-		return 0;
-
-	/* Enable MSI-X */
-	if (!nicvf_enable_msix(nic))
-		return 1;
-
-	sprintf(nic->irq_name[irq], "%s Mbox", "NICVF");
-	/* Register Misc interrupt */
-	ret = request_irq(nic->msix_entries[irq].vector,
-			  nicvf_misc_intr_handler, 0, nic->irq_name[irq], nic);
-
-	if (ret)
-		return ret;
-	nic->irq_allocated[irq] = true;
-
-	/* Enable mailbox interrupt */
-	nicvf_enable_intr(nic, NICVF_INTR_MBOX, 0);
-
-	/* Check if VF is able to communicate with PF */
-	if (!nicvf_check_pf_ready(nic)) {
-		nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
-		nicvf_unregister_interrupts(nic);
-		return 1;
-	}
-
-	return 0;
-}
-
-static netdev_tx_t nicvf_xmit(struct sk_buff *skb, struct net_device *netdev)
-{
-	struct nicvf *nic = netdev_priv(netdev);
-	int qid = skb_get_queue_mapping(skb);
-	struct netdev_queue *txq = netdev_get_tx_queue(netdev, qid);
-
-	/* Check for minimum packet length */
-	if (skb->len <= ETH_HLEN) {
-		dev_kfree_skb(skb);
-		return NETDEV_TX_OK;
-	}
-
-	if (!netif_tx_queue_stopped(txq) && !nicvf_sq_append_skb(nic, skb)) {
-		netif_tx_stop_queue(txq);
-		nic->drv_stats.txq_stop++;
-		if (netif_msg_tx_err(nic))
-			netdev_warn(netdev,
-				    "%s: Transmit ring full, stopping SQ%d\n",
-				    netdev->name, qid);
-		return NETDEV_TX_BUSY;
-	}
-
-	return NETDEV_TX_OK;
-}
-
-static inline void nicvf_free_cq_poll(struct nicvf *nic)
-{
-	struct nicvf_cq_poll *cq_poll = NULL;
-	int qidx;
-
-	for (qidx = 0; qidx < nic->qs->cq_cnt; qidx++) {
-		cq_poll = nic->napi[qidx];
-		if (!cq_poll)
-			continue;
-		nic->napi[qidx] = NULL;
-		kfree(cq_poll);
-	}
-}
-
-int nicvf_stop(struct net_device *netdev)
-{
-	int irq, qidx;
-	struct nicvf *nic = netdev_priv(netdev);
-	struct queue_set *qs = nic->qs;
-	struct nicvf_cq_poll *cq_poll = NULL;
-	union nic_mbx mbx = {};
-
-	mbx.msg.msg = NIC_MBOX_MSG_SHUTDOWN;
-	nicvf_send_msg_to_pf(nic, &mbx);
-
-	netif_carrier_off(netdev);
-	netif_tx_stop_all_queues(nic->netdev);
-
-#ifdef	VNIC_MULTI_QSET_SUPPORT
-	/* Teardown secondary qsets first */
-	if (!nic->sqs_mode) {
-		for (qidx = 0; qidx < nic->sqs_count; qidx++) {
-			if (!nic->snicvf[qidx])
-				continue;
-			nicvf_stop(nic->snicvf[qidx]->netdev);
-			nic->snicvf[qidx] = NULL;
-		}
-	}
-#endif
-
-	/* Disable RBDR & QS error interrupts */
-	for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) {
-		nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx);
-		nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx);
-	}
-	nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0);
-	nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0);
-
-	/* Wait for pending IRQ handlers to finish */
-	for (irq = 0; irq < nic->num_vec; irq++)
-		synchronize_irq(nic->msix_entries[irq].vector);
-
-	tasklet_kill(&nic->rbdr_task);
-	tasklet_kill(&nic->qs_err_task);
-	if (nic->rb_work_scheduled)
-		cancel_delayed_work_sync(&nic->rbdr_work);
-
-	for (qidx = 0; qidx < nic->qs->cq_cnt; qidx++) {
-		cq_poll = nic->napi[qidx];
-		if (!cq_poll)
-			continue;
-		napi_synchronize(&cq_poll->napi);
-		/* CQ intr is enabled while napi_complete,
-		 * so disable it now
-		 */
-		nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
-		nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx);
-		napi_disable(&cq_poll->napi);
-		netif_napi_del(&cq_poll->napi);
-	}
-
-	netif_tx_disable(netdev);
-
-	/* Free resources */
-	nicvf_config_data_transfer(nic, false);
-
-	/* Disable HW Qset */
-	nicvf_qset_config(nic, false);
-
-	/* disable mailbox interrupt */
-	nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
-
-	nicvf_unregister_interrupts(nic);
-
-	nicvf_free_cq_poll(nic);
-
-#ifdef	VNIC_MULTI_QSET_SUPPORT
-	/* Clear multiqset info */
+	nic = device_get_softc(dev);
+	nic->dev = dev;
 	nic->pnicvf = nic;
-	nic->sqs_count = 0;
-#endif
 
-	return 0;
-}
+	NICVF_CORE_LOCK_INIT(nic);
 
-int nicvf_open(struct net_device *netdev)
-{
-	int err, qidx;
-	struct nicvf *nic = netdev_priv(netdev);
-	struct queue_set *qs = nic->qs;
-	struct nicvf_cq_poll *cq_poll = NULL;
-
-	nic->mtu = netdev->mtu;
-
-	netif_carrier_off(netdev);
-
-	err = nicvf_register_misc_interrupt(nic);
-	if (err)
-		return err;
-
-	/* Register NAPI handler for processing CQEs */
-	for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
-		cq_poll = kzalloc(sizeof(*cq_poll), GFP_KERNEL);
-		if (!cq_poll) {
-			err = -ENOMEM;
-			goto napi_del;
-		}
-		cq_poll->cq_idx = qidx;
-		cq_poll->nicvf = nic;
-		netif_napi_add(netdev, &cq_poll->napi, nicvf_poll,
-			       NAPI_POLL_WEIGHT);
-		napi_enable(&cq_poll->napi);
-		nic->napi[qidx] = cq_poll;
+	rid = VNIC_VF_REG_RID;
+	nic->reg_base = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
+	    RF_ACTIVE);
+	if (nic->reg_base == NULL) {
+		device_printf(dev, "Could not allocate registers memory\n");
+		return (ENXIO);
 	}
 
-	/* Check if we got MAC address from PF or else generate a radom MAC */
-	if (is_zero_ether_addr(netdev->dev_addr)) {
-		eth_hw_addr_random(netdev);
-		nicvf_hw_set_mac_addr(nic, netdev);
+	qcount = MAX_CMP_QUEUES_PER_QS;
+	nic->max_queues = qcount;
+
+	err = nicvf_set_qset_resources(nic);
+	if (err != 0)
+		goto err_free_res;
+
+	/* Check if PF is alive and get MAC address for this VF */
+	err = nicvf_allocate_misc_interrupt(nic);
+	if (err != 0)
+		goto err_free_res;
+
+	NICVF_CORE_LOCK(nic);
+	err = nicvf_enable_misc_interrupt(nic);
+	NICVF_CORE_UNLOCK(nic);
+	if (err != 0)
+		goto err_release_intr;
+
+	err = nicvf_allocate_net_interrupts(nic);
+	if (err != 0) {
+		device_printf(dev,
+		    "Could not allocate network interface interrupts\n");
+		goto err_free_ifnet;
 	}
 
-	/* Init tasklet for handling Qset err interrupt */
-	tasklet_init(&nic->qs_err_task, nicvf_handle_qs_err,
-		     (unsigned long)nic);
-
-	/* Init RBDR tasklet which will refill RBDR */
-	tasklet_init(&nic->rbdr_task, nicvf_rbdr_task,
-		     (unsigned long)nic);
-	INIT_DELAYED_WORK(&nic->rbdr_work, nicvf_rbdr_work);
+	/* If no MAC address was obtained we generate random one */
+	if (memcmp(nic->hwaddr, zeromac, ETHER_ADDR_LEN) == 0) {
+		nicvf_hw_addr_random(hwaddr);
+		memcpy(nic->hwaddr, hwaddr, ETHER_ADDR_LEN);
+		NICVF_CORE_LOCK(nic);
+		nicvf_hw_set_mac_addr(nic, hwaddr);
+		NICVF_CORE_UNLOCK(nic);
+	}
 
 	/* Configure CPI alorithm */
-	nic->cpi_alg = cpi_alg;
-	if (!nic->sqs_mode)
-		nicvf_config_cpi(nic);
+	nic->cpi_alg = CPI_ALG_NONE;
+	NICVF_CORE_LOCK(nic);
+	nicvf_config_cpi(nic);
+	NICVF_CORE_UNLOCK(nic);
 
-#ifdef VNIC_MULTI_QSET_SUPPORT
-	nicvf_request_sqs(nic);
-	if (nic->sqs_mode)
-		nicvf_get_primary_vf_struct(nic);
+	err = nicvf_setup_ifnet(nic);
+	if (err != 0) {
+		device_printf(dev, "Could not set-up ifnet\n");
+		goto err_release_intr;
+	}
+
+	err = nicvf_setup_ifmedia(nic);
+	if (err != 0) {
+		device_printf(dev, "Could not set-up ifmedia\n");
+		goto err_free_ifnet;
+	}
+
+	mtx_init(&nic->stats_mtx, "VNIC stats", NULL, MTX_DEF);
+	callout_init_mtx(&nic->stats_callout, &nic->stats_mtx, 0);
+
+	ether_ifattach(nic->ifp, nic->hwaddr);
+
+	return (0);
+
+err_free_ifnet:
+	if_free(nic->ifp);
+err_release_intr:
+	nicvf_release_all_interrupts(nic);
+err_free_res:
+	bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(nic->reg_base),
+	    nic->reg_base);
+
+	return (err);
+}
+
+static int
+nicvf_detach(device_t dev)
+{
+	struct nicvf *nic;
+
+	nic = device_get_softc(dev);
+
+	NICVF_CORE_LOCK(nic);
+	/* Shut down the port and release ring resources */
+	nicvf_stop_locked(nic);
+	/* Release stats lock */
+	mtx_destroy(&nic->stats_mtx);
+	/* Release interrupts */
+	nicvf_release_all_interrupts(nic);
+	/* Release memory resource */
+	if (nic->reg_base != NULL) {
+		bus_release_resource(dev, SYS_RES_MEMORY,
+		    rman_get_rid(nic->reg_base), nic->reg_base);
+	}
+
+	/* Remove all ifmedia configurations */
+	ifmedia_removeall(&nic->if_media);
+	/* Free this ifnet */
+	if_free(nic->ifp);
+	NICVF_CORE_UNLOCK(nic);
+	/* Finally destroy the lock */
+	NICVF_CORE_LOCK_DESTROY(nic);
+
+	return (0);
+}
+
+static void
+nicvf_hw_addr_random(uint8_t *hwaddr)
+{
+	uint32_t rnd;
+	uint8_t addr[ETHER_ADDR_LEN];
+
+	/*
+	 * Create randomized MAC address.
+	 * Set 'bsd' + random 24 low-order bits.
+	 */
+	rnd = arc4random() & 0x00ffffff;
+	addr[0] = 'b';
+	addr[1] = 's';
+	addr[2] = 'd';
+	addr[3] = rnd >> 16;
+	addr[4] = rnd >> 8;
+	addr[5] = rnd >> 0;
+
+	memcpy(hwaddr, addr, ETHER_ADDR_LEN);
+}
+
+static int
+nicvf_setup_ifnet(struct nicvf *nic)
+{
+	struct ifnet *ifp;
+
+	ifp = if_alloc(IFT_ETHER);
+	if (ifp == NULL) {
+		device_printf(nic->dev, "Could not allocate ifnet structure\n");
+		return (ENOMEM);
+	}
+
+	nic->ifp = ifp;
+
+	if_setsoftc(ifp, nic);
+	if_initname(ifp, device_get_name(nic->dev), device_get_unit(nic->dev));
+	if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX);
+
+	if_settransmitfn(ifp, nicvf_if_transmit);
+	if_setqflushfn(ifp, nicvf_if_qflush);
+	if_setioctlfn(ifp, nicvf_if_ioctl);
+	if_setinitfn(ifp, nicvf_if_init);
+	if_setgetcounterfn(ifp, nicvf_if_getcounter);
+
+	/* Set send queue len to number to default maximum */
+	if_setsendqlen(ifp, IFQ_MAXLEN);
+	if_setsendqready(ifp);
+	if_setmtu(ifp, ETHERMTU);
+
+	if_setcapabilities(ifp, IFCAP_VLAN_MTU);
+#ifdef DEVICE_POLLING
+#error "DEVICE_POLLING not supported in VNIC driver yet"
+	if_setcapabilitiesbit(ifp, IFCAP_POLLING, 0);
+#endif
+	if_setcapenable(ifp, if_getcapabilities(ifp));
+	if_setmtu(ifp, ETHERMTU);
+
+	return (0);
+}
+
+static int
+nicvf_setup_ifmedia(struct nicvf *nic)
+{
+
+	ifmedia_init(&nic->if_media, IFM_IMASK, nicvf_media_change,
+	    nicvf_media_status);
+
+	/*
+	 * Advertise availability of all possible connection types,
+	 * even though not all are possible at the same time.
+	 */
+
+	ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_10_T | IFM_FDX),
+	    0, NULL);
+	ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_100_TX | IFM_FDX),
+	    0, NULL);
+	ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_1000_T | IFM_FDX),
+	    0, NULL);
+	ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_10G_SR | IFM_FDX),
+	    0, NULL);
+	ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_40G_CR4 | IFM_FDX),
+	    0, NULL);
+	ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_AUTO | IFM_FDX),
+	    0, NULL);
+
+	ifmedia_set(&nic->if_media, (IFM_ETHER | IFM_AUTO | IFM_FDX));
+
+	return (0);
+}
+
+static int
+nicvf_if_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
+{
+	struct nicvf *nic;
+	struct ifreq *ifr;
+	uint32_t flags;
+	int mask, err;
+#if defined(INET) || defined(INET6)
+	struct ifaddr *ifa;
+	boolean_t avoid_reset = FALSE;
 #endif
 
-#ifdef	VNIC_RSS_SUPPORT
-	/* Configure receive side scaling */
-	if (!nic->sqs_mode)
-		nicvf_rss_init(nic);
+	nic = if_getsoftc(ifp);
+	ifr = (struct ifreq *)data;
+#if defined(INET) || defined(INET6)
+	ifa = (struct ifaddr *)data;
+#endif
+	err = 0;
+	switch (cmd) {
+	case SIOCSIFADDR:
+#ifdef INET
+		if (ifa->ifa_addr->sa_family == AF_INET)
+			avoid_reset = TRUE;
+#endif
+#ifdef INET6
+		if (ifa->ifa_addr->sa_family == AF_INET6)
+			avoid_reset = TRUE;
 #endif
 
-	err = nicvf_register_interrupts(nic);
-	if (err)
-		goto cleanup;
+#if defined(INET) || defined(INET6)
+		/* Avoid reinitialization unless it's necessary */
+		if (avoid_reset) {
+			ifp->if_flags |= IFF_UP;
+			if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))
+				nicvf_if_init(nic);
+#ifdef INET
+			if (!(if_getflags(ifp) & IFF_NOARP))
+				arp_ifinit(ifp, ifa);
+#endif
+
+			return (0);
+		}
+#endif
+		err = ether_ioctl(ifp, cmd, data);
+		break;
+	case SIOCSIFMTU:
+		/*
+		 * ARM64TODO: Needs to be implemented.
+		 * Currently ETHERMTU is set by default.
+		 */
+		err = ether_ioctl(ifp, cmd, data);
+		break;
+	case SIOCSIFFLAGS:
+		NICVF_CORE_LOCK(nic);
+		if (if_getflags(ifp) & IFF_UP) {
+			if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
+				flags = ifp->if_flags ^ nic->if_flags;
+				if ((nic->if_flags & ifp->if_flags) &
+				    IFF_PROMISC) {
+					/* Change promiscous mode */
+#if 0
+					/* ARM64TODO */
+					nicvf_set_promiscous(nic);
+#endif
+				}
+
+				if ((nic->if_flags ^ ifp->if_flags) &
+				    IFF_ALLMULTI) {
+					/* Change multicasting settings */
+#if 0
+					/* ARM64TODO */
+					nicvf_set_multicast(nic);
+#endif
+				}
+			} else {
+				nicvf_if_init_locked(nic);
+			}
+		} else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
+			nicvf_stop_locked(nic);
+
+		nic->if_flags = ifp->if_flags;
+		NICVF_CORE_UNLOCK(nic);
+		break;
+
+	case SIOCADDMULTI:
+	case SIOCDELMULTI:
+		if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
+#if 0
+			NICVF_CORE_LOCK(nic);
+			/* ARM64TODO */
+			nicvf_set_multicast(nic);
+			NICVF_CORE_UNLOCK(nic);
+#endif
+		}
+		break;
+
+	case SIOCSIFMEDIA:
+	case SIOCGIFMEDIA:
+		err = ifmedia_ioctl(ifp, ifr, &nic->if_media, cmd);
+		break;
+
+	case SIOCSIFCAP:
+		mask = ifp->if_capenable ^ ifr->ifr_reqcap;
+		if (mask & IFCAP_VLAN_MTU) {
+			/* No work to do except acknowledge the change took. */
+			ifp->if_capenable ^= IFCAP_VLAN_MTU;
+		}
+		break;
+
+	default:
+		err = ether_ioctl(ifp, cmd, data);
+		break;
+	}
+
+	return (err);
+}
+
+static void
+nicvf_if_init_locked(struct nicvf *nic)
+{
+	struct queue_set *qs = nic->qs;
+	struct ifnet *ifp;
+	int qidx;
+	int err;
+	caddr_t if_addr;
+
+	NICVF_CORE_LOCK_ASSERT(nic);
+	ifp = nic->ifp;
+
+	if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0)
+		nicvf_stop_locked(nic);
+
+	err = nicvf_enable_misc_interrupt(nic);
+	if (err != 0) {
+		if_printf(ifp, "Could not reenable Mbox interrupt\n");
+		return;
+	}
+
+	/* Get the latest MAC address */
+	if_addr = if_getlladdr(ifp);
+	/* Update MAC address if changed */
+	if (memcmp(nic->hwaddr, if_addr, ETHER_ADDR_LEN) != 0) {
+		memcpy(nic->hwaddr, if_addr, ETHER_ADDR_LEN);
+		nicvf_hw_set_mac_addr(nic, if_addr);
+	}
 
 	/* Initialize the queues */
 	err = nicvf_init_resources(nic);
-	if (err)
-		goto cleanup;
+	if (err != 0)
+		goto error;
 
 	/* Make sure queue initialization is written */
 	wmb();
 
-	nicvf_reg_write(nic, NIC_VF_INT, -1);
+	nicvf_reg_write(nic, NIC_VF_INT, ~0UL);
 	/* Enable Qset err interrupt */
 	nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0);
 
@@ -1249,115 +562,819 @@ int nicvf_open(struct net_device *netdev)
 	nic->drv_stats.txq_stop = 0;
 	nic->drv_stats.txq_wake = 0;
 
-	netif_carrier_on(netdev);
-	netif_tx_start_all_queues(netdev);
+	/* Activate network interface */
+	if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
 
-	return 0;
-cleanup:
-	nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
-	nicvf_unregister_interrupts(nic);
-	tasklet_kill(&nic->qs_err_task);
-	tasklet_kill(&nic->rbdr_task);
-napi_del:
-	for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
-		cq_poll = nic->napi[qidx];
-		if (!cq_poll)
-			continue;
-		napi_disable(&cq_poll->napi);
-		netif_napi_del(&cq_poll->napi);
+	/* Schedule callout to update stats */
+	callout_reset(&nic->stats_callout, hz, nicvf_tick_stats, nic);
+
+	return;
+
+error:
+	/* Something went very wrong. Disable this ifnet for good */
+	if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
+}
+
+static void
+nicvf_if_init(void *if_softc)
+{
+	struct nicvf *nic = if_softc;
+
+	NICVF_CORE_LOCK(nic);
+	nicvf_if_init_locked(nic);
+	NICVF_CORE_UNLOCK(nic);
+}
+
+static int
+nicvf_if_transmit(struct ifnet *ifp, struct mbuf *mbuf)
+{
+	struct nicvf *nic = if_getsoftc(ifp);
+	struct queue_set *qs = nic->qs;
+	struct snd_queue *sq;
+	int qidx;
+	int err = 0;
+
+
+	if (__predict_false(qs == NULL)) {
+		panic("%s: missing queue set for %s", __func__,
+		    device_get_nameunit(nic->dev));
 	}
-	nicvf_free_cq_poll(nic);
-	return err;
+
+	/* Select queue */
+	if (M_HASHTYPE_GET(mbuf) != M_HASHTYPE_NONE)
+		qidx = mbuf->m_pkthdr.flowid % qs->sq_cnt;
+	else
+		qidx = curcpu % qs->sq_cnt;
+
+	sq = &qs->sq[qidx];
+
+	if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
+	    IFF_DRV_RUNNING) {
+		if (mbuf != NULL)
+			err = drbr_enqueue(ifp, sq->br, mbuf);
+		return (err);
+	}
+
+	if (mbuf != NULL) {
+		err = drbr_enqueue(ifp, sq->br, mbuf);
+		if (err != 0)
+			return (err);
+	}
+
+	taskqueue_enqueue(sq->snd_taskq, &sq->snd_task);
+
+	return (0);
 }
 
-static int nicvf_update_hw_max_frs(struct nicvf *nic, int mtu)
+static void
+nicvf_if_qflush(struct ifnet *ifp)
+{
+	struct nicvf *nic;
+	struct queue_set *qs;
+	struct snd_queue *sq;
+	struct mbuf *mbuf;
+	size_t idx;
+
+	nic = if_getsoftc(ifp);
+	qs = nic->qs;
+
+	for (idx = 0; idx < qs->sq_cnt; idx++) {
+		sq = &qs->sq[idx];
+		NICVF_TX_LOCK(sq);
+		while ((mbuf = buf_ring_dequeue_sc(sq->br)) != NULL)
+			m_freem(mbuf);
+		NICVF_TX_UNLOCK(sq);
+	}
+	if_qflush(ifp);
+}
+
+static uint64_t
+nicvf_if_getcounter(struct ifnet *ifp, ift_counter cnt)
+{
+	struct nicvf *nic;
+	struct nicvf_hw_stats *hw_stats;
+	struct nicvf_drv_stats *drv_stats;
+
+	nic = if_getsoftc(ifp);
+	hw_stats = &nic->hw_stats;
+	drv_stats = &nic->drv_stats;
+
+	switch (cnt) {
+	case IFCOUNTER_IPACKETS:
+		return (drv_stats->rx_frames_ok);
+	case IFCOUNTER_OPACKETS:
+		return (drv_stats->tx_frames_ok);
+	case IFCOUNTER_IBYTES:
+		return (hw_stats->rx_bytes);
+	case IFCOUNTER_OBYTES:
+		return (hw_stats->tx_bytes_ok);
+	case IFCOUNTER_IMCASTS:
+		return (hw_stats->rx_mcast_frames);
+	case IFCOUNTER_COLLISIONS:
+		return (0);
+	case IFCOUNTER_IQDROPS:
+		return (drv_stats->rx_drops);
+	case IFCOUNTER_OQDROPS:
+		return (drv_stats->tx_drops);
+	default:
+		return (if_get_counter_default(ifp, cnt));
+	}
+
+}
+
+static void
+nicvf_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
+{
+	struct nicvf *nic = if_getsoftc(ifp);
+
+	NICVF_CORE_LOCK(nic);
+
+	ifmr->ifm_status = IFM_AVALID;
+	ifmr->ifm_active = IFM_ETHER;
+
+	if (nic->link_up) {
+		/* Device attached to working network */
+		ifmr->ifm_status |= IFM_ACTIVE;
+	}
+
+	switch (nic->speed) {
+	case SPEED_10:
+		ifmr->ifm_active |= IFM_10_T;
+		break;
+	case SPEED_100:
+		ifmr->ifm_active |= IFM_100_TX;
+		break;
+	case SPEED_1000:
+		ifmr->ifm_active |= IFM_1000_T;
+		break;
+	case SPEED_10000:
+		ifmr->ifm_active |= IFM_10G_SR;
+		break;
+	case SPEED_40000:
+		ifmr->ifm_active |= IFM_40G_CR4;
+		break;
+	default:
+		ifmr->ifm_active |= IFM_AUTO;
+		break;
+	}
+
+	if (nic->duplex)
+		ifmr->ifm_active |= IFM_FDX;
+	else
+		ifmr->ifm_active |= IFM_HDX;
+
+	NICVF_CORE_UNLOCK(nic);
+}
+
+static int
+nicvf_media_change(struct ifnet *ifp __unused)
+{
+
+	return (0);
+}
+
+/* Register read/write APIs */
+void
+nicvf_reg_write(struct nicvf *nic, bus_space_handle_t offset, uint64_t val)
+{
+
+	bus_write_8(nic->reg_base, offset, val);
+}
+
+uint64_t
+nicvf_reg_read(struct nicvf *nic, uint64_t offset)
+{
+
+	return (bus_read_8(nic->reg_base, offset));
+}
+
+void
+nicvf_queue_reg_write(struct nicvf *nic, bus_space_handle_t offset,
+    uint64_t qidx, uint64_t val)
+{
+
+	bus_write_8(nic->reg_base, offset + (qidx << NIC_Q_NUM_SHIFT), val);
+}
+
+uint64_t
+nicvf_queue_reg_read(struct nicvf *nic, bus_space_handle_t offset,
+    uint64_t qidx)
+{
+
+	return (bus_read_8(nic->reg_base, offset + (qidx << NIC_Q_NUM_SHIFT)));
+}
+
+/* VF -> PF mailbox communication */
+static void
+nicvf_write_to_mbx(struct nicvf *nic, union nic_mbx *mbx)
+{
+	uint64_t *msg = (uint64_t *)mbx;
+
+	nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 0, msg[0]);
+	nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 8, msg[1]);
+}
+
+int
+nicvf_send_msg_to_pf(struct nicvf *nic, union nic_mbx *mbx)
+{
+	int timeout = NIC_MBOX_MSG_TIMEOUT * 10;
+	int sleep = 2;
+
+	NICVF_CORE_LOCK_ASSERT(nic);
+
+	nic->pf_acked = FALSE;
+	nic->pf_nacked = FALSE;
+
+	nicvf_write_to_mbx(nic, mbx);
+
+	/* Wait for previous message to be acked, timeout 2sec */
+	while (!nic->pf_acked) {
+		if (nic->pf_nacked)
+			return (EINVAL);
+
+		DELAY(sleep * 1000);
+
+		if (nic->pf_acked)
+			break;
+		timeout -= sleep;
+		if (!timeout) {
+			device_printf(nic->dev,
+				   "PF didn't ack to mbox msg %d from VF%d\n",
+				   (mbx->msg.msg & 0xFF), nic->vf_id);
+
+			return (EBUSY);
+		}
+	}
+	return (0);
+}
+
+/*
+ * Checks if VF is able to comminicate with PF
+ * and also gets the VNIC number this VF is associated to.
+ */
+static int
+nicvf_check_pf_ready(struct nicvf *nic)
 {
 	union nic_mbx mbx = {};
 
-	mbx.frs.msg = NIC_MBOX_MSG_SET_MAX_FRS;
-	mbx.frs.max_frs = mtu;
-	mbx.frs.vf_id = nic->vf_id;
+	mbx.msg.msg = NIC_MBOX_MSG_READY;
+	if (nicvf_send_msg_to_pf(nic, &mbx)) {
+		device_printf(nic->dev,
+			   "PF didn't respond to READY msg\n");
+		return 0;
+	}
 
-	return nicvf_send_msg_to_pf(nic, &mbx);
+	return 1;
 }
 
-static int nicvf_change_mtu(struct net_device *netdev, int new_mtu)
+static void
+nicvf_read_bgx_stats(struct nicvf *nic, struct bgx_stats_msg *bgx)
 {
-	struct nicvf *nic = netdev_priv(netdev);
 
-	if (new_mtu > NIC_HW_MAX_FRS)
-		return -EINVAL;
-
-	if (new_mtu < NIC_HW_MIN_FRS)
-		return -EINVAL;
-
-	if (nicvf_update_hw_max_frs(nic, new_mtu))
-		return -EINVAL;
-	netdev->mtu = new_mtu;
-	nic->mtu = new_mtu;
-
-	return 0;
+	if (bgx->rx)
+		nic->bgx_stats.rx_stats[bgx->idx] = bgx->stats;
+	else
+		nic->bgx_stats.tx_stats[bgx->idx] = bgx->stats;
 }
 
-static int nicvf_set_mac_address(struct net_device *netdev, void *p)
+static void
+nicvf_handle_mbx_intr(struct nicvf *nic)
 {
-	struct sockaddr *addr = p;
-	struct nicvf *nic = netdev_priv(netdev);
+	union nic_mbx mbx = {};
+	uint64_t *mbx_data;
+	uint64_t mbx_addr;
+	int i;
 
-	if (!is_valid_ether_addr(addr->sa_data))
-		return -EADDRNOTAVAIL;
+	mbx_addr = NIC_VF_PF_MAILBOX_0_1;
+	mbx_data = (uint64_t *)&mbx;
 
-	memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
+	for (i = 0; i < NIC_PF_VF_MAILBOX_SIZE; i++) {
+		*mbx_data = nicvf_reg_read(nic, mbx_addr);
+		mbx_data++;
+		mbx_addr += sizeof(uint64_t);
+	}
 
-	if (nic->msix_enabled)
-		if (nicvf_hw_set_mac_addr(nic, netdev))
-			return -EBUSY;
-
-	return 0;
+	switch (mbx.msg.msg) {
+	case NIC_MBOX_MSG_READY:
+		nic->pf_acked = TRUE;
+		nic->vf_id = mbx.nic_cfg.vf_id & 0x7F;
+		nic->tns_mode = mbx.nic_cfg.tns_mode & 0x7F;
+		nic->node = mbx.nic_cfg.node_id;
+		memcpy(nic->hwaddr, mbx.nic_cfg.mac_addr, ETHER_ADDR_LEN);
+		nic->loopback_supported = mbx.nic_cfg.loopback_supported;
+		nic->link_up = FALSE;
+		nic->duplex = 0;
+		nic->speed = 0;
+		break;
+	case NIC_MBOX_MSG_ACK:
+		nic->pf_acked = TRUE;
+		break;
+	case NIC_MBOX_MSG_NACK:
+		nic->pf_nacked = TRUE;
+		break;
+	case NIC_MBOX_MSG_BGX_STATS:
+		nicvf_read_bgx_stats(nic, &mbx.bgx_stats);
+		nic->pf_acked = TRUE;
+		break;
+	case NIC_MBOX_MSG_BGX_LINK_CHANGE:
+		nic->pf_acked = TRUE;
+		nic->link_up = mbx.link_status.link_up;
+		nic->duplex = mbx.link_status.duplex;
+		nic->speed = mbx.link_status.speed;
+		if (nic->link_up) {
+			if_setbaudrate(nic->ifp, nic->speed * 1000000);
+			if_link_state_change(nic->ifp, LINK_STATE_UP);
+		} else {
+			if_setbaudrate(nic->ifp, 0);
+			if_link_state_change(nic->ifp, LINK_STATE_DOWN);
+		}
+		break;
+	default:
+		device_printf(nic->dev,
+			   "Invalid message from PF, msg 0x%x\n", mbx.msg.msg);
+		break;
+	}
+	nicvf_clear_intr(nic, NICVF_INTR_MBOX, 0);
 }
 
-void nicvf_update_lmac_stats(struct nicvf *nic)
+static int
+nicvf_hw_set_mac_addr(struct nicvf *nic, uint8_t *hwaddr)
 {
-	int stat = 0;
 	union nic_mbx mbx = {};
 
-	if (!netif_running(nic->netdev))
+	mbx.mac.msg = NIC_MBOX_MSG_SET_MAC;
+	mbx.mac.vf_id = nic->vf_id;
+	memcpy(mbx.mac.mac_addr, hwaddr, ETHER_ADDR_LEN);
+
+	return (nicvf_send_msg_to_pf(nic, &mbx));
+}
+
+static void
+nicvf_config_cpi(struct nicvf *nic)
+{
+	union nic_mbx mbx = {};
+
+	mbx.cpi_cfg.msg = NIC_MBOX_MSG_CPI_CFG;
+	mbx.cpi_cfg.vf_id = nic->vf_id;
+	mbx.cpi_cfg.cpi_alg = nic->cpi_alg;
+	mbx.cpi_cfg.rq_cnt = nic->qs->rq_cnt;
+
+	nicvf_send_msg_to_pf(nic, &mbx);
+}
+
+static int
+nicvf_init_resources(struct nicvf *nic)
+{
+	int err;
+	union nic_mbx mbx = {};
+
+	mbx.msg.msg = NIC_MBOX_MSG_CFG_DONE;
+
+	/* Enable Qset */
+	nicvf_qset_config(nic, TRUE);
+
+	/* Initialize queues and HW for data transfer */
+	err = nicvf_config_data_transfer(nic, TRUE);
+	if (err) {
+		device_printf(nic->dev,
+		    "Failed to alloc/config VF's QSet resources\n");
+		return (err);
+	}
+
+	/* Send VF config done msg to PF */
+	nicvf_write_to_mbx(nic, &mbx);
+
+	return (0);
+}
+
+static void
+nicvf_misc_intr_handler(void *arg)
+{
+	struct nicvf *nic = (struct nicvf *)arg;
+	uint64_t intr;
+
+	intr = nicvf_reg_read(nic, NIC_VF_INT);
+	/* Check for spurious interrupt */
+	if (!(intr & NICVF_INTR_MBOX_MASK))
 		return;
 
-	mbx.bgx_stats.msg = NIC_MBOX_MSG_BGX_STATS;
-	mbx.bgx_stats.vf_id = nic->vf_id;
-	/* Rx stats */
-	mbx.bgx_stats.rx = 1;
-	while (stat < BGX_RX_STATS_COUNT) {
-		mbx.bgx_stats.idx = stat;
-		if (nicvf_send_msg_to_pf(nic, &mbx))
-			return;
-		stat++;
-	}
+	nicvf_handle_mbx_intr(nic);
+}
 
-	stat = 0;
+static int
+nicvf_intr_handler(void *arg)
+{
+	struct nicvf *nic;
+	struct cmp_queue *cq;
+	int qidx;
 
-	/* Tx stats */
-	mbx.bgx_stats.rx = 0;
-	while (stat < BGX_TX_STATS_COUNT) {
-		mbx.bgx_stats.idx = stat;
-		if (nicvf_send_msg_to_pf(nic, &mbx))
-			return;
-		stat++;
+	cq = (struct cmp_queue *)arg;
+	nic = cq->nic;
+	qidx = cq->idx;
+
+	/* Disable interrupts */
+	nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
+
+	taskqueue_enqueue(cq->cmp_taskq, &cq->cmp_task);
+
+	/* Clear interrupt */
+	nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx);
+
+	return (FILTER_HANDLED);
+}
+
+static void
+nicvf_rbdr_intr_handler(void *arg)
+{
+	struct nicvf *nic;
+	struct queue_set *qs;
+	struct rbdr *rbdr;
+	int qidx;
+
+	nic = (struct nicvf *)arg;
+
+	/* Disable RBDR interrupt and schedule softirq */
+	for (qidx = 0; qidx < nic->qs->rbdr_cnt; qidx++) {
+		if (!nicvf_is_intr_enabled(nic, NICVF_INTR_RBDR, qidx))
+			continue;
+		nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx);
+
+		qs = nic->qs;
+		rbdr = &qs->rbdr[qidx];
+		taskqueue_enqueue(rbdr->rbdr_taskq, &rbdr->rbdr_task_nowait);
+		/* Clear interrupt */
+		nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx);
 	}
 }
 
-void nicvf_update_stats(struct nicvf *nic)
+static void
+nicvf_qs_err_intr_handler(void *arg)
+{
+	struct nicvf *nic = (struct nicvf *)arg;
+	struct queue_set *qs = nic->qs;
+
+	/* Disable Qset err interrupt and schedule softirq */
+	nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0);
+	taskqueue_enqueue(qs->qs_err_taskq, &qs->qs_err_task);
+	nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0);
+
+}
+
+static int
+nicvf_enable_msix(struct nicvf *nic)
+{
+	struct pci_devinfo *dinfo;
+	int rid, count;
+	int ret;
+
+	dinfo = device_get_ivars(nic->dev);
+	rid = dinfo->cfg.msix.msix_table_bar;
+	nic->msix_table_res =
+	    bus_alloc_resource_any(nic->dev, SYS_RES_MEMORY, &rid, RF_ACTIVE);
+	if (nic->msix_table_res == NULL) {
+		device_printf(nic->dev,
+		    "Could not allocate memory for MSI-X table\n");
+		return (ENXIO);
+	}
+
+	count = nic->num_vec = NIC_VF_MSIX_VECTORS;
+
+	ret = pci_alloc_msix(nic->dev, &count);
+	if ((ret != 0) || (count != nic->num_vec)) {
+		device_printf(nic->dev,
+		    "Request for #%d msix vectors failed, error: %d\n",
+		    nic->num_vec, ret);
+		return (ret);
+	}
+
+	nic->msix_enabled = 1;
+	return (0);
+}
+
+static void
+nicvf_disable_msix(struct nicvf *nic)
+{
+
+	if (nic->msix_enabled) {
+		pci_release_msi(nic->dev);
+		nic->msix_enabled = 0;
+		nic->num_vec = 0;
+	}
+}
+
+static void
+nicvf_release_all_interrupts(struct nicvf *nic)
+{
+	struct resource *res;
+	int irq;
+	int err;
+
+	/* Free registered interrupts */
+	for (irq = 0; irq < nic->num_vec; irq++) {
+		res = nic->msix_entries[irq].irq_res;
+		if (res == NULL)
+			continue;
+		/* Teardown interrupt first */
+		if (nic->msix_entries[irq].handle != NULL) {
+			err = bus_teardown_intr(nic->dev,
+			    nic->msix_entries[irq].irq_res,
+			    nic->msix_entries[irq].handle);
+			KASSERT(err == 0,
+			    ("ERROR: Unable to teardown interrupt %d", irq));
+			nic->msix_entries[irq].handle = NULL;
+		}
+
+		bus_release_resource(nic->dev, SYS_RES_IRQ,
+			    rman_get_rid(res), nic->msix_entries[irq].irq_res);
+		nic->msix_entries[irq].irq_res = NULL;
+	}
+	/* Disable MSI-X */
+	nicvf_disable_msix(nic);
+}
+
+/*
+ * Initialize MSIX vectors and register MISC interrupt.
+ * Send READY message to PF to check if its alive
+ */
+static int
+nicvf_allocate_misc_interrupt(struct nicvf *nic)
+{
+	struct resource *res;
+	int irq, rid;
+	int ret = 0;
+
+	/* Return if mailbox interrupt is already registered */
+	if (nic->msix_enabled)
+		return (0);
+
+	/* Enable MSI-X */
+	if (nicvf_enable_msix(nic) != 0)
+		return (ENXIO);
+
+	irq = NICVF_INTR_ID_MISC;
+	rid = irq + 1;
+	nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev,
+	    SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE));
+	if (nic->msix_entries[irq].irq_res == NULL) {
+		device_printf(nic->dev,
+		    "Could not allocate Mbox interrupt for VF%d\n",
+		    device_get_unit(nic->dev));
+		return (ENXIO);
+	}
+
+	ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res,
+	    (INTR_MPSAFE | INTR_TYPE_MISC), NULL, nicvf_misc_intr_handler, nic,
+	    &nic->msix_entries[irq].handle);
+	if (ret != 0) {
+		res = nic->msix_entries[irq].irq_res;
+		bus_release_resource(nic->dev, SYS_RES_IRQ,
+			    rman_get_rid(res), res);
+		nic->msix_entries[irq].irq_res = NULL;
+		return (ret);
+	}
+
+	return (0);
+}
+
+static int
+nicvf_enable_misc_interrupt(struct nicvf *nic)
+{
+
+	/* Enable mailbox interrupt */
+	nicvf_enable_intr(nic, NICVF_INTR_MBOX, 0);
+
+	/* Check if VF is able to communicate with PF */
+	if (!nicvf_check_pf_ready(nic)) {
+		nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
+		return (ENXIO);
+	}
+
+	return (0);
+}
+
+static void
+nicvf_release_net_interrupts(struct nicvf *nic)
+{
+	struct resource *res;
+	int irq;
+	int err;
+
+	for_each_cq_irq(irq) {
+		res = nic->msix_entries[irq].irq_res;
+		if (res == NULL)
+			continue;
+		/* Teardown active interrupts first */
+		if (nic->msix_entries[irq].handle != NULL) {
+			err = bus_teardown_intr(nic->dev,
+			    nic->msix_entries[irq].irq_res,
+			    nic->msix_entries[irq].handle);
+			KASSERT(err == 0,
+			    ("ERROR: Unable to teardown CQ interrupt %d",
+			    (irq - NICVF_INTR_ID_CQ)));
+			if (err != 0)
+				continue;
+		}
+
+		/* Release resource */
+		bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res),
+		    res);
+		nic->msix_entries[irq].irq_res = NULL;
+	}
+
+	for_each_rbdr_irq(irq) {
+		res = nic->msix_entries[irq].irq_res;
+		if (res == NULL)
+			continue;
+		/* Teardown active interrupts first */
+		if (nic->msix_entries[irq].handle != NULL) {
+			err = bus_teardown_intr(nic->dev,
+			    nic->msix_entries[irq].irq_res,
+			    nic->msix_entries[irq].handle);
+			KASSERT(err == 0,
+			    ("ERROR: Unable to teardown RDBR interrupt %d",
+			    (irq - NICVF_INTR_ID_RBDR)));
+			if (err != 0)
+				continue;
+		}
+
+		/* Release resource */
+		bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res),
+		    res);
+		nic->msix_entries[irq].irq_res = NULL;
+	}
+
+	irq = NICVF_INTR_ID_QS_ERR;
+	res = nic->msix_entries[irq].irq_res;
+	if (res != NULL) {
+		/* Teardown active interrupts first */
+		if (nic->msix_entries[irq].handle != NULL) {
+			err = bus_teardown_intr(nic->dev,
+			    nic->msix_entries[irq].irq_res,
+			    nic->msix_entries[irq].handle);
+			KASSERT(err == 0,
+			    ("ERROR: Unable to teardown QS Error interrupt %d",
+			    irq));
+			if (err != 0)
+				return;
+		}
+
+		/* Release resource */
+		bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res),
+		    res);
+		nic->msix_entries[irq].irq_res = NULL;
+	}
+}
+
+static int
+nicvf_allocate_net_interrupts(struct nicvf *nic)
+{
+	int irq, rid;
+	int qidx;
+	int ret = 0;
+
+	/* MSI-X must be configured by now */
+	if (!nic->msix_enabled) {
+		device_printf(nic->dev, "Cannot alloacte queue interrups. "
+		    "MSI-X interrupts disabled.\n");
+		return (ENXIO);
+	}
+
+	/* Register CQ interrupts */
+	for_each_cq_irq(irq) {
+		if (irq >= (NICVF_INTR_ID_CQ + nic->qs->cq_cnt))
+			break;
+
+		qidx = irq - NICVF_INTR_ID_CQ;
+		rid = irq + 1;
+		nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev,
+		    SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE));
+		if (nic->msix_entries[irq].irq_res == NULL) {
+			device_printf(nic->dev,
+			    "Could not allocate CQ interrupt %d for VF%d\n",
+			    (irq - NICVF_INTR_ID_CQ), device_get_unit(nic->dev));
+			ret = ENXIO;
+			goto error;
+		}
+		ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res,
+		    (INTR_MPSAFE | INTR_TYPE_NET), nicvf_intr_handler,
+		    NULL, &nic->qs->cq[qidx], &nic->msix_entries[irq].handle);
+		if (ret != 0) {
+			device_printf(nic->dev,
+			    "Could not setup CQ interrupt %d for VF%d\n",
+			    (irq - NICVF_INTR_ID_CQ), device_get_unit(nic->dev));
+			goto error;
+		}
+	}
+
+	/* Register RBDR interrupt */
+	for_each_rbdr_irq(irq) {
+		if (irq >= (NICVF_INTR_ID_RBDR + nic->qs->rbdr_cnt))
+			break;
+
+		rid = irq + 1;
+		nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev,
+		    SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE));
+		if (nic->msix_entries[irq].irq_res == NULL) {
+			device_printf(nic->dev,
+			    "Could not allocate RBDR interrupt %d for VF%d\n",
+			    (irq - NICVF_INTR_ID_RBDR),
+			    device_get_unit(nic->dev));
+			ret = ENXIO;
+			goto error;
+		}
+		ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res,
+		    (INTR_MPSAFE | INTR_TYPE_NET), NULL,
+		    nicvf_rbdr_intr_handler, nic,
+		    &nic->msix_entries[irq].handle);
+		if (ret != 0) {
+			device_printf(nic->dev,
+			    "Could not setup RBDR interrupt %d for VF%d\n",
+			    (irq - NICVF_INTR_ID_RBDR),
+			    device_get_unit(nic->dev));
+			goto error;
+		}
+	}
+
+	/* Register QS error interrupt */
+	irq = NICVF_INTR_ID_QS_ERR;
+	rid = irq + 1;
+	nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev,
+	    SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE));
+	if (nic->msix_entries[irq].irq_res == NULL) {
+		device_printf(nic->dev,
+		    "Could not allocate QS Error interrupt for VF%d\n",
+		    device_get_unit(nic->dev));
+		ret = ENXIO;
+		goto error;
+	}
+	ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res,
+	    (INTR_MPSAFE | INTR_TYPE_NET), NULL, nicvf_qs_err_intr_handler,
+	    nic, &nic->msix_entries[irq].handle);
+	if (ret != 0) {
+		device_printf(nic->dev,
+		    "Could not setup QS Error interrupt for VF%d\n",
+		    device_get_unit(nic->dev));
+		goto error;
+	}
+
+	return (0);
+error:
+	nicvf_release_net_interrupts(nic);
+	return (ret);
+}
+
+static int
+nicvf_stop_locked(struct nicvf *nic)
+{
+	struct ifnet *ifp;
+	int qidx;
+	struct queue_set *qs = nic->qs;
+	union nic_mbx mbx = {};
+
+	NICVF_CORE_LOCK_ASSERT(nic);
+	/* Stop callout. Can block here since holding SX lock */
+	callout_drain(&nic->stats_callout);
+
+	ifp = nic->ifp;
+
+	mbx.msg.msg = NIC_MBOX_MSG_SHUTDOWN;
+	nicvf_send_msg_to_pf(nic, &mbx);
+
+	/* Disable RBDR & QS error interrupts */
+	for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) {
+		nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx);
+		nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx);
+	}
+	nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0);
+	nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0);
+
+	/* Deactivate network interface */
+	if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
+
+	/* Free resources */
+	nicvf_config_data_transfer(nic, FALSE);
+
+	/* Disable HW Qset */
+	nicvf_qset_config(nic, FALSE);
+
+	/* disable mailbox interrupt */
+	nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
+
+	return (0);
+}
+
+static void
+nicvf_update_stats(struct nicvf *nic)
 {
 	int qidx;
 	struct nicvf_hw_stats *stats = &nic->hw_stats;
 	struct nicvf_drv_stats *drv_stats = &nic->drv_stats;
 	struct queue_set *qs = nic->qs;
 
-#define GET_RX_STATS(reg) \
-	nicvf_reg_read(nic, NIC_VNIC_RX_STAT_0_13 | (reg << 3))
+#define	GET_RX_STATS(reg) \
+    nicvf_reg_read(nic, NIC_VNIC_RX_STAT_0_13 | ((reg) << 3))
 #define GET_TX_STATS(reg) \
-	nicvf_reg_read(nic, NIC_VNIC_TX_STAT_0_4 | (reg << 3))
+    nicvf_reg_read(nic, NIC_VNIC_TX_STAT_0_4 | ((reg) << 3))
 
 	stats->rx_bytes = GET_RX_STATS(RX_OCTS);
 	stats->rx_ucast_frames = GET_RX_STATS(RX_UCAST);
@@ -1381,10 +1398,8 @@ void nicvf_update_stats(struct nicvf *nic)
 	stats->tx_drops = GET_TX_STATS(TX_DROP);
 
 	drv_stats->tx_frames_ok = stats->tx_ucast_frames_ok +
-				  stats->tx_bcast_frames_ok +
-				  stats->tx_mcast_frames_ok;
-	drv_stats->rx_drops = stats->rx_drop_red +
-			      stats->rx_drop_overrun;
+	    stats->tx_bcast_frames_ok + stats->tx_mcast_frames_ok;
+	drv_stats->rx_drops = stats->rx_drop_red + stats->rx_drop_overrun;
 	drv_stats->tx_drops = stats->tx_drops;
 
 	/* Update RQ and SQ stats */
@@ -1394,274 +1409,15 @@ void nicvf_update_stats(struct nicvf *nic)
 		nicvf_update_sq_stats(nic, qidx);
 }
 
-static struct rtnl_link_stats64 *nicvf_get_stats64(struct net_device *netdev,
-					    struct rtnl_link_stats64 *stats)
+static void
+nicvf_tick_stats(void *arg)
 {
-	struct nicvf *nic = netdev_priv(netdev);
-	struct nicvf_hw_stats *hw_stats = &nic->hw_stats;
-	struct nicvf_drv_stats *drv_stats = &nic->drv_stats;
+	struct nicvf *nic;
 
+	nic = (struct nicvf *)arg;
+
+	/* Read the statistics */
 	nicvf_update_stats(nic);
 
-	stats->rx_bytes = hw_stats->rx_bytes;
-	stats->rx_packets = drv_stats->rx_frames_ok;
-	stats->rx_dropped = drv_stats->rx_drops;
-	stats->multicast = hw_stats->rx_mcast_frames;
-
-	stats->tx_bytes = hw_stats->tx_bytes_ok;
-	stats->tx_packets = drv_stats->tx_frames_ok;
-	stats->tx_dropped = drv_stats->tx_drops;
-
-	return stats;
+	callout_reset(&nic->stats_callout, hz, nicvf_tick_stats, nic);
 }
-
-static void nicvf_tx_timeout(struct net_device *dev)
-{
-	struct nicvf *nic = netdev_priv(dev);
-
-	if (netif_msg_tx_err(nic))
-		netdev_warn(dev, "%s: Transmit timed out, resetting\n",
-			    dev->name);
-
-	schedule_work(&nic->reset_task);
-}
-
-static void nicvf_reset_task(struct work_struct *work)
-{
-	struct nicvf *nic;
-
-	nic = container_of(work, struct nicvf, reset_task);
-
-	if (!netif_running(nic->netdev))
-		return;
-
-	nicvf_stop(nic->netdev);
-	nicvf_open(nic->netdev);
-	nic->netdev->trans_start = jiffies;
-}
-
-static int nicvf_config_loopback(struct nicvf *nic,
-				 netdev_features_t features)
-{
-	union nic_mbx mbx = {};
-
-	mbx.lbk.msg = NIC_MBOX_MSG_LOOPBACK;
-	mbx.lbk.vf_id = nic->vf_id;
-	mbx.lbk.enable = (features & NETIF_F_LOOPBACK) != 0;
-
-	return nicvf_send_msg_to_pf(nic, &mbx);
-}
-
-static netdev_features_t nicvf_fix_features(struct net_device *netdev,
-					    netdev_features_t features)
-{
-	struct nicvf *nic = netdev_priv(netdev);
-
-	if ((features & NETIF_F_LOOPBACK) &&
-	    netif_running(netdev) && !nic->loopback_supported)
-		features &= ~NETIF_F_LOOPBACK;
-
-	return features;
-}
-
-static int nicvf_set_features(struct net_device *netdev,
-			      netdev_features_t features)
-{
-	struct nicvf *nic = netdev_priv(netdev);
-	netdev_features_t changed = features ^ netdev->features;
-
-	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
-		nicvf_config_vlan_stripping(nic, features);
-
-	if ((changed & NETIF_F_LOOPBACK) && netif_running(netdev))
-		return nicvf_config_loopback(nic, features);
-
-	return 0;
-}
-
-static const struct net_device_ops nicvf_netdev_ops = {
-	.ndo_open		= nicvf_open,
-	.ndo_stop		= nicvf_stop,
-	.ndo_start_xmit		= nicvf_xmit,
-	.ndo_change_mtu		= nicvf_change_mtu,
-	.ndo_set_mac_address	= nicvf_set_mac_address,
-	.ndo_get_stats64	= nicvf_get_stats64,
-	.ndo_tx_timeout         = nicvf_tx_timeout,
-	.ndo_fix_features       = nicvf_fix_features,
-	.ndo_set_features       = nicvf_set_features,
-};
-
-static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
-{
-	struct device *dev = &pdev->dev;
-	struct net_device *netdev;
-	struct nicvf *nic;
-	int    err, qcount;
-
-	err = pci_enable_device(pdev);
-	if (err) {
-		dev_err(dev, "Failed to enable PCI device\n");
-		return err;
-	}
-
-	err = pci_request_regions(pdev, DRV_NAME);
-	if (err) {
-		dev_err(dev, "PCI request regions failed 0x%x\n", err);
-		goto err_disable_device;
-	}
-
-	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
-	if (err) {
-		dev_err(dev, "Unable to get usable DMA configuration\n");
-		goto err_release_regions;
-	}
-
-	err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
-	if (err) {
-		dev_err(dev, "unable to get 48-bit DMA for consistent allocations\n");
-		goto err_release_regions;
-	}
-
-	qcount = MAX_CMP_QUEUES_PER_QS;
-
-#ifdef VNIC_MULTI_QSET_SUPPORT
-	/* Restrict multiqset support only for host bound VFs */
-	if (pdev->is_virtfn) {
-		/* Set max number of queues per VF */
-		qcount = roundup(num_online_cpus(), MAX_CMP_QUEUES_PER_QS);
-		qcount = min(qcount,
-			     (MAX_SQS_PER_VF + 1) * MAX_CMP_QUEUES_PER_QS);
-	}
-#endif
-
-	netdev = alloc_etherdev_mqs(sizeof(struct nicvf), qcount, qcount);
-	if (!netdev) {
-		err = -ENOMEM;
-		goto err_release_regions;
-	}
-
-	pci_set_drvdata(pdev, netdev);
-
-	SET_NETDEV_DEV(netdev, &pdev->dev);
-
-	nic = netdev_priv(netdev);
-	nic->netdev = netdev;
-	nic->pdev = pdev;
-	nic->pnicvf = nic;
-	nic->max_queues = qcount;
-
-	/* MAP VF's configuration registers */
-	nic->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0);
-	if (!nic->reg_base) {
-		dev_err(dev, "Cannot map config register space, aborting\n");
-		err = -ENOMEM;
-		goto err_free_netdev;
-	}
-
-	err = nicvf_set_qset_resources(nic);
-	if (err)
-		goto err_free_netdev;
-
-	/* Check if PF is alive and get MAC address for this VF */
-	err = nicvf_register_misc_interrupt(nic);
-	if (err)
-		goto err_free_netdev;
-
-#ifdef VNIC_MULTI_QSET_SUPPORT
-	nicvf_send_vf_struct(nic);
-
-	/* Check if this VF is in QS only mode */
-	if (nic->sqs_mode)
-		return 0;
-#endif
-
-	err = nicvf_set_real_num_queues(netdev, nic->tx_queues, nic->rx_queues);
-	if (err)
-		goto err_unregister_interrupts;
-
-	netdev->hw_features = (NETIF_F_RXCSUM | NETIF_F_IP_CSUM | NETIF_F_SG |
-			       NETIF_F_GRO |
-			       NETIF_F_HW_VLAN_CTAG_RX);
-#ifdef	VNIC_RSS_SUPPORT
-	netdev->hw_features |= NETIF_F_RXHASH;
-#endif
-
-	netdev->features |= netdev->hw_features;
-	netdev->hw_features |= NETIF_F_LOOPBACK;
-
-	netdev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM;
-
-	netdev->netdev_ops = &nicvf_netdev_ops;
-	netdev->watchdog_timeo = NICVF_TX_TIMEOUT;
-
-	INIT_WORK(&nic->reset_task, nicvf_reset_task);
-
-	err = register_netdev(netdev);
-	if (err) {
-		dev_err(dev, "Failed to register netdevice\n");
-		goto err_unregister_interrupts;
-	}
-
-	nic->msg_enable = debug;
-
-	nicvf_set_ethtool_ops(netdev);
-
-	return 0;
-
-err_unregister_interrupts:
-	nicvf_unregister_interrupts(nic);
-err_free_netdev:
-	pci_set_drvdata(pdev, NULL);
-	free_netdev(netdev);
-err_release_regions:
-	pci_release_regions(pdev);
-err_disable_device:
-	pci_disable_device(pdev);
-	return err;
-}
-
-static void nicvf_remove(struct pci_dev *pdev)
-{
-	struct net_device *netdev = pci_get_drvdata(pdev);
-	struct nicvf *nic = netdev_priv(netdev);
-	struct net_device *pnetdev = nic->pnicvf->netdev;
-
-	/* Check if this Qset is assigned to different VF.
-	 * If yes, clean primary and all secondary Qsets.
-	 */
-	if (pnetdev && (pnetdev->reg_state == NETREG_REGISTERED))
-		unregister_netdev(pnetdev);
-	nicvf_unregister_interrupts(nic);
-	pci_set_drvdata(pdev, NULL);
-	free_netdev(netdev);
-	pci_release_regions(pdev);
-	pci_disable_device(pdev);
-}
-
-static void nicvf_shutdown(struct pci_dev *pdev)
-{
-	nicvf_remove(pdev);
-}
-
-static struct pci_driver nicvf_driver = {
-	.name = DRV_NAME,
-	.id_table = nicvf_id_table,
-	.probe = nicvf_probe,
-	.remove = nicvf_remove,
-	.shutdown = nicvf_shutdown,
-};
-
-static int __init nicvf_init_module(void)
-{
-	pr_info("%s, ver %s\n", DRV_NAME, DRV_VERSION);
-
-	return pci_register_driver(&nicvf_driver);
-}
-
-static void __exit nicvf_cleanup_module(void)
-{
-	pci_unregister_driver(&nicvf_driver);
-}
-
-module_init(nicvf_init_module);
-module_exit(nicvf_cleanup_module);
diff --git a/sys/dev/vnic/nicvf_queues.c b/sys/dev/vnic/nicvf_queues.c
index 9fa902403fc5..4c5d28086b36 100644
--- a/sys/dev/vnic/nicvf_queues.c
+++ b/sys/dev/vnic/nicvf_queues.c
@@ -26,257 +26,520 @@
  * $FreeBSD$
  *
  */
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
 
-#include <linux/pci.h>
-#include <linux/netdevice.h>
-#include <linux/ip.h>
-#include <linux/etherdevice.h>
-#include <net/ip.h>
-#include <net/tso.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bitset.h>
+#include <sys/bitstring.h>
+#include <sys/buf_ring.h>
+#include <sys/bus.h>
+#include <sys/endian.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/rman.h>
+#include <sys/pciio.h>
+#include <sys/pcpu.h>
+#include <sys/proc.h>
+#include <sys/sockio.h>
+#include <sys/socket.h>
+#include <sys/stdatomic.h>
+#include <sys/cpuset.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/smp.h>
+#include <sys/taskqueue.h>
 
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/bus.h>
+#include <machine/vmparam.h>
+
+#include <net/ethernet.h>
+#include <net/if.h>
+#include <net/if_var.h>
+#include <net/if_media.h>
+#include <net/ifq.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#include "thunder_bgx.h"
 #include "nic_reg.h"
 #include "nic.h"
 #include "q_struct.h"
 #include "nicvf_queues.h"
 
+#define	DEBUG
+#undef DEBUG
+
+#ifdef DEBUG
+#define	dprintf(dev, fmt, ...)	device_printf(dev, fmt, ##__VA_ARGS__)
+#else
+#define	dprintf(dev, fmt, ...)
+#endif
+
+MALLOC_DECLARE(M_NICVF);
+
+static void nicvf_free_snd_queue(struct nicvf *, struct snd_queue *);
+static int nicvf_tx_mbuf_locked(struct snd_queue *, struct mbuf *);
+static struct mbuf * nicvf_get_rcv_mbuf(struct nicvf *, struct cqe_rx_t *);
+static void nicvf_sq_disable(struct nicvf *, int);
+static void nicvf_sq_enable(struct nicvf *, struct snd_queue *, int);
+static void nicvf_put_sq_desc(struct snd_queue *, int);
+static void nicvf_cmp_queue_config(struct nicvf *, struct queue_set *, int,
+    boolean_t);
+static void nicvf_sq_free_used_descs(struct nicvf *, struct snd_queue *, int);
+
+static void nicvf_rbdr_task(void *, int);
+static void nicvf_rbdr_task_nowait(void *, int);
+
 struct rbuf_info {
-	struct page *page;
-	void	*data;
-	u64	offset;
+	bus_dma_tag_t	dmat;
+	bus_dmamap_t	dmap;
+	struct mbuf *	mbuf;
 };
 
-#define GET_RBUF_INFO(x) ((struct rbuf_info *)(x - NICVF_RCV_BUF_ALIGN_BYTES))
+#define GET_RBUF_INFO(x) ((struct rbuf_info *)((x) - NICVF_RCV_BUF_ALIGN_BYTES))
 
 /* Poll a register for a specific value */
 static int nicvf_poll_reg(struct nicvf *nic, int qidx,
-			  u64 reg, int bit_pos, int bits, int val)
+			  uint64_t reg, int bit_pos, int bits, int val)
 {
-	u64 bit_mask;
-	u64 reg_val;
+	uint64_t bit_mask;
+	uint64_t reg_val;
 	int timeout = 10;
 
-	bit_mask = (1ULL << bits) - 1;
+	bit_mask = (1UL << bits) - 1;
 	bit_mask = (bit_mask << bit_pos);
 
 	while (timeout) {
 		reg_val = nicvf_queue_reg_read(nic, reg, qidx);
 		if (((reg_val & bit_mask) >> bit_pos) == val)
-			return 0;
-		usleep_range(1000, 2000);
+			return (0);
+
+		DELAY(1000);
 		timeout--;
 	}
-	netdev_err(nic->netdev, "Poll on reg 0x%llx failed\n", reg);
-	return 1;
+	device_printf(nic->dev, "Poll on reg 0x%lx failed\n", reg);
+	return (ETIMEDOUT);
+}
+
+/* Callback for bus_dmamap_load() */
+static void
+nicvf_dmamap_q_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
+{
+	bus_addr_t *paddr;
+
+	KASSERT(nseg == 1, ("wrong number of segments, should be 1"));
+	paddr = arg;
+	*paddr = segs->ds_addr;
 }
 
 /* Allocate memory for a queue's descriptors */
-static int nicvf_alloc_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem,
-				  int q_len, int desc_size, int align_bytes)
+static int
+nicvf_alloc_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem,
+    int q_len, int desc_size, int align_bytes)
 {
-	dmem->q_len = q_len;
-	dmem->size = (desc_size * q_len) + align_bytes;
-	/* Save address, need it while freeing */
-	dmem->unalign_base = dma_zalloc_coherent(&nic->pdev->dev, dmem->size,
-						&dmem->dma, GFP_KERNEL);
-	if (!dmem->unalign_base)
-		return -ENOMEM;
+	int err, err_dmat;
 
-	/* Align memory address for 'align_bytes' */
-	dmem->phys_base = NICVF_ALIGNED_ADDR((u64)dmem->dma, align_bytes);
-	dmem->base = dmem->unalign_base + (dmem->phys_base - dmem->dma);
-	return 0;
+	/* Create DMA tag first */
+	err = bus_dma_tag_create(
+	    bus_get_dma_tag(nic->dev),		/* parent tag */
+	    align_bytes,			/* alignment */
+	    0,					/* boundary */
+	    BUS_SPACE_MAXADDR,			/* lowaddr */
+	    BUS_SPACE_MAXADDR,			/* highaddr */
+	    NULL, NULL,				/* filtfunc, filtfuncarg */
+	    (q_len * desc_size),		/* maxsize */
+	    1,					/* nsegments */
+	    (q_len * desc_size),		/* maxsegsize */
+	    0,					/* flags */
+	    NULL, NULL,				/* lockfunc, lockfuncarg */
+	    &dmem->dmat);			/* dmat */
+
+	if (err != 0) {
+		device_printf(nic->dev,
+		    "Failed to create busdma tag for descriptors ring\n");
+		return (err);
+	}
+
+	/* Allocate segment of continuous DMA safe memory */
+	err = bus_dmamem_alloc(
+	    dmem->dmat,				/* DMA tag */
+	    &dmem->base,			/* virtual address */
+	    (BUS_DMA_NOWAIT | BUS_DMA_ZERO),	/* flags */
+	    &dmem->dmap);			/* DMA map */
+	if (err != 0) {
+		device_printf(nic->dev, "Failed to allocate DMA safe memory for"
+		    "descriptors ring\n");
+		goto dmamem_fail;
+	}
+
+	err = bus_dmamap_load(
+	    dmem->dmat,
+	    dmem->dmap,
+	    dmem->base,
+	    (q_len * desc_size),		/* allocation size */
+	    nicvf_dmamap_q_cb,			/* map to DMA address cb. */
+	    &dmem->phys_base,			/* physical address */
+	    BUS_DMA_NOWAIT);
+	if (err != 0) {
+		device_printf(nic->dev,
+		    "Cannot load DMA map of descriptors ring\n");
+		goto dmamap_fail;
+	}
+
+	dmem->q_len = q_len;
+	dmem->size = (desc_size * q_len);
+
+	return (0);
+
+dmamap_fail:
+	bus_dmamem_free(dmem->dmat, dmem->base, dmem->dmap);
+	dmem->phys_base = 0;
+dmamem_fail:
+	err_dmat = bus_dma_tag_destroy(dmem->dmat);
+	dmem->base = NULL;
+	KASSERT(err_dmat == 0,
+	    ("%s: Trying to destroy BUSY DMA tag", __func__));
+
+	return (err);
 }
 
 /* Free queue's descriptor memory */
-static void nicvf_free_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem)
+static void
+nicvf_free_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem)
 {
-	if (!dmem)
+	int err;
+
+	if ((dmem == NULL) || (dmem->base == NULL))
 		return;
 
-	dma_free_coherent(&nic->pdev->dev, dmem->size,
-			  dmem->unalign_base, dmem->dma);
-	dmem->unalign_base = NULL;
+	/* Unload a map */
+	bus_dmamap_sync(dmem->dmat, dmem->dmap, BUS_DMASYNC_POSTREAD);
+	bus_dmamap_unload(dmem->dmat, dmem->dmap);
+	/* Free DMA memory */
+	bus_dmamem_free(dmem->dmat, dmem->base, dmem->dmap);
+	/* Destroy DMA tag */
+	err = bus_dma_tag_destroy(dmem->dmat);
+
+	KASSERT(err == 0,
+	    ("%s: Trying to destroy BUSY DMA tag", __func__));
+
+	dmem->phys_base = 0;
 	dmem->base = NULL;
 }
 
-/* Allocate buffer for packet reception
+/*
+ * Allocate buffer for packet reception
  * HW returns memory address where packet is DMA'ed but not a pointer
  * into RBDR ring, so save buffer address at the start of fragment and
  * align the start address to a cache aligned address
  */
-static inline int nicvf_alloc_rcv_buffer(struct nicvf *nic, gfp_t gfp,
-					 u32 buf_len, u64 **rbuf)
+static __inline int
+nicvf_alloc_rcv_buffer(struct nicvf *nic, struct rbdr *rbdr,
+    bus_dmamap_t dmap, int mflags, uint32_t buf_len, bus_addr_t *rbuf)
 {
-	u64 data;
+	struct mbuf *mbuf;
 	struct rbuf_info *rinfo;
-	int order = get_order(buf_len);
+	bus_dma_segment_t segs[1];
+	int nsegs;
+	int err;
 
-	/* Check if request can be accomodated in previous allocated page */
-	if (nic->rb_page) {
-		if ((nic->rb_page_offset + buf_len + buf_len) >
-		    (PAGE_SIZE << order)) {
-			nic->rb_page = NULL;
-		} else {
-			nic->rb_page_offset += buf_len;
-			get_page(nic->rb_page);
-		}
+	mbuf = m_getjcl(mflags, MT_DATA, M_PKTHDR, MCLBYTES);
+	if (mbuf == NULL)
+		return (ENOMEM);
+
+	/*
+	 * The length is equal to the actual length + one 128b line
+	 * used as a room for rbuf_info structure.
+	 */
+	mbuf->m_len = mbuf->m_pkthdr.len = buf_len;
+
+	err = bus_dmamap_load_mbuf_sg(rbdr->rbdr_buff_dmat, dmap, mbuf, segs,
+	    &nsegs, BUS_DMA_NOWAIT);
+	if (err != 0) {
+		device_printf(nic->dev,
+		    "Failed to map mbuf into DMA visible memory, err: %d\n",
+		    err);
+		m_freem(mbuf);
+		bus_dmamap_destroy(rbdr->rbdr_buff_dmat, dmap);
+		return (err);
 	}
+	if (nsegs != 1)
+		panic("Unexpected number of DMA segments for RB: %d", nsegs);
+	/*
+	 * Now use the room for rbuf_info structure
+	 * and adjust mbuf data and length.
+	 */
+	rinfo = (struct rbuf_info *)mbuf->m_data;
+	m_adj(mbuf, NICVF_RCV_BUF_ALIGN_BYTES);
 
-	/* Allocate a new page */
-	if (!nic->rb_page) {
-		nic->rb_page = alloc_pages(gfp | __GFP_COMP | __GFP_NOWARN,
-					   order);
-		if (!nic->rb_page) {
-			netdev_err(nic->netdev,
-				   "Failed to allocate new rcv buffer\n");
-			return -ENOMEM;
-		}
-		nic->rb_page_offset = 0;
-	}
+	rinfo->dmat = rbdr->rbdr_buff_dmat;
+	rinfo->dmap = dmap;
+	rinfo->mbuf = mbuf;
 
-	data = (u64)page_address(nic->rb_page) + nic->rb_page_offset;
+	*rbuf = segs[0].ds_addr + NICVF_RCV_BUF_ALIGN_BYTES;
 
-	/* Align buffer addr to cache line i.e 128 bytes */
-	rinfo = (struct rbuf_info *)(data + NICVF_RCV_BUF_ALIGN_LEN(data));
-	/* Save page address for reference updation */
-	rinfo->page = nic->rb_page;
-	/* Store start address for later retrieval */
-	rinfo->data = (void *)data;
-	/* Store alignment offset */
-	rinfo->offset = NICVF_RCV_BUF_ALIGN_LEN(data);
-
-	data += rinfo->offset;
-
-	/* Give next aligned address to hw for DMA */
-	*rbuf = (u64 *)(data + NICVF_RCV_BUF_ALIGN_BYTES);
-	return 0;
+	return (0);
 }
 
-/* Retrieve actual buffer start address and build skb for received packet */
-static struct sk_buff *nicvf_rb_ptr_to_skb(struct nicvf *nic,
-					   u64 rb_ptr, int len)
+/* Retrieve mbuf for received packet */
+static struct mbuf *
+nicvf_rb_ptr_to_mbuf(struct nicvf *nic, bus_addr_t rb_ptr)
 {
-	struct sk_buff *skb;
+	struct mbuf *mbuf;
 	struct rbuf_info *rinfo;
 
-	rb_ptr = (u64)phys_to_virt(rb_ptr);
 	/* Get buffer start address and alignment offset */
-	rinfo = GET_RBUF_INFO(rb_ptr);
+	rinfo = GET_RBUF_INFO(PHYS_TO_DMAP(rb_ptr));
 
-	/* Now build an skb to give to stack */
-	skb = build_skb(rinfo->data, RCV_FRAG_LEN);
-	if (!skb) {
-		put_page(rinfo->page);
-		return NULL;
+	/* Now retrieve mbuf to give to stack */
+	mbuf = rinfo->mbuf;
+	if (__predict_false(mbuf == NULL)) {
+		panic("%s: Received packet fragment with NULL mbuf",
+		    device_get_nameunit(nic->dev));
 	}
+	/*
+	 * Clear the mbuf in the descriptor to indicate
+	 * that this slot is processed and free to use.
+	 */
+	rinfo->mbuf = NULL;
 
-	/* Set correct skb->data */
-	skb_reserve(skb, rinfo->offset + NICVF_RCV_BUF_ALIGN_BYTES);
+	bus_dmamap_sync(rinfo->dmat, rinfo->dmap, BUS_DMASYNC_POSTREAD);
+	bus_dmamap_unload(rinfo->dmat, rinfo->dmap);
 
-	prefetch((void *)rb_ptr);
-	return skb;
+	return (mbuf);
 }
 
 /* Allocate RBDR ring and populate receive buffers */
-static int  nicvf_init_rbdr(struct nicvf *nic, struct rbdr *rbdr,
-			    int ring_len, int buf_size)
+static int
+nicvf_init_rbdr(struct nicvf *nic, struct rbdr *rbdr, int ring_len,
+    int buf_size, int qidx)
 {
-	int idx;
-	u64 *rbuf;
+	bus_dmamap_t dmap;
+	bus_addr_t rbuf;
 	struct rbdr_entry_t *desc;
+	int idx;
 	int err;
 
+	/* Allocate rbdr descriptors ring */
 	err = nicvf_alloc_q_desc_mem(nic, &rbdr->dmem, ring_len,
-				     sizeof(struct rbdr_entry_t),
-				     NICVF_RCV_BUF_ALIGN_BYTES);
-	if (err)
-		return err;
+	    sizeof(struct rbdr_entry_t), NICVF_RCV_BUF_ALIGN_BYTES);
+	if (err != 0) {
+		device_printf(nic->dev,
+		    "Failed to create RBDR descriptors ring\n");
+		return (err);
+	}
 
 	rbdr->desc = rbdr->dmem.base;
-	/* Buffer size has to be in multiples of 128 bytes */
-	rbdr->dma_size = buf_size;
-	rbdr->enable = true;
+	/*
+	 * Buffer size has to be in multiples of 128 bytes.
+	 * Make room for metadata of size of one line (128 bytes).
+	 */
+	rbdr->dma_size = buf_size - NICVF_RCV_BUF_ALIGN_BYTES;
+	rbdr->enable = TRUE;
 	rbdr->thresh = RBDR_THRESH;
+	rbdr->nic = nic;
+	rbdr->idx = qidx;
+
+	/*
+	 * Create DMA tag for Rx buffers.
+	 * Each map created using this tag is intended to store Rx payload for
+	 * one fragment and one header structure containing rbuf_info (thus
+	 * additional 128 byte line since RB must be a multiple of 128 byte
+	 * cache line).
+	 */
+	if (buf_size > MCLBYTES) {
+		device_printf(nic->dev,
+		    "Buffer size to large for mbuf cluster\n");
+		return (EINVAL);
+	}
+	err = bus_dma_tag_create(
+	    bus_get_dma_tag(nic->dev),		/* parent tag */
+	    NICVF_RCV_BUF_ALIGN_BYTES,		/* alignment */
+	    0,					/* boundary */
+	    DMAP_MAX_PHYSADDR,			/* lowaddr */
+	    DMAP_MIN_PHYSADDR,			/* highaddr */
+	    NULL, NULL,				/* filtfunc, filtfuncarg */
+	    roundup2(buf_size, MCLBYTES),	/* maxsize */
+	    1,					/* nsegments */
+	    roundup2(buf_size, MCLBYTES),	/* maxsegsize */
+	    0,					/* flags */
+	    NULL, NULL,				/* lockfunc, lockfuncarg */
+	    &rbdr->rbdr_buff_dmat);		/* dmat */
+
+	if (err != 0) {
+		device_printf(nic->dev,
+		    "Failed to create busdma tag for RBDR buffers\n");
+		return (err);
+	}
+
+	rbdr->rbdr_buff_dmaps = malloc(sizeof(*rbdr->rbdr_buff_dmaps) *
+	    ring_len, M_NICVF, (M_WAITOK | M_ZERO));
 
-	nic->rb_page = NULL;
 	for (idx = 0; idx < ring_len; idx++) {
-		err = nicvf_alloc_rcv_buffer(nic, GFP_KERNEL, RCV_FRAG_LEN,
-					     &rbuf);
-		if (err)
-			return err;
+		err = bus_dmamap_create(rbdr->rbdr_buff_dmat, 0, &dmap);
+		if (err != 0) {
+			device_printf(nic->dev,
+			    "Failed to create DMA map for RB\n");
+			return (err);
+		}
+		rbdr->rbdr_buff_dmaps[idx] = dmap;
+
+		err = nicvf_alloc_rcv_buffer(nic, rbdr, dmap, M_WAITOK,
+		    DMA_BUFFER_LEN, &rbuf);
+		if (err != 0)
+			return (err);
 
 		desc = GET_RBDR_DESC(rbdr, idx);
-		desc->buf_addr = virt_to_phys(rbuf) >> NICVF_RCV_BUF_ALIGN;
+		desc->buf_addr = (rbuf >> NICVF_RCV_BUF_ALIGN);
 	}
-	return 0;
+
+	/* Allocate taskqueue */
+	TASK_INIT(&rbdr->rbdr_task, 0, nicvf_rbdr_task, rbdr);
+	TASK_INIT(&rbdr->rbdr_task_nowait, 0, nicvf_rbdr_task_nowait, rbdr);
+	rbdr->rbdr_taskq = taskqueue_create_fast("nicvf_rbdr_taskq", M_WAITOK,
+	    taskqueue_thread_enqueue, &rbdr->rbdr_taskq);
+	taskqueue_start_threads(&rbdr->rbdr_taskq, 1, PI_NET, "%s: rbdr_taskq",
+	    device_get_nameunit(nic->dev));
+
+	return (0);
 }
 
 /* Free RBDR ring and its receive buffers */
-static void nicvf_free_rbdr(struct nicvf *nic, struct rbdr *rbdr)
+static void
+nicvf_free_rbdr(struct nicvf *nic, struct rbdr *rbdr)
 {
-	int head, tail;
-	u64 buf_addr;
+	struct mbuf *mbuf;
+	struct queue_set *qs;
 	struct rbdr_entry_t *desc;
 	struct rbuf_info *rinfo;
+	bus_addr_t buf_addr;
+	int head, tail, idx;
+	int err;
 
-	if (!rbdr)
+	qs = nic->qs;
+
+	if ((qs == NULL) || (rbdr == NULL))
 		return;
 
-	rbdr->enable = false;
-	if (!rbdr->dmem.base)
-		return;
+	rbdr->enable = FALSE;
+	if (rbdr->rbdr_taskq != NULL) {
+		/* Remove tasks */
+		while (taskqueue_cancel(rbdr->rbdr_taskq,
+		    &rbdr->rbdr_task_nowait, NULL) != 0) {
+			/* Finish the nowait task first */
+			taskqueue_drain(rbdr->rbdr_taskq,
+			    &rbdr->rbdr_task_nowait);
+		}
+		taskqueue_free(rbdr->rbdr_taskq);
+		rbdr->rbdr_taskq = NULL;
 
-	head = rbdr->head;
-	tail = rbdr->tail;
-
-	/* Free SKBs */
-	while (head != tail) {
-		desc = GET_RBDR_DESC(rbdr, head);
-		buf_addr = desc->buf_addr << NICVF_RCV_BUF_ALIGN;
-		rinfo = GET_RBUF_INFO((u64)phys_to_virt(buf_addr));
-		put_page(rinfo->page);
-		head++;
-		head &= (rbdr->dmem.q_len - 1);
+		while (taskqueue_cancel(taskqueue_thread,
+		    &rbdr->rbdr_task, NULL) != 0) {
+			/* Now finish the sleepable task */
+			taskqueue_drain(taskqueue_thread, &rbdr->rbdr_task);
+		}
+	}
+
+	/*
+	 * Free all of the memory under the RB descriptors.
+	 * There are assumptions here:
+	 * 1. Corresponding RBDR is disabled
+	 *    - it is safe to operate using head and tail indexes
+	 * 2. All bffers that were received are properly freed by
+	 *    the receive handler
+	 *    - there is no need to unload DMA map and free MBUF for other
+	 *      descriptors than unused ones
+	 */
+	if (rbdr->rbdr_buff_dmat != NULL) {
+		head = rbdr->head;
+		tail = rbdr->tail;
+		while (head != tail) {
+			desc = GET_RBDR_DESC(rbdr, head);
+			buf_addr = desc->buf_addr << NICVF_RCV_BUF_ALIGN;
+			rinfo = GET_RBUF_INFO(PHYS_TO_DMAP(buf_addr));
+			bus_dmamap_unload(rbdr->rbdr_buff_dmat, rinfo->dmap);
+			mbuf = rinfo->mbuf;
+			/* This will destroy everything including rinfo! */
+			m_freem(mbuf);
+			head++;
+			head &= (rbdr->dmem.q_len - 1);
+		}
+		/* Free tail descriptor */
+		desc = GET_RBDR_DESC(rbdr, tail);
+		buf_addr = desc->buf_addr << NICVF_RCV_BUF_ALIGN;
+		rinfo = GET_RBUF_INFO(PHYS_TO_DMAP(buf_addr));
+		bus_dmamap_unload(rbdr->rbdr_buff_dmat, rinfo->dmap);
+		mbuf = rinfo->mbuf;
+		/* This will destroy everything including rinfo! */
+		m_freem(mbuf);
+
+		/* Destroy DMA maps */
+		for (idx = 0; idx < qs->rbdr_len; idx++) {
+			if (rbdr->rbdr_buff_dmaps[idx] == NULL)
+				continue;
+			err = bus_dmamap_destroy(rbdr->rbdr_buff_dmat,
+			    rbdr->rbdr_buff_dmaps[idx]);
+			KASSERT(err == 0,
+			    ("%s: Could not destroy DMA map for RB, desc: %d",
+			    __func__, idx));
+			rbdr->rbdr_buff_dmaps[idx] = NULL;
+		}
+
+		/* Now destroy the tag */
+		err = bus_dma_tag_destroy(rbdr->rbdr_buff_dmat);
+		KASSERT(err == 0,
+		    ("%s: Trying to destroy BUSY DMA tag", __func__));
+
+		rbdr->head = 0;
+		rbdr->tail = 0;
 	}
-	/* Free SKB of tail desc */
-	desc = GET_RBDR_DESC(rbdr, tail);
-	buf_addr = desc->buf_addr << NICVF_RCV_BUF_ALIGN;
-	rinfo = GET_RBUF_INFO((u64)phys_to_virt(buf_addr));
-	put_page(rinfo->page);
 
 	/* Free RBDR ring */
 	nicvf_free_q_desc_mem(nic, &rbdr->dmem);
 }
 
-/* Refill receive buffer descriptors with new buffers.
+/*
+ * Refill receive buffer descriptors with new buffers.
  */
-static void nicvf_refill_rbdr(struct nicvf *nic, gfp_t gfp)
+static int
+nicvf_refill_rbdr(struct rbdr *rbdr, int mflags)
 {
-	struct queue_set *qs = nic->qs;
-	int rbdr_idx = qs->rbdr_cnt;
+	struct nicvf *nic;
+	struct queue_set *qs;
+	int rbdr_idx;
 	int tail, qcount;
 	int refill_rb_cnt;
-	struct rbdr *rbdr;
 	struct rbdr_entry_t *desc;
-	u64 *rbuf;
-	int new_rb = 0;
+	bus_dmamap_t dmap;
+	bus_addr_t rbuf;
+	boolean_t rb_alloc_fail;
+	int new_rb;
+
+	rb_alloc_fail = TRUE;
+	new_rb = 0;
+	nic = rbdr->nic;
+	qs = nic->qs;
+	rbdr_idx = rbdr->idx;
 
-refill:
-	if (!rbdr_idx)
-		return;
-	rbdr_idx--;
-	rbdr = &qs->rbdr[rbdr_idx];
 	/* Check if it's enabled */
 	if (!rbdr->enable)
-		goto next_rbdr;
+		return (0);
 
 	/* Get no of desc's to be refilled */
 	qcount = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_STATUS0, rbdr_idx);
 	qcount &= 0x7FFFF;
 	/* Doorbell can be ringed with a max of ring size minus 1 */
-	if (qcount >= (qs->rbdr_len - 1))
-		goto next_rbdr;
-	else
+	if (qcount >= (qs->rbdr_len - 1)) {
+		rb_alloc_fail = FALSE;
+		goto out;
+	} else
 		refill_rb_cnt = qs->rbdr_len - qcount - 1;
 
 	/* Start filling descs from tail */
@@ -285,137 +548,597 @@ refill:
 		tail++;
 		tail &= (rbdr->dmem.q_len - 1);
 
-		if (nicvf_alloc_rcv_buffer(nic, gfp, RCV_FRAG_LEN, &rbuf))
+		dmap = rbdr->rbdr_buff_dmaps[tail];
+		if (nicvf_alloc_rcv_buffer(nic, rbdr, dmap, mflags,
+		    DMA_BUFFER_LEN, &rbuf)) {
+			/* Something went wrong. Resign */
 			break;
-
+		}
 		desc = GET_RBDR_DESC(rbdr, tail);
-		desc->buf_addr = virt_to_phys(rbuf) >> NICVF_RCV_BUF_ALIGN;
+		desc->buf_addr = (rbuf >> NICVF_RCV_BUF_ALIGN);
 		refill_rb_cnt--;
 		new_rb++;
 	}
 
 	/* make sure all memory stores are done before ringing doorbell */
-	smp_wmb();
+	wmb();
 
 	/* Check if buffer allocation failed */
-	if (refill_rb_cnt)
-		nic->rb_alloc_fail = true;
-	else
-		nic->rb_alloc_fail = false;
+	if (refill_rb_cnt == 0)
+		rb_alloc_fail = FALSE;
 
 	/* Notify HW */
 	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_DOOR,
 			      rbdr_idx, new_rb);
-next_rbdr:
-	/* Re-enable RBDR interrupts only if buffer allocation is success */
-	if (!nic->rb_alloc_fail && rbdr->enable)
+out:
+	if (!rb_alloc_fail) {
+		/*
+		 * Re-enable RBDR interrupts only
+		 * if buffer allocation is success.
+		 */
 		nicvf_enable_intr(nic, NICVF_INTR_RBDR, rbdr_idx);
 
-	if (rbdr_idx)
-		goto refill;
+		return (0);
+	}
+
+	return (ENOMEM);
 }
 
-/* Alloc rcv buffers in non-atomic mode for better success */
-void nicvf_rbdr_work(struct work_struct *work)
+/* Refill RBs even if sleep is needed to reclaim memory */
+static void
+nicvf_rbdr_task(void *arg, int pending)
 {
-	struct nicvf *nic = container_of(work, struct nicvf, rbdr_work.work);
+	struct rbdr *rbdr;
+	int err;
 
-	nicvf_refill_rbdr(nic, GFP_KERNEL);
-	if (nic->rb_alloc_fail)
-		schedule_delayed_work(&nic->rbdr_work, msecs_to_jiffies(10));
-	else
-		nic->rb_work_scheduled = false;
-}
+	rbdr = (struct rbdr *)arg;
 
-/* In Softirq context, alloc rcv buffers in atomic mode */
-void nicvf_rbdr_task(unsigned long data)
-{
-	struct nicvf *nic = (struct nicvf *)data;
-
-	nicvf_refill_rbdr(nic, GFP_ATOMIC);
-	if (nic->rb_alloc_fail) {
-		nic->rb_work_scheduled = true;
-		schedule_delayed_work(&nic->rbdr_work, msecs_to_jiffies(10));
+	err = nicvf_refill_rbdr(rbdr, M_WAITOK);
+	if (__predict_false(err != 0)) {
+		panic("%s: Failed to refill RBs even when sleep enabled",
+		    __func__);
 	}
 }
 
+/* Refill RBs as soon as possible without waiting */
+static void
+nicvf_rbdr_task_nowait(void *arg, int pending)
+{
+	struct rbdr *rbdr;
+	int err;
+
+	rbdr = (struct rbdr *)arg;
+
+	err = nicvf_refill_rbdr(rbdr, M_NOWAIT);
+	if (err != 0) {
+		/*
+		 * Schedule another, sleepable kernel thread
+		 * that will for sure refill the buffers.
+		 */
+		taskqueue_enqueue(taskqueue_thread, &rbdr->rbdr_task);
+	}
+}
+
+static int
+nicvf_rcv_pkt_handler(struct nicvf *nic, struct cmp_queue *cq,
+    struct cqe_rx_t *cqe_rx, int cqe_type)
+{
+	struct mbuf *mbuf;
+	int rq_idx;
+	int err = 0;
+
+	rq_idx = cqe_rx->rq_idx;
+
+	/* Check for errors */
+	err = nicvf_check_cqe_rx_errs(nic, cq, cqe_rx);
+	if (err && !cqe_rx->rb_cnt)
+		return (0);
+
+	mbuf = nicvf_get_rcv_mbuf(nic, cqe_rx);
+	if (mbuf == NULL) {
+		dprintf(nic->dev, "Packet not received\n");
+		return (0);
+	}
+
+	/* If error packet */
+	if (err != 0) {
+		m_freem(mbuf);
+		return (0);
+	}
+
+	/*
+	 * Push this packet to the stack later to avoid
+	 * unlocking completion task in the middle of work.
+	 */
+	err = buf_ring_enqueue(cq->rx_br, mbuf);
+	if (err != 0) {
+		/*
+		 * Failed to enqueue this mbuf.
+		 * We don't drop it, just schedule another task.
+		 */
+		return (err);
+	}
+
+	return (0);
+}
+
+static int
+nicvf_snd_pkt_handler(struct nicvf *nic, struct cmp_queue *cq,
+    struct cqe_send_t *cqe_tx, int cqe_type)
+{
+	bus_dmamap_t dmap;
+	struct mbuf *mbuf;
+	struct snd_queue *sq;
+	struct sq_hdr_subdesc *hdr;
+
+	mbuf = NULL;
+	sq = &nic->qs->sq[cqe_tx->sq_idx];
+	/* Avoid blocking here since we hold a non-sleepable NICVF_CMP_LOCK */
+	if (NICVF_TX_TRYLOCK(sq) == 0)
+		return (EAGAIN);
+
+	hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, cqe_tx->sqe_ptr);
+	if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER) {
+		NICVF_TX_UNLOCK(sq);
+		return (0);
+	}
+
+	dprintf(nic->dev,
+	    "%s Qset #%d SQ #%d SQ ptr #%d subdesc count %d\n",
+	    __func__, cqe_tx->sq_qs, cqe_tx->sq_idx,
+	    cqe_tx->sqe_ptr, hdr->subdesc_cnt);
+
+	dmap = (bus_dmamap_t)sq->snd_buff[cqe_tx->sqe_ptr].dmap;
+	bus_dmamap_unload(sq->snd_buff_dmat, dmap);
+
+	mbuf = (struct mbuf *)sq->snd_buff[cqe_tx->sqe_ptr].mbuf;
+	if (mbuf != NULL) {
+		m_freem(mbuf);
+		sq->snd_buff[cqe_tx->sqe_ptr].mbuf = NULL;
+	}
+
+	nicvf_check_cqe_tx_errs(nic, cq, cqe_tx);
+	nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1);
+
+	NICVF_TX_UNLOCK(sq);
+	return (0);
+}
+
+static int
+nicvf_cq_intr_handler(struct nicvf *nic, uint8_t cq_idx)
+{
+	struct mbuf *mbuf;
+	struct ifnet *ifp;
+	int processed_cqe, work_done = 0, tx_done = 0;
+	int cqe_count, cqe_head;
+	struct queue_set *qs = nic->qs;
+	struct cmp_queue *cq = &qs->cq[cq_idx];
+	struct cqe_rx_t *cq_desc;
+	int cmp_err;
+
+	NICVF_CMP_LOCK(cq);
+	cmp_err = 0;
+	processed_cqe = 0;
+	/* Get no of valid CQ entries to process */
+	cqe_count = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS, cq_idx);
+	cqe_count &= CQ_CQE_COUNT;
+	if (cqe_count == 0)
+		goto out;
+
+	/* Get head of the valid CQ entries */
+	cqe_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD, cq_idx) >> 9;
+	cqe_head &= 0xFFFF;
+
+	dprintf(nic->dev, "%s CQ%d cqe_count %d cqe_head %d\n",
+	    __func__, cq_idx, cqe_count, cqe_head);
+	while (processed_cqe < cqe_count) {
+		/* Get the CQ descriptor */
+		cq_desc = (struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head);
+		cqe_head++;
+		cqe_head &= (cq->dmem.q_len - 1);
+
+		dprintf(nic->dev, "CQ%d cq_desc->cqe_type %d\n", cq_idx,
+		    cq_desc->cqe_type);
+		switch (cq_desc->cqe_type) {
+		case CQE_TYPE_RX:
+			cmp_err = nicvf_rcv_pkt_handler(nic, cq, cq_desc,
+			    CQE_TYPE_RX);
+			if (__predict_false(cmp_err != 0)) {
+				/*
+				 * Ups. Cannot finish now.
+				 * Let's try again later.
+				 */
+				goto done;
+			}
+			work_done++;
+			break;
+		case CQE_TYPE_SEND:
+			cmp_err = nicvf_snd_pkt_handler(nic, cq,
+			    (void *)cq_desc, CQE_TYPE_SEND);
+			if (__predict_false(cmp_err != 0)) {
+				/*
+				 * Ups. Cannot finish now.
+				 * Let's try again later.
+				 */
+				goto done;
+			}
+
+			tx_done++;
+			break;
+		case CQE_TYPE_INVALID:
+		case CQE_TYPE_RX_SPLIT:
+		case CQE_TYPE_RX_TCP:
+		case CQE_TYPE_SEND_PTP:
+			/* Ignore for now */
+			break;
+		}
+		processed_cqe++;
+	}
+done:
+	dprintf(nic->dev,
+	    "%s CQ%d processed_cqe %d work_done %d\n",
+	    __func__, cq_idx, processed_cqe, work_done);
+
+	/* Ring doorbell to inform H/W to reuse processed CQEs */
+	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_DOOR, cq_idx, processed_cqe);
+
+	if ((tx_done > 0) &&
+	    ((if_getdrvflags(nic->ifp) & IFF_DRV_RUNNING) != 0)) {
+		/* Reenable TXQ if its stopped earlier due to SQ full */
+		if_setdrvflagbits(nic->ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
+	}
+out:
+	NICVF_CMP_UNLOCK(cq);
+
+	ifp = nic->ifp;
+	/* Push received MBUFs to the stack */
+	while (!buf_ring_empty(cq->rx_br)) {
+		mbuf = buf_ring_dequeue_mc(cq->rx_br);
+		if (__predict_true(mbuf != NULL))
+			(*ifp->if_input)(ifp, mbuf);
+	}
+
+	return (cmp_err);
+}
+
+/*
+ * Qset error interrupt handler
+ *
+ * As of now only CQ errors are handled
+ */
+static void
+nicvf_qs_err_task(void *arg, int pending)
+{
+	struct nicvf *nic;
+	struct queue_set *qs;
+	int qidx;
+	uint64_t status;
+	boolean_t enable = TRUE;
+
+	nic = (struct nicvf *)arg;
+	qs = nic->qs;
+
+	/* Deactivate network interface */
+	if_setdrvflagbits(nic->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
+
+	/* Check if it is CQ err */
+	for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
+		status = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS,
+		    qidx);
+		if ((status & CQ_ERR_MASK) == 0)
+			continue;
+		/* Process already queued CQEs and reconfig CQ */
+		nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
+		nicvf_sq_disable(nic, qidx);
+		(void)nicvf_cq_intr_handler(nic, qidx);
+		nicvf_cmp_queue_config(nic, qs, qidx, enable);
+		nicvf_sq_free_used_descs(nic, &qs->sq[qidx], qidx);
+		nicvf_sq_enable(nic, &qs->sq[qidx], qidx);
+		nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx);
+	}
+
+	if_setdrvflagbits(nic->ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
+	/* Re-enable Qset error interrupt */
+	nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0);
+}
+
+static void
+nicvf_cmp_task(void *arg, int pending)
+{
+	uint64_t cq_head;
+	struct cmp_queue *cq;
+	struct nicvf *nic;
+	int cmp_err;
+
+	cq = (struct cmp_queue *)arg;
+	nic = cq->nic;
+
+	/* Handle CQ descriptors */
+	cmp_err = nicvf_cq_intr_handler(nic, cq->idx);
+	/* Re-enable interrupts */
+	cq_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD, cq->idx);
+	nicvf_clear_intr(nic, NICVF_INTR_CQ, cq->idx);
+	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_HEAD, cq->idx, cq_head);
+
+	if (__predict_false(cmp_err != 0)) {
+		/*
+		 * Schedule another thread here since we did not
+		 * process the entire CQ due to Tx or Rx CQ parse error.
+		 */
+		taskqueue_enqueue(cq->cmp_taskq, &cq->cmp_task);
+
+	}
+
+	/* Reenable interrupt (previously disabled in nicvf_intr_handler() */
+	nicvf_enable_intr(nic, NICVF_INTR_CQ, cq->idx);
+
+}
+
 /* Initialize completion queue */
-static int nicvf_init_cmp_queue(struct nicvf *nic,
-				struct cmp_queue *cq, int q_len)
+static int
+nicvf_init_cmp_queue(struct nicvf *nic, struct cmp_queue *cq, int q_len,
+    int qidx)
 {
 	int err;
 
+	/* Initizalize lock */
+	snprintf(cq->mtx_name, sizeof(cq->mtx_name), "%s: CQ(%d) lock",
+	    device_get_nameunit(nic->dev), qidx);
+	mtx_init(&cq->mtx, cq->mtx_name, NULL, MTX_DEF);
+
 	err = nicvf_alloc_q_desc_mem(nic, &cq->dmem, q_len, CMP_QUEUE_DESC_SIZE,
 				     NICVF_CQ_BASE_ALIGN_BYTES);
-	if (err)
-		return err;
+
+	if (err != 0) {
+		device_printf(nic->dev,
+		    "Could not allocate DMA memory for CQ\n");
+		return (err);
+	}
 
 	cq->desc = cq->dmem.base;
 	cq->thresh = CMP_QUEUE_CQE_THRESH;
+	cq->nic = nic;
+	cq->idx = qidx;
 	nic->cq_coalesce_usecs = (CMP_QUEUE_TIMER_THRESH * 0.05) - 1;
 
-	return 0;
+	cq->rx_br = buf_ring_alloc(CMP_QUEUE_LEN * 8, M_DEVBUF, M_WAITOK,
+	    &cq->mtx);
+
+	/* Allocate taskqueue */
+	TASK_INIT(&cq->cmp_task, 0, nicvf_cmp_task, cq);
+	cq->cmp_taskq = taskqueue_create_fast("nicvf_cmp_taskq", M_WAITOK,
+	    taskqueue_thread_enqueue, &cq->cmp_taskq);
+	taskqueue_start_threads(&cq->cmp_taskq, 1, PI_NET, "%s: cmp_taskq(%d)",
+	    device_get_nameunit(nic->dev), qidx);
+
+	return (0);
 }
 
-static void nicvf_free_cmp_queue(struct nicvf *nic, struct cmp_queue *cq)
+static void
+nicvf_free_cmp_queue(struct nicvf *nic, struct cmp_queue *cq)
 {
-	if (!cq)
-		return;
-	if (!cq->dmem.base)
-		return;
 
+	if (cq == NULL)
+		return;
+	/*
+	 * The completion queue itself should be disabled by now
+	 * (ref. nicvf_snd_queue_config()).
+	 * Ensure that it is safe to disable it or panic.
+	 */
+	if (cq->enable)
+		panic("%s: Trying to free working CQ(%d)", __func__, cq->idx);
+
+	if (cq->cmp_taskq != NULL) {
+		/* Remove task */
+		while (taskqueue_cancel(cq->cmp_taskq, &cq->cmp_task, NULL) != 0)
+			taskqueue_drain(cq->cmp_taskq, &cq->cmp_task);
+
+		taskqueue_free(cq->cmp_taskq);
+		cq->cmp_taskq = NULL;
+	}
+	/*
+	 * Completion interrupt will possibly enable interrupts again
+	 * so disable interrupting now after we finished processing
+	 * completion task. It is safe to do so since the corresponding CQ
+	 * was already disabled.
+	 */
+	nicvf_disable_intr(nic, NICVF_INTR_CQ, cq->idx);
+	nicvf_clear_intr(nic, NICVF_INTR_CQ, cq->idx);
+
+	NICVF_CMP_LOCK(cq);
 	nicvf_free_q_desc_mem(nic, &cq->dmem);
+	drbr_free(cq->rx_br, M_DEVBUF);
+	NICVF_CMP_UNLOCK(cq);
+	mtx_destroy(&cq->mtx);
+	memset(cq->mtx_name, 0, sizeof(cq->mtx_name));
+}
+
+static void
+nicvf_snd_task(void *arg, int pending)
+{
+	struct snd_queue *sq = (struct snd_queue *)arg;
+	struct mbuf *mbuf;
+
+	NICVF_TX_LOCK(sq);
+	while (1) {
+		mbuf = drbr_dequeue(NULL, sq->br);
+		if (mbuf == NULL)
+			break;
+
+		if (nicvf_tx_mbuf_locked(sq, mbuf) != 0) {
+			/* XXX ARM64TODO: Increase Tx drop counter */
+			m_freem(mbuf);
+			break;
+		}
+	}
+	NICVF_TX_UNLOCK(sq);
 }
 
 /* Initialize transmit queue */
-static int nicvf_init_snd_queue(struct nicvf *nic,
-				struct snd_queue *sq, int q_len)
+static int
+nicvf_init_snd_queue(struct nicvf *nic, struct snd_queue *sq, int q_len,
+    int qidx)
 {
+	size_t i;
 	int err;
 
+	/* Initizalize TX lock for this queue */
+	snprintf(sq->mtx_name, sizeof(sq->mtx_name), "%s: SQ(%d) lock",
+	    device_get_nameunit(nic->dev), qidx);
+	mtx_init(&sq->mtx, sq->mtx_name, NULL, MTX_DEF);
+
+	NICVF_TX_LOCK(sq);
+	/* Allocate buffer ring */
+	sq->br = buf_ring_alloc(q_len / MIN_SQ_DESC_PER_PKT_XMIT, M_DEVBUF,
+	    M_NOWAIT, &sq->mtx);
+	if (sq->br == NULL) {
+		device_printf(nic->dev,
+		    "ERROR: Could not set up buf ring for SQ(%d)\n", qidx);
+		err = ENOMEM;
+		goto error;
+	}
+
+	/* Allocate DMA memory for Tx descriptors */
 	err = nicvf_alloc_q_desc_mem(nic, &sq->dmem, q_len, SND_QUEUE_DESC_SIZE,
 				     NICVF_SQ_BASE_ALIGN_BYTES);
-	if (err)
-		return err;
+	if (err != 0) {
+		device_printf(nic->dev,
+		    "Could not allocate DMA memory for SQ\n");
+		goto error;
+	}
 
 	sq->desc = sq->dmem.base;
-	sq->skbuff = kcalloc(q_len, sizeof(u64), GFP_KERNEL);
-	if (!sq->skbuff)
-		return -ENOMEM;
-	sq->head = 0;
-	sq->tail = 0;
-	atomic_set(&sq->free_cnt, q_len - 1);
+	sq->head = sq->tail = 0;
+	atomic_store_rel_int(&sq->free_cnt, q_len - 1);
 	sq->thresh = SND_QUEUE_THRESH;
+	sq->idx = qidx;
+	sq->nic = nic;
 
-	/* Preallocate memory for TSO segment's header */
-	sq->tso_hdrs = dma_alloc_coherent(&nic->pdev->dev,
-					  q_len * TSO_HEADER_SIZE,
-					  &sq->tso_hdrs_phys, GFP_KERNEL);
-	if (!sq->tso_hdrs)
-		return -ENOMEM;
+	/*
+	 * Allocate DMA maps for Tx buffers
+	 */
 
-	return 0;
+	/* Create DMA tag first */
+	err = bus_dma_tag_create(
+	    bus_get_dma_tag(nic->dev),		/* parent tag */
+	    1,					/* alignment */
+	    0,					/* boundary */
+	    BUS_SPACE_MAXADDR,			/* lowaddr */
+	    BUS_SPACE_MAXADDR,			/* highaddr */
+	    NULL, NULL,				/* filtfunc, filtfuncarg */
+	    NICVF_TXBUF_MAXSIZE,		/* maxsize */
+	    NICVF_TXBUF_NSEGS,			/* nsegments */
+	    MCLBYTES,				/* maxsegsize */
+	    0,					/* flags */
+	    NULL, NULL,				/* lockfunc, lockfuncarg */
+	    &sq->snd_buff_dmat);		/* dmat */
+
+	if (err != 0) {
+		device_printf(nic->dev,
+		    "Failed to create busdma tag for Tx buffers\n");
+		goto error;
+	}
+
+	/* Allocate send buffers array */
+	sq->snd_buff = malloc(sizeof(*sq->snd_buff) * q_len, M_NICVF,
+	    (M_NOWAIT | M_ZERO));
+	if (sq->snd_buff == NULL) {
+		device_printf(nic->dev,
+		    "Could not allocate memory for Tx buffers array\n");
+		err = ENOMEM;
+		goto error;
+	}
+
+	/* Now populate maps */
+	for (i = 0; i < q_len; i++) {
+		err = bus_dmamap_create(sq->snd_buff_dmat, 0,
+		    &sq->snd_buff[i].dmap);
+		if (err != 0) {
+			device_printf(nic->dev,
+			    "Failed to create DMA maps for Tx buffers\n");
+			goto error;
+		}
+	}
+	NICVF_TX_UNLOCK(sq);
+
+	/* Allocate taskqueue */
+	TASK_INIT(&sq->snd_task, 0, nicvf_snd_task, sq);
+	sq->snd_taskq = taskqueue_create_fast("nicvf_snd_taskq", M_WAITOK,
+	    taskqueue_thread_enqueue, &sq->snd_taskq);
+	taskqueue_start_threads(&sq->snd_taskq, 1, PI_NET, "%s: snd_taskq(%d)",
+	    device_get_nameunit(nic->dev), qidx);
+
+	return (0);
+error:
+	NICVF_TX_UNLOCK(sq);
+	return (err);
 }
 
-static void nicvf_free_snd_queue(struct nicvf *nic, struct snd_queue *sq)
+static void
+nicvf_free_snd_queue(struct nicvf *nic, struct snd_queue *sq)
 {
-	if (!sq)
-		return;
-	if (!sq->dmem.base)
+	struct queue_set *qs = nic->qs;
+	size_t i;
+	int err;
+
+	if (sq == NULL)
 		return;
 
-	if (sq->tso_hdrs)
-		dma_free_coherent(&nic->pdev->dev,
-				  sq->dmem.q_len * TSO_HEADER_SIZE,
-				  sq->tso_hdrs, sq->tso_hdrs_phys);
+	if (sq->snd_taskq != NULL) {
+		/* Remove task */
+		while (taskqueue_cancel(sq->snd_taskq, &sq->snd_task, NULL) != 0)
+			taskqueue_drain(sq->snd_taskq, &sq->snd_task);
 
-	kfree(sq->skbuff);
-	nicvf_free_q_desc_mem(nic, &sq->dmem);
+		taskqueue_free(sq->snd_taskq);
+		sq->snd_taskq = NULL;
+	}
+
+	NICVF_TX_LOCK(sq);
+	if (sq->snd_buff_dmat != NULL) {
+		if (sq->snd_buff != NULL) {
+			for (i = 0; i < qs->sq_len; i++) {
+				m_freem(sq->snd_buff[i].mbuf);
+				sq->snd_buff[i].mbuf = NULL;
+
+				bus_dmamap_unload(sq->snd_buff_dmat,
+				    sq->snd_buff[i].dmap);
+				err = bus_dmamap_destroy(sq->snd_buff_dmat,
+				    sq->snd_buff[i].dmap);
+				/*
+				 * If bus_dmamap_destroy fails it can cause
+				 * random panic later if the tag is also
+				 * destroyed in the process.
+				 */
+				KASSERT(err == 0,
+				    ("%s: Could not destroy DMA map for SQ",
+				    __func__));
+			}
+		}
+
+		free(sq->snd_buff, M_NICVF);
+
+		err = bus_dma_tag_destroy(sq->snd_buff_dmat);
+		KASSERT(err == 0,
+		    ("%s: Trying to destroy BUSY DMA tag", __func__));
+	}
+
+	/* Free private driver ring for this send queue */
+	if (sq->br != NULL)
+		drbr_free(sq->br, M_DEVBUF);
+
+	if (sq->dmem.base != NULL)
+		nicvf_free_q_desc_mem(nic, &sq->dmem);
+
+	NICVF_TX_UNLOCK(sq);
+	/* Destroy Tx lock */
+	mtx_destroy(&sq->mtx);
+	memset(sq->mtx_name, 0, sizeof(sq->mtx_name));
 }
 
-static void nicvf_reclaim_snd_queue(struct nicvf *nic,
-				    struct queue_set *qs, int qidx)
+static void
+nicvf_reclaim_snd_queue(struct nicvf *nic, struct queue_set *qs, int qidx)
 {
+
 	/* Disable send queue */
 	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, 0);
 	/* Check if SQ is stopped */
@@ -425,8 +1148,8 @@ static void nicvf_reclaim_snd_queue(struct nicvf *nic,
 	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, NICVF_SQ_RESET);
 }
 
-static void nicvf_reclaim_rcv_queue(struct nicvf *nic,
-				    struct queue_set *qs, int qidx)
+static void
+nicvf_reclaim_rcv_queue(struct nicvf *nic, struct queue_set *qs, int qidx)
 {
 	union nic_mbx mbx = {};
 
@@ -435,9 +1158,10 @@ static void nicvf_reclaim_rcv_queue(struct nicvf *nic,
 	nicvf_send_msg_to_pf(nic, &mbx);
 }
 
-static void nicvf_reclaim_cmp_queue(struct nicvf *nic,
-				    struct queue_set *qs, int qidx)
+static void
+nicvf_reclaim_cmp_queue(struct nicvf *nic, struct queue_set *qs, int qidx)
 {
+
 	/* Disable timer threshold (doesn't get reset upon CQ reset */
 	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG2, qidx, 0);
 	/* Disable completion queue */
@@ -446,27 +1170,27 @@ static void nicvf_reclaim_cmp_queue(struct nicvf *nic,
 	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, NICVF_CQ_RESET);
 }
 
-static void nicvf_reclaim_rbdr(struct nicvf *nic,
-			       struct rbdr *rbdr, int qidx)
+static void
+nicvf_reclaim_rbdr(struct nicvf *nic, struct rbdr *rbdr, int qidx)
 {
-	u64 tmp, fifo_state;
+	uint64_t tmp, fifo_state;
 	int timeout = 10;
 
 	/* Save head and tail pointers for feeing up buffers */
-	rbdr->head = nicvf_queue_reg_read(nic,
-					  NIC_QSET_RBDR_0_1_HEAD,
-					  qidx) >> 3;
-	rbdr->tail = nicvf_queue_reg_read(nic,
-					  NIC_QSET_RBDR_0_1_TAIL,
-					  qidx) >> 3;
+	rbdr->head =
+	    nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_HEAD, qidx) >> 3;
+	rbdr->tail =
+	    nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_TAIL, qidx) >> 3;
 
-	/* If RBDR FIFO is in 'FAIL' state then do a reset first
+	/*
+	 * If RBDR FIFO is in 'FAIL' state then do a reset first
 	 * before relaiming.
 	 */
 	fifo_state = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_STATUS0, qidx);
-	if (((fifo_state >> 62) & 0x03) == 0x3)
+	if (((fifo_state >> 62) & 0x03) == 0x3) {
 		nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG,
-				      qidx, NICVF_RBDR_RESET);
+		    qidx, NICVF_RBDR_RESET);
+	}
 
 	/* Disable RBDR */
 	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx, 0);
@@ -474,20 +1198,20 @@ static void nicvf_reclaim_rbdr(struct nicvf *nic,
 		return;
 	while (1) {
 		tmp = nicvf_queue_reg_read(nic,
-					   NIC_QSET_RBDR_0_1_PREFETCH_STATUS,
-					   qidx);
+		    NIC_QSET_RBDR_0_1_PREFETCH_STATUS, qidx);
 		if ((tmp & 0xFFFFFFFF) == ((tmp >> 32) & 0xFFFFFFFF))
 			break;
-		usleep_range(1000, 2000);
+
+		DELAY(1000);
 		timeout--;
 		if (!timeout) {
-			netdev_err(nic->netdev,
-				   "Failed polling on prefetch status\n");
+			device_printf(nic->dev,
+			    "Failed polling on prefetch status\n");
 			return;
 		}
 	}
-	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG,
-			      qidx, NICVF_RBDR_RESET);
+	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx,
+	    NICVF_RBDR_RESET);
 
 	if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x02))
 		return;
@@ -496,38 +1220,13 @@ static void nicvf_reclaim_rbdr(struct nicvf *nic,
 		return;
 }
 
-void nicvf_config_vlan_stripping(struct nicvf *nic, netdev_features_t features)
-{
-	u64 rq_cfg;
-#ifdef VNIC_MULTI_QSET_SUPPORT
-	int sqs = 0;
-#endif
-
-	rq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_RQ_GEN_CFG, 0);
-
-	/* Enable first VLAN stripping */
-	if (features & NETIF_F_HW_VLAN_CTAG_RX)
-		rq_cfg |= (1ULL << 25);
-	else
-		rq_cfg &= ~(1ULL << 25);
-	nicvf_queue_reg_write(nic, NIC_QSET_RQ_GEN_CFG, 0, rq_cfg);
-
-#ifdef VNIC_MULTI_QSET_SUPPORT
-	/* Configure Secondary Qsets, if any */
-	for (sqs = 0; sqs < nic->sqs_count; sqs++)
-		if (nic->snicvf[sqs])
-			nicvf_queue_reg_write(nic->snicvf[sqs],
-					      NIC_QSET_RQ_GEN_CFG, 0, rq_cfg);
-#endif
-}
-
 /* Configures receive queue */
-static void nicvf_rcv_queue_config(struct nicvf *nic, struct queue_set *qs,
-				   int qidx, bool enable)
+static void
+nicvf_rcv_queue_config(struct nicvf *nic, struct queue_set *qs,
+    int qidx, bool enable)
 {
 	union nic_mbx mbx = {};
 	struct rcv_queue *rq;
-	struct cmp_queue *cq;
 	struct rq_cfg rq_cfg;
 
 	rq = &qs->rq[qidx];
@@ -555,35 +1254,36 @@ static void nicvf_rcv_queue_config(struct nicvf *nic, struct queue_set *qs,
 	mbx.rq.qs_num = qs->vnic_id;
 	mbx.rq.rq_num = qidx;
 	mbx.rq.cfg = (rq->caching << 26) | (rq->cq_qs << 19) |
-			  (rq->cq_idx << 16) | (rq->cont_rbdr_qs << 9) |
-			  (rq->cont_qs_rbdr_idx << 8) |
-			  (rq->start_rbdr_qs << 1) | (rq->start_qs_rbdr_idx);
+	    (rq->cq_idx << 16) | (rq->cont_rbdr_qs << 9) |
+	    (rq->cont_qs_rbdr_idx << 8) | (rq->start_rbdr_qs << 1) |
+	    (rq->start_qs_rbdr_idx);
 	nicvf_send_msg_to_pf(nic, &mbx);
 
 	mbx.rq.msg = NIC_MBOX_MSG_RQ_BP_CFG;
-	mbx.rq.cfg = (1ULL << 63) | (1ULL << 62) | (qs->vnic_id << 0);
+	mbx.rq.cfg = (1UL << 63) | (1UL << 62) | (qs->vnic_id << 0);
 	nicvf_send_msg_to_pf(nic, &mbx);
 
-	/* RQ drop config
+	/*
+	 * RQ drop config
 	 * Enable CQ drop to reserve sufficient CQEs for all tx packets
 	 */
 	mbx.rq.msg = NIC_MBOX_MSG_RQ_DROP_CFG;
-	mbx.rq.cfg = (1ULL << 62) | (RQ_CQ_DROP << 8);
+	mbx.rq.cfg = (1UL << 62) | (RQ_CQ_DROP << 8);
 	nicvf_send_msg_to_pf(nic, &mbx);
 
 	nicvf_queue_reg_write(nic, NIC_QSET_RQ_GEN_CFG, 0, 0x00);
-	if (!nic->sqs_mode)
-		nicvf_config_vlan_stripping(nic, nic->netdev->features);
 
 	/* Enable Receive queue */
 	rq_cfg.ena = 1;
 	rq_cfg.tcp_ena = 0;
-	nicvf_queue_reg_write(nic, NIC_QSET_RQ_0_7_CFG, qidx, *(u64 *)&rq_cfg);
+	nicvf_queue_reg_write(nic, NIC_QSET_RQ_0_7_CFG, qidx,
+	    *(uint64_t *)&rq_cfg);
 }
 
 /* Configures completion queue */
-void nicvf_cmp_queue_config(struct nicvf *nic, struct queue_set *qs,
-			    int qidx, bool enable)
+static void
+nicvf_cmp_queue_config(struct nicvf *nic, struct queue_set *qs,
+    int qidx, boolean_t enable)
 {
 	struct cmp_queue *cq;
 	struct cq_cfg cq_cfg;
@@ -599,13 +1299,9 @@ void nicvf_cmp_queue_config(struct nicvf *nic, struct queue_set *qs,
 	/* Reset completion queue */
 	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, NICVF_CQ_RESET);
 
-	if (!cq->enable)
-		return;
-
-	spin_lock_init(&cq->lock);
 	/* Set completion queue base address */
-	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_BASE,
-			      qidx, (u64)(cq->dmem.phys_base));
+	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_BASE, qidx,
+	    (uint64_t)(cq->dmem.phys_base));
 
 	/* Enable Completion queue */
 	cq_cfg.ena = 1;
@@ -613,17 +1309,18 @@ void nicvf_cmp_queue_config(struct nicvf *nic, struct queue_set *qs,
 	cq_cfg.caching = 0;
 	cq_cfg.qsize = CMP_QSIZE;
 	cq_cfg.avg_con = 0;
-	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, *(u64 *)&cq_cfg);
+	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, *(uint64_t *)&cq_cfg);
 
 	/* Set threshold value for interrupt generation */
 	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_THRESH, qidx, cq->thresh);
-	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG2,
-			      qidx, nic->cq_coalesce_usecs);
+	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG2, qidx,
+	    nic->cq_coalesce_usecs);
 }
 
 /* Configures transmit queue */
-static void nicvf_snd_queue_config(struct nicvf *nic, struct queue_set *qs,
-				   int qidx, bool enable)
+static void
+nicvf_snd_queue_config(struct nicvf *nic, struct queue_set *qs, int qidx,
+    boolean_t enable)
 {
 	union nic_mbx mbx = {};
 	struct snd_queue *sq;
@@ -652,8 +1349,8 @@ static void nicvf_snd_queue_config(struct nicvf *nic, struct queue_set *qs,
 	nicvf_send_msg_to_pf(nic, &mbx);
 
 	/* Set queue base address */
-	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_BASE,
-			      qidx, (u64)(sq->dmem.phys_base));
+	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_BASE, qidx,
+	    (uint64_t)(sq->dmem.phys_base));
 
 	/* Enable send queue  & set queue size */
 	sq_cfg.ena = 1;
@@ -661,15 +1358,16 @@ static void nicvf_snd_queue_config(struct nicvf *nic, struct queue_set *qs,
 	sq_cfg.ldwb = 0;
 	sq_cfg.qsize = SND_QSIZE;
 	sq_cfg.tstmp_bgx_intf = 0;
-	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, *(u64 *)&sq_cfg);
+	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, *(uint64_t *)&sq_cfg);
 
 	/* Set threshold value for interrupt generation */
 	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_THRESH, qidx, sq->thresh);
 }
 
 /* Configures receive buffer descriptor ring */
-static void nicvf_rbdr_config(struct nicvf *nic, struct queue_set *qs,
-			      int qidx, bool enable)
+static void
+nicvf_rbdr_config(struct nicvf *nic, struct queue_set *qs, int qidx,
+    boolean_t enable)
 {
 	struct rbdr *rbdr;
 	struct rbdr_cfg rbdr_cfg;
@@ -680,8 +1378,8 @@ static void nicvf_rbdr_config(struct nicvf *nic, struct queue_set *qs,
 		return;
 
 	/* Set descriptor base address */
-	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_BASE,
-			      qidx, (u64)(rbdr->dmem.phys_base));
+	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_BASE, qidx,
+	    (uint64_t)(rbdr->dmem.phys_base));
 
 	/* Enable RBDR  & set queue size */
 	/* Buffer size should be in multiples of 128 bytes */
@@ -691,28 +1389,30 @@ static void nicvf_rbdr_config(struct nicvf *nic, struct queue_set *qs,
 	rbdr_cfg.qsize = RBDR_SIZE;
 	rbdr_cfg.avg_con = 0;
 	rbdr_cfg.lines = rbdr->dma_size / 128;
-	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG,
-			      qidx, *(u64 *)&rbdr_cfg);
+	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx,
+	    *(uint64_t *)&rbdr_cfg);
 
 	/* Notify HW */
-	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_DOOR,
-			      qidx, qs->rbdr_len - 1);
+	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_DOOR, qidx,
+	    qs->rbdr_len - 1);
 
 	/* Set threshold value for interrupt generation */
-	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_THRESH,
-			      qidx, rbdr->thresh - 1);
+	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_THRESH, qidx,
+	    rbdr->thresh - 1);
 }
 
 /* Requests PF to assign and enable Qset */
-void nicvf_qset_config(struct nicvf *nic, bool enable)
+void
+nicvf_qset_config(struct nicvf *nic, boolean_t enable)
 {
 	union nic_mbx mbx = {};
-	struct queue_set *qs = nic->qs;
+	struct queue_set *qs;
 	struct qs_cfg *qs_cfg;
 
-	if (!qs) {
-		netdev_warn(nic->netdev,
-			    "Qset is still not allocated, don't init queues\n");
+	qs = nic->qs;
+	if (qs == NULL) {
+		device_printf(nic->dev,
+		    "Qset is still not allocated, don't init queues\n");
 		return;
 	}
 
@@ -722,27 +1422,37 @@ void nicvf_qset_config(struct nicvf *nic, bool enable)
 	/* Send a mailbox msg to PF to config Qset */
 	mbx.qs.msg = NIC_MBOX_MSG_QS_CFG;
 	mbx.qs.num = qs->vnic_id;
-#ifdef VNIC_MULTI_QSET_SUPPORT
-	mbx.qs.sqs_count = nic->sqs_count;
-#endif
 
 	mbx.qs.cfg = 0;
 	qs_cfg = (struct qs_cfg *)&mbx.qs.cfg;
 	if (qs->enable) {
 		qs_cfg->ena = 1;
-#ifdef __BIG_ENDIAN
-		qs_cfg->be = 1;
-#endif
 		qs_cfg->vnic = qs->vnic_id;
 	}
 	nicvf_send_msg_to_pf(nic, &mbx);
 }
 
-static void nicvf_free_resources(struct nicvf *nic)
+static void
+nicvf_free_resources(struct nicvf *nic)
 {
 	int qidx;
-	struct queue_set *qs = nic->qs;
+	struct queue_set *qs;
 
+	qs = nic->qs;
+	/*
+	 * Remove QS error task first since it has to be dead
+	 * to safely free completion queue tasks.
+	 */
+	if (qs->qs_err_taskq != NULL) {
+		/* Shut down QS error tasks */
+		while (taskqueue_cancel(qs->qs_err_taskq,
+		    &qs->qs_err_task,  NULL) != 0) {
+			taskqueue_drain(qs->qs_err_taskq, &qs->qs_err_task);
+
+		}
+		taskqueue_free(qs->qs_err_taskq);
+		qs->qs_err_taskq = NULL;
+	}
 	/* Free receive buffer descriptor ring */
 	for (qidx = 0; qidx < qs->rbdr_cnt; qidx++)
 		nicvf_free_rbdr(nic, &qs->rbdr[qidx]);
@@ -756,52 +1466,57 @@ static void nicvf_free_resources(struct nicvf *nic)
 		nicvf_free_snd_queue(nic, &qs->sq[qidx]);
 }
 
-static int nicvf_alloc_resources(struct nicvf *nic)
+static int
+nicvf_alloc_resources(struct nicvf *nic)
 {
-	int qidx;
 	struct queue_set *qs = nic->qs;
+	int qidx;
 
 	/* Alloc receive buffer descriptor ring */
 	for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) {
 		if (nicvf_init_rbdr(nic, &qs->rbdr[qidx], qs->rbdr_len,
-				    DMA_BUFFER_LEN))
+				    DMA_BUFFER_LEN, qidx))
 			goto alloc_fail;
 	}
 
 	/* Alloc send queue */
 	for (qidx = 0; qidx < qs->sq_cnt; qidx++) {
-		if (nicvf_init_snd_queue(nic, &qs->sq[qidx], qs->sq_len))
+		if (nicvf_init_snd_queue(nic, &qs->sq[qidx], qs->sq_len, qidx))
 			goto alloc_fail;
 	}
 
 	/* Alloc completion queue */
 	for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
-		if (nicvf_init_cmp_queue(nic, &qs->cq[qidx], qs->cq_len))
+		if (nicvf_init_cmp_queue(nic, &qs->cq[qidx], qs->cq_len, qidx))
 			goto alloc_fail;
 	}
 
-	return 0;
+	/* Allocate QS error taskqueue */
+	TASK_INIT(&qs->qs_err_task, 0, nicvf_qs_err_task, nic);
+	qs->qs_err_taskq = taskqueue_create_fast("nicvf_qs_err_taskq", M_WAITOK,
+	    taskqueue_thread_enqueue, &qs->qs_err_taskq);
+	taskqueue_start_threads(&qs->qs_err_taskq, 1, PI_NET, "%s: qs_taskq",
+	    device_get_nameunit(nic->dev));
+
+	return (0);
 alloc_fail:
 	nicvf_free_resources(nic);
-	return -ENOMEM;
+	return (ENOMEM);
 }
 
-int nicvf_set_qset_resources(struct nicvf *nic)
+int
+nicvf_set_qset_resources(struct nicvf *nic)
 {
 	struct queue_set *qs;
 
-	qs = devm_kzalloc(&nic->pdev->dev, sizeof(*qs), GFP_KERNEL);
-	if (!qs)
-		return -ENOMEM;
+	qs = malloc(sizeof(*qs), M_NICVF, (M_ZERO | M_WAITOK));
 	nic->qs = qs;
 
 	/* Set count of each queue */
 	qs->rbdr_cnt = RBDR_CNT;
-#ifdef VNIC_RSS_SUPPORT
-	qs->rq_cnt = RCV_QUEUE_CNT;
-#else
+	/* With no RSS we stay with single RQ */
 	qs->rq_cnt = 1;
-#endif
+
 	qs->sq_cnt = SND_QUEUE_CNT;
 	qs->cq_cnt = CMP_QUEUE_CNT;
 
@@ -813,21 +1528,23 @@ int nicvf_set_qset_resources(struct nicvf *nic)
 	nic->rx_queues = qs->rq_cnt;
 	nic->tx_queues = qs->sq_cnt;
 
-	return 0;
+	return (0);
 }
 
-int nicvf_config_data_transfer(struct nicvf *nic, bool enable)
+int
+nicvf_config_data_transfer(struct nicvf *nic, boolean_t enable)
 {
-	bool disable = false;
-	struct queue_set *qs = nic->qs;
+	boolean_t disable = FALSE;
+	struct queue_set *qs;
 	int qidx;
 
-	if (!qs)
-		return 0;
+	qs = nic->qs;
+	if (qs == NULL)
+		return (0);
 
 	if (enable) {
-		if (nicvf_alloc_resources(nic))
-			return -ENOMEM;
+		if (nicvf_alloc_resources(nic) != 0)
+			return (ENOMEM);
 
 		for (qidx = 0; qidx < qs->sq_cnt; qidx++)
 			nicvf_snd_queue_config(nic, qs, qidx, enable);
@@ -850,42 +1567,48 @@ int nicvf_config_data_transfer(struct nicvf *nic, bool enable)
 		nicvf_free_resources(nic);
 	}
 
-	return 0;
+	return (0);
 }
 
-/* Get a free desc from SQ
+/*
+ * Get a free desc from SQ
  * returns descriptor ponter & descriptor number
  */
-static inline int nicvf_get_sq_desc(struct snd_queue *sq, int desc_cnt)
+static __inline int
+nicvf_get_sq_desc(struct snd_queue *sq, int desc_cnt)
 {
 	int qentry;
 
 	qentry = sq->tail;
-	atomic_sub(desc_cnt, &sq->free_cnt);
+	atomic_subtract_int(&sq->free_cnt, desc_cnt);
 	sq->tail += desc_cnt;
 	sq->tail &= (sq->dmem.q_len - 1);
 
-	return qentry;
+	return (qentry);
 }
 
 /* Free descriptor back to SQ for future use */
-void nicvf_put_sq_desc(struct snd_queue *sq, int desc_cnt)
+static void
+nicvf_put_sq_desc(struct snd_queue *sq, int desc_cnt)
 {
-	atomic_add(desc_cnt, &sq->free_cnt);
+
+	atomic_add_int(&sq->free_cnt, desc_cnt);
 	sq->head += desc_cnt;
 	sq->head &= (sq->dmem.q_len - 1);
 }
 
-static inline int nicvf_get_nxt_sqentry(struct snd_queue *sq, int qentry)
+static __inline int
+nicvf_get_nxt_sqentry(struct snd_queue *sq, int qentry)
 {
 	qentry++;
 	qentry &= (sq->dmem.q_len - 1);
-	return qentry;
+	return (qentry);
 }
 
-void nicvf_sq_enable(struct nicvf *nic, struct snd_queue *sq, int qidx)
+static void
+nicvf_sq_enable(struct nicvf *nic, struct snd_queue *sq, int qidx)
 {
-	u64 sq_cfg;
+	uint64_t sq_cfg;
 
 	sq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_CFG, qidx);
 	sq_cfg |= NICVF_SQ_EN;
@@ -894,23 +1617,24 @@ void nicvf_sq_enable(struct nicvf *nic, struct snd_queue *sq, int qidx)
 	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR, qidx, 0);
 }
 
-void nicvf_sq_disable(struct nicvf *nic, int qidx)
+static void
+nicvf_sq_disable(struct nicvf *nic, int qidx)
 {
-	u64 sq_cfg;
+	uint64_t sq_cfg;
 
 	sq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_CFG, qidx);
 	sq_cfg &= ~NICVF_SQ_EN;
 	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, sq_cfg);
 }
 
-void nicvf_sq_free_used_descs(struct net_device *netdev, struct snd_queue *sq,
-			      int qidx)
+static void
+nicvf_sq_free_used_descs(struct nicvf *nic, struct snd_queue *sq, int qidx)
 {
-	u64 head, tail;
-	struct sk_buff *skb;
-	struct nicvf *nic = netdev_priv(netdev);
+	uint64_t head, tail;
+	struct snd_buff *snd_buff;
 	struct sq_hdr_subdesc *hdr;
 
+	NICVF_TX_LOCK(sq);
 	head = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_HEAD, qidx) >> 4;
 	tail = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_TAIL, qidx) >> 4;
 	while (sq->head != head) {
@@ -919,44 +1643,29 @@ void nicvf_sq_free_used_descs(struct net_device *netdev, struct snd_queue *sq,
 			nicvf_put_sq_desc(sq, 1);
 			continue;
 		}
-		skb = (struct sk_buff *)sq->skbuff[sq->head];
-		if (skb)
-			dev_kfree_skb_any(skb);
-		atomic64_add(1, (atomic64_t *)&netdev->stats.tx_packets);
-		atomic64_add(hdr->tot_len,
-			     (atomic64_t *)&netdev->stats.tx_bytes);
+		snd_buff = &sq->snd_buff[sq->head];
+		if (snd_buff->mbuf != NULL) {
+			bus_dmamap_unload(sq->snd_buff_dmat, snd_buff->dmap);
+			m_freem(snd_buff->mbuf);
+			sq->snd_buff[sq->head].mbuf = NULL;
+		}
 		nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1);
 	}
+	NICVF_TX_UNLOCK(sq);
 }
 
-/* Get the number of SQ descriptors needed to xmit this skb */
-static int nicvf_sq_subdesc_required(struct nicvf *nic, struct sk_buff *skb)
-{
-	int subdesc_cnt = MIN_SQ_DESC_PER_PKT_XMIT;
-
-	if (skb_shinfo(skb)->gso_size) {
-		subdesc_cnt = nicvf_tso_count_subdescs(skb);
-		return subdesc_cnt;
-	}
-
-	if (skb_shinfo(skb)->nr_frags)
-		subdesc_cnt += skb_shinfo(skb)->nr_frags;
-
-	return subdesc_cnt;
-}
-
-/* Add SQ HEADER subdescriptor.
+/*
+ * Add SQ HEADER subdescriptor.
  * First subdescriptor for every send descriptor.
  */
-static inline void
+static __inline void
 nicvf_sq_add_hdr_subdesc(struct snd_queue *sq, int qentry,
-			 int subdesc_cnt, struct sk_buff *skb, int len)
+			 int subdesc_cnt, struct mbuf *mbuf, int len)
 {
-	int proto;
 	struct sq_hdr_subdesc *hdr;
 
 	hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, qentry);
-	sq->skbuff[qentry] = (u64)skb;
+	sq->snd_buff[qentry].mbuf = mbuf;
 
 	memset(hdr, 0, SND_QUEUE_DESC_SIZE);
 	hdr->subdesc_type = SQ_DESC_TYPE_HEADER;
@@ -966,32 +1675,15 @@ nicvf_sq_add_hdr_subdesc(struct snd_queue *sq, int qentry,
 	hdr->subdesc_cnt = subdesc_cnt;
 	hdr->tot_len = len;
 
-	/* Offload checksum calculation to HW */
-	if (skb->ip_summed == CHECKSUM_PARTIAL) {
-		hdr->csum_l3 = 1; /* Enable IP csum calculation */
-		hdr->l3_offset = skb_network_offset(skb);
-		hdr->l4_offset = skb_transport_offset(skb);
-
-		proto = ip_hdr(skb)->protocol;
-		switch (proto) {
-		case IPPROTO_TCP:
-			hdr->csum_l4 = SEND_L4_CSUM_TCP;
-			break;
-		case IPPROTO_UDP:
-			hdr->csum_l4 = SEND_L4_CSUM_UDP;
-			break;
-		case IPPROTO_SCTP:
-			hdr->csum_l4 = SEND_L4_CSUM_SCTP;
-			break;
-		}
-	}
+	/* ARM64TODO: Implement HW checksums calculation */
 }
 
-/* SQ GATHER subdescriptor
+/*
+ * SQ GATHER subdescriptor
  * Must follow HDR descriptor
  */
 static inline void nicvf_sq_add_gather_subdesc(struct snd_queue *sq, int qentry,
-					       int size, u64 data)
+					       int size, uint64_t data)
 {
 	struct sq_gather_subdesc *gather;
 
@@ -1005,173 +1697,152 @@ static inline void nicvf_sq_add_gather_subdesc(struct snd_queue *sq, int qentry,
 	gather->addr = data;
 }
 
-/* Append an skb to a SQ for packet transfer. */
-int nicvf_sq_append_skb(struct nicvf *nic, struct sk_buff *skb)
+/* Put an mbuf to a SQ for packet transfer. */
+static int
+nicvf_tx_mbuf_locked(struct snd_queue *sq, struct mbuf *mbuf)
 {
-	int i, size;
-	int subdesc_cnt;
-	int sq_num, qentry;
-	struct queue_set *qs;
-	struct snd_queue *sq;
+	bus_dma_segment_t segs[256];
+	struct snd_buff *snd_buff;
+	size_t seg;
+	int nsegs, qentry;
+	int subdesc_cnt = MIN_SQ_DESC_PER_PKT_XMIT - 1;
+	int err;
 
-	sq_num = skb_get_queue_mapping(skb);
-#ifdef VNIC_MULTI_QSET_SUPPORT
-	if (sq_num >= MAX_SND_QUEUES_PER_QS) {
-		/* Get secondary Qset's SQ structure */
-		i = sq_num / MAX_SND_QUEUES_PER_QS;
-		if (!nic->snicvf[i - 1]) {
-			netdev_warn(nic->netdev,
-				    "Secondary Qset#%d's ptr not initialized\n",
-				    i - 1);
-			return 1;
-		}
-		nic = (struct nicvf *)nic->snicvf[i - 1];
-		sq_num = sq_num % MAX_SND_QUEUES_PER_QS;
+	NICVF_TX_LOCK_ASSERT(sq);
+
+	if (sq->free_cnt == 0)
+		return (ENOBUFS);
+
+	snd_buff = &sq->snd_buff[sq->tail];
+
+	err = bus_dmamap_load_mbuf_sg(sq->snd_buff_dmat, snd_buff->dmap,
+	    mbuf, segs, &nsegs, BUS_DMA_NOWAIT);
+	if (err != 0) {
+		/* ARM64TODO: Add mbuf defragmenting if we lack maps */
+		return (err);
 	}
-#endif
 
-	qs = nic->qs;
-	sq = &qs->sq[sq_num];
+	/* Set how many subdescriptors is required */
+	subdesc_cnt += nsegs;
 
-	subdesc_cnt = nicvf_sq_subdesc_required(nic, skb);
-	if (subdesc_cnt > atomic_read(&sq->free_cnt))
-		goto append_fail;
+	if (subdesc_cnt > sq->free_cnt) {
+		/* ARM64TODO: Add mbuf defragmentation if we lack descriptors */
+		bus_dmamap_unload(sq->snd_buff_dmat, snd_buff->dmap);
+		return (ENOBUFS);
+	}
 
 	qentry = nicvf_get_sq_desc(sq, subdesc_cnt);
 
 	/* Add SQ header subdesc */
-	nicvf_sq_add_hdr_subdesc(sq, qentry, subdesc_cnt - 1, skb, skb->len);
+	nicvf_sq_add_hdr_subdesc(sq, qentry, subdesc_cnt - 1, mbuf,
+	    mbuf->m_pkthdr.len);
 
 	/* Add SQ gather subdescs */
-	qentry = nicvf_get_nxt_sqentry(sq, qentry);
-	size = skb_is_nonlinear(skb) ? skb_headlen(skb) : skb->len;
-	nicvf_sq_add_gather_subdesc(sq, qentry, size, virt_to_phys(skb->data));
-
-	/* Check for scattered buffer */
-	if (!skb_is_nonlinear(skb))
-		goto doorbell;
-
-	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
-		const struct skb_frag_struct *frag;
-
-		frag = &skb_shinfo(skb)->frags[i];
-
+	for (seg = 0; seg < nsegs; seg++) {
 		qentry = nicvf_get_nxt_sqentry(sq, qentry);
-		size = skb_frag_size(frag);
-		nicvf_sq_add_gather_subdesc(sq, qentry, size,
-					    virt_to_phys(
-					    skb_frag_address(frag)));
+		nicvf_sq_add_gather_subdesc(sq, qentry, segs[seg].ds_len,
+		    segs[seg].ds_addr);
 	}
 
-doorbell:
 	/* make sure all memory stores are done before ringing doorbell */
-	smp_wmb();
+	bus_dmamap_sync(sq->dmem.dmat, sq->dmem.dmap, BUS_DMASYNC_PREWRITE);
 
+	dprintf(sq->nic->dev, "%s: sq->idx: %d, subdesc_cnt: %d\n",
+	    __func__, sq->idx, subdesc_cnt);
 	/* Inform HW to xmit new packet */
-	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR,
-			      sq_num, subdesc_cnt);
-	return 1;
-
-append_fail:
-	/* Use original PCI dev for debug log */
-	nic = nic->pnicvf;
-	netdev_dbg(nic->netdev, "Not enough SQ descriptors to xmit pkt\n");
-	return 0;
+	nicvf_queue_reg_write(sq->nic, NIC_QSET_SQ_0_7_DOOR,
+	    sq->idx, subdesc_cnt);
+	return (0);
 }
 
-static inline unsigned frag_num(unsigned i)
+static __inline u_int
+frag_num(u_int i)
 {
-#ifdef __BIG_ENDIAN
-	return (i & ~3) + 3 - (i & 3);
+#if BYTE_ORDER == BIG_ENDIAN
+	return ((i & ~3) + 3 - (i & 3));
 #else
-	return i;
+	return (i);
 #endif
 }
 
-/* Returns SKB for a received packet */
-struct sk_buff *nicvf_get_rcv_skb(struct nicvf *nic, struct cqe_rx_t *cqe_rx)
+/* Returns MBUF for a received packet */
+struct mbuf *
+nicvf_get_rcv_mbuf(struct nicvf *nic, struct cqe_rx_t *cqe_rx)
 {
 	int frag;
 	int payload_len = 0;
-	struct sk_buff *skb = NULL;
-	struct sk_buff *skb_frag = NULL;
-	struct sk_buff *prev_frag = NULL;
-	u16 *rb_lens = NULL;
-	u64 *rb_ptrs = NULL;
+	struct mbuf *mbuf;
+	struct mbuf *mbuf_frag;
+	uint16_t *rb_lens = NULL;
+	uint64_t *rb_ptrs = NULL;
 
-	rb_lens = (void *)cqe_rx + (3 * sizeof(u64));
-	rb_ptrs = (void *)cqe_rx + (6 * sizeof(u64));
+	mbuf = NULL;
+	rb_lens = (uint16_t *)((uint8_t *)cqe_rx + (3 * sizeof(uint64_t)));
+	rb_ptrs = (uint64_t *)((uint8_t *)cqe_rx + (6 * sizeof(uint64_t)));
 
-	netdev_dbg(nic->netdev, "%s rb_cnt %d rb0_ptr %llx rb0_sz %d\n",
-		   __func__, cqe_rx->rb_cnt, cqe_rx->rb0_ptr, cqe_rx->rb0_sz);
+	dprintf(nic->dev, "%s rb_cnt %d rb0_ptr %lx rb0_sz %d\n",
+	    __func__, cqe_rx->rb_cnt, cqe_rx->rb0_ptr, cqe_rx->rb0_sz);
 
 	for (frag = 0; frag < cqe_rx->rb_cnt; frag++) {
 		payload_len = rb_lens[frag_num(frag)];
-		if (!frag) {
+		if (frag == 0) {
 			/* First fragment */
-			skb = nicvf_rb_ptr_to_skb(nic,
-						  *rb_ptrs - cqe_rx->align_pad,
-						  payload_len);
-			if (!skb)
-				return NULL;
-			skb_reserve(skb, cqe_rx->align_pad);
-			skb_put(skb, payload_len);
+			mbuf = nicvf_rb_ptr_to_mbuf(nic,
+			    (*rb_ptrs - cqe_rx->align_pad));
+			mbuf->m_len = payload_len;
+			mbuf->m_data += cqe_rx->align_pad;
+			if_setrcvif(mbuf, nic->ifp);
 		} else {
 			/* Add fragments */
-			skb_frag = nicvf_rb_ptr_to_skb(nic, *rb_ptrs,
-						       payload_len);
-			if (!skb_frag) {
-				dev_kfree_skb(skb);
-				return NULL;
-			}
-
-			if (!skb_shinfo(skb)->frag_list)
-				skb_shinfo(skb)->frag_list = skb_frag;
-			else
-				prev_frag->next = skb_frag;
-
-			prev_frag = skb_frag;
-			skb->len += payload_len;
-			skb->data_len += payload_len;
-			skb_frag->len = payload_len;
+			mbuf_frag = nicvf_rb_ptr_to_mbuf(nic, *rb_ptrs);
+			m_append(mbuf, payload_len, mbuf_frag->m_data);
+			m_freem(mbuf_frag);
 		}
 		/* Next buffer pointer */
 		rb_ptrs++;
 	}
-	return skb;
+
+	if (__predict_true(mbuf != NULL)) {
+		m_fixhdr(mbuf);
+		mbuf->m_pkthdr.flowid = cqe_rx->rq_idx;
+		M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE);
+	}
+
+	return (mbuf);
 }
 
 /* Enable interrupt */
-void nicvf_enable_intr(struct nicvf *nic, int int_type, int q_idx)
+void
+nicvf_enable_intr(struct nicvf *nic, int int_type, int q_idx)
 {
-	u64 reg_val;
+	uint64_t reg_val;
 
 	reg_val = nicvf_reg_read(nic, NIC_VF_ENA_W1S);
 
 	switch (int_type) {
 	case NICVF_INTR_CQ:
-		reg_val |= ((1ULL << q_idx) << NICVF_INTR_CQ_SHIFT);
+		reg_val |= ((1UL << q_idx) << NICVF_INTR_CQ_SHIFT);
 		break;
 	case NICVF_INTR_SQ:
-		reg_val |= ((1ULL << q_idx) << NICVF_INTR_SQ_SHIFT);
+		reg_val |= ((1UL << q_idx) << NICVF_INTR_SQ_SHIFT);
 		break;
 	case NICVF_INTR_RBDR:
-		reg_val |= ((1ULL << q_idx) << NICVF_INTR_RBDR_SHIFT);
+		reg_val |= ((1UL << q_idx) << NICVF_INTR_RBDR_SHIFT);
 		break;
 	case NICVF_INTR_PKT_DROP:
-		reg_val |= (1ULL << NICVF_INTR_PKT_DROP_SHIFT);
+		reg_val |= (1UL << NICVF_INTR_PKT_DROP_SHIFT);
 		break;
 	case NICVF_INTR_TCP_TIMER:
-		reg_val |= (1ULL << NICVF_INTR_TCP_TIMER_SHIFT);
+		reg_val |= (1UL << NICVF_INTR_TCP_TIMER_SHIFT);
 		break;
 	case NICVF_INTR_MBOX:
-		reg_val |= (1ULL << NICVF_INTR_MBOX_SHIFT);
+		reg_val |= (1UL << NICVF_INTR_MBOX_SHIFT);
 		break;
 	case NICVF_INTR_QS_ERR:
-		reg_val |= (1ULL << NICVF_INTR_QS_ERR_SHIFT);
+		reg_val |= (1UL << NICVF_INTR_QS_ERR_SHIFT);
 		break;
 	default:
-		netdev_err(nic->netdev,
+		device_printf(nic->dev,
 			   "Failed to enable interrupt: unknown type\n");
 		break;
 	}
@@ -1180,34 +1851,35 @@ void nicvf_enable_intr(struct nicvf *nic, int int_type, int q_idx)
 }
 
 /* Disable interrupt */
-void nicvf_disable_intr(struct nicvf *nic, int int_type, int q_idx)
+void
+nicvf_disable_intr(struct nicvf *nic, int int_type, int q_idx)
 {
-	u64 reg_val = 0;
+	uint64_t reg_val = 0;
 
 	switch (int_type) {
 	case NICVF_INTR_CQ:
-		reg_val |= ((1ULL << q_idx) << NICVF_INTR_CQ_SHIFT);
+		reg_val |= ((1UL << q_idx) << NICVF_INTR_CQ_SHIFT);
 		break;
 	case NICVF_INTR_SQ:
-		reg_val |= ((1ULL << q_idx) << NICVF_INTR_SQ_SHIFT);
+		reg_val |= ((1UL << q_idx) << NICVF_INTR_SQ_SHIFT);
 		break;
 	case NICVF_INTR_RBDR:
-		reg_val |= ((1ULL << q_idx) << NICVF_INTR_RBDR_SHIFT);
+		reg_val |= ((1UL << q_idx) << NICVF_INTR_RBDR_SHIFT);
 		break;
 	case NICVF_INTR_PKT_DROP:
-		reg_val |= (1ULL << NICVF_INTR_PKT_DROP_SHIFT);
+		reg_val |= (1UL << NICVF_INTR_PKT_DROP_SHIFT);
 		break;
 	case NICVF_INTR_TCP_TIMER:
-		reg_val |= (1ULL << NICVF_INTR_TCP_TIMER_SHIFT);
+		reg_val |= (1UL << NICVF_INTR_TCP_TIMER_SHIFT);
 		break;
 	case NICVF_INTR_MBOX:
-		reg_val |= (1ULL << NICVF_INTR_MBOX_SHIFT);
+		reg_val |= (1UL << NICVF_INTR_MBOX_SHIFT);
 		break;
 	case NICVF_INTR_QS_ERR:
-		reg_val |= (1ULL << NICVF_INTR_QS_ERR_SHIFT);
+		reg_val |= (1UL << NICVF_INTR_QS_ERR_SHIFT);
 		break;
 	default:
-		netdev_err(nic->netdev,
+		device_printf(nic->dev,
 			   "Failed to disable interrupt: unknown type\n");
 		break;
 	}
@@ -1216,34 +1888,35 @@ void nicvf_disable_intr(struct nicvf *nic, int int_type, int q_idx)
 }
 
 /* Clear interrupt */
-void nicvf_clear_intr(struct nicvf *nic, int int_type, int q_idx)
+void
+nicvf_clear_intr(struct nicvf *nic, int int_type, int q_idx)
 {
-	u64 reg_val = 0;
+	uint64_t reg_val = 0;
 
 	switch (int_type) {
 	case NICVF_INTR_CQ:
-		reg_val = ((1ULL << q_idx) << NICVF_INTR_CQ_SHIFT);
+		reg_val = ((1UL << q_idx) << NICVF_INTR_CQ_SHIFT);
 		break;
 	case NICVF_INTR_SQ:
-		reg_val = ((1ULL << q_idx) << NICVF_INTR_SQ_SHIFT);
+		reg_val = ((1UL << q_idx) << NICVF_INTR_SQ_SHIFT);
 		break;
 	case NICVF_INTR_RBDR:
-		reg_val = ((1ULL << q_idx) << NICVF_INTR_RBDR_SHIFT);
+		reg_val = ((1UL << q_idx) << NICVF_INTR_RBDR_SHIFT);
 		break;
 	case NICVF_INTR_PKT_DROP:
-		reg_val = (1ULL << NICVF_INTR_PKT_DROP_SHIFT);
+		reg_val = (1UL << NICVF_INTR_PKT_DROP_SHIFT);
 		break;
 	case NICVF_INTR_TCP_TIMER:
-		reg_val = (1ULL << NICVF_INTR_TCP_TIMER_SHIFT);
+		reg_val = (1UL << NICVF_INTR_TCP_TIMER_SHIFT);
 		break;
 	case NICVF_INTR_MBOX:
-		reg_val = (1ULL << NICVF_INTR_MBOX_SHIFT);
+		reg_val = (1UL << NICVF_INTR_MBOX_SHIFT);
 		break;
 	case NICVF_INTR_QS_ERR:
-		reg_val |= (1ULL << NICVF_INTR_QS_ERR_SHIFT);
+		reg_val |= (1UL << NICVF_INTR_QS_ERR_SHIFT);
 		break;
 	default:
-		netdev_err(nic->netdev,
+		device_printf(nic->dev,
 			   "Failed to clear interrupt: unknown type\n");
 		break;
 	}
@@ -1252,22 +1925,23 @@ void nicvf_clear_intr(struct nicvf *nic, int int_type, int q_idx)
 }
 
 /* Check if interrupt is enabled */
-int nicvf_is_intr_enabled(struct nicvf *nic, int int_type, int q_idx)
+int
+nicvf_is_intr_enabled(struct nicvf *nic, int int_type, int q_idx)
 {
-	u64 reg_val;
-	u64 mask = 0xff;
+	uint64_t reg_val;
+	uint64_t mask = 0xff;
 
 	reg_val = nicvf_reg_read(nic, NIC_VF_ENA_W1S);
 
 	switch (int_type) {
 	case NICVF_INTR_CQ:
-		mask = ((1ULL << q_idx) << NICVF_INTR_CQ_SHIFT);
+		mask = ((1UL << q_idx) << NICVF_INTR_CQ_SHIFT);
 		break;
 	case NICVF_INTR_SQ:
-		mask = ((1ULL << q_idx) << NICVF_INTR_SQ_SHIFT);
+		mask = ((1UL << q_idx) << NICVF_INTR_SQ_SHIFT);
 		break;
 	case NICVF_INTR_RBDR:
-		mask = ((1ULL << q_idx) << NICVF_INTR_RBDR_SHIFT);
+		mask = ((1UL << q_idx) << NICVF_INTR_RBDR_SHIFT);
 		break;
 	case NICVF_INTR_PKT_DROP:
 		mask = NICVF_INTR_PKT_DROP_MASK;
@@ -1282,7 +1956,7 @@ int nicvf_is_intr_enabled(struct nicvf *nic, int int_type, int q_idx)
 		mask = NICVF_INTR_QS_ERR_MASK;
 		break;
 	default:
-		netdev_err(nic->netdev,
+		device_printf(nic->dev,
 			   "Failed to check interrupt enable: unknown type\n");
 		break;
 	}
@@ -1290,7 +1964,8 @@ int nicvf_is_intr_enabled(struct nicvf *nic, int int_type, int q_idx)
 	return (reg_val & mask);
 }
 
-void nicvf_update_rq_stats(struct nicvf *nic, int rq_idx)
+void
+nicvf_update_rq_stats(struct nicvf *nic, int rq_idx)
 {
 	struct rcv_queue *rq;
 
@@ -1303,7 +1978,8 @@ void nicvf_update_rq_stats(struct nicvf *nic, int rq_idx)
 	rq->stats.pkts = GET_RQ_STATS(RQ_SQ_STATS_PKTS);
 }
 
-void nicvf_update_sq_stats(struct nicvf *nic, int sq_idx)
+void
+nicvf_update_sq_stats(struct nicvf *nic, int sq_idx)
 {
 	struct snd_queue *sq;
 
@@ -1317,23 +1993,18 @@ void nicvf_update_sq_stats(struct nicvf *nic, int sq_idx)
 }
 
 /* Check for errors in the receive cmp.queue entry */
-int nicvf_check_cqe_rx_errs(struct nicvf *nic,
-			    struct cmp_queue *cq, struct cqe_rx_t *cqe_rx)
+int
+nicvf_check_cqe_rx_errs(struct nicvf *nic, struct cmp_queue *cq,
+    struct cqe_rx_t *cqe_rx)
 {
 	struct nicvf_hw_stats *stats = &nic->hw_stats;
 	struct nicvf_drv_stats *drv_stats = &nic->drv_stats;
 
 	if (!cqe_rx->err_level && !cqe_rx->err_opcode) {
 		drv_stats->rx_frames_ok++;
-		return 0;
+		return (0);
 	}
 
-	if (netif_msg_rx_err(nic))
-		netdev_err(nic->netdev,
-			   "%s: RX error CQE err_level 0x%x err_opcode 0x%x\n",
-			   nic->netdev->name,
-			   cqe_rx->err_level, cqe_rx->err_opcode);
-
 	switch (cqe_rx->err_opcode) {
 	case CQ_RX_ERROP_RE_PARTIAL:
 		stats->rx_bgx_truncated_pkts++;
@@ -1409,19 +2080,20 @@ int nicvf_check_cqe_rx_errs(struct nicvf *nic,
 		break;
 	}
 
-	return 1;
+	return (1);
 }
 
 /* Check for errors in the send cmp.queue entry */
-int nicvf_check_cqe_tx_errs(struct nicvf *nic,
-			    struct cmp_queue *cq, struct cqe_send_t *cqe_tx)
+int
+nicvf_check_cqe_tx_errs(struct nicvf *nic, struct cmp_queue *cq,
+    struct cqe_send_t *cqe_tx)
 {
 	struct cmp_queue_stats *stats = &cq->stats;
 
 	switch (cqe_tx->send_status) {
 	case CQ_TX_ERROP_GOOD:
 		stats->tx.good++;
-		return 0;
+		return (0);
 	case CQ_TX_ERROP_DESC_FAULT:
 		stats->tx.desc_fault++;
 		break;
@@ -1463,5 +2135,5 @@ int nicvf_check_cqe_tx_errs(struct nicvf *nic,
 		break;
 	}
 
-	return 1;
+	return (1);
 }
diff --git a/sys/dev/vnic/nicvf_queues.h b/sys/dev/vnic/nicvf_queues.h
index d5842e928453..7dd197860933 100644
--- a/sys/dev/vnic/nicvf_queues.h
+++ b/sys/dev/vnic/nicvf_queues.h
@@ -28,16 +28,15 @@
  */
 
 #ifndef NICVF_QUEUES_H
-#define NICVF_QUEUES_H
+#define	NICVF_QUEUES_H
 
-#include <linux/netdevice.h>
 #include "q_struct.h"
 
-#define MAX_QUEUE_SET			128
-#define MAX_RCV_QUEUES_PER_QS		8
-#define MAX_RCV_BUF_DESC_RINGS_PER_QS	2
-#define MAX_SND_QUEUES_PER_QS		8
-#define MAX_CMP_QUEUES_PER_QS		8
+#define	MAX_QUEUE_SET			128
+#define	MAX_RCV_QUEUES_PER_QS		8
+#define	MAX_RCV_BUF_DESC_RINGS_PER_QS	2
+#define	MAX_SND_QUEUES_PER_QS		8
+#define	MAX_CMP_QUEUES_PER_QS		8
 
 /* VF's queue interrupt ranges */
 #define	NICVF_INTR_ID_CQ		0
@@ -47,100 +46,103 @@
 #define	NICVF_INTR_ID_QS_ERR		19
 
 #define	for_each_cq_irq(irq)	\
-	for (irq = NICVF_INTR_ID_CQ; irq < NICVF_INTR_ID_SQ; irq++)
+	for ((irq) = NICVF_INTR_ID_CQ; (irq) < NICVF_INTR_ID_SQ; (irq)++)
 #define	for_each_sq_irq(irq)	\
-	for (irq = NICVF_INTR_ID_SQ; irq < NICVF_INTR_ID_RBDR; irq++)
+	for ((irq) = NICVF_INTR_ID_SQ; (irq) < NICVF_INTR_ID_RBDR; (irq)++)
 #define	for_each_rbdr_irq(irq)	\
-	for (irq = NICVF_INTR_ID_RBDR; irq < NICVF_INTR_ID_MISC; irq++)
+	for ((irq) = NICVF_INTR_ID_RBDR; (irq) < NICVF_INTR_ID_MISC; (irq)++)
 
-#define RBDR_SIZE0		0ULL /* 8K entries */
-#define RBDR_SIZE1		1ULL /* 16K entries */
-#define RBDR_SIZE2		2ULL /* 32K entries */
-#define RBDR_SIZE3		3ULL /* 64K entries */
-#define RBDR_SIZE4		4ULL /* 126K entries */
-#define RBDR_SIZE5		5ULL /* 256K entries */
-#define RBDR_SIZE6		6ULL /* 512K entries */
+#define	RBDR_SIZE0		0UL /* 8K entries */
+#define	RBDR_SIZE1		1UL /* 16K entries */
+#define	RBDR_SIZE2		2UL /* 32K entries */
+#define	RBDR_SIZE3		3UL /* 64K entries */
+#define	RBDR_SIZE4		4UL /* 126K entries */
+#define	RBDR_SIZE5		5UL /* 256K entries */
+#define	RBDR_SIZE6		6UL /* 512K entries */
 
-#define SND_QUEUE_SIZE0		0ULL /* 1K entries */
-#define SND_QUEUE_SIZE1		1ULL /* 2K entries */
-#define SND_QUEUE_SIZE2		2ULL /* 4K entries */
-#define SND_QUEUE_SIZE3		3ULL /* 8K entries */
-#define SND_QUEUE_SIZE4		4ULL /* 16K entries */
-#define SND_QUEUE_SIZE5		5ULL /* 32K entries */
-#define SND_QUEUE_SIZE6		6ULL /* 64K entries */
+#define	SND_QUEUE_SIZE0		0UL /* 1K entries */
+#define	SND_QUEUE_SIZE1		1UL /* 2K entries */
+#define	SND_QUEUE_SIZE2		2UL /* 4K entries */
+#define	SND_QUEUE_SIZE3		3UL /* 8K entries */
+#define	SND_QUEUE_SIZE4		4UL /* 16K entries */
+#define	SND_QUEUE_SIZE5		5UL /* 32K entries */
+#define	SND_QUEUE_SIZE6		6UL /* 64K entries */
 
-#define CMP_QUEUE_SIZE0		0ULL /* 1K entries */
-#define CMP_QUEUE_SIZE1		1ULL /* 2K entries */
-#define CMP_QUEUE_SIZE2		2ULL /* 4K entries */
-#define CMP_QUEUE_SIZE3		3ULL /* 8K entries */
-#define CMP_QUEUE_SIZE4		4ULL /* 16K entries */
-#define CMP_QUEUE_SIZE5		5ULL /* 32K entries */
-#define CMP_QUEUE_SIZE6		6ULL /* 64K entries */
+#define	CMP_QUEUE_SIZE0		0UL /* 1K entries */
+#define	CMP_QUEUE_SIZE1		1UL /* 2K entries */
+#define	CMP_QUEUE_SIZE2		2UL /* 4K entries */
+#define	CMP_QUEUE_SIZE3		3UL /* 8K entries */
+#define	CMP_QUEUE_SIZE4		4UL /* 16K entries */
+#define	CMP_QUEUE_SIZE5		5UL /* 32K entries */
+#define	CMP_QUEUE_SIZE6		6UL /* 64K entries */
 
 /* Default queue count per QS, its lengths and threshold values */
-#define RBDR_CNT		1
-#define RCV_QUEUE_CNT		8
-#define SND_QUEUE_CNT		8
-#define CMP_QUEUE_CNT		8 /* Max of RCV and SND qcount */
+#define	RBDR_CNT		1
+#define	RCV_QUEUE_CNT		8
+#define	SND_QUEUE_CNT		8
+#define	CMP_QUEUE_CNT		8 /* Max of RCV and SND qcount */
 
-#define SND_QSIZE		SND_QUEUE_SIZE2
-#define SND_QUEUE_LEN		(1ULL << (SND_QSIZE + 10))
-#define MAX_SND_QUEUE_LEN	(1ULL << (SND_QUEUE_SIZE6 + 10))
-#define SND_QUEUE_THRESH	2ULL
-#define MIN_SQ_DESC_PER_PKT_XMIT	2
+#define	SND_QSIZE		SND_QUEUE_SIZE2
+#define	SND_QUEUE_LEN		(1UL << (SND_QSIZE + 10))
+#define	MAX_SND_QUEUE_LEN	(1UL << (SND_QUEUE_SIZE6 + 10))
+#define	SND_QUEUE_THRESH	2UL
+#define	MIN_SQ_DESC_PER_PKT_XMIT	2
 /* Since timestamp not enabled, otherwise 2 */
-#define MAX_CQE_PER_PKT_XMIT		1
+#define	MAX_CQE_PER_PKT_XMIT		1
 
-/* Keep CQ and SQ sizes same, if timestamping
+/*
+ * Keep CQ and SQ sizes same, if timestamping
  * is enabled this equation will change.
  */
-#define CMP_QSIZE		CMP_QUEUE_SIZE2
-#define CMP_QUEUE_LEN		(1ULL << (CMP_QSIZE + 10))
-#define CMP_QUEUE_CQE_THRESH	0
-#define CMP_QUEUE_TIMER_THRESH	220 /* 10usec */
+#define	CMP_QSIZE		CMP_QUEUE_SIZE2
+#define	CMP_QUEUE_LEN		(1UL << (CMP_QSIZE + 10))
+#define	CMP_QUEUE_CQE_THRESH	0
+#define	CMP_QUEUE_TIMER_THRESH	220 /* 10usec */
 
-#define RBDR_SIZE		RBDR_SIZE0
-#define RCV_BUF_COUNT		(1ULL << (RBDR_SIZE + 13))
-#define MAX_RCV_BUF_COUNT	(1ULL << (RBDR_SIZE6 + 13))
-#define RBDR_THRESH		(RCV_BUF_COUNT / 2)
-#define DMA_BUFFER_LEN		2048 /* In multiples of 128bytes */
-#define RCV_FRAG_LEN	(SKB_DATA_ALIGN(DMA_BUFFER_LEN + NET_SKB_PAD) + \
-			 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + \
-			 (NICVF_RCV_BUF_ALIGN_BYTES * 2))
-#define RCV_DATA_OFFSET		NICVF_RCV_BUF_ALIGN_BYTES
+#define	RBDR_SIZE		RBDR_SIZE0
+#define	RCV_BUF_COUNT		(1UL << (RBDR_SIZE + 13))
+#define	MAX_RCV_BUF_COUNT	(1UL << (RBDR_SIZE6 + 13))
+#define	RBDR_THRESH		(RCV_BUF_COUNT / 2)
+#define	DMA_BUFFER_LEN		2048 /* In multiples of 128bytes */
 
-#define MAX_CQES_FOR_TX		((SND_QUEUE_LEN / MIN_SQ_DESC_PER_PKT_XMIT) * \
-				 MAX_CQE_PER_PKT_XMIT)
+#define	MAX_CQES_FOR_TX		\
+    ((SND_QUEUE_LEN / MIN_SQ_DESC_PER_PKT_XMIT) * MAX_CQE_PER_PKT_XMIT)
 /* Calculate number of CQEs to reserve for all SQEs.
  * Its 1/256th level of CQ size.
  * '+ 1' to account for pipelining
  */
-#define RQ_CQ_DROP		((256 / (CMP_QUEUE_LEN / \
-				 (CMP_QUEUE_LEN - MAX_CQES_FOR_TX))) + 1)
+#define	RQ_CQ_DROP		\
+    ((256 / (CMP_QUEUE_LEN / (CMP_QUEUE_LEN - MAX_CQES_FOR_TX))) + 1)
 
 /* Descriptor size in bytes */
-#define SND_QUEUE_DESC_SIZE	16
-#define CMP_QUEUE_DESC_SIZE	512
+#define	SND_QUEUE_DESC_SIZE	16
+#define	CMP_QUEUE_DESC_SIZE	512
 
 /* Buffer / descriptor alignments */
-#define NICVF_RCV_BUF_ALIGN		7
-#define NICVF_RCV_BUF_ALIGN_BYTES	(1ULL << NICVF_RCV_BUF_ALIGN)
-#define NICVF_CQ_BASE_ALIGN_BYTES	512  /* 9 bits */
-#define NICVF_SQ_BASE_ALIGN_BYTES	128  /* 7 bits */
+#define	NICVF_RCV_BUF_ALIGN		7
+#define	NICVF_RCV_BUF_ALIGN_BYTES	(1UL << NICVF_RCV_BUF_ALIGN)
+#define	NICVF_CQ_BASE_ALIGN_BYTES	512  /* 9 bits */
+#define	NICVF_SQ_BASE_ALIGN_BYTES	128  /* 7 bits */
+
+#define	NICVF_ALIGNED_ADDR(addr, align_bytes)	\
+    roundup2((addr), (align_bytes))
+#define	NICVF_ADDR_ALIGN_LEN(addr, bytes)	\
+    (NICVF_ALIGNED_ADDR((addr), (bytes)) - (bytes))
+#define	NICVF_RCV_BUF_ALIGN_LEN(addr)		\
+    (NICVF_ALIGNED_ADDR((addr), NICVF_RCV_BUF_ALIGN_BYTES) - (addr))
+
+#define	NICVF_TXBUF_MAXSIZE	9212	/* Total max payload without TSO */
+#define	NICVF_TXBUF_NSEGS	256	/* Single command is at most 256 buffers
+					   (hdr + 255 subcmds) */
 
-#define NICVF_ALIGNED_ADDR(ADDR, ALIGN_BYTES)	ALIGN(ADDR, ALIGN_BYTES)
-#define NICVF_ADDR_ALIGN_LEN(ADDR, BYTES)\
-	(NICVF_ALIGNED_ADDR(ADDR, BYTES) - BYTES)
-#define NICVF_RCV_BUF_ALIGN_LEN(X)\
-	(NICVF_ALIGNED_ADDR(X, NICVF_RCV_BUF_ALIGN_BYTES) - X)
 
 /* Queue enable/disable */
-#define NICVF_SQ_EN		BIT_ULL(19)
+#define	NICVF_SQ_EN		(1UL << 19)
 
 /* Queue reset */
-#define NICVF_CQ_RESET		BIT_ULL(41)
-#define NICVF_SQ_RESET		BIT_ULL(17)
-#define NICVF_RBDR_RESET	BIT_ULL(43)
+#define	NICVF_CQ_RESET		(1UL << 41)
+#define	NICVF_SQ_RESET		(1UL << 17)
+#define	NICVF_RBDR_RESET	(1UL << 43)
 
 enum CQ_RX_ERRLVL_E {
 	CQ_ERRLVL_MAC,
@@ -203,22 +205,22 @@ enum CQ_TX_ERROP_E {
 
 struct cmp_queue_stats {
 	struct tx_stats {
-		u64 good;
-		u64 desc_fault;
-		u64 hdr_cons_err;
-		u64 subdesc_err;
-		u64 imm_size_oflow;
-		u64 data_seq_err;
-		u64 mem_seq_err;
-		u64 lock_viol;
-		u64 data_fault;
-		u64 tstmp_conflict;
-		u64 tstmp_timeout;
-		u64 mem_fault;
-		u64 csum_overlap;
-		u64 csum_overflow;
+		uint64_t good;
+		uint64_t desc_fault;
+		uint64_t hdr_cons_err;
+		uint64_t subdesc_err;
+		uint64_t imm_size_oflow;
+		uint64_t data_seq_err;
+		uint64_t mem_seq_err;
+		uint64_t lock_viol;
+		uint64_t data_fault;
+		uint64_t tstmp_conflict;
+		uint64_t tstmp_timeout;
+		uint64_t mem_fault;
+		uint64_t csum_overlap;
+		uint64_t csum_overflow;
 	} tx;
-} ____cacheline_aligned_in_smp;
+} __aligned(CACHE_LINE_SIZE);
 
 enum RQ_SQ_STATS {
 	RQ_SQ_STATS_OCTS,
@@ -226,141 +228,173 @@ enum RQ_SQ_STATS {
 };
 
 struct rx_tx_queue_stats {
-	u64	bytes;
-	u64	pkts;
-} ____cacheline_aligned_in_smp;
+	uint64_t	bytes;
+	uint64_t	pkts;
+} __aligned(CACHE_LINE_SIZE);
 
 struct q_desc_mem {
-	dma_addr_t	dma;
-	u64		size;
-	u16		q_len;
-	dma_addr_t	phys_base;
+	bus_dma_tag_t	dmat;
+	bus_dmamap_t	dmap;
 	void		*base;
-	void		*unalign_base;
+	bus_addr_t	phys_base;
+	uint64_t	size;
+	uint16_t	q_len;
 };
 
 struct rbdr {
-	bool		enable;
-	u32		dma_size;
-	u32		frag_len;
-	u32		thresh;		/* Threshold level for interrupt */
-	void		*desc;
-	u32		head;
-	u32		tail;
-	struct q_desc_mem   dmem;
-} ____cacheline_aligned_in_smp;
+	boolean_t		enable;
+	uint32_t		dma_size;
+	uint32_t		frag_len;
+	uint32_t		thresh;		/* Threshold level for interrupt */
+	void			*desc;
+	uint32_t		head;
+	uint32_t		tail;
+	struct q_desc_mem	dmem;
+
+	struct nicvf		*nic;
+	int			idx;
+
+	struct task		rbdr_task;
+	struct task		rbdr_task_nowait;
+	struct taskqueue	*rbdr_taskq;
+
+	bus_dma_tag_t		rbdr_buff_dmat;
+	bus_dmamap_t		*rbdr_buff_dmaps;
+} __aligned(CACHE_LINE_SIZE);
 
 struct rcv_queue {
-	bool		enable;
+	boolean_t	enable;
 	struct	rbdr	*rbdr_start;
 	struct	rbdr	*rbdr_cont;
-	bool		en_tcp_reassembly;
-	u8		cq_qs;  /* CQ's QS to which this RQ is assigned */
-	u8		cq_idx; /* CQ index (0 to 7) in the QS */
-	u8		cont_rbdr_qs;      /* Continue buffer ptrs - QS num */
-	u8		cont_qs_rbdr_idx;  /* RBDR idx in the cont QS */
-	u8		start_rbdr_qs;     /* First buffer ptrs - QS num */
-	u8		start_qs_rbdr_idx; /* RBDR idx in the above QS */
-	u8		caching;
+	boolean_t	en_tcp_reassembly;
+	uint8_t		cq_qs;  /* CQ's QS to which this RQ is assigned */
+	uint8_t		cq_idx; /* CQ index (0 to 7) in the QS */
+	uint8_t		cont_rbdr_qs;      /* Continue buffer ptrs - QS num */
+	uint8_t		cont_qs_rbdr_idx;  /* RBDR idx in the cont QS */
+	uint8_t		start_rbdr_qs;     /* First buffer ptrs - QS num */
+	uint8_t		start_qs_rbdr_idx; /* RBDR idx in the above QS */
+	uint8_t		caching;
 	struct		rx_tx_queue_stats stats;
-} ____cacheline_aligned_in_smp;
+} __aligned(CACHE_LINE_SIZE);
 
 struct cmp_queue {
-	bool		enable;
-	u16		thresh;
-	spinlock_t	lock;  /* lock to serialize processing CQEs */
-	void		*desc;
-	struct q_desc_mem   dmem;
+	boolean_t		enable;
+	uint16_t		thresh;
+
+	struct nicvf		*nic;
+	int			idx;	/* This queue index */
+
+	struct buf_ring		*rx_br;	/* Reception buf ring */
+	struct mtx		mtx;	/* lock to serialize processing CQEs */
+	char			mtx_name[32];
+
+	struct task		cmp_task;
+	struct taskqueue	*cmp_taskq;
+
+	void			*desc;
+	struct q_desc_mem	dmem;
 	struct cmp_queue_stats	stats;
-	int		irq;
-} ____cacheline_aligned_in_smp;
+	int			irq;
+} __aligned(CACHE_LINE_SIZE);
+
+struct snd_buff {
+	bus_dmamap_t	dmap;
+	struct mbuf	*mbuf;
+};
 
 struct snd_queue {
-	bool		enable;
-	u8		cq_qs;  /* CQ's QS to which this SQ is pointing */
-	u8		cq_idx; /* CQ index (0 to 7) in the above QS */
-	u16		thresh;
-	atomic_t	free_cnt;
-	u32		head;
-	u32		tail;
-	u64		*skbuff;
-	void		*desc;
+	boolean_t		enable;
+	uint8_t			cq_qs;  /* CQ's QS to which this SQ is pointing */
+	uint8_t			cq_idx; /* CQ index (0 to 7) in the above QS */
+	uint16_t		thresh;
+	volatile int		free_cnt;
+	uint32_t		head;
+	uint32_t		tail;
+	uint64_t		*skbuff;
+	void			*desc;
 
-	struct q_desc_mem   dmem;
+	struct nicvf		*nic;
+	int			idx;	/* This queue index */
+
+	bus_dma_tag_t		snd_buff_dmat;
+	struct snd_buff		*snd_buff;
+
+	struct buf_ring		*br;	/* Transmission buf ring */
+	struct mtx		mtx;
+	char			mtx_name[32];
+
+	struct task		snd_task;
+	struct taskqueue	*snd_taskq;
+
+	struct q_desc_mem	dmem;
 	struct rx_tx_queue_stats stats;
-} ____cacheline_aligned_in_smp;
+} __aligned(CACHE_LINE_SIZE);
 
 struct queue_set {
-	bool		enable;
-	bool		be_en;
-	u8		vnic_id;
-	u8		rq_cnt;
-	u8		cq_cnt;
-	u64		cq_len;
-	u8		sq_cnt;
-	u64		sq_len;
-	u8		rbdr_cnt;
-	u64		rbdr_len;
+	boolean_t	enable;
+	boolean_t	be_en;
+	uint8_t		vnic_id;
+	uint8_t		rq_cnt;
+	uint8_t		cq_cnt;
+	uint64_t	cq_len;
+	uint8_t		sq_cnt;
+	uint64_t	sq_len;
+	uint8_t		rbdr_cnt;
+	uint64_t	rbdr_len;
 	struct	rcv_queue	rq[MAX_RCV_QUEUES_PER_QS];
 	struct	cmp_queue	cq[MAX_CMP_QUEUES_PER_QS];
 	struct	snd_queue	sq[MAX_SND_QUEUES_PER_QS];
 	struct	rbdr		rbdr[MAX_RCV_BUF_DESC_RINGS_PER_QS];
-} ____cacheline_aligned_in_smp;
 
-#define GET_RBDR_DESC(RING, idx)\
-		(&(((struct rbdr_entry_t *)((RING)->desc))[idx]))
-#define GET_SQ_DESC(RING, idx)\
-		(&(((struct sq_hdr_subdesc *)((RING)->desc))[idx]))
-#define GET_CQ_DESC(RING, idx)\
-		(&(((union cq_desc_t *)((RING)->desc))[idx]))
+	struct task		qs_err_task;
+	struct taskqueue	*qs_err_taskq;
+} __aligned(CACHE_LINE_SIZE);
+
+#define	GET_RBDR_DESC(RING, idx)				\
+    (&(((struct rbdr_entry_t *)((RING)->desc))[(idx)]))
+#define	GET_SQ_DESC(RING, idx)					\
+    (&(((struct sq_hdr_subdesc *)((RING)->desc))[(idx)]))
+#define	GET_CQ_DESC(RING, idx)					\
+    (&(((union cq_desc_t *)((RING)->desc))[(idx)]))
 
 /* CQ status bits */
-#define	CQ_WR_FULL	BIT(26)
-#define	CQ_WR_DISABLE	BIT(25)
-#define	CQ_WR_FAULT	BIT(24)
+#define	CQ_WR_FUL	(1UL << 26)
+#define	CQ_WR_DISABLE	(1UL << 25)
+#define	CQ_WR_FAULT	(1UL << 24)
 #define	CQ_CQE_COUNT	(0xFFFF << 0)
 
-#define	CQ_ERR_MASK	(CQ_WR_FULL | CQ_WR_DISABLE | CQ_WR_FAULT)
+#define	CQ_ERR_MASK	(CQ_WR_FUL | CQ_WR_DISABLE | CQ_WR_FAULT)
 
-void nicvf_config_vlan_stripping(struct nicvf *nic,
-				 netdev_features_t features);
-int nicvf_set_qset_resources(struct nicvf *nic);
-int nicvf_config_data_transfer(struct nicvf *nic, bool enable);
-void nicvf_qset_config(struct nicvf *nic, bool enable);
-void nicvf_cmp_queue_config(struct nicvf *nic, struct queue_set *qs,
-			    int qidx, bool enable);
+#define	NICVF_TX_LOCK(sq)		mtx_lock(&(sq)->mtx)
+#define	NICVF_TX_TRYLOCK(sq)		mtx_trylock(&(sq)->mtx)
+#define	NICVF_TX_UNLOCK(sq)		mtx_unlock(&(sq)->mtx)
+#define	NICVF_TX_LOCK_ASSERT(sq)	mtx_assert(&(sq)->mtx, MA_OWNED)
 
-void nicvf_sq_enable(struct nicvf *nic, struct snd_queue *sq, int qidx);
-void nicvf_sq_disable(struct nicvf *nic, int qidx);
-void nicvf_put_sq_desc(struct snd_queue *sq, int desc_cnt);
-void nicvf_sq_free_used_descs(struct net_device *netdev,
-			      struct snd_queue *sq, int qidx);
-int nicvf_sq_append_skb(struct nicvf *nic, struct sk_buff *skb);
+#define	NICVF_CMP_LOCK(cq)		mtx_lock(&(cq)->mtx)
+#define	NICVF_CMP_UNLOCK(cq)		mtx_unlock(&(cq)->mtx)
 
-struct sk_buff *nicvf_get_rcv_skb(struct nicvf *nic, struct cqe_rx_t *cqe_rx);
-void nicvf_rbdr_task(unsigned long data);
-void nicvf_rbdr_work(struct work_struct *work);
+int nicvf_set_qset_resources(struct nicvf *);
+int nicvf_config_data_transfer(struct nicvf *, boolean_t);
+void nicvf_qset_config(struct nicvf *, boolean_t);
 
-void nicvf_enable_intr(struct nicvf *nic, int int_type, int q_idx);
-void nicvf_disable_intr(struct nicvf *nic, int int_type, int q_idx);
-void nicvf_clear_intr(struct nicvf *nic, int int_type, int q_idx);
-int nicvf_is_intr_enabled(struct nicvf *nic, int int_type, int q_idx);
+void nicvf_enable_intr(struct nicvf *, int, int);
+void nicvf_disable_intr(struct nicvf *, int, int);
+void nicvf_clear_intr(struct nicvf *, int, int);
+int nicvf_is_intr_enabled(struct nicvf *, int, int);
 
 /* Register access APIs */
-void nicvf_reg_write(struct nicvf *nic, u64 offset, u64 val);
-u64  nicvf_reg_read(struct nicvf *nic, u64 offset);
-void nicvf_qset_reg_write(struct nicvf *nic, u64 offset, u64 val);
-u64 nicvf_qset_reg_read(struct nicvf *nic, u64 offset);
-void nicvf_queue_reg_write(struct nicvf *nic, u64 offset,
-			   u64 qidx, u64 val);
-u64  nicvf_queue_reg_read(struct nicvf *nic,
-			  u64 offset, u64 qidx);
+void nicvf_reg_write(struct nicvf *, uint64_t, uint64_t);
+uint64_t nicvf_reg_read(struct nicvf *, uint64_t);
+void nicvf_qset_reg_write(struct nicvf *, uint64_t, uint64_t);
+uint64_t nicvf_qset_reg_read(struct nicvf *, uint64_t);
+void nicvf_queue_reg_write(struct nicvf *, uint64_t, uint64_t, uint64_t);
+uint64_t nicvf_queue_reg_read(struct nicvf *, uint64_t, uint64_t);
 
 /* Stats */
-void nicvf_update_rq_stats(struct nicvf *nic, int rq_idx);
-void nicvf_update_sq_stats(struct nicvf *nic, int sq_idx);
-int nicvf_check_cqe_rx_errs(struct nicvf *nic,
-			    struct cmp_queue *cq, struct cqe_rx_t *cqe_rx);
-int nicvf_check_cqe_tx_errs(struct nicvf *nic,
-			    struct cmp_queue *cq, struct cqe_send_t *cqe_tx);
+void nicvf_update_rq_stats(struct nicvf *, int);
+void nicvf_update_sq_stats(struct nicvf *, int);
+int nicvf_check_cqe_rx_errs(struct nicvf *, struct cmp_queue *,
+    struct cqe_rx_t *);
+int nicvf_check_cqe_tx_errs(struct nicvf *,struct cmp_queue *,
+    struct cqe_send_t *);
 #endif /* NICVF_QUEUES_H */
diff --git a/sys/dev/vnic/q_struct.h b/sys/dev/vnic/q_struct.h
index 6d33c7b097e2..471cc4fada49 100644
--- a/sys/dev/vnic/q_struct.h
+++ b/sys/dev/vnic/q_struct.h
@@ -30,6 +30,8 @@
 #ifndef Q_STRUCT_H
 #define Q_STRUCT_H
 
+#define	__LITTLE_ENDIAN_BITFIELD
+
 /* Load transaction types for reading segment bytes specified by
  * NIC_SEND_GATHER_S[LD_TYPE].
  */
@@ -191,185 +193,185 @@ enum cqe_rx_err_opcode {
 
 struct cqe_rx_t {
 #if defined(__BIG_ENDIAN_BITFIELD)
-	u64   cqe_type:4; /* W0 */
-	u64   stdn_fault:1;
-	u64   rsvd0:1;
-	u64   rq_qs:7;
-	u64   rq_idx:3;
-	u64   rsvd1:12;
-	u64   rss_alg:4;
-	u64   rsvd2:4;
-	u64   rb_cnt:4;
-	u64   vlan_found:1;
-	u64   vlan_stripped:1;
-	u64   vlan2_found:1;
-	u64   vlan2_stripped:1;
-	u64   l4_type:4;
-	u64   l3_type:4;
-	u64   l2_present:1;
-	u64   err_level:3;
-	u64   err_opcode:8;
+	uint64_t   cqe_type:4; /* W0 */
+	uint64_t   stdn_fault:1;
+	uint64_t   rsvd0:1;
+	uint64_t   rq_qs:7;
+	uint64_t   rq_idx:3;
+	uint64_t   rsvd1:12;
+	uint64_t   rss_alg:4;
+	uint64_t   rsvd2:4;
+	uint64_t   rb_cnt:4;
+	uint64_t   vlan_found:1;
+	uint64_t   vlan_stripped:1;
+	uint64_t   vlan2_found:1;
+	uint64_t   vlan2_stripped:1;
+	uint64_t   l4_type:4;
+	uint64_t   l3_type:4;
+	uint64_t   l2_present:1;
+	uint64_t   err_level:3;
+	uint64_t   err_opcode:8;
 
-	u64   pkt_len:16; /* W1 */
-	u64   l2_ptr:8;
-	u64   l3_ptr:8;
-	u64   l4_ptr:8;
-	u64   cq_pkt_len:8;
-	u64   align_pad:3;
-	u64   rsvd3:1;
-	u64   chan:12;
+	uint64_t   pkt_len:16; /* W1 */
+	uint64_t   l2_ptr:8;
+	uint64_t   l3_ptr:8;
+	uint64_t   l4_ptr:8;
+	uint64_t   cq_pkt_len:8;
+	uint64_t   align_pad:3;
+	uint64_t   rsvd3:1;
+	uint64_t   chan:12;
 
-	u64   rss_tag:32; /* W2 */
-	u64   vlan_tci:16;
-	u64   vlan_ptr:8;
-	u64   vlan2_ptr:8;
+	uint64_t   rss_tag:32; /* W2 */
+	uint64_t   vlan_tci:16;
+	uint64_t   vlan_ptr:8;
+	uint64_t   vlan2_ptr:8;
 
-	u64   rb3_sz:16; /* W3 */
-	u64   rb2_sz:16;
-	u64   rb1_sz:16;
-	u64   rb0_sz:16;
+	uint64_t   rb3_sz:16; /* W3 */
+	uint64_t   rb2_sz:16;
+	uint64_t   rb1_sz:16;
+	uint64_t   rb0_sz:16;
 
-	u64   rb7_sz:16; /* W4 */
-	u64   rb6_sz:16;
-	u64   rb5_sz:16;
-	u64   rb4_sz:16;
+	uint64_t   rb7_sz:16; /* W4 */
+	uint64_t   rb6_sz:16;
+	uint64_t   rb5_sz:16;
+	uint64_t   rb4_sz:16;
 
-	u64   rb11_sz:16; /* W5 */
-	u64   rb10_sz:16;
-	u64   rb9_sz:16;
-	u64   rb8_sz:16;
+	uint64_t   rb11_sz:16; /* W5 */
+	uint64_t   rb10_sz:16;
+	uint64_t   rb9_sz:16;
+	uint64_t   rb8_sz:16;
 #elif defined(__LITTLE_ENDIAN_BITFIELD)
-	u64   err_opcode:8;
-	u64   err_level:3;
-	u64   l2_present:1;
-	u64   l3_type:4;
-	u64   l4_type:4;
-	u64   vlan2_stripped:1;
-	u64   vlan2_found:1;
-	u64   vlan_stripped:1;
-	u64   vlan_found:1;
-	u64   rb_cnt:4;
-	u64   rsvd2:4;
-	u64   rss_alg:4;
-	u64   rsvd1:12;
-	u64   rq_idx:3;
-	u64   rq_qs:7;
-	u64   rsvd0:1;
-	u64   stdn_fault:1;
-	u64   cqe_type:4; /* W0 */
-	u64   chan:12;
-	u64   rsvd3:1;
-	u64   align_pad:3;
-	u64   cq_pkt_len:8;
-	u64   l4_ptr:8;
-	u64   l3_ptr:8;
-	u64   l2_ptr:8;
-	u64   pkt_len:16; /* W1 */
-	u64   vlan2_ptr:8;
-	u64   vlan_ptr:8;
-	u64   vlan_tci:16;
-	u64   rss_tag:32; /* W2 */
-	u64   rb0_sz:16;
-	u64   rb1_sz:16;
-	u64   rb2_sz:16;
-	u64   rb3_sz:16; /* W3 */
-	u64   rb4_sz:16;
-	u64   rb5_sz:16;
-	u64   rb6_sz:16;
-	u64   rb7_sz:16; /* W4 */
-	u64   rb8_sz:16;
-	u64   rb9_sz:16;
-	u64   rb10_sz:16;
-	u64   rb11_sz:16; /* W5 */
+	uint64_t   err_opcode:8;
+	uint64_t   err_level:3;
+	uint64_t   l2_present:1;
+	uint64_t   l3_type:4;
+	uint64_t   l4_type:4;
+	uint64_t   vlan2_stripped:1;
+	uint64_t   vlan2_found:1;
+	uint64_t   vlan_stripped:1;
+	uint64_t   vlan_found:1;
+	uint64_t   rb_cnt:4;
+	uint64_t   rsvd2:4;
+	uint64_t   rss_alg:4;
+	uint64_t   rsvd1:12;
+	uint64_t   rq_idx:3;
+	uint64_t   rq_qs:7;
+	uint64_t   rsvd0:1;
+	uint64_t   stdn_fault:1;
+	uint64_t   cqe_type:4; /* W0 */
+	uint64_t   chan:12;
+	uint64_t   rsvd3:1;
+	uint64_t   align_pad:3;
+	uint64_t   cq_pkt_len:8;
+	uint64_t   l4_ptr:8;
+	uint64_t   l3_ptr:8;
+	uint64_t   l2_ptr:8;
+	uint64_t   pkt_len:16; /* W1 */
+	uint64_t   vlan2_ptr:8;
+	uint64_t   vlan_ptr:8;
+	uint64_t   vlan_tci:16;
+	uint64_t   rss_tag:32; /* W2 */
+	uint64_t   rb0_sz:16;
+	uint64_t   rb1_sz:16;
+	uint64_t   rb2_sz:16;
+	uint64_t   rb3_sz:16; /* W3 */
+	uint64_t   rb4_sz:16;
+	uint64_t   rb5_sz:16;
+	uint64_t   rb6_sz:16;
+	uint64_t   rb7_sz:16; /* W4 */
+	uint64_t   rb8_sz:16;
+	uint64_t   rb9_sz:16;
+	uint64_t   rb10_sz:16;
+	uint64_t   rb11_sz:16; /* W5 */
 #endif
-	u64   rb0_ptr:64;
-	u64   rb1_ptr:64;
-	u64   rb2_ptr:64;
-	u64   rb3_ptr:64;
-	u64   rb4_ptr:64;
-	u64   rb5_ptr:64;
-	u64   rb6_ptr:64;
-	u64   rb7_ptr:64;
-	u64   rb8_ptr:64;
-	u64   rb9_ptr:64;
-	u64   rb10_ptr:64;
-	u64   rb11_ptr:64;
+	uint64_t   rb0_ptr:64;
+	uint64_t   rb1_ptr:64;
+	uint64_t   rb2_ptr:64;
+	uint64_t   rb3_ptr:64;
+	uint64_t   rb4_ptr:64;
+	uint64_t   rb5_ptr:64;
+	uint64_t   rb6_ptr:64;
+	uint64_t   rb7_ptr:64;
+	uint64_t   rb8_ptr:64;
+	uint64_t   rb9_ptr:64;
+	uint64_t   rb10_ptr:64;
+	uint64_t   rb11_ptr:64;
 };
 
 struct cqe_rx_tcp_err_t {
 #if defined(__BIG_ENDIAN_BITFIELD)
-	u64   cqe_type:4; /* W0 */
-	u64   rsvd0:60;
+	uint64_t   cqe_type:4; /* W0 */
+	uint64_t   rsvd0:60;
 
-	u64   rsvd1:4; /* W1 */
-	u64   partial_first:1;
-	u64   rsvd2:27;
-	u64   rbdr_bytes:8;
-	u64   rsvd3:24;
+	uint64_t   rsvd1:4; /* W1 */
+	uint64_t   partial_first:1;
+	uint64_t   rsvd2:27;
+	uint64_t   rbdr_bytes:8;
+	uint64_t   rsvd3:24;
 #elif defined(__LITTLE_ENDIAN_BITFIELD)
-	u64   rsvd0:60;
-	u64   cqe_type:4;
+	uint64_t   rsvd0:60;
+	uint64_t   cqe_type:4;
 
-	u64   rsvd3:24;
-	u64   rbdr_bytes:8;
-	u64   rsvd2:27;
-	u64   partial_first:1;
-	u64   rsvd1:4;
+	uint64_t   rsvd3:24;
+	uint64_t   rbdr_bytes:8;
+	uint64_t   rsvd2:27;
+	uint64_t   partial_first:1;
+	uint64_t   rsvd1:4;
 #endif
 };
 
 struct cqe_rx_tcp_t {
 #if defined(__BIG_ENDIAN_BITFIELD)
-	u64   cqe_type:4; /* W0 */
-	u64   rsvd0:52;
-	u64   cq_tcp_status:8;
+	uint64_t   cqe_type:4; /* W0 */
+	uint64_t   rsvd0:52;
+	uint64_t   cq_tcp_status:8;
 
-	u64   rsvd1:32; /* W1 */
-	u64   tcp_cntx_bytes:8;
-	u64   rsvd2:8;
-	u64   tcp_err_bytes:16;
+	uint64_t   rsvd1:32; /* W1 */
+	uint64_t   tcp_cntx_bytes:8;
+	uint64_t   rsvd2:8;
+	uint64_t   tcp_err_bytes:16;
 #elif defined(__LITTLE_ENDIAN_BITFIELD)
-	u64   cq_tcp_status:8;
-	u64   rsvd0:52;
-	u64   cqe_type:4; /* W0 */
+	uint64_t   cq_tcp_status:8;
+	uint64_t   rsvd0:52;
+	uint64_t   cqe_type:4; /* W0 */
 
-	u64   tcp_err_bytes:16;
-	u64   rsvd2:8;
-	u64   tcp_cntx_bytes:8;
-	u64   rsvd1:32; /* W1 */
+	uint64_t   tcp_err_bytes:16;
+	uint64_t   rsvd2:8;
+	uint64_t   tcp_cntx_bytes:8;
+	uint64_t   rsvd1:32; /* W1 */
 #endif
 };
 
 struct cqe_send_t {
 #if defined(__BIG_ENDIAN_BITFIELD)
-	u64   cqe_type:4; /* W0 */
-	u64   rsvd0:4;
-	u64   sqe_ptr:16;
-	u64   rsvd1:4;
-	u64   rsvd2:10;
-	u64   sq_qs:7;
-	u64   sq_idx:3;
-	u64   rsvd3:8;
-	u64   send_status:8;
+	uint64_t   cqe_type:4; /* W0 */
+	uint64_t   rsvd0:4;
+	uint64_t   sqe_ptr:16;
+	uint64_t   rsvd1:4;
+	uint64_t   rsvd2:10;
+	uint64_t   sq_qs:7;
+	uint64_t   sq_idx:3;
+	uint64_t   rsvd3:8;
+	uint64_t   send_status:8;
 
-	u64   ptp_timestamp:64; /* W1 */
+	uint64_t   ptp_timestamp:64; /* W1 */
 #elif defined(__LITTLE_ENDIAN_BITFIELD)
-	u64   send_status:8;
-	u64   rsvd3:8;
-	u64   sq_idx:3;
-	u64   sq_qs:7;
-	u64   rsvd2:10;
-	u64   rsvd1:4;
-	u64   sqe_ptr:16;
-	u64   rsvd0:4;
-	u64   cqe_type:4; /* W0 */
+	uint64_t   send_status:8;
+	uint64_t   rsvd3:8;
+	uint64_t   sq_idx:3;
+	uint64_t   sq_qs:7;
+	uint64_t   rsvd2:10;
+	uint64_t   rsvd1:4;
+	uint64_t   sqe_ptr:16;
+	uint64_t   rsvd0:4;
+	uint64_t   cqe_type:4; /* W0 */
 
-	u64   ptp_timestamp:64; /* W1 */
+	uint64_t   ptp_timestamp:64; /* W1 */
 #endif
 };
 
 union cq_desc_t {
-	u64    u[64];
+	uint64_t    u[64];
 	struct cqe_send_t snd_hdr;
 	struct cqe_rx_t rx_hdr;
 	struct cqe_rx_tcp_t rx_tcp_hdr;
@@ -378,54 +380,54 @@ union cq_desc_t {
 
 struct rbdr_entry_t {
 #if defined(__BIG_ENDIAN_BITFIELD)
-	u64   rsvd0:15;
-	u64   buf_addr:42;
-	u64   cache_align:7;
+	uint64_t   rsvd0:15;
+	uint64_t   buf_addr:42;
+	uint64_t   cache_align:7;
 #elif defined(__LITTLE_ENDIAN_BITFIELD)
-	u64   cache_align:7;
-	u64   buf_addr:42;
-	u64   rsvd0:15;
+	uint64_t   cache_align:7;
+	uint64_t   buf_addr:42;
+	uint64_t   rsvd0:15;
 #endif
 };
 
 /* TCP reassembly context */
 struct rbe_tcp_cnxt_t {
 #if defined(__BIG_ENDIAN_BITFIELD)
-	u64   tcp_pkt_cnt:12;
-	u64   rsvd1:4;
-	u64   align_hdr_bytes:4;
-	u64   align_ptr_bytes:4;
-	u64   ptr_bytes:16;
-	u64   rsvd2:24;
-	u64   cqe_type:4;
-	u64   rsvd0:54;
-	u64   tcp_end_reason:2;
-	u64   tcp_status:4;
+	uint64_t   tcp_pkt_cnt:12;
+	uint64_t   rsvd1:4;
+	uint64_t   align_hdr_bytes:4;
+	uint64_t   align_ptr_bytes:4;
+	uint64_t   ptr_bytes:16;
+	uint64_t   rsvd2:24;
+	uint64_t   cqe_type:4;
+	uint64_t   rsvd0:54;
+	uint64_t   tcp_end_reason:2;
+	uint64_t   tcp_status:4;
 #elif defined(__LITTLE_ENDIAN_BITFIELD)
-	u64   tcp_status:4;
-	u64   tcp_end_reason:2;
-	u64   rsvd0:54;
-	u64   cqe_type:4;
-	u64   rsvd2:24;
-	u64   ptr_bytes:16;
-	u64   align_ptr_bytes:4;
-	u64   align_hdr_bytes:4;
-	u64   rsvd1:4;
-	u64   tcp_pkt_cnt:12;
+	uint64_t   tcp_status:4;
+	uint64_t   tcp_end_reason:2;
+	uint64_t   rsvd0:54;
+	uint64_t   cqe_type:4;
+	uint64_t   rsvd2:24;
+	uint64_t   ptr_bytes:16;
+	uint64_t   align_ptr_bytes:4;
+	uint64_t   align_hdr_bytes:4;
+	uint64_t   rsvd1:4;
+	uint64_t   tcp_pkt_cnt:12;
 #endif
 };
 
 /* Always Big endian */
 struct rx_hdr_t {
-	u64   opaque:32;
-	u64   rss_flow:8;
-	u64   skip_length:6;
-	u64   disable_rss:1;
-	u64   disable_tcp_reassembly:1;
-	u64   nodrop:1;
-	u64   dest_alg:2;
-	u64   rsvd0:2;
-	u64   dest_rq:11;
+	uint64_t   opaque:32;
+	uint64_t   rss_flow:8;
+	uint64_t   skip_length:6;
+	uint64_t   disable_rss:1;
+	uint64_t   disable_tcp_reassembly:1;
+	uint64_t   nodrop:1;
+	uint64_t   dest_alg:2;
+	uint64_t   rsvd0:2;
+	uint64_t   dest_rq:11;
 };
 
 enum send_l4_csum_type {
@@ -472,247 +474,247 @@ enum sq_subdesc_type {
 
 struct sq_crc_subdesc {
 #if defined(__BIG_ENDIAN_BITFIELD)
-	u64    rsvd1:32;
-	u64    crc_ival:32;
-	u64    subdesc_type:4;
-	u64    crc_alg:2;
-	u64    rsvd0:10;
-	u64    crc_insert_pos:16;
-	u64    hdr_start:16;
-	u64    crc_len:16;
+	uint64_t    rsvd1:32;
+	uint64_t    crc_ival:32;
+	uint64_t    subdesc_type:4;
+	uint64_t    crc_alg:2;
+	uint64_t    rsvd0:10;
+	uint64_t    crc_insert_pos:16;
+	uint64_t    hdr_start:16;
+	uint64_t    crc_len:16;
 #elif defined(__LITTLE_ENDIAN_BITFIELD)
-	u64    crc_len:16;
-	u64    hdr_start:16;
-	u64    crc_insert_pos:16;
-	u64    rsvd0:10;
-	u64    crc_alg:2;
-	u64    subdesc_type:4;
-	u64    crc_ival:32;
-	u64    rsvd1:32;
+	uint64_t    crc_len:16;
+	uint64_t    hdr_start:16;
+	uint64_t    crc_insert_pos:16;
+	uint64_t    rsvd0:10;
+	uint64_t    crc_alg:2;
+	uint64_t    subdesc_type:4;
+	uint64_t    crc_ival:32;
+	uint64_t    rsvd1:32;
 #endif
 };
 
 struct sq_gather_subdesc {
 #if defined(__BIG_ENDIAN_BITFIELD)
-	u64    subdesc_type:4; /* W0 */
-	u64    ld_type:2;
-	u64    rsvd0:42;
-	u64    size:16;
+	uint64_t    subdesc_type:4; /* W0 */
+	uint64_t    ld_type:2;
+	uint64_t    rsvd0:42;
+	uint64_t    size:16;
 
-	u64    rsvd1:15; /* W1 */
-	u64    addr:49;
+	uint64_t    rsvd1:15; /* W1 */
+	uint64_t    addr:49;
 #elif defined(__LITTLE_ENDIAN_BITFIELD)
-	u64    size:16;
-	u64    rsvd0:42;
-	u64    ld_type:2;
-	u64    subdesc_type:4; /* W0 */
+	uint64_t    size:16;
+	uint64_t    rsvd0:42;
+	uint64_t    ld_type:2;
+	uint64_t    subdesc_type:4; /* W0 */
 
-	u64    addr:49;
-	u64    rsvd1:15; /* W1 */
+	uint64_t    addr:49;
+	uint64_t    rsvd1:15; /* W1 */
 #endif
 };
 
 /* SQ immediate subdescriptor */
 struct sq_imm_subdesc {
 #if defined(__BIG_ENDIAN_BITFIELD)
-	u64    subdesc_type:4; /* W0 */
-	u64    rsvd0:46;
-	u64    len:14;
+	uint64_t    subdesc_type:4; /* W0 */
+	uint64_t    rsvd0:46;
+	uint64_t    len:14;
 
-	u64    data:64; /* W1 */
+	uint64_t    data:64; /* W1 */
 #elif defined(__LITTLE_ENDIAN_BITFIELD)
-	u64    len:14;
-	u64    rsvd0:46;
-	u64    subdesc_type:4; /* W0 */
+	uint64_t    len:14;
+	uint64_t    rsvd0:46;
+	uint64_t    subdesc_type:4; /* W0 */
 
-	u64    data:64; /* W1 */
+	uint64_t    data:64; /* W1 */
 #endif
 };
 
 struct sq_mem_subdesc {
 #if defined(__BIG_ENDIAN_BITFIELD)
-	u64    subdesc_type:4; /* W0 */
-	u64    mem_alg:4;
-	u64    mem_dsz:2;
-	u64    wmem:1;
-	u64    rsvd0:21;
-	u64    offset:32;
+	uint64_t    subdesc_type:4; /* W0 */
+	uint64_t    mem_alg:4;
+	uint64_t    mem_dsz:2;
+	uint64_t    wmem:1;
+	uint64_t    rsvd0:21;
+	uint64_t    offset:32;
 
-	u64    rsvd1:15; /* W1 */
-	u64    addr:49;
+	uint64_t    rsvd1:15; /* W1 */
+	uint64_t    addr:49;
 #elif defined(__LITTLE_ENDIAN_BITFIELD)
-	u64    offset:32;
-	u64    rsvd0:21;
-	u64    wmem:1;
-	u64    mem_dsz:2;
-	u64    mem_alg:4;
-	u64    subdesc_type:4; /* W0 */
+	uint64_t    offset:32;
+	uint64_t    rsvd0:21;
+	uint64_t    wmem:1;
+	uint64_t    mem_dsz:2;
+	uint64_t    mem_alg:4;
+	uint64_t    subdesc_type:4; /* W0 */
 
-	u64    addr:49;
-	u64    rsvd1:15; /* W1 */
+	uint64_t    addr:49;
+	uint64_t    rsvd1:15; /* W1 */
 #endif
 };
 
 struct sq_hdr_subdesc {
 #if defined(__BIG_ENDIAN_BITFIELD)
-	u64    subdesc_type:4;
-	u64    tso:1;
-	u64    post_cqe:1; /* Post CQE on no error also */
-	u64    dont_send:1;
-	u64    tstmp:1;
-	u64    subdesc_cnt:8;
-	u64    csum_l4:2;
-	u64    csum_l3:1;
-	u64    rsvd0:5;
-	u64    l4_offset:8;
-	u64    l3_offset:8;
-	u64    rsvd1:4;
-	u64    tot_len:20; /* W0 */
+	uint64_t    subdesc_type:4;
+	uint64_t    tso:1;
+	uint64_t    post_cqe:1; /* Post CQE on no error also */
+	uint64_t    dont_send:1;
+	uint64_t    tstmp:1;
+	uint64_t    subdesc_cnt:8;
+	uint64_t    csum_l4:2;
+	uint64_t    csum_l3:1;
+	uint64_t    rsvd0:5;
+	uint64_t    l4_offset:8;
+	uint64_t    l3_offset:8;
+	uint64_t    rsvd1:4;
+	uint64_t    tot_len:20; /* W0 */
 
-	u64    tso_sdc_cont:8;
-	u64    tso_sdc_first:8;
-	u64    tso_l4_offset:8;
-	u64    tso_flags_last:12;
-	u64    tso_flags_first:12;
-	u64    rsvd2:2;
-	u64    tso_max_paysize:14; /* W1 */
+	uint64_t    tso_sdc_cont:8;
+	uint64_t    tso_sdc_first:8;
+	uint64_t    tso_l4_offset:8;
+	uint64_t    tso_flags_last:12;
+	uint64_t    tso_flags_first:12;
+	uint64_t    rsvd2:2;
+	uint64_t    tso_max_paysize:14; /* W1 */
 #elif defined(__LITTLE_ENDIAN_BITFIELD)
-	u64    tot_len:20;
-	u64    rsvd1:4;
-	u64    l3_offset:8;
-	u64    l4_offset:8;
-	u64    rsvd0:5;
-	u64    csum_l3:1;
-	u64    csum_l4:2;
-	u64    subdesc_cnt:8;
-	u64    tstmp:1;
-	u64    dont_send:1;
-	u64    post_cqe:1; /* Post CQE on no error also */
-	u64    tso:1;
-	u64    subdesc_type:4; /* W0 */
+	uint64_t    tot_len:20;
+	uint64_t    rsvd1:4;
+	uint64_t    l3_offset:8;
+	uint64_t    l4_offset:8;
+	uint64_t    rsvd0:5;
+	uint64_t    csum_l3:1;
+	uint64_t    csum_l4:2;
+	uint64_t    subdesc_cnt:8;
+	uint64_t    tstmp:1;
+	uint64_t    dont_send:1;
+	uint64_t    post_cqe:1; /* Post CQE on no error also */
+	uint64_t    tso:1;
+	uint64_t    subdesc_type:4; /* W0 */
 
-	u64    tso_max_paysize:14;
-	u64    rsvd2:2;
-	u64    tso_flags_first:12;
-	u64    tso_flags_last:12;
-	u64    tso_l4_offset:8;
-	u64    tso_sdc_first:8;
-	u64    tso_sdc_cont:8; /* W1 */
+	uint64_t    tso_max_paysize:14;
+	uint64_t    rsvd2:2;
+	uint64_t    tso_flags_first:12;
+	uint64_t    tso_flags_last:12;
+	uint64_t    tso_l4_offset:8;
+	uint64_t    tso_sdc_first:8;
+	uint64_t    tso_sdc_cont:8; /* W1 */
 #endif
 };
 
 /* Queue config register formats */
 struct rq_cfg {
 #if defined(__BIG_ENDIAN_BITFIELD)
-	u64 reserved_2_63:62;
-	u64 ena:1;
-	u64 tcp_ena:1;
+	uint64_t reserved_2_63:62;
+	uint64_t ena:1;
+	uint64_t tcp_ena:1;
 #elif defined(__LITTLE_ENDIAN_BITFIELD)
-	u64 tcp_ena:1;
-	u64 ena:1;
-	u64 reserved_2_63:62;
+	uint64_t tcp_ena:1;
+	uint64_t ena:1;
+	uint64_t reserved_2_63:62;
 #endif
 };
 
 struct cq_cfg {
 #if defined(__BIG_ENDIAN_BITFIELD)
-	u64 reserved_43_63:21;
-	u64 ena:1;
-	u64 reset:1;
-	u64 caching:1;
-	u64 reserved_35_39:5;
-	u64 qsize:3;
-	u64 reserved_25_31:7;
-	u64 avg_con:9;
-	u64 reserved_0_15:16;
+	uint64_t reserved_43_63:21;
+	uint64_t ena:1;
+	uint64_t reset:1;
+	uint64_t caching:1;
+	uint64_t reserved_35_39:5;
+	uint64_t qsize:3;
+	uint64_t reserved_25_31:7;
+	uint64_t avg_con:9;
+	uint64_t reserved_0_15:16;
 #elif defined(__LITTLE_ENDIAN_BITFIELD)
-	u64 reserved_0_15:16;
-	u64 avg_con:9;
-	u64 reserved_25_31:7;
-	u64 qsize:3;
-	u64 reserved_35_39:5;
-	u64 caching:1;
-	u64 reset:1;
-	u64 ena:1;
-	u64 reserved_43_63:21;
+	uint64_t reserved_0_15:16;
+	uint64_t avg_con:9;
+	uint64_t reserved_25_31:7;
+	uint64_t qsize:3;
+	uint64_t reserved_35_39:5;
+	uint64_t caching:1;
+	uint64_t reset:1;
+	uint64_t ena:1;
+	uint64_t reserved_43_63:21;
 #endif
 };
 
 struct sq_cfg {
 #if defined(__BIG_ENDIAN_BITFIELD)
-	u64 reserved_20_63:44;
-	u64 ena:1;
-	u64 reserved_18_18:1;
-	u64 reset:1;
-	u64 ldwb:1;
-	u64 reserved_11_15:5;
-	u64 qsize:3;
-	u64 reserved_3_7:5;
-	u64 tstmp_bgx_intf:3;
+	uint64_t reserved_20_63:44;
+	uint64_t ena:1;
+	uint64_t reserved_18_18:1;
+	uint64_t reset:1;
+	uint64_t ldwb:1;
+	uint64_t reserved_11_15:5;
+	uint64_t qsize:3;
+	uint64_t reserved_3_7:5;
+	uint64_t tstmp_bgx_intf:3;
 #elif defined(__LITTLE_ENDIAN_BITFIELD)
-	u64 tstmp_bgx_intf:3;
-	u64 reserved_3_7:5;
-	u64 qsize:3;
-	u64 reserved_11_15:5;
-	u64 ldwb:1;
-	u64 reset:1;
-	u64 reserved_18_18:1;
-	u64 ena:1;
-	u64 reserved_20_63:44;
+	uint64_t tstmp_bgx_intf:3;
+	uint64_t reserved_3_7:5;
+	uint64_t qsize:3;
+	uint64_t reserved_11_15:5;
+	uint64_t ldwb:1;
+	uint64_t reset:1;
+	uint64_t reserved_18_18:1;
+	uint64_t ena:1;
+	uint64_t reserved_20_63:44;
 #endif
 };
 
 struct rbdr_cfg {
 #if defined(__BIG_ENDIAN_BITFIELD)
-	u64 reserved_45_63:19;
-	u64 ena:1;
-	u64 reset:1;
-	u64 ldwb:1;
-	u64 reserved_36_41:6;
-	u64 qsize:4;
-	u64 reserved_25_31:7;
-	u64 avg_con:9;
-	u64 reserved_12_15:4;
-	u64 lines:12;
+	uint64_t reserved_45_63:19;
+	uint64_t ena:1;
+	uint64_t reset:1;
+	uint64_t ldwb:1;
+	uint64_t reserved_36_41:6;
+	uint64_t qsize:4;
+	uint64_t reserved_25_31:7;
+	uint64_t avg_con:9;
+	uint64_t reserved_12_15:4;
+	uint64_t lines:12;
 #elif defined(__LITTLE_ENDIAN_BITFIELD)
-	u64 lines:12;
-	u64 reserved_12_15:4;
-	u64 avg_con:9;
-	u64 reserved_25_31:7;
-	u64 qsize:4;
-	u64 reserved_36_41:6;
-	u64 ldwb:1;
-	u64 reset:1;
-	u64 ena: 1;
-	u64 reserved_45_63:19;
+	uint64_t lines:12;
+	uint64_t reserved_12_15:4;
+	uint64_t avg_con:9;
+	uint64_t reserved_25_31:7;
+	uint64_t qsize:4;
+	uint64_t reserved_36_41:6;
+	uint64_t ldwb:1;
+	uint64_t reset:1;
+	uint64_t ena: 1;
+	uint64_t reserved_45_63:19;
 #endif
 };
 
 struct qs_cfg {
 #if defined(__BIG_ENDIAN_BITFIELD)
-	u64 reserved_32_63:32;
-	u64 ena:1;
-	u64 reserved_27_30:4;
-	u64 sq_ins_ena:1;
-	u64 sq_ins_pos:6;
-	u64 lock_ena:1;
-	u64 lock_viol_cqe_ena:1;
-	u64 send_tstmp_ena:1;
-	u64 be:1;
-	u64 reserved_7_15:9;
-	u64 vnic:7;
+	uint64_t reserved_32_63:32;
+	uint64_t ena:1;
+	uint64_t reserved_27_30:4;
+	uint64_t sq_ins_ena:1;
+	uint64_t sq_ins_pos:6;
+	uint64_t lock_ena:1;
+	uint64_t lock_viol_cqe_ena:1;
+	uint64_t send_tstmp_ena:1;
+	uint64_t be:1;
+	uint64_t reserved_7_15:9;
+	uint64_t vnic:7;
 #elif defined(__LITTLE_ENDIAN_BITFIELD)
-	u64 vnic:7;
-	u64 reserved_7_15:9;
-	u64 be:1;
-	u64 send_tstmp_ena:1;
-	u64 lock_viol_cqe_ena:1;
-	u64 lock_ena:1;
-	u64 sq_ins_pos:6;
-	u64 sq_ins_ena:1;
-	u64 reserved_27_30:4;
-	u64 ena:1;
-	u64 reserved_32_63:32;
+	uint64_t vnic:7;
+	uint64_t reserved_7_15:9;
+	uint64_t be:1;
+	uint64_t send_tstmp_ena:1;
+	uint64_t lock_viol_cqe_ena:1;
+	uint64_t lock_ena:1;
+	uint64_t sq_ins_pos:6;
+	uint64_t sq_ins_ena:1;
+	uint64_t reserved_27_30:4;
+	uint64_t ena:1;
+	uint64_t reserved_32_63:32;
 #endif
 };
 
diff --git a/sys/dev/vnic/thunder_bgx.c b/sys/dev/vnic/thunder_bgx.c
index 3dabd59668c8..32d97c871cf6 100644
--- a/sys/dev/vnic/thunder_bgx.c
+++ b/sys/dev/vnic/thunder_bgx.c
@@ -26,155 +26,274 @@
  * $FreeBSD$
  *
  */
+#include "opt_platform.h"
 
-#include <linux/acpi.h>
-#include <linux/module.h>
-#include <linux/interrupt.h>
-#include <linux/pci.h>
-#include <linux/netdevice.h>
-#include <linux/etherdevice.h>
-#include <linux/phy.h>
-#include <linux/of.h>
-#include <linux/of_mdio.h>
-#include <linux/of_net.h>
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
 
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bitset.h>
+#include <sys/bitstring.h>
+#include <sys/bus.h>
+#include <sys/endian.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/rman.h>
+#include <sys/pciio.h>
+#include <sys/pcpu.h>
+#include <sys/proc.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
+#include <sys/cpuset.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+
+#include <net/ethernet.h>
+#include <net/if.h>
+#include <net/if_media.h>
+
+#include <machine/bus.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#include "thunder_bgx.h"
+#include "thunder_bgx_var.h"
 #include "nic_reg.h"
 #include "nic.h"
-#include "thunder_bgx.h"
+
+#include "lmac_if.h"
+
+#define	THUNDER_BGX_DEVSTR	"ThunderX BGX Ethernet I/O Interface"
+
+static MALLOC_DEFINE(M_BGX, "thunder_bgx", "ThunderX BGX dynamic memory");
+
+#define BGX_NODE_ID_MASK	0x1
+#define BGX_NODE_ID_SHIFT	24
 
 #define DRV_NAME	"thunder-BGX"
 #define DRV_VERSION	"1.0"
 
-struct lmac {
-	struct bgx		*bgx;
-	int			dmac;
-	u8			mac[ETH_ALEN];
-	bool			link_up;
-	int			lmacid; /* ID within BGX */
-	int			lmacid_bd; /* ID on board */
-	struct net_device       netdev;
-	struct phy_device       *phydev;
-	unsigned int            last_duplex;
-	unsigned int            last_link;
-	unsigned int            last_speed;
-	bool			is_sgmii;
-	struct delayed_work	dwork;
-	struct workqueue_struct *check_link;
-};
-
-struct bgx {
-	u8			bgx_id;
-	u8			qlm_mode;
-	struct	lmac		lmac[MAX_LMAC_PER_BGX];
-	int			lmac_count;
-	int                     lmac_type;
-	int                     lane_to_sds;
-	int			use_training;
-	void __iomem		*reg_base;
-	struct pci_dev		*pdev;
-};
+static int bgx_init_phy(struct bgx *);
 
 static struct bgx *bgx_vnic[MAX_BGX_THUNDER];
-static int lmac_count; /* Total no of LMACs in system */
+static int lmac_count __unused; /* Total no of LMACs in system */
 
 static int bgx_xaui_check_link(struct lmac *lmac);
+static void bgx_get_qlm_mode(struct bgx *);
+static void bgx_init_hw(struct bgx *);
+static int bgx_lmac_enable(struct bgx *, uint8_t);
+static void bgx_lmac_disable(struct bgx *, uint8_t);
 
-/* Supported devices */
-static const struct pci_device_id bgx_id_table[] = {
-	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_BGX) },
-	{ 0, }  /* end of table */
+static int thunder_bgx_probe(device_t);
+static int thunder_bgx_attach(device_t);
+static int thunder_bgx_detach(device_t);
+
+static device_method_t thunder_bgx_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		thunder_bgx_probe),
+	DEVMETHOD(device_attach,	thunder_bgx_attach),
+	DEVMETHOD(device_detach,	thunder_bgx_detach),
+
+	DEVMETHOD_END,
 };
 
-MODULE_AUTHOR("Cavium Inc");
-MODULE_DESCRIPTION("Cavium Thunder BGX/MAC Driver");
-MODULE_VERSION(DRV_VERSION);
-MODULE_DEVICE_TABLE(pci, bgx_id_table);
+static driver_t thunder_bgx_driver = {
+	"bgx",
+	thunder_bgx_methods,
+	sizeof(struct lmac),
+};
 
-/* The Cavium ThunderX network controller can *only* be found in SoCs
- * containing the ThunderX ARM64 CPU implementation.  All accesses to the device
- * registers on this platform are implicitly strongly ordered with respect
- * to memory accesses. So writeq_relaxed() and readq_relaxed() are safe to use
- * with no memory barriers in this driver.  The readq()/writeq() functions add
- * explicit ordering operation which in this case are redundant, and only
- * add overhead.
- */
+static devclass_t thunder_bgx_devclass;
+
+DRIVER_MODULE(thunder_bgx, pci, thunder_bgx_driver, thunder_bgx_devclass, 0, 0);
+MODULE_DEPEND(thunder_bgx, pci, 1, 1, 1);
+MODULE_DEPEND(thunder_bgx, ether, 1, 1, 1);
+MODULE_DEPEND(thunder_bgx, octeon_mdio, 1, 1, 1);
+
+static int
+thunder_bgx_probe(device_t dev)
+{
+	uint16_t vendor_id;
+	uint16_t device_id;
+
+	vendor_id = pci_get_vendor(dev);
+	device_id = pci_get_device(dev);
+
+	if (vendor_id == PCI_VENDOR_ID_CAVIUM &&
+	    device_id == PCI_DEVICE_ID_THUNDER_BGX) {
+		device_set_desc(dev, THUNDER_BGX_DEVSTR);
+		return (BUS_PROBE_DEFAULT);
+	}
+
+	return (ENXIO);
+}
+
+static int
+thunder_bgx_attach(device_t dev)
+{
+	struct bgx *bgx;
+	uint8_t lmac;
+	int err;
+	int rid;
+
+	bgx = malloc(sizeof(*bgx), M_BGX, (M_WAITOK | M_ZERO));
+	bgx->dev = dev;
+	/* Enable bus mastering */
+	pci_enable_busmaster(dev);
+	/* Allocate resources - configuration registers */
+	rid = PCIR_BAR(PCI_CFG_REG_BAR_NUM);
+	bgx->reg_base = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
+	    RF_ACTIVE);
+	if (bgx->reg_base == NULL) {
+		device_printf(dev, "Could not allocate CSR memory space\n");
+		err = ENXIO;
+		goto err_disable_device;
+	}
+
+	bgx->bgx_id = (rman_get_start(bgx->reg_base) >> BGX_NODE_ID_SHIFT) &
+	    BGX_NODE_ID_MASK;
+	bgx->bgx_id += nic_get_node_id(bgx->reg_base) * MAX_BGX_PER_CN88XX;
+
+	bgx_vnic[bgx->bgx_id] = bgx;
+	bgx_get_qlm_mode(bgx);
+
+	err = bgx_init_phy(bgx);
+	if (err != 0)
+		goto err_free_res;
+
+	bgx_init_hw(bgx);
+
+	/* Enable all LMACs */
+	for (lmac = 0; lmac < bgx->lmac_count; lmac++) {
+		err = bgx_lmac_enable(bgx, lmac);
+		if (err) {
+			device_printf(dev, "BGX%d failed to enable lmac%d\n",
+				bgx->bgx_id, lmac);
+			goto err_free_res;
+		}
+	}
+
+	return (0);
+
+err_free_res:
+	bgx_vnic[bgx->bgx_id] = NULL;
+	bus_release_resource(dev, SYS_RES_MEMORY,
+	    rman_get_rid(bgx->reg_base), bgx->reg_base);
+err_disable_device:
+	free(bgx, M_BGX);
+	pci_disable_busmaster(dev);
+
+	return (err);
+}
+
+static int
+thunder_bgx_detach(device_t dev)
+{
+	struct lmac *lmac;
+	struct bgx *bgx;
+	uint8_t lmacid;
+
+	lmac = device_get_softc(dev);
+	bgx = lmac->bgx;
+	/* Disable all LMACs */
+	for (lmacid = 0; lmacid < bgx->lmac_count; lmacid++)
+		bgx_lmac_disable(bgx, lmacid);
+
+	return (0);
+}
 
 /* Register read/write APIs */
-static u64 bgx_reg_read(struct bgx *bgx, u8 lmac, u64 offset)
+static uint64_t
+bgx_reg_read(struct bgx *bgx, uint8_t lmac, uint64_t offset)
 {
-	void __iomem *addr = bgx->reg_base + ((u32)lmac << 20) + offset;
+	bus_space_handle_t addr;
 
-	return readq_relaxed(addr);
+	addr = ((uint32_t)lmac << 20) + offset;
+
+	return (bus_read_8(bgx->reg_base, addr));
 }
 
-static void bgx_reg_write(struct bgx *bgx, u8 lmac, u64 offset, u64 val)
+static void
+bgx_reg_write(struct bgx *bgx, uint8_t lmac, uint64_t offset, uint64_t val)
 {
-	void __iomem *addr = bgx->reg_base + ((u32)lmac << 20) + offset;
+	bus_space_handle_t addr;
 
-	writeq_relaxed(val, addr);
+	addr = ((uint32_t)lmac << 20) + offset;
+
+	bus_write_8(bgx->reg_base, addr, val);
 }
 
-static void bgx_reg_modify(struct bgx *bgx, u8 lmac, u64 offset, u64 val)
+static void
+bgx_reg_modify(struct bgx *bgx, uint8_t lmac, uint64_t offset, uint64_t val)
 {
-	void __iomem *addr = bgx->reg_base + ((u32)lmac << 20) + offset;
+	bus_space_handle_t addr;
 
-	writeq_relaxed(val | readq_relaxed(addr), addr);
+	addr = ((uint32_t)lmac << 20) + offset;
+
+	bus_write_8(bgx->reg_base, addr, val | bus_read_8(bgx->reg_base, addr));
 }
 
-static int bgx_poll_reg(struct bgx *bgx, u8 lmac, u64 reg, u64 mask, bool zero)
+static int
+bgx_poll_reg(struct bgx *bgx, uint8_t lmac, uint64_t reg, uint64_t mask,
+    boolean_t zero)
 {
 	int timeout = 100;
-	u64 reg_val;
+	uint64_t reg_val;
 
 	while (timeout) {
 		reg_val = bgx_reg_read(bgx, lmac, reg);
 		if (zero && !(reg_val & mask))
-			return 0;
+			return (0);
 		if (!zero && (reg_val & mask))
-			return 0;
-		usleep_range(1000, 2000);
+			return (0);
+
+		DELAY(1000);
 		timeout--;
 	}
-	return 1;
+	return (ETIMEDOUT);
 }
 
 /* Return number of BGX present in HW */
-unsigned bgx_get_map(int node)
+u_int
+bgx_get_map(int node)
 {
 	int i;
-	unsigned map = 0;
+	u_int map = 0;
 
 	for (i = 0; i < MAX_BGX_PER_CN88XX; i++) {
 		if (bgx_vnic[(node * MAX_BGX_PER_CN88XX) + i])
 			map |= (1 << i);
 	}
 
-	return map;
+	return (map);
 }
-EXPORT_SYMBOL(bgx_get_map);
 
 /* Return number of LMAC configured for this BGX */
-int bgx_get_lmac_count(int node, int bgx_idx)
+int
+bgx_get_lmac_count(int node, int bgx_idx)
 {
 	struct bgx *bgx;
 
 	bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx];
-	if (bgx)
-		return bgx->lmac_count;
+	if (bgx != NULL)
+		return (bgx->lmac_count);
 
-	return 0;
+	return (0);
 }
-EXPORT_SYMBOL(bgx_get_lmac_count);
 
 /* Returns the current link status of LMAC */
-void bgx_get_lmac_link_state(int node, int bgx_idx, int lmacid, void *status)
+void
+bgx_get_lmac_link_state(int node, int bgx_idx, int lmacid, void *status)
 {
 	struct bgx_link_status *link = (struct bgx_link_status *)status;
 	struct bgx *bgx;
 	struct lmac *lmac;
 
 	bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx];
-	if (!bgx)
+	if (bgx == NULL)
 		return;
 
 	lmac = &bgx->lmac[lmacid];
@@ -182,36 +301,36 @@ void bgx_get_lmac_link_state(int node, int bgx_idx, int lmacid, void *status)
 	link->duplex = lmac->last_duplex;
 	link->speed = lmac->last_speed;
 }
-EXPORT_SYMBOL(bgx_get_lmac_link_state);
 
-const u8 *bgx_get_lmac_mac(int node, int bgx_idx, int lmacid)
+const uint8_t
+*bgx_get_lmac_mac(int node, int bgx_idx, int lmacid)
 {
 	struct bgx *bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx];
 
-	if (bgx)
-		return bgx->lmac[lmacid].mac;
+	if (bgx != NULL)
+		return (bgx->lmac[lmacid].mac);
 
-	return NULL;
+	return (NULL);
 }
-EXPORT_SYMBOL(bgx_get_lmac_mac);
 
-void bgx_set_lmac_mac(int node, int bgx_idx, int lmacid, const u8 *mac)
+void
+bgx_set_lmac_mac(int node, int bgx_idx, int lmacid, const uint8_t *mac)
 {
 	struct bgx *bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx];
 
-	if (!bgx)
+	if (bgx == NULL)
 		return;
 
-	ether_addr_copy(bgx->lmac[lmacid].mac, mac);
+	memcpy(bgx->lmac[lmacid].mac, mac, ETHER_ADDR_LEN);
 }
-EXPORT_SYMBOL(bgx_set_lmac_mac);
 
-static void bgx_sgmii_change_link_state(struct lmac *lmac)
+static void
+bgx_sgmii_change_link_state(struct lmac *lmac)
 {
 	struct bgx *bgx = lmac->bgx;
-	u64 cmr_cfg;
-	u64 port_cfg = 0;
-	u64 misc_ctl = 0;
+	uint64_t cmr_cfg;
+	uint64_t port_cfg = 0;
+	uint64_t misc_ctl = 0;
 
 	cmr_cfg = bgx_reg_read(bgx, lmac->lmacid, BGX_CMRX_CFG);
 	cmr_cfg &= ~CMR_EN;
@@ -274,31 +393,37 @@ static void bgx_sgmii_change_link_state(struct lmac *lmac)
 	bgx_reg_write(bgx, lmac->lmacid, BGX_CMRX_CFG, cmr_cfg);
 }
 
-static void bgx_lmac_handler(struct net_device *netdev)
+static void
+bgx_lmac_handler(void *arg)
 {
-	struct lmac *lmac = container_of(netdev, struct lmac, netdev);
-	struct phy_device *phydev = lmac->phydev;
+	struct lmac *lmac;
+	int link, duplex, speed;
 	int link_changed = 0;
+	int err;
 
-	if (!lmac)
-		return;
+	lmac = (struct lmac *)arg;
 
-	if (!phydev->link && lmac->last_link)
+	err = LMAC_MEDIA_STATUS(lmac->phy_if_dev, lmac->lmacid,
+	    &link, &duplex, &speed);
+	if (err != 0)
+		goto out;
+
+	if (!link && lmac->last_link)
 		link_changed = -1;
 
-	if (phydev->link &&
-	    (lmac->last_duplex != phydev->duplex ||
-	     lmac->last_link != phydev->link ||
-	     lmac->last_speed != phydev->speed)) {
+	if (link &&
+	    (lmac->last_duplex != duplex ||
+	     lmac->last_link != link ||
+	     lmac->last_speed != speed)) {
 			link_changed = 1;
 	}
 
-	lmac->last_link = phydev->link;
-	lmac->last_speed = phydev->speed;
-	lmac->last_duplex = phydev->duplex;
+	lmac->last_link = link;
+	lmac->last_speed = speed;
+	lmac->last_duplex = duplex;
 
 	if (!link_changed)
-		return;
+		goto out;
 
 	if (link_changed > 0)
 		lmac->link_up = true;
@@ -309,49 +434,54 @@ static void bgx_lmac_handler(struct net_device *netdev)
 		bgx_sgmii_change_link_state(lmac);
 	else
 		bgx_xaui_check_link(lmac);
+
+out:
+	callout_reset(&lmac->check_link, hz * 2, bgx_lmac_handler, lmac);
 }
 
-u64 bgx_get_rx_stats(int node, int bgx_idx, int lmac, int idx)
+uint64_t
+bgx_get_rx_stats(int node, int bgx_idx, int lmac, int idx)
 {
 	struct bgx *bgx;
 
 	bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx];
-	if (!bgx)
-		return 0;
+	if (bgx == NULL)
+		return (0);
 
 	if (idx > 8)
-		lmac = 0;
-	return bgx_reg_read(bgx, lmac, BGX_CMRX_RX_STAT0 + (idx * 8));
+		lmac = (0);
+	return (bgx_reg_read(bgx, lmac, BGX_CMRX_RX_STAT0 + (idx * 8)));
 }
-EXPORT_SYMBOL(bgx_get_rx_stats);
 
-u64 bgx_get_tx_stats(int node, int bgx_idx, int lmac, int idx)
+uint64_t
+bgx_get_tx_stats(int node, int bgx_idx, int lmac, int idx)
 {
 	struct bgx *bgx;
 
 	bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx];
-	if (!bgx)
-		return 0;
+	if (bgx == NULL)
+		return (0);
 
-	return bgx_reg_read(bgx, lmac, BGX_CMRX_TX_STAT0 + (idx * 8));
+	return (bgx_reg_read(bgx, lmac, BGX_CMRX_TX_STAT0 + (idx * 8)));
 }
-EXPORT_SYMBOL(bgx_get_tx_stats);
 
-static void bgx_flush_dmac_addrs(struct bgx *bgx, int lmac)
+static void
+bgx_flush_dmac_addrs(struct bgx *bgx, int lmac)
 {
-	u64 offset;
+	uint64_t offset;
 
 	while (bgx->lmac[lmac].dmac > 0) {
-		offset = ((bgx->lmac[lmac].dmac - 1) * sizeof(u64)) +
-			(lmac * MAX_DMAC_PER_LMAC * sizeof(u64));
+		offset = ((bgx->lmac[lmac].dmac - 1) * sizeof(uint64_t)) +
+		    (lmac * MAX_DMAC_PER_LMAC * sizeof(uint64_t));
 		bgx_reg_write(bgx, 0, BGX_CMR_RX_DMACX_CAM + offset, 0);
 		bgx->lmac[lmac].dmac--;
 	}
 }
 
-void bgx_add_dmac_addr(u64 dmac, int node, int bgx_idx, int lmac)
+void
+bgx_add_dmac_addr(uint64_t dmac, int node, int bgx_idx, int lmac)
 {
-	u64 offset;
+	uint64_t offset;
 	struct bgx *bgx;
 
 #ifdef BGX_IN_PROMISCUOUS_MODE
@@ -362,44 +492,44 @@ void bgx_add_dmac_addr(u64 dmac, int node, int bgx_idx, int lmac)
 	bgx = bgx_vnic[bgx_idx];
 
 	if (!bgx) {
-		dev_err(&bgx->pdev->dev,
-			"BGX%d not yet initialized, ignoring DMAC addition\n",
-			bgx_idx);
+		device_printf(bgx->dev,
+		    "BGX%d not yet initialized, ignoring DMAC addition\n",
+		    bgx_idx);
 		return;
 	}
 
-	dmac = dmac | (1ULL << 48) | ((u64)lmac << 49); /* Enable DMAC */
+	dmac = dmac | (1UL << 48) | ((uint64_t)lmac << 49); /* Enable DMAC */
 	if (bgx->lmac[lmac].dmac == MAX_DMAC_PER_LMAC) {
-		dev_err(&bgx->pdev->dev,
-			"Max DMAC filters for LMAC%d reached, ignoring\n",
-			lmac);
+		device_printf(bgx->dev,
+		    "Max DMAC filters for LMAC%d reached, ignoring\n",
+		    lmac);
 		return;
 	}
 
 	if (bgx->lmac[lmac].dmac == MAX_DMAC_PER_LMAC_TNS_BYPASS_MODE)
 		bgx->lmac[lmac].dmac = 1;
 
-	offset = (bgx->lmac[lmac].dmac * sizeof(u64)) +
-		(lmac * MAX_DMAC_PER_LMAC * sizeof(u64));
+	offset = (bgx->lmac[lmac].dmac * sizeof(uint64_t)) +
+	    (lmac * MAX_DMAC_PER_LMAC * sizeof(uint64_t));
 	bgx_reg_write(bgx, 0, BGX_CMR_RX_DMACX_CAM + offset, dmac);
 	bgx->lmac[lmac].dmac++;
 
 	bgx_reg_write(bgx, lmac, BGX_CMRX_RX_DMAC_CTL,
-		      (CAM_ACCEPT << 3) | (MCAST_MODE_CAM_FILTER << 1)
-		      | (BCAST_ACCEPT << 0));
+	    (CAM_ACCEPT << 3) | (MCAST_MODE_CAM_FILTER << 1) |
+	    (BCAST_ACCEPT << 0));
 }
-EXPORT_SYMBOL(bgx_add_dmac_addr);
 
 /* Configure BGX LMAC in internal loopback mode */
-void bgx_lmac_internal_loopback(int node, int bgx_idx,
-				int lmac_idx, bool enable)
+void
+bgx_lmac_internal_loopback(int node, int bgx_idx,
+    int lmac_idx, boolean_t enable)
 {
 	struct bgx *bgx;
 	struct lmac *lmac;
-	u64    cfg;
+	uint64_t cfg;
 
 	bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx];
-	if (!bgx)
+	if (bgx == NULL)
 		return;
 
 	lmac = &bgx->lmac[lmac_idx];
@@ -419,11 +549,11 @@ void bgx_lmac_internal_loopback(int node, int bgx_idx,
 		bgx_reg_write(bgx, lmac_idx, BGX_SPUX_CONTROL1, cfg);
 	}
 }
-EXPORT_SYMBOL(bgx_lmac_internal_loopback);
 
-static int bgx_lmac_sgmii_init(struct bgx *bgx, int lmacid)
+static int
+bgx_lmac_sgmii_init(struct bgx *bgx, int lmacid)
 {
-	u64 cfg;
+	uint64_t cfg;
 
 	bgx_reg_modify(bgx, lmacid, BGX_GMP_GMI_TXX_THRESH, 0x30);
 	/* max packet size */
@@ -440,9 +570,9 @@ static int bgx_lmac_sgmii_init(struct bgx *bgx, int lmacid)
 	/* PCS reset */
 	bgx_reg_modify(bgx, lmacid, BGX_GMP_PCS_MRX_CTL, PCS_MRX_CTL_RESET);
 	if (bgx_poll_reg(bgx, lmacid, BGX_GMP_PCS_MRX_CTL,
-			 PCS_MRX_CTL_RESET, true)) {
-		dev_err(&bgx->pdev->dev, "BGX PCS reset not completed\n");
-		return -1;
+	    PCS_MRX_CTL_RESET, TRUE) != 0) {
+		device_printf(bgx->dev, "BGX PCS reset not completed\n");
+		return (ENXIO);
 	}
 
 	/* power down, reset autoneg, autoneg enable */
@@ -452,23 +582,25 @@ static int bgx_lmac_sgmii_init(struct bgx *bgx, int lmacid)
 	bgx_reg_write(bgx, lmacid, BGX_GMP_PCS_MRX_CTL, cfg);
 
 	if (bgx_poll_reg(bgx, lmacid, BGX_GMP_PCS_MRX_STATUS,
-			 PCS_MRX_STATUS_AN_CPT, false)) {
-		dev_err(&bgx->pdev->dev, "BGX AN_CPT not completed\n");
-		return -1;
+	    PCS_MRX_STATUS_AN_CPT, FALSE) != 0) {
+		device_printf(bgx->dev, "BGX AN_CPT not completed\n");
+		return (ENXIO);
 	}
 
-	return 0;
+	return (0);
 }
 
-static int bgx_lmac_xaui_init(struct bgx *bgx, int lmacid, int lmac_type)
+static int
+bgx_lmac_xaui_init(struct bgx *bgx, int lmacid, int lmac_type)
 {
-	u64 cfg;
+	uint64_t cfg;
 
 	/* Reset SPU */
 	bgx_reg_modify(bgx, lmacid, BGX_SPUX_CONTROL1, SPU_CTL_RESET);
-	if (bgx_poll_reg(bgx, lmacid, BGX_SPUX_CONTROL1, SPU_CTL_RESET, true)) {
-		dev_err(&bgx->pdev->dev, "BGX SPU reset not completed\n");
-		return -1;
+	if (bgx_poll_reg(bgx, lmacid, BGX_SPUX_CONTROL1,
+	    SPU_CTL_RESET, TRUE) != 0) {
+		device_printf(bgx->dev, "BGX SPU reset not completed\n");
+		return (ENXIO);
 	}
 
 	/* Disable LMAC */
@@ -478,12 +610,13 @@ static int bgx_lmac_xaui_init(struct bgx *bgx, int lmacid, int lmac_type)
 
 	bgx_reg_modify(bgx, lmacid, BGX_SPUX_CONTROL1, SPU_CTL_LOW_POWER);
 	/* Set interleaved running disparity for RXAUI */
-	if (bgx->lmac_type != BGX_MODE_RXAUI)
+	if (bgx->lmac_type != BGX_MODE_RXAUI) {
 		bgx_reg_modify(bgx, lmacid,
-			       BGX_SPUX_MISC_CONTROL, SPU_MISC_CTL_RX_DIS);
-	else
+		    BGX_SPUX_MISC_CONTROL, SPU_MISC_CTL_RX_DIS);
+	} else {
 		bgx_reg_modify(bgx, lmacid, BGX_SPUX_MISC_CONTROL,
-			       SPU_MISC_CTL_RX_DIS | SPU_MISC_CTL_INTLV_RDISP);
+		    SPU_MISC_CTL_RX_DIS | SPU_MISC_CTL_INTLV_RDISP);
+	}
 
 	/* clear all interrupts */
 	cfg = bgx_reg_read(bgx, lmacid, BGX_SMUX_RX_INT);
@@ -498,8 +631,8 @@ static int bgx_lmac_xaui_init(struct bgx *bgx, int lmacid, int lmac_type)
 		bgx_reg_write(bgx, lmacid, BGX_SPUX_BR_PMD_LD_CUP, 0x00);
 		bgx_reg_write(bgx, lmacid, BGX_SPUX_BR_PMD_LD_REP, 0x00);
 		/* training enable */
-		bgx_reg_modify(bgx, lmacid,
-			       BGX_SPUX_BR_PMD_CRTL, SPU_PMD_CRTL_TRAIN_EN);
+		bgx_reg_modify(bgx, lmacid, BGX_SPUX_BR_PMD_CRTL,
+		    SPU_PMD_CRTL_TRAIN_EN);
 	}
 
 	/* Append FCS to each packet */
@@ -522,7 +655,7 @@ static int bgx_lmac_xaui_init(struct bgx *bgx, int lmacid, int lmac_type)
 		cfg |= (1 << 24);
 	else
 		cfg &= ~((1 << 23) | (1 << 24));
-	cfg = cfg & (~((1ULL << 25) | (1ULL << 22) | (1ULL << 12)));
+	cfg = cfg & (~((1UL << 25) | (1UL << 22) | (1UL << 12)));
 	bgx_reg_write(bgx, lmacid, BGX_SPUX_AN_ADV, cfg);
 
 	cfg = bgx_reg_read(bgx, 0, BGX_SPU_DBG_CONTROL);
@@ -546,116 +679,122 @@ static int bgx_lmac_xaui_init(struct bgx *bgx, int lmacid, int lmac_type)
 	/* max packet size */
 	bgx_reg_modify(bgx, lmacid, BGX_SMUX_RX_JABBER, MAX_FRAME_SIZE);
 
-	return 0;
+	return (0);
 }
 
-static int bgx_xaui_check_link(struct lmac *lmac)
+static int
+bgx_xaui_check_link(struct lmac *lmac)
 {
 	struct bgx *bgx = lmac->bgx;
 	int lmacid = lmac->lmacid;
 	int lmac_type = bgx->lmac_type;
-	u64 cfg;
+	uint64_t cfg;
 
 	bgx_reg_modify(bgx, lmacid, BGX_SPUX_MISC_CONTROL, SPU_MISC_CTL_RX_DIS);
 	if (bgx->use_training) {
 		cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_INT);
-		if (!(cfg & (1ull << 13))) {
-			cfg = (1ull << 13) | (1ull << 14);
+		if ((cfg & (1UL << 13)) == 0) {
+			cfg = (1UL << 13) | (1UL << 14);
 			bgx_reg_write(bgx, lmacid, BGX_SPUX_INT, cfg);
 			cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_BR_PMD_CRTL);
-			cfg |= (1ull << 0);
+			cfg |= (1UL << 0);
 			bgx_reg_write(bgx, lmacid, BGX_SPUX_BR_PMD_CRTL, cfg);
-			return -1;
+			return (ENXIO);
 		}
 	}
 
 	/* wait for PCS to come out of reset */
-	if (bgx_poll_reg(bgx, lmacid, BGX_SPUX_CONTROL1, SPU_CTL_RESET, true)) {
-		dev_err(&bgx->pdev->dev, "BGX SPU reset not completed\n");
-		return -1;
+	if (bgx_poll_reg(bgx, lmacid, BGX_SPUX_CONTROL1,
+	    SPU_CTL_RESET, TRUE) != 0) {
+		device_printf(bgx->dev, "BGX SPU reset not completed\n");
+		return (ENXIO);
 	}
 
 	if ((lmac_type == BGX_MODE_10G_KR) || (lmac_type == BGX_MODE_XFI) ||
 	    (lmac_type == BGX_MODE_40G_KR) || (lmac_type == BGX_MODE_XLAUI)) {
 		if (bgx_poll_reg(bgx, lmacid, BGX_SPUX_BR_STATUS1,
-				 SPU_BR_STATUS_BLK_LOCK, false)) {
-			dev_err(&bgx->pdev->dev,
-				"SPU_BR_STATUS_BLK_LOCK not completed\n");
-			return -1;
+		    SPU_BR_STATUS_BLK_LOCK, FALSE)) {
+			device_printf(bgx->dev,
+			    "SPU_BR_STATUS_BLK_LOCK not completed\n");
+			return (ENXIO);
 		}
 	} else {
 		if (bgx_poll_reg(bgx, lmacid, BGX_SPUX_BX_STATUS,
-				 SPU_BX_STATUS_RX_ALIGN, false)) {
-			dev_err(&bgx->pdev->dev,
-				"SPU_BX_STATUS_RX_ALIGN not completed\n");
-			return -1;
+		    SPU_BX_STATUS_RX_ALIGN, FALSE) != 0) {
+			device_printf(bgx->dev,
+			    "SPU_BX_STATUS_RX_ALIGN not completed\n");
+			return (ENXIO);
 		}
 	}
 
 	/* Clear rcvflt bit (latching high) and read it back */
 	bgx_reg_modify(bgx, lmacid, BGX_SPUX_STATUS2, SPU_STATUS2_RCVFLT);
 	if (bgx_reg_read(bgx, lmacid, BGX_SPUX_STATUS2) & SPU_STATUS2_RCVFLT) {
-		dev_err(&bgx->pdev->dev, "Receive fault, retry training\n");
+		device_printf(bgx->dev, "Receive fault, retry training\n");
 		if (bgx->use_training) {
 			cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_INT);
-			if (!(cfg & (1ull << 13))) {
-				cfg = (1ull << 13) | (1ull << 14);
+			if ((cfg & (1UL << 13)) == 0) {
+				cfg = (1UL << 13) | (1UL << 14);
 				bgx_reg_write(bgx, lmacid, BGX_SPUX_INT, cfg);
 				cfg = bgx_reg_read(bgx, lmacid,
-						   BGX_SPUX_BR_PMD_CRTL);
-				cfg |= (1ull << 0);
+				    BGX_SPUX_BR_PMD_CRTL);
+				cfg |= (1UL << 0);
 				bgx_reg_write(bgx, lmacid,
-					      BGX_SPUX_BR_PMD_CRTL, cfg);
-				return -1;
+				    BGX_SPUX_BR_PMD_CRTL, cfg);
+				return (ENXIO);
 			}
 		}
-		return -1;
+		return (ENXIO);
 	}
 
 	/* Wait for MAC RX to be ready */
 	if (bgx_poll_reg(bgx, lmacid, BGX_SMUX_RX_CTL,
-			 SMU_RX_CTL_STATUS, true)) {
-		dev_err(&bgx->pdev->dev, "SMU RX link not okay\n");
-		return -1;
+	    SMU_RX_CTL_STATUS, TRUE) != 0) {
+		device_printf(bgx->dev, "SMU RX link not okay\n");
+		return (ENXIO);
 	}
 
 	/* Wait for BGX RX to be idle */
-	if (bgx_poll_reg(bgx, lmacid, BGX_SMUX_CTL, SMU_CTL_RX_IDLE, false)) {
-		dev_err(&bgx->pdev->dev, "SMU RX not idle\n");
-		return -1;
+	if (bgx_poll_reg(bgx, lmacid, BGX_SMUX_CTL,
+	    SMU_CTL_RX_IDLE, FALSE) != 0) {
+		device_printf(bgx->dev, "SMU RX not idle\n");
+		return (ENXIO);
 	}
 
 	/* Wait for BGX TX to be idle */
-	if (bgx_poll_reg(bgx, lmacid, BGX_SMUX_CTL, SMU_CTL_TX_IDLE, false)) {
-		dev_err(&bgx->pdev->dev, "SMU TX not idle\n");
-		return -1;
+	if (bgx_poll_reg(bgx, lmacid, BGX_SMUX_CTL,
+	    SMU_CTL_TX_IDLE, FALSE) != 0) {
+		device_printf(bgx->dev, "SMU TX not idle\n");
+		return (ENXIO);
 	}
 
-	if (bgx_reg_read(bgx, lmacid, BGX_SPUX_STATUS2) & SPU_STATUS2_RCVFLT) {
-		dev_err(&bgx->pdev->dev, "Receive fault\n");
-		return -1;
+	if ((bgx_reg_read(bgx, lmacid, BGX_SPUX_STATUS2) &
+	    SPU_STATUS2_RCVFLT) != 0) {
+		device_printf(bgx->dev, "Receive fault\n");
+		return (ENXIO);
 	}
 
 	/* Receive link is latching low. Force it high and verify it */
 	bgx_reg_modify(bgx, lmacid, BGX_SPUX_STATUS1, SPU_STATUS1_RCV_LNK);
 	if (bgx_poll_reg(bgx, lmacid, BGX_SPUX_STATUS1,
-			 SPU_STATUS1_RCV_LNK, false)) {
-		dev_err(&bgx->pdev->dev, "SPU receive link down\n");
-		return -1;
+	    SPU_STATUS1_RCV_LNK, FALSE) != 0) {
+		device_printf(bgx->dev, "SPU receive link down\n");
+		return (ENXIO);
 	}
 
 	cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_MISC_CONTROL);
 	cfg &= ~SPU_MISC_CTL_RX_DIS;
 	bgx_reg_write(bgx, lmacid, BGX_SPUX_MISC_CONTROL, cfg);
-	return 0;
+	return (0);
 }
 
-static void bgx_poll_for_link(struct work_struct *work)
+static void
+bgx_poll_for_link(void *arg)
 {
 	struct lmac *lmac;
-	u64 link;
+	uint64_t link;
 
-	lmac = container_of(work, struct lmac, dwork.work);
+	lmac = (struct lmac *)arg;
 
 	/* Receive link is latching low. Force it high and verify it */
 	bgx_reg_modify(lmac->bgx, lmac->lmacid,
@@ -681,21 +820,22 @@ static void bgx_poll_for_link(struct work_struct *work)
 			bgx_xaui_check_link(lmac);
 	}
 
-	queue_delayed_work(lmac->check_link, &lmac->dwork, HZ * 2);
+	callout_reset(&lmac->check_link, hz * 2, bgx_poll_for_link, lmac);
 }
 
-static int bgx_lmac_enable(struct bgx *bgx, u8 lmacid)
+static int
+bgx_lmac_enable(struct bgx *bgx, uint8_t lmacid)
 {
-	u64 dmac_bcast = (1ULL << 48) - 1;
+	uint64_t __unused dmac_bcast = (1UL << 48) - 1;
 	struct lmac *lmac;
-	u64 cfg;
+	uint64_t cfg;
 
 	lmac = &bgx->lmac[lmacid];
 	lmac->bgx = bgx;
 
 	if (bgx->lmac_type == BGX_MODE_SGMII) {
 		lmac->is_sgmii = 1;
-		if (bgx_lmac_sgmii_init(bgx, lmacid))
+		if (bgx_lmac_sgmii_init(bgx, lmacid) != 0)
 			return -1;
 	} else {
 		lmac->is_sgmii = 0;
@@ -705,12 +845,12 @@ static int bgx_lmac_enable(struct bgx *bgx, u8 lmacid)
 
 	if (lmac->is_sgmii) {
 		cfg = bgx_reg_read(bgx, lmacid, BGX_GMP_GMI_TXX_APPEND);
-		cfg |= ((1ull << 2) | (1ull << 1)); /* FCS and PAD */
+		cfg |= ((1UL << 2) | (1UL << 1)); /* FCS and PAD */
 		bgx_reg_modify(bgx, lmacid, BGX_GMP_GMI_TXX_APPEND, cfg);
 		bgx_reg_write(bgx, lmacid, BGX_GMP_GMI_TXX_MIN_PKT, 60 - 1);
 	} else {
 		cfg = bgx_reg_read(bgx, lmacid, BGX_SMUX_TX_APPEND);
-		cfg |= ((1ull << 2) | (1ull << 1)); /* FCS and PAD */
+		cfg |= ((1UL << 2) | (1UL << 1)); /* FCS and PAD */
 		bgx_reg_modify(bgx, lmacid, BGX_SMUX_TX_APPEND, cfg);
 		bgx_reg_write(bgx, lmacid, BGX_SMUX_TX_MIN_PKT, 60 + 4);
 	}
@@ -726,44 +866,49 @@ static int bgx_lmac_enable(struct bgx *bgx, u8 lmacid)
 	bgx_add_dmac_addr(dmac_bcast, 0, bgx->bgx_id, lmacid);
 
 	if ((bgx->lmac_type != BGX_MODE_XFI) &&
+	    (bgx->lmac_type != BGX_MODE_XAUI) &&
 	    (bgx->lmac_type != BGX_MODE_XLAUI) &&
 	    (bgx->lmac_type != BGX_MODE_40G_KR) &&
 	    (bgx->lmac_type != BGX_MODE_10G_KR)) {
-		if (!lmac->phydev)
-			return -ENODEV;
+		if (lmac->phy_if_dev == NULL) {
+			device_printf(bgx->dev,
+			    "LMAC%d missing interface to PHY\n", lmacid);
+			return (ENXIO);
+		}
 
-		lmac->phydev->dev_flags = 0;
-
-		if (phy_connect_direct(&lmac->netdev, lmac->phydev,
-				       bgx_lmac_handler,
-				       PHY_INTERFACE_MODE_SGMII))
-			return -ENODEV;
-
-		phy_start_aneg(lmac->phydev);
+		if (LMAC_PHY_CONNECT(lmac->phy_if_dev, lmac->phyaddr,
+		    lmacid) != 0) {
+			device_printf(bgx->dev,
+			    "LMAC%d could not connect to PHY\n", lmacid);
+			return (ENXIO);
+		}
+		mtx_init(&lmac->check_link_mtx, "BGX link poll", NULL, MTX_DEF);
+		callout_init_mtx(&lmac->check_link, &lmac->check_link_mtx, 0);
+		mtx_lock(&lmac->check_link_mtx);
+		bgx_lmac_handler(lmac);
+		mtx_unlock(&lmac->check_link_mtx);
 	} else {
-		lmac->check_link = alloc_workqueue("check_link", WQ_UNBOUND |
-						   WQ_MEM_RECLAIM, 1);
-		if (!lmac->check_link)
-			return -ENOMEM;
-		INIT_DELAYED_WORK(&lmac->dwork, bgx_poll_for_link);
-		queue_delayed_work(lmac->check_link, &lmac->dwork, 0);
+		mtx_init(&lmac->check_link_mtx, "BGX link poll", NULL, MTX_DEF);
+		callout_init_mtx(&lmac->check_link, &lmac->check_link_mtx, 0);
+		mtx_lock(&lmac->check_link_mtx);
+		bgx_poll_for_link(lmac);
+		mtx_unlock(&lmac->check_link_mtx);
 	}
 
-	return 0;
+	return (0);
 }
 
-static void bgx_lmac_disable(struct bgx *bgx, u8 lmacid)
+static void
+bgx_lmac_disable(struct bgx *bgx, uint8_t lmacid)
 {
 	struct lmac *lmac;
-	u64 cmrx_cfg;
+	uint64_t cmrx_cfg;
 
 	lmac = &bgx->lmac[lmacid];
-	if (lmac->check_link) {
-		/* Destroy work queue */
-		cancel_delayed_work(&lmac->dwork);
-		flush_workqueue(lmac->check_link);
-		destroy_workqueue(lmac->check_link);
-	}
+
+	/* Stop callout */
+	callout_drain(&lmac->check_link);
+	mtx_destroy(&lmac->check_link_mtx);
 
 	cmrx_cfg = bgx_reg_read(bgx, lmacid, BGX_CMRX_CFG);
 	cmrx_cfg &= ~(1 << 15);
@@ -774,15 +919,25 @@ static void bgx_lmac_disable(struct bgx *bgx, u8 lmacid)
 	    (bgx->lmac_type != BGX_MODE_XLAUI) &&
 	    (bgx->lmac_type != BGX_MODE_40G_KR) &&
 	    (bgx->lmac_type != BGX_MODE_10G_KR)) {
-		if (lmac->phydev)
-			phy_disconnect(lmac->phydev);
+		if (lmac->phy_if_dev == NULL) {
+			device_printf(bgx->dev,
+			    "LMAC%d missing interface to PHY\n", lmacid);
+			return;
+		}
+		if (LMAC_PHY_DISCONNECT(lmac->phy_if_dev, lmac->phyaddr,
+		    lmacid) != 0) {
+			device_printf(bgx->dev,
+			    "LMAC%d could not disconnect PHY\n", lmacid);
+			return;
+		}
+		lmac->phy_if_dev = NULL;
 	}
-	lmac->phydev = NULL;
 }
 
-static void bgx_set_num_ports(struct bgx *bgx)
+static void
+bgx_set_num_ports(struct bgx *bgx)
 {
-	u64 lmac_count;
+	uint64_t lmac_count;
 
 	switch (bgx->qlm_mode) {
 	case QLM_MODE_SGMII:
@@ -827,7 +982,8 @@ static void bgx_set_num_ports(struct bgx *bgx)
 		break;
 	}
 
-	/* Check if low level firmware has programmed LMAC count
+	/*
+	 * Check if low level firmware has programmed LMAC count
 	 * based on board type, if yes consider that otherwise
 	 * the default static values
 	 */
@@ -836,7 +992,8 @@ static void bgx_set_num_ports(struct bgx *bgx)
 		bgx->lmac_count = lmac_count;
 }
 
-static void bgx_init_hw(struct bgx *bgx)
+static void
+bgx_init_hw(struct bgx *bgx)
 {
 	int i;
 
@@ -844,7 +1001,7 @@ static void bgx_init_hw(struct bgx *bgx)
 
 	bgx_reg_modify(bgx, 0, BGX_CMR_GLOBAL_CFG, CMR_GLOBAL_CFG_FCS_STRIP);
 	if (bgx_reg_read(bgx, 0, BGX_CMR_BIST_STATUS))
-		dev_err(&bgx->pdev->dev, "BGX%d BIST failed\n", bgx->bgx_id);
+		device_printf(bgx->dev, "BGX%d BIST failed\n", bgx->bgx_id);
 
 	/* Set lmac type and lane2serdes mapping */
 	for (i = 0; i < bgx->lmac_count; i++) {
@@ -854,11 +1011,11 @@ static void bgx_init_hw(struct bgx *bgx)
 			else
 				bgx->lane_to_sds = 0x04;
 			bgx_reg_write(bgx, i, BGX_CMRX_CFG,
-				      (bgx->lmac_type << 8) | bgx->lane_to_sds);
+			    (bgx->lmac_type << 8) | bgx->lane_to_sds);
 			continue;
 		}
 		bgx_reg_write(bgx, i, BGX_CMRX_CFG,
-			      (bgx->lmac_type << 8) | (bgx->lane_to_sds + i));
+		    (bgx->lmac_type << 8) | (bgx->lane_to_sds + i));
 		bgx->lmac[i].lmacid_bd = lmac_count;
 		lmac_count++;
 	}
@@ -867,10 +1024,11 @@ static void bgx_init_hw(struct bgx *bgx)
 	bgx_reg_write(bgx, 0, BGX_CMR_RX_LMACS, bgx->lmac_count);
 
 	/* Set the backpressure AND mask */
-	for (i = 0; i < bgx->lmac_count; i++)
+	for (i = 0; i < bgx->lmac_count; i++) {
 		bgx_reg_modify(bgx, 0, BGX_CMR_CHAN_MSK_AND,
-			       ((1ULL << MAX_BGX_CHANS_PER_LMAC) - 1) <<
-			       (i * MAX_BGX_CHANS_PER_LMAC));
+		    ((1UL << MAX_BGX_CHANS_PER_LMAC) - 1) <<
+		    (i * MAX_BGX_CHANS_PER_LMAC));
+	}
 
 	/* Disable all MAC filtering */
 	for (i = 0; i < RX_DMAC_COUNT; i++)
@@ -881,9 +1039,10 @@ static void bgx_init_hw(struct bgx *bgx)
 		bgx_reg_write(bgx, 0, BGX_CMR_RX_STREERING + (i * 8), 0x00);
 }
 
-static void bgx_get_qlm_mode(struct bgx *bgx)
+static void
+bgx_get_qlm_mode(struct bgx *bgx)
 {
-	struct device *dev = &bgx->pdev->dev;
+	device_t dev = bgx->dev;;
 	int lmac_type;
 	int train_en;
 
@@ -894,319 +1053,83 @@ static void bgx_get_qlm_mode(struct bgx *bgx)
 	lmac_type = (lmac_type >> 8) & 0x07;
 
 	train_en = bgx_reg_read(bgx, 0, BGX_SPUX_BR_PMD_CRTL) &
-				SPU_PMD_CRTL_TRAIN_EN;
+	    SPU_PMD_CRTL_TRAIN_EN;
 
 	switch (lmac_type) {
 	case BGX_MODE_SGMII:
 		bgx->qlm_mode = QLM_MODE_SGMII;
-		dev_info(dev, "BGX%d QLM mode: SGMII\n", bgx->bgx_id);
+		if (bootverbose) {
+			device_printf(dev, "BGX%d QLM mode: SGMII\n",
+			    bgx->bgx_id);
+		}
 		break;
 	case BGX_MODE_XAUI:
 		bgx->qlm_mode = QLM_MODE_XAUI_1X4;
-		dev_info(dev, "BGX%d QLM mode: XAUI\n", bgx->bgx_id);
+		if (bootverbose) {
+			device_printf(dev, "BGX%d QLM mode: XAUI\n",
+			    bgx->bgx_id);
+		}
 		break;
 	case BGX_MODE_RXAUI:
 		bgx->qlm_mode = QLM_MODE_RXAUI_2X2;
-		dev_info(dev, "BGX%d QLM mode: RXAUI\n", bgx->bgx_id);
+		if (bootverbose) {
+			device_printf(dev, "BGX%d QLM mode: RXAUI\n",
+			    bgx->bgx_id);
+		}
 		break;
 	case BGX_MODE_XFI:
 		if (!train_en) {
 			bgx->qlm_mode = QLM_MODE_XFI_4X1;
-			dev_info(dev, "BGX%d QLM mode: XFI\n", bgx->bgx_id);
+			if (bootverbose) {
+				device_printf(dev, "BGX%d QLM mode: XFI\n",
+				    bgx->bgx_id);
+			}
 		} else {
 			bgx->qlm_mode = QLM_MODE_10G_KR_4X1;
-			dev_info(dev, "BGX%d QLM mode: 10G_KR\n", bgx->bgx_id);
+			if (bootverbose) {
+				device_printf(dev, "BGX%d QLM mode: 10G_KR\n",
+				    bgx->bgx_id);
+			}
 		}
 		break;
 	case BGX_MODE_XLAUI:
 		if (!train_en) {
 			bgx->qlm_mode = QLM_MODE_XLAUI_1X4;
-			dev_info(dev, "BGX%d QLM mode: XLAUI\n", bgx->bgx_id);
+			if (bootverbose) {
+				device_printf(dev, "BGX%d QLM mode: XLAUI\n",
+				    bgx->bgx_id);
+			}
 		} else {
 			bgx->qlm_mode = QLM_MODE_40G_KR4_1X4;
-			dev_info(dev, "BGX%d QLM mode: 40G_KR4\n", bgx->bgx_id);
+			if (bootverbose) {
+				device_printf(dev, "BGX%d QLM mode: 40G_KR4\n",
+				    bgx->bgx_id);
+			}
 		}
 		break;
 	default:
 		bgx->qlm_mode = QLM_MODE_SGMII;
-		dev_info(dev, "BGX%d QLM default mode: SGMII\n", bgx->bgx_id);
-	}
-}
-
-#ifdef CONFIG_ACPI
-
-static int bgx_match_phy_id(struct device *dev, void *data)
-{
-	struct phy_device *phydev = to_phy_device(dev);
-	u32 *phy_id = data;
-
-	if (phydev->addr == *phy_id)
-		return 1;
-
-	return 0;
-}
-
-static const char *addr_propnames[] = {
-	"mac-address",
-	"local-mac-address",
-	"address",
-};
-
-static int acpi_get_mac_address(struct acpi_device *adev, u8 *dst)
-{
-	u64 mac;
-	int i;
-	int ret;
-
-	for (i = 0; i < ARRAY_SIZE(addr_propnames); i++) {
-		ret = acpi_dev_prop_read_single(adev, addr_propnames[i],
-						DEV_PROP_U64, &mac);
-		if (ret)
-			continue;
-
-		if (mac & (~0ULL << 48))
-			continue;	/* more than 6 bytes */
-
-		if (!is_valid_ether_addr((u8 *)&mac))
-			continue;
-
-		ether_addr_copy(dst, (u8 *)&mac);
-
-		return 0;
-	}
-
-	return ret ? ret : -EINVAL;
-}
-
-static acpi_status bgx_acpi_register_phy(acpi_handle handle,
-					u32 lvl, void *context, void **rv)
-{
-	struct acpi_reference_args args;
-	struct bgx *bgx = context;
-	struct acpi_device *adev;
-	struct device *phy_dev;
-	u32 phy_id;
-
-	if (acpi_bus_get_device(handle, &adev))
-		return AE_OK;
-
-	if (acpi_dev_get_property_reference(adev, "phy-handle", 0, &args))
-		return AE_OK;
-
-	if (acpi_dev_prop_read_single(args.adev, "phy-channel", DEV_PROP_U32,
-					&phy_id))
-		return AE_OK;
-
-	phy_dev = bus_find_device(&mdio_bus_type, NULL, (void *)&phy_id,
-				  bgx_match_phy_id);
-	if (!phy_dev)
-		return AE_OK;
-
-	SET_NETDEV_DEV(&bgx->lmac[bgx->lmac_count].netdev, &bgx->pdev->dev);
-	bgx->lmac[bgx->lmac_count].phydev = to_phy_device(phy_dev);
-
-	acpi_get_mac_address(adev, bgx->lmac[bgx->lmac_count].mac);
-
-	bgx->lmac[bgx->lmac_count].lmacid = bgx->lmac_count;
-	bgx->lmac_count++;
-
-	return AE_OK;
-}
-
-static acpi_status bgx_acpi_match_id(acpi_handle handle, u32 lvl,
-				void *context, void **ret_val)
-{
-	struct acpi_buffer string = { ACPI_ALLOCATE_BUFFER, NULL };
-	struct bgx *bgx = context;
-	char bgx_sel[5];
-
-	snprintf(bgx_sel, 5, "BGX%d", bgx->bgx_id);
-	if (ACPI_FAILURE(acpi_get_name(handle, ACPI_SINGLE_NAME, &string))) {
-		pr_warn("Invalid link device\n");
-		return AE_OK;
-	}
-
-	if (strncmp(string.pointer, bgx_sel, 4))
-		return AE_OK;
-
-	acpi_walk_namespace(ACPI_TYPE_DEVICE, handle, 1,
-			    bgx_acpi_register_phy, NULL, bgx, NULL);
-
-	kfree(string.pointer);
-	return AE_CTRL_TERMINATE;
-}
-
-static int bgx_init_acpi_phy(struct bgx *bgx)
-{
-	acpi_get_devices(NULL, bgx_acpi_match_id, bgx, (void **)NULL);
-	return 0;
-}
-
-#else
-
-static int bgx_init_acpi_phy(struct bgx *bgx)
-{
-	return -ENODEV;
-}
-
-#endif /* CONFIG_ACPI */
-
-#if IS_ENABLED(CONFIG_OF_MDIO)
-
-static int bgx_init_of_phy(struct bgx *bgx)
-{
-	struct device_node *np;
-	struct device_node *np_child;
-	u8 lmac = 0;
-	char bgx_sel[5];
-	const char *mac;
-
-	/* Get BGX node from DT */
-	snprintf(bgx_sel, 5, "bgx%d", bgx->bgx_id);
-	np = of_find_node_by_name(NULL, bgx_sel);
-	if (!np)
-		return -ENODEV;
-
-	for_each_child_of_node(np, np_child) {
-		struct device_node *phy_np = of_parse_phandle(np_child,
-							      "phy-handle", 0);
-		if (!phy_np)
-			continue;
-		bgx->lmac[lmac].phydev = of_phy_find_device(phy_np);
-
-		mac = of_get_mac_address(np_child);
-		if (mac)
-			ether_addr_copy(bgx->lmac[lmac].mac, mac);
-
-		SET_NETDEV_DEV(&bgx->lmac[lmac].netdev, &bgx->pdev->dev);
-		bgx->lmac[lmac].lmacid = lmac;
-		lmac++;
-		if (lmac == MAX_LMAC_PER_BGX)
-			break;
-	}
-	return 0;
-}
-
-#else
-
-static int bgx_init_of_phy(struct bgx *bgx)
-{
-	return -ENODEV;
-}
-
-#endif /* CONFIG_OF_MDIO */
-
-static int bgx_init_phy(struct bgx *bgx)
-{
-	int err = bgx_init_of_phy(bgx);
-
-	if (err != -ENODEV)
-		return err;
-
-	return bgx_init_acpi_phy(bgx);
-}
-
-static int bgx_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
-{
-	int err;
-	struct device *dev = &pdev->dev;
-	struct bgx *bgx = NULL;
-	u8 lmac;
-
-	bgx = devm_kzalloc(dev, sizeof(*bgx), GFP_KERNEL);
-	if (!bgx)
-		return -ENOMEM;
-	bgx->pdev = pdev;
-
-	pci_set_drvdata(pdev, bgx);
-
-	err = pci_enable_device(pdev);
-	if (err) {
-		dev_err(dev, "Failed to enable PCI device\n");
-		pci_set_drvdata(pdev, NULL);
-		return err;
-	}
-
-	err = pci_request_regions(pdev, DRV_NAME);
-	if (err) {
-		dev_err(dev, "PCI request regions failed 0x%x\n", err);
-		goto err_disable_device;
-	}
-
-	/* MAP configuration registers */
-	bgx->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0);
-	if (!bgx->reg_base) {
-		dev_err(dev, "BGX: Cannot map CSR memory space, aborting\n");
-		err = -ENOMEM;
-		goto err_release_regions;
-	}
-	bgx->bgx_id = (pci_resource_start(pdev, PCI_CFG_REG_BAR_NUM) >> 24) & 1;
-	bgx->bgx_id += nic_get_node_id(pdev) * MAX_BGX_PER_CN88XX;
-
-	bgx_vnic[bgx->bgx_id] = bgx;
-	bgx_get_qlm_mode(bgx);
-
-	err = bgx_init_phy(bgx);
-	if (err)
-		goto err_enable;
-
-	bgx_init_hw(bgx);
-
-	/* Enable all LMACs */
-	for (lmac = 0; lmac < bgx->lmac_count; lmac++) {
-		err = bgx_lmac_enable(bgx, lmac);
-		if (err) {
-			dev_err(dev, "BGX%d failed to enable lmac%d\n",
-				bgx->bgx_id, lmac);
-			goto err_enable;
+		if (bootverbose) {
+			device_printf(dev, "BGX%d QLM default mode: SGMII\n",
+			    bgx->bgx_id);
 		}
 	}
-
-	return 0;
-
-err_enable:
-	bgx_vnic[bgx->bgx_id] = NULL;
-err_release_regions:
-	pci_release_regions(pdev);
-err_disable_device:
-	pci_disable_device(pdev);
-	pci_set_drvdata(pdev, NULL);
-	return err;
 }
 
-static void bgx_remove(struct pci_dev *pdev)
+static int
+bgx_init_phy(struct bgx *bgx)
 {
-	struct bgx *bgx = pci_get_drvdata(pdev);
-	u8 lmac;
+	int err;
 
-	/* Disable all LMACs */
-	for (lmac = 0; lmac < bgx->lmac_count; lmac++)
-		bgx_lmac_disable(bgx, lmac);
-
-	bgx_vnic[bgx->bgx_id] = NULL;
-	pci_release_regions(pdev);
-	pci_disable_device(pdev);
-	pci_set_drvdata(pdev, NULL);
+	/* By default we fail */
+	err = ENXIO;
+#ifdef FDT
+	err = bgx_fdt_init_phy(bgx);
+#endif
+#ifdef ACPI
+	if (err != 0) {
+		/* ARM64TODO: Add ACPI function here */
+	}
+#endif
+	return (err);
 }
-
-static struct pci_driver bgx_driver = {
-	.name = DRV_NAME,
-	.id_table = bgx_id_table,
-	.probe = bgx_probe,
-	.remove = bgx_remove,
-};
-
-static int __init bgx_init_module(void)
-{
-	pr_info("%s, ver %s\n", DRV_NAME, DRV_VERSION);
-
-	return pci_register_driver(&bgx_driver);
-}
-
-static void __exit bgx_cleanup_module(void)
-{
-	pci_unregister_driver(&bgx_driver);
-}
-
-module_init(bgx_init_module);
-module_exit(bgx_cleanup_module);
diff --git a/sys/dev/vnic/thunder_bgx.h b/sys/dev/vnic/thunder_bgx.h
index b1c761866b55..374f44c3b4eb 100644
--- a/sys/dev/vnic/thunder_bgx.h
+++ b/sys/dev/vnic/thunder_bgx.h
@@ -28,169 +28,169 @@
  */
 
 #ifndef THUNDER_BGX_H
-#define THUNDER_BGX_H
+#define	THUNDER_BGX_H
 
-#define    MAX_BGX_THUNDER			8 /* Max 4 nodes, 2 per node */
-#define    MAX_BGX_PER_CN88XX			2
-#define    MAX_LMAC_PER_BGX			4
-#define    MAX_BGX_CHANS_PER_LMAC		16
-#define    MAX_DMAC_PER_LMAC			8
-#define    MAX_FRAME_SIZE			9216
+#define	MAX_BGX_THUNDER			8 /* Max 4 nodes, 2 per node */
+#define	MAX_BGX_PER_CN88XX		2
+#define	MAX_LMAC_PER_BGX		4
+#define	MAX_BGX_CHANS_PER_LMAC		16
+#define	MAX_DMAC_PER_LMAC		8
+#define	MAX_FRAME_SIZE			9216
 
-#define    MAX_DMAC_PER_LMAC_TNS_BYPASS_MODE	2
+#define	MAX_DMAC_PER_LMAC_TNS_BYPASS_MODE	2
 
-#define    MAX_LMAC	(MAX_BGX_PER_CN88XX * MAX_LMAC_PER_BGX)
+#define	MAX_LMAC	(MAX_BGX_PER_CN88XX * MAX_LMAC_PER_BGX)
 
 /* Registers */
-#define BGX_CMRX_CFG			0x00
-#define  CMR_PKT_TX_EN				BIT_ULL(13)
-#define  CMR_PKT_RX_EN				BIT_ULL(14)
-#define  CMR_EN					BIT_ULL(15)
-#define BGX_CMR_GLOBAL_CFG		0x08
-#define  CMR_GLOBAL_CFG_FCS_STRIP		BIT_ULL(6)
-#define BGX_CMRX_RX_ID_MAP		0x60
-#define BGX_CMRX_RX_STAT0		0x70
-#define BGX_CMRX_RX_STAT1		0x78
-#define BGX_CMRX_RX_STAT2		0x80
-#define BGX_CMRX_RX_STAT3		0x88
-#define BGX_CMRX_RX_STAT4		0x90
-#define BGX_CMRX_RX_STAT5		0x98
-#define BGX_CMRX_RX_STAT6		0xA0
-#define BGX_CMRX_RX_STAT7		0xA8
-#define BGX_CMRX_RX_STAT8		0xB0
-#define BGX_CMRX_RX_STAT9		0xB8
-#define BGX_CMRX_RX_STAT10		0xC0
-#define BGX_CMRX_RX_BP_DROP		0xC8
-#define BGX_CMRX_RX_DMAC_CTL		0x0E8
-#define BGX_CMR_RX_DMACX_CAM		0x200
-#define  RX_DMACX_CAM_EN			BIT_ULL(48)
-#define  RX_DMACX_CAM_LMACID(x)			(x << 49)
-#define  RX_DMAC_COUNT				32
-#define BGX_CMR_RX_STREERING		0x300
-#define  RX_TRAFFIC_STEER_RULE_COUNT		8
-#define BGX_CMR_CHAN_MSK_AND		0x450
-#define BGX_CMR_BIST_STATUS		0x460
-#define BGX_CMR_RX_LMACS		0x468
-#define BGX_CMRX_TX_STAT0		0x600
-#define BGX_CMRX_TX_STAT1		0x608
-#define BGX_CMRX_TX_STAT2		0x610
-#define BGX_CMRX_TX_STAT3		0x618
-#define BGX_CMRX_TX_STAT4		0x620
-#define BGX_CMRX_TX_STAT5		0x628
-#define BGX_CMRX_TX_STAT6		0x630
-#define BGX_CMRX_TX_STAT7		0x638
-#define BGX_CMRX_TX_STAT8		0x640
-#define BGX_CMRX_TX_STAT9		0x648
-#define BGX_CMRX_TX_STAT10		0x650
-#define BGX_CMRX_TX_STAT11		0x658
-#define BGX_CMRX_TX_STAT12		0x660
-#define BGX_CMRX_TX_STAT13		0x668
-#define BGX_CMRX_TX_STAT14		0x670
-#define BGX_CMRX_TX_STAT15		0x678
-#define BGX_CMRX_TX_STAT16		0x680
-#define BGX_CMRX_TX_STAT17		0x688
-#define BGX_CMR_TX_LMACS		0x1000
+#define	BGX_CMRX_CFG			0x00
+#define	 CMR_PKT_TX_EN				(1UL << 13)
+#define	 CMR_PKT_RX_EN				(1UL << 14)
+#define	 CMR_EN					(1UL << 15)
+#define	BGX_CMR_GLOBAL_CFG		0x08
+#define	 CMR_GLOBAL_CFG_FCS_STRIP		(1UL << 6)
+#define	BGX_CMRX_RX_ID_MAP		0x60
+#define	BGX_CMRX_RX_STAT0		0x70
+#define	BGX_CMRX_RX_STAT1		0x78
+#define	BGX_CMRX_RX_STAT2		0x80
+#define	BGX_CMRX_RX_STAT3		0x88
+#define	BGX_CMRX_RX_STAT4		0x90
+#define	BGX_CMRX_RX_STAT5		0x98
+#define	BGX_CMRX_RX_STAT6		0xA0
+#define	BGX_CMRX_RX_STAT7		0xA8
+#define	BGX_CMRX_RX_STAT8		0xB0
+#define	BGX_CMRX_RX_STAT9		0xB8
+#define	BGX_CMRX_RX_STAT10		0xC0
+#define	BGX_CMRX_RX_BP_DROP		0xC8
+#define	BGX_CMRX_RX_DMAC_CTL		0x0E8
+#define	BGX_CMR_RX_DMACX_CAM		0x200
+#define	 RX_DMACX_CAM_EN			(1UL << 48)
+#define	 RX_DMACX_CAM_LMACID(x)			(x << 49)
+#define	 RX_DMAC_COUNT				32
+#define	BGX_CMR_RX_STREERING		0x300
+#define	 RX_TRAFFIC_STEER_RULE_COUNT		8
+#define	BGX_CMR_CHAN_MSK_AND		0x450
+#define	BGX_CMR_BIST_STATUS		0x460
+#define	BGX_CMR_RX_LMACS		0x468
+#define	BGX_CMRX_TX_STAT0		0x600
+#define	BGX_CMRX_TX_STAT1		0x608
+#define	BGX_CMRX_TX_STAT2		0x610
+#define	BGX_CMRX_TX_STAT3		0x618
+#define	BGX_CMRX_TX_STAT4		0x620
+#define	BGX_CMRX_TX_STAT5		0x628
+#define	BGX_CMRX_TX_STAT6		0x630
+#define	BGX_CMRX_TX_STAT7		0x638
+#define	BGX_CMRX_TX_STAT8		0x640
+#define	BGX_CMRX_TX_STAT9		0x648
+#define	BGX_CMRX_TX_STAT10		0x650
+#define	BGX_CMRX_TX_STAT11		0x658
+#define	BGX_CMRX_TX_STAT12		0x660
+#define	BGX_CMRX_TX_STAT13		0x668
+#define	BGX_CMRX_TX_STAT14		0x670
+#define	BGX_CMRX_TX_STAT15		0x678
+#define	BGX_CMRX_TX_STAT16		0x680
+#define	BGX_CMRX_TX_STAT17		0x688
+#define	BGX_CMR_TX_LMACS		0x1000
 
-#define BGX_SPUX_CONTROL1		0x10000
-#define  SPU_CTL_LOW_POWER			BIT_ULL(11)
-#define  SPU_CTL_LOOPBACK			BIT_ULL(14)
-#define  SPU_CTL_RESET				BIT_ULL(15)
-#define BGX_SPUX_STATUS1		0x10008
-#define  SPU_STATUS1_RCV_LNK			BIT_ULL(2)
-#define BGX_SPUX_STATUS2		0x10020
-#define  SPU_STATUS2_RCVFLT			BIT_ULL(10)
-#define BGX_SPUX_BX_STATUS		0x10028
-#define  SPU_BX_STATUS_RX_ALIGN			BIT_ULL(12)
-#define BGX_SPUX_BR_STATUS1		0x10030
-#define  SPU_BR_STATUS_BLK_LOCK			BIT_ULL(0)
-#define  SPU_BR_STATUS_RCV_LNK			BIT_ULL(12)
-#define BGX_SPUX_BR_PMD_CRTL		0x10068
-#define  SPU_PMD_CRTL_TRAIN_EN			BIT_ULL(1)
-#define BGX_SPUX_BR_PMD_LP_CUP		0x10078
-#define BGX_SPUX_BR_PMD_LD_CUP		0x10088
-#define BGX_SPUX_BR_PMD_LD_REP		0x10090
-#define BGX_SPUX_FEC_CONTROL		0x100A0
-#define  SPU_FEC_CTL_FEC_EN			BIT_ULL(0)
-#define  SPU_FEC_CTL_ERR_EN			BIT_ULL(1)
-#define BGX_SPUX_AN_CONTROL		0x100C8
-#define  SPU_AN_CTL_AN_EN			BIT_ULL(12)
-#define  SPU_AN_CTL_XNP_EN			BIT_ULL(13)
-#define BGX_SPUX_AN_ADV			0x100D8
-#define BGX_SPUX_MISC_CONTROL		0x10218
-#define  SPU_MISC_CTL_INTLV_RDISP		BIT_ULL(10)
-#define  SPU_MISC_CTL_RX_DIS			BIT_ULL(12)
-#define BGX_SPUX_INT			0x10220	/* +(0..3) << 20 */
-#define BGX_SPUX_INT_W1S		0x10228
-#define BGX_SPUX_INT_ENA_W1C		0x10230
-#define BGX_SPUX_INT_ENA_W1S		0x10238
-#define BGX_SPU_DBG_CONTROL		0x10300
-#define  SPU_DBG_CTL_AN_ARB_LINK_CHK_EN		BIT_ULL(18)
-#define  SPU_DBG_CTL_AN_NONCE_MCT_DIS		BIT_ULL(29)
+#define	BGX_SPUX_CONTROL1		0x10000
+#define	 SPU_CTL_LOW_POWER			(1UL << 11)
+#define	 SPU_CTL_LOOPBACK			(1UL << 14)
+#define	 SPU_CTL_RESET				(1UL << 15)
+#define	BGX_SPUX_STATUS1		0x10008
+#define	 SPU_STATUS1_RCV_LNK			(1UL << 2)
+#define	BGX_SPUX_STATUS2		0x10020
+#define	 SPU_STATUS2_RCVFLT			(1UL << 10)
+#define	BGX_SPUX_BX_STATUS		0x10028
+#define	 SPU_BX_STATUS_RX_ALIGN			(1UL << 12)
+#define	BGX_SPUX_BR_STATUS1		0x10030
+#define	 SPU_BR_STATUS_BLK_LOCK			(1UL << 0)
+#define	 SPU_BR_STATUS_RCV_LNK			(1UL << 12)
+#define	BGX_SPUX_BR_PMD_CRTL		0x10068
+#define	 SPU_PMD_CRTL_TRAIN_EN			(1UL << 1)
+#define	BGX_SPUX_BR_PMD_LP_CUP		0x10078
+#define	BGX_SPUX_BR_PMD_LD_CUP		0x10088
+#define	BGX_SPUX_BR_PMD_LD_REP		0x10090
+#define	BGX_SPUX_FEC_CONTROL		0x100A0
+#define	 SPU_FEC_CTL_FEC_EN			(1UL << 0)
+#define	 SPU_FEC_CTL_ERR_EN			(1UL << 1)
+#define	BGX_SPUX_AN_CONTROL		0x100C8
+#define	 SPU_AN_CTL_AN_EN			(1UL << 12)
+#define	 SPU_AN_CTL_XNP_EN			(1UL << 13)
+#define	BGX_SPUX_AN_ADV			0x100D8
+#define	BGX_SPUX_MISC_CONTROL		0x10218
+#define	 SPU_MISC_CTL_INTLV_RDISP		(1UL << 10)
+#define	 SPU_MISC_CTL_RX_DIS			(1UL << 12)
+#define	BGX_SPUX_INT			0x10220	/* +(0..3) << 20 */
+#define	BGX_SPUX_INT_W1S		0x10228
+#define	BGX_SPUX_INT_ENA_W1C		0x10230
+#define	BGX_SPUX_INT_ENA_W1S		0x10238
+#define	BGX_SPU_DBG_CONTROL		0x10300
+#define	 SPU_DBG_CTL_AN_ARB_LINK_CHK_EN		(1UL << 18)
+#define	 SPU_DBG_CTL_AN_NONCE_MCT_DIS		(1UL << 29)
 
-#define BGX_SMUX_RX_INT			0x20000
-#define BGX_SMUX_RX_JABBER		0x20030
-#define BGX_SMUX_RX_CTL			0x20048
-#define  SMU_RX_CTL_STATUS			(3ull << 0)
-#define BGX_SMUX_TX_APPEND		0x20100
-#define  SMU_TX_APPEND_FCS_D			BIT_ULL(2)
-#define BGX_SMUX_TX_MIN_PKT		0x20118
-#define BGX_SMUX_TX_INT			0x20140
-#define BGX_SMUX_TX_CTL			0x20178
-#define  SMU_TX_CTL_DIC_EN			BIT_ULL(0)
-#define  SMU_TX_CTL_UNI_EN			BIT_ULL(1)
-#define  SMU_TX_CTL_LNK_STATUS			(3ull << 4)
-#define BGX_SMUX_TX_THRESH		0x20180
-#define BGX_SMUX_CTL			0x20200
-#define  SMU_CTL_RX_IDLE			BIT_ULL(0)
-#define  SMU_CTL_TX_IDLE			BIT_ULL(1)
+#define	BGX_SMUX_RX_INT			0x20000
+#define	BGX_SMUX_RX_JABBER		0x20030
+#define	BGX_SMUX_RX_CTL			0x20048
+#define	 SMU_RX_CTL_STATUS			(3UL << 0)
+#define	BGX_SMUX_TX_APPEND		0x20100
+#define	 SMU_TX_APPEND_FCS_D			(1UL << 2)
+#define	BGX_SMUX_TX_MIN_PKT		0x20118
+#define	BGX_SMUX_TX_INT			0x20140
+#define	BGX_SMUX_TX_CTL			0x20178
+#define	 SMU_TX_CTL_DIC_EN			(1UL << 0)
+#define	 SMU_TX_CTL_UNI_EN			(1UL << 1)
+#define	 SMU_TX_CTL_LNK_STATUS			(3UL << 4)
+#define	BGX_SMUX_TX_THRESH		0x20180
+#define	BGX_SMUX_CTL			0x20200
+#define	 SMU_CTL_RX_IDLE			(1UL << 0)
+#define	 SMU_CTL_TX_IDLE			(1UL << 1)
 
-#define BGX_GMP_PCS_MRX_CTL		0x30000
-#define	 PCS_MRX_CTL_RST_AN			BIT_ULL(9)
-#define	 PCS_MRX_CTL_PWR_DN			BIT_ULL(11)
-#define	 PCS_MRX_CTL_AN_EN			BIT_ULL(12)
-#define	 PCS_MRX_CTL_LOOPBACK1			BIT_ULL(14)
-#define	 PCS_MRX_CTL_RESET			BIT_ULL(15)
-#define BGX_GMP_PCS_MRX_STATUS		0x30008
-#define	 PCS_MRX_STATUS_AN_CPT			BIT_ULL(5)
-#define BGX_GMP_PCS_ANX_AN_RESULTS	0x30020
-#define BGX_GMP_PCS_SGM_AN_ADV		0x30068
-#define BGX_GMP_PCS_MISCX_CTL		0x30078
-#define  PCS_MISC_CTL_GMX_ENO			BIT_ULL(11)
-#define  PCS_MISC_CTL_SAMP_PT_MASK	0x7Full
-#define BGX_GMP_GMI_PRTX_CFG		0x38020
-#define  GMI_PORT_CFG_SPEED			BIT_ULL(1)
-#define  GMI_PORT_CFG_DUPLEX			BIT_ULL(2)
-#define  GMI_PORT_CFG_SLOT_TIME			BIT_ULL(3)
-#define  GMI_PORT_CFG_SPEED_MSB			BIT_ULL(8)
-#define BGX_GMP_GMI_RXX_JABBER		0x38038
-#define BGX_GMP_GMI_TXX_THRESH		0x38210
-#define BGX_GMP_GMI_TXX_APPEND		0x38218
-#define BGX_GMP_GMI_TXX_SLOT		0x38220
-#define BGX_GMP_GMI_TXX_BURST		0x38228
-#define BGX_GMP_GMI_TXX_MIN_PKT		0x38240
-#define BGX_GMP_GMI_TXX_SGMII_CTL	0x38300
+#define	BGX_GMP_PCS_MRX_CTL		0x30000
+#define	 PCS_MRX_CTL_RST_AN			(1UL << 9)
+#define	 PCS_MRX_CTL_PWR_DN			(1UL << 11)
+#define	 PCS_MRX_CTL_AN_EN			(1UL << 12)
+#define	 PCS_MRX_CTL_LOOPBACK1			(1UL << 14)
+#define	 PCS_MRX_CTL_RESET			(1UL << 15)
+#define	BGX_GMP_PCS_MRX_STATUS		0x30008
+#define	 PCS_MRX_STATUS_AN_CPT			(1UL << 5)
+#define	BGX_GMP_PCS_ANX_AN_RESULTS	0x30020
+#define	BGX_GMP_PCS_SGM_AN_ADV		0x30068
+#define	BGX_GMP_PCS_MISCX_CTL		0x30078
+#define	 PCS_MISC_CTL_GMX_ENO			(1UL << 11)
+#define	 PCS_MISC_CTL_SAMP_PT_MASK	0x7FUL
+#define	BGX_GMP_GMI_PRTX_CFG		0x38020
+#define	 GMI_PORT_CFG_SPEED			(1UL << 1)
+#define	 GMI_PORT_CFG_DUPLEX			(1UL << 2)
+#define	 GMI_PORT_CFG_SLOT_TIME			(1UL << 3)
+#define	 GMI_PORT_CFG_SPEED_MSB			(1UL << 8)
+#define	BGX_GMP_GMI_RXX_JABBER		0x38038
+#define	BGX_GMP_GMI_TXX_THRESH		0x38210
+#define	BGX_GMP_GMI_TXX_APPEND		0x38218
+#define	BGX_GMP_GMI_TXX_SLOT		0x38220
+#define	BGX_GMP_GMI_TXX_BURST		0x38228
+#define	BGX_GMP_GMI_TXX_MIN_PKT		0x38240
+#define	BGX_GMP_GMI_TXX_SGMII_CTL	0x38300
 
 #define BGX_MSIX_VEC_0_29_ADDR		0x400000 /* +(0..29) << 4 */
 #define BGX_MSIX_VEC_0_29_CTL		0x400008
 #define BGX_MSIX_PBA_0			0x4F0000
 
 /* MSI-X interrupts */
-#define BGX_MSIX_VECTORS	30
-#define BGX_LMAC_VEC_OFFSET	7
-#define BGX_MSIX_VEC_SHIFT	4
+#define	BGX_MSIX_VECTORS	30
+#define	BGX_LMAC_VEC_OFFSET	7
+#define	BGX_MSIX_VEC_SHIFT	4
 
-#define CMRX_INT		0
-#define SPUX_INT		1
-#define SMUX_RX_INT		2
-#define SMUX_TX_INT		3
-#define GMPX_PCS_INT		4
-#define GMPX_GMI_RX_INT		5
-#define GMPX_GMI_TX_INT		6
-#define CMR_MEM_INT		28
-#define SPU_MEM_INT		29
+#define	CMRX_INT		0
+#define	SPUX_INT		1
+#define	SMUX_RX_INT		2
+#define	SMUX_TX_INT		3
+#define	GMPX_PCS_INT		4
+#define	GMPX_GMI_RX_INT		5
+#define	GMPX_GMI_TX_INT		6
+#define	CMR_MEM_INT		28
+#define	SPU_MEM_INT		29
 
-#define LMAC_INTR_LINK_UP	BIT(0)
-#define LMAC_INTR_LINK_DOWN	BIT(1)
+#define	LMAC_INTR_LINK_UP	(1 << 0)
+#define	LMAC_INTR_LINK_DOWN	(1 << 1)
 
 /*  RX_DMAC_CTL configuration*/
 enum MCAST_MODE {
@@ -200,29 +200,29 @@ enum MCAST_MODE {
 		RSVD
 };
 
-#define BCAST_ACCEPT	1
-#define CAM_ACCEPT	1
+#define	BCAST_ACCEPT	1
+#define	CAM_ACCEPT	1
 
 void octeon_mdiobus_force_mod_depencency(void);
-void bgx_add_dmac_addr(u64 dmac, int node, int bgx_idx, int lmac);
+void bgx_add_dmac_addr(uint64_t dmac, int node, int bgx_idx, int lmac);
 unsigned bgx_get_map(int node);
 int bgx_get_lmac_count(int node, int bgx);
-const u8 *bgx_get_lmac_mac(int node, int bgx_idx, int lmacid);
-void bgx_set_lmac_mac(int node, int bgx_idx, int lmacid, const u8 *mac);
+const uint8_t *bgx_get_lmac_mac(int node, int bgx_idx, int lmacid);
+void bgx_set_lmac_mac(int node, int bgx_idx, int lmacid, const uint8_t *mac);
 void bgx_get_lmac_link_state(int node, int bgx_idx, int lmacid, void *status);
 void bgx_lmac_internal_loopback(int node, int bgx_idx,
-				int lmac_idx, bool enable);
-u64 bgx_get_rx_stats(int node, int bgx_idx, int lmac, int idx);
-u64 bgx_get_tx_stats(int node, int bgx_idx, int lmac, int idx);
-#define BGX_RX_STATS_COUNT 11
-#define BGX_TX_STATS_COUNT 18
+    int lmac_idx, boolean_t enable);
+uint64_t bgx_get_rx_stats(int node, int bgx_idx, int lmac, int idx);
+uint64_t bgx_get_tx_stats(int node, int bgx_idx, int lmac, int idx);
+#define	BGX_RX_STATS_COUNT 11
+#define	BGX_TX_STATS_COUNT 18
 
 struct bgx_stats {
-	u64 rx_stats[BGX_RX_STATS_COUNT];
-	u64 tx_stats[BGX_TX_STATS_COUNT];
+	uint64_t rx_stats[BGX_RX_STATS_COUNT];
+	uint64_t tx_stats[BGX_TX_STATS_COUNT];
 };
 
-#define BGX_IN_PROMISCUOUS_MODE 1
+#define	BGX_IN_PROMISCUOUS_MODE 1
 
 enum LMAC_TYPE {
 	BGX_MODE_SGMII = 0, /* 1 lane, 1.250 Gbaud */
diff --git a/sys/dev/vnic/thunder_bgx_fdt.c b/sys/dev/vnic/thunder_bgx_fdt.c
new file mode 100644
index 000000000000..1560262c8cc0
--- /dev/null
+++ b/sys/dev/vnic/thunder_bgx_fdt.c
@@ -0,0 +1,207 @@
+/*-
+ * Copyright (c) 2015 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by Semihalf under
+ * the sponsorship of the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bitset.h>
+#include <sys/bitstring.h>
+#include <sys/bus.h>
+#include <sys/endian.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/rman.h>
+#include <sys/pciio.h>
+#include <sys/pcpu.h>
+#include <sys/proc.h>
+#include <sys/socket.h>
+#include <sys/cpuset.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+
+#include <net/ethernet.h>
+#include <net/if.h>
+#include <net/if_media.h>
+
+#include <dev/ofw/openfirm.h>
+#include <dev/mii/miivar.h>
+
+#include "thunder_bgx.h"
+#include "thunder_bgx_var.h"
+
+#define	CONN_TYPE_MAXLEN	16
+#define	CONN_TYPE_OFFSET	2
+
+int bgx_fdt_init_phy(struct bgx *);
+
+static void
+bgx_fdt_get_macaddr(phandle_t phy, uint8_t *hwaddr)
+{
+	uint8_t addr[ETHER_ADDR_LEN];
+
+	if (OF_getprop(phy, "local-mac-address", addr, ETHER_ADDR_LEN) == -1) {
+		/* Missing MAC address should be marked by clearing it */
+		memset(hwaddr, 0, ETHER_ADDR_LEN);
+	} else
+		memcpy(hwaddr, addr, ETHER_ADDR_LEN);
+}
+
+static boolean_t
+bgx_fdt_phy_mode_match(struct bgx *bgx, char *qlm_mode, size_t size)
+{
+
+	size -= CONN_TYPE_OFFSET;
+
+	switch (bgx->qlm_mode) {
+	case QLM_MODE_SGMII:
+		if (strncmp(&qlm_mode[CONN_TYPE_OFFSET], "sgmii", size) == 0)
+			return (TRUE);
+		break;
+	case QLM_MODE_XAUI_1X4:
+		if (strncmp(&qlm_mode[CONN_TYPE_OFFSET], "xaui", size) == 0)
+			return (TRUE);
+		if (strncmp(&qlm_mode[CONN_TYPE_OFFSET], "dxaui", size) == 0)
+			return (TRUE);
+		break;
+	case QLM_MODE_RXAUI_2X2:
+		if (strncmp(&qlm_mode[CONN_TYPE_OFFSET], "raui", size) == 0)
+			return (TRUE);
+		break;
+	case QLM_MODE_XFI_4X1:
+		if (strncmp(&qlm_mode[CONN_TYPE_OFFSET], "xfi", size) == 0)
+			return (TRUE);
+		break;
+	case QLM_MODE_XLAUI_1X4:
+		if (strncmp(&qlm_mode[CONN_TYPE_OFFSET], "xlaui", size) == 0)
+			return (TRUE);
+		break;
+	case QLM_MODE_10G_KR_4X1:
+		if (strncmp(&qlm_mode[CONN_TYPE_OFFSET], "xfi-10g-kr", size) == 0)
+			return (TRUE);
+		break;
+	case QLM_MODE_40G_KR4_1X4:
+		if (strncmp(&qlm_mode[CONN_TYPE_OFFSET], "xlaui-40g-kr", size) == 0)
+			return (TRUE);
+		break;
+	default:
+		return (FALSE);
+	}
+
+	return (FALSE);
+}
+
+int
+bgx_fdt_init_phy(struct bgx *bgx)
+{
+	phandle_t node, child;
+	phandle_t phy, mdio;
+	uint8_t lmac;
+	char bgx_sel[6];
+	char qlm_mode[CONN_TYPE_MAXLEN];
+	const char *mac;
+
+	(void)mac;
+
+	lmac = 0;
+	/* Get BGX node from DT */
+	snprintf(bgx_sel, 6, "/bgx%d", bgx->bgx_id);
+	node = OF_finddevice(bgx_sel);
+	if (node == 0 || node == -1) {
+		device_printf(bgx->dev,
+		    "Could not find %s node in FDT\n", bgx_sel);
+		return (ENXIO);
+	}
+
+	for (child = OF_child(node); child > 0; child = OF_peer(child)) {
+		if (OF_getprop(child, "qlm-mode", qlm_mode,
+		    sizeof(qlm_mode)) <= 0) {
+			/* Missing qlm-mode, skipping */
+			continue;
+		}
+
+		if (!bgx_fdt_phy_mode_match(bgx, qlm_mode, sizeof(qlm_mode))) {
+			/*
+			 * Connection type not match with BGX mode.
+			 */
+			continue;
+		}
+
+		if (OF_getencprop(child, "phy-handle", &phy,
+		    sizeof(phy)) <= 0) {
+			if (bootverbose) {
+				device_printf(bgx->dev,
+				    "No phy-handle in PHY node. Skipping...\n");
+			}
+			continue;
+		}
+
+		/* Acquire PHY address */
+		phy = OF_node_from_xref(phy);
+		if (OF_getencprop(phy, "reg", &bgx->lmac[lmac].phyaddr,
+		    sizeof(bgx->lmac[lmac].phyaddr)) <= 0) {
+			if (bootverbose) {
+				device_printf(bgx->dev,
+				    "Could not retrieve PHY address\n");
+			}
+			bgx->lmac[lmac].phyaddr = MII_PHY_ANY;
+		}
+
+		/*
+		 * Get PHY interface (MDIO bus) device.
+		 * Driver must be already attached.
+		 */
+		mdio = OF_parent(phy);
+		bgx->lmac[lmac].phy_if_dev =
+		    OF_device_from_xref(OF_xref_from_node(mdio));
+		if (bgx->lmac[lmac].phy_if_dev == NULL) {
+			if (bootverbose) {
+				device_printf(bgx->dev,
+				    "Could not find interface to PHY\n");
+			}
+			continue;
+		}
+
+		/* Get mac address from FDT */
+		bgx_fdt_get_macaddr(phy, bgx->lmac[lmac].mac);
+
+		bgx->lmac[lmac].lmacid = lmac;
+		lmac++;
+		if (lmac == MAX_LMAC_PER_BGX)
+			break;
+	}
+	if (lmac == 0) {
+		device_printf(bgx->dev, "Could not find matching PHY\n");
+		return (ENXIO);
+	}
+
+	return (0);
+}
diff --git a/sys/dev/vnic/thunder_bgx_var.h b/sys/dev/vnic/thunder_bgx_var.h
new file mode 100644
index 000000000000..bfb1ef0ccbb5
--- /dev/null
+++ b/sys/dev/vnic/thunder_bgx_var.h
@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2015 Cavium Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ *
+ */
+
+#ifndef __THUNDER_BGX_VAR_H__
+#define	__THUNDER_BGX_VAR_H__
+
+struct lmac {
+	struct bgx		*bgx;
+	int			dmac;
+	uint8_t			mac[ETHER_ADDR_LEN];
+	boolean_t		link_up;
+	int			lmacid; /* ID within BGX */
+	int			lmacid_bd; /* ID on board */
+	device_t		phy_if_dev;
+	int			phyaddr;
+	unsigned int            last_duplex;
+	unsigned int            last_link;
+	unsigned int            last_speed;
+	boolean_t		is_sgmii;
+	struct callout		check_link;
+	struct mtx		check_link_mtx;
+};
+
+struct bgx {
+	device_t		dev;
+	struct resource *	reg_base;
+
+	uint8_t			bgx_id;
+	enum qlm_mode		qlm_mode;
+	struct	lmac		lmac[MAX_LMAC_PER_BGX];
+	int			lmac_count;
+	int                     lmac_type;
+	int                     lane_to_sds;
+	int			use_training;
+};
+
+#ifdef FDT
+extern int bgx_fdt_init_phy(struct bgx *);
+#endif
+
+#endif

From 654dadae0c1369dcb505cb5078d7722bbcc675ae Mon Sep 17 00:00:00 2001
From: zbb <zbb@FreeBSD.org>
Date: Sun, 18 Oct 2015 22:10:08 +0000
Subject: [PATCH 365/380] Introduce driver for Cavium's ThunderX MDIO

This commit adds support for MDIO present in the ThunderX SoC.
From the FDT point of view it is compatible with "octeon-3860-mdio"
however only C22 mode is used.
The code also implements lmac_if interface functions.

Obtained from: Semihalf
Sponsored by:  The FreeBSD Foundation
---
 sys/conf/files.arm64            |   2 +
 sys/dev/vnic/thunder_mdio.c     | 512 ++++++++++++++++++++++++++++++++
 sys/dev/vnic/thunder_mdio_fdt.c |  95 ++++++
 sys/dev/vnic/thunder_mdio_var.h |  63 ++++
 4 files changed, 672 insertions(+)
 create mode 100644 sys/dev/vnic/thunder_mdio.c
 create mode 100644 sys/dev/vnic/thunder_mdio_fdt.c
 create mode 100644 sys/dev/vnic/thunder_mdio_var.h

diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64
index 698c053166f1..a49a1108993f 100644
--- a/sys/conf/files.arm64
+++ b/sys/conf/files.arm64
@@ -73,6 +73,8 @@ dev/vnic/nicvf_main.c		optional	vnic pci pci_iov
 dev/vnic/nicvf_queues.c		optional	vnic pci pci_iov
 dev/vnic/thunder_bgx_fdt.c	optional	vnic fdt
 dev/vnic/thunder_bgx.c		optional	vnic pci
+dev/vnic/thunder_mdio_fdt.c	optional	vnic fdt
+dev/vnic/thunder_mdio.c		optional	vnic
 dev/vnic/lmac_if.m		optional	vnic
 kern/kern_clocksource.c		standard
 kern/subr_dummy_vdso_tc.c	standard
diff --git a/sys/dev/vnic/thunder_mdio.c b/sys/dev/vnic/thunder_mdio.c
new file mode 100644
index 000000000000..81576238a803
--- /dev/null
+++ b/sys/dev/vnic/thunder_mdio.c
@@ -0,0 +1,512 @@
+/*-
+ * Copyright (c) 2015 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by Semihalf under
+ * the sponsorship of the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/resource.h>
+#include <sys/rman.h>
+#include <sys/socket.h>
+#include <sys/queue.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+
+#include <net/if.h>
+#include <net/if_media.h>
+#include <net/if_types.h>
+#include <net/if_var.h>
+
+#include <dev/mii/mii.h>
+#include <dev/mii/miivar.h>
+
+#include "thunder_mdio_var.h"
+
+#include "lmac_if.h"
+#include "miibus_if.h"
+
+#define	REG_BASE_RID	0
+
+#define	SMI_CMD				0x00
+#define	 SMI_CMD_PHY_REG_ADR_SHIFT	(0)
+#define	 SMI_CMD_PHY_REG_ADR_MASK	(0x1FUL << SMI_CMD_PHY_REG_ADR_SHIFT)
+#define	 SMI_CMD_PHY_ADR_SHIFT		(8)
+#define	 SMI_CMD_PHY_ADR_MASK		(0x1FUL << SMI_CMD_PHY_ADR_SHIFT)
+#define	 SMI_CMD_PHY_OP_MASK		(0x3UL << 16)
+#define	 SMI_CMD_PHY_OP_C22_READ	(0x1UL << 16)
+#define	 SMI_CMD_PHY_OP_C22_WRITE	(0x0UL << 16)
+#define	 SMI_CMD_PHY_OP_C45_READ	(0x3UL << 16)
+#define	 SMI_CMD_PHY_OP_C45_WRITE	(0x1UL << 16)
+#define	 SMI_CMD_PHY_OP_C45_ADDR	(0x0UL << 16)
+
+#define	SMI_WR_DAT			0x08
+#define	 SMI_WR_DAT_PENDING		(1UL << 17)
+#define	 SMI_WR_DAT_VAL			(1UL << 16)
+#define	 SMI_WR_DAT_DAT_MASK		(0xFFFFUL << 0)
+
+#define	SMI_RD_DAT			0x10
+#define	 SMI_RD_DAT_PENDING		(1UL << 17)
+#define	 SMI_RD_DAT_VAL			(1UL << 16)
+#define	 SMI_RD_DAT_DAT_MASK		(0xFFFFUL << 0)
+
+#define	SMI_CLK				0x18
+#define	 SMI_CLK_PREAMBLE		(1UL << 12)
+#define	 SMI_CLK_MODE			(1UL << 24)
+
+#define	SMI_EN				0x20
+#define	 SMI_EN_EN			(1UL << 0)	/* Enabele interface */
+
+#define	SMI_DRV_CTL			0x28
+
+static int thunder_mdio_detach(device_t);
+
+static int thunder_mdio_read(device_t, int, int);
+static int thunder_mdio_write(device_t, int, int, int);
+
+static int thunder_ifmedia_change_stub(struct ifnet *);
+static void thunder_ifmedia_status_stub(struct ifnet *, struct ifmediareq *);
+
+static int thunder_mdio_media_status(device_t, int, int *, int *, int *);
+static int thunder_mdio_media_change(device_t, int, int, int, int);
+static int thunder_mdio_phy_connect(device_t, int, int);
+static int thunder_mdio_phy_disconnect(device_t, int, int);
+
+static device_method_t thunder_mdio_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_detach,	thunder_mdio_detach),
+	/* LMAC interface */
+	DEVMETHOD(lmac_media_status,	thunder_mdio_media_status),
+	DEVMETHOD(lmac_media_change,	thunder_mdio_media_change),
+	DEVMETHOD(lmac_phy_connect,	thunder_mdio_phy_connect),
+	DEVMETHOD(lmac_phy_disconnect,	thunder_mdio_phy_disconnect),
+	/* MII interface */
+	DEVMETHOD(miibus_readreg,	thunder_mdio_read),
+	DEVMETHOD(miibus_writereg,	thunder_mdio_write),
+
+	/* End */
+	DEVMETHOD_END
+};
+
+DEFINE_CLASS_0(thunder_mdio, thunder_mdio_driver, thunder_mdio_methods,
+    sizeof(struct thunder_mdio_softc));
+
+DRIVER_MODULE(miibus, thunder_mdio, miibus_driver, miibus_devclass, 0, 0);
+MODULE_DEPEND(thunder_mdio, ether, 1, 1, 1);
+MODULE_DEPEND(thunder_mdio, miibus, 1, 1, 1);
+
+MALLOC_DEFINE(M_THUNDER_MDIO, "ThunderX MDIO",
+    "Cavium ThunderX MDIO dynamic memory");
+
+#define	MDIO_LOCK_INIT(sc, name)			\
+    mtx_init(&(sc)->mtx, name, NULL, MTX_DEF)
+
+#define	MDIO_LOCK_DESTROY(sc)				\
+    mtx_destroy(&(sc)->mtx)
+
+#define	MDIO_LOCK(sc)	mtx_lock(&(sc)->mtx)
+#define	MDIO_UNLOCK(sc)	mtx_unlock(&(sc)->mtx)
+
+#define	MDIO_LOCK_ASSERT(sc)				\
+    mtx_assert(&(sc)->mtx, MA_OWNED)
+
+
+#define	mdio_reg_read(sc, reg)				\
+    bus_read_8((sc)->reg_base, (reg))
+
+#define	mdio_reg_write(sc, reg, val)			\
+    bus_write_8((sc)->reg_base, (reg), (val))
+
+int
+thunder_mdio_attach(device_t dev)
+{
+	struct thunder_mdio_softc *sc;
+	int rid;
+
+	sc = device_get_softc(dev);
+	sc->dev = dev;
+
+	/* Allocate memory resources */
+	rid = REG_BASE_RID;
+	sc->reg_base = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
+	    RF_ACTIVE);
+	if (sc->reg_base == NULL) {
+		device_printf(dev, "Could not allocate memory\n");
+		return (ENXIO);
+	}
+
+	TAILQ_INIT(&sc->phy_desc_head);
+	MDIO_LOCK_INIT(sc, "ThunderX MDIO lock");
+
+	/* Enable SMI/MDIO interface */
+	mdio_reg_write(sc, SMI_EN, SMI_EN_EN);
+
+	return (0);
+}
+
+static int
+thunder_mdio_detach(device_t dev)
+{
+	struct thunder_mdio_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	if (sc->reg_base != NULL) {
+		bus_release_resource(dev, SYS_RES_MEMORY, REG_BASE_RID,
+		    sc->reg_base);
+	}
+
+	return (0);
+}
+
+static __inline void
+thunder_mdio_set_mode(struct thunder_mdio_softc *sc,
+    enum thunder_mdio_mode mode)
+{
+	uint64_t smi_clk;
+
+	if (sc->mode == mode)
+		return;
+
+	/* Set mode, IEEE CLAUSE 22 or IEEE CAUSE 45 */
+	smi_clk = mdio_reg_read(sc, SMI_CLK);
+	if (mode == MODE_IEEE_C22)
+		smi_clk &= ~SMI_CLK_MODE;
+	else
+		smi_clk |= SMI_CLK_MODE;
+	/* Enable sending 32 bit preable on SMI transactions */
+	smi_clk |= SMI_CLK_PREAMBLE;
+	/* Saved setings */
+	mdio_reg_write(sc, SMI_CLK, smi_clk);
+	sc->mode = mode;
+}
+
+static int
+thunder_mdio_c45_addr(struct thunder_mdio_softc *sc, int phy, int reg)
+{
+	uint64_t smi_cmd, smi_wr_dat;
+	ssize_t timeout;
+
+	thunder_mdio_set_mode(sc, MODE_IEEE_C45);
+
+	/* Prepare data for transmission */
+	mdio_reg_write(sc, SMI_WR_DAT, reg & SMI_WR_DAT_DAT_MASK);
+	/*
+	 * Assemble command
+	 */
+	smi_cmd = 0;
+	/* Set opcode */
+	smi_cmd |= SMI_CMD_PHY_OP_C45_WRITE;
+
+	/* Set PHY address */
+	smi_cmd |= ((phy << SMI_CMD_PHY_ADR_SHIFT) & SMI_CMD_PHY_ADR_MASK);
+	/* Set PHY register offset */
+	smi_cmd |= ((reg << SMI_CMD_PHY_REG_ADR_SHIFT) &
+	    SMI_CMD_PHY_REG_ADR_MASK);
+
+	mdio_reg_write(sc, SMI_CMD, smi_cmd);
+	for (timeout = 1000; timeout > 0; timeout--) {
+		smi_wr_dat = mdio_reg_read(sc, SMI_WR_DAT);
+		if (smi_wr_dat & SMI_WR_DAT_PENDING)
+			DELAY(1000);
+		else
+			break;
+	}
+
+	if (timeout <= 0)
+		return (EIO);
+	else {
+		/* Return 0 on success */
+		return (0);
+	}
+}
+
+static int
+thunder_mdio_read(device_t dev, int phy, int reg)
+{
+	struct thunder_mdio_softc *sc;
+	uint64_t smi_cmd, smi_rd_dat;
+	ssize_t timeout;
+	int err;
+
+	sc = device_get_softc(dev);
+
+	/* XXX Always C22 - for <= 1Gbps only */
+	thunder_mdio_set_mode(sc, MODE_IEEE_C22);
+
+	/*
+	 * Assemble command
+	 */
+	smi_cmd = 0;
+	/* Set opcode */
+	if (sc->mode == MODE_IEEE_C22)
+		smi_cmd |= SMI_CMD_PHY_OP_C22_READ;
+	else {
+		smi_cmd |= SMI_CMD_PHY_OP_C45_READ;
+		err = thunder_mdio_c45_addr(sc, phy, reg);
+		if (err != 0)
+			return (err);
+
+		reg = (reg >> 16) & 0x1F;
+	}
+
+	/* Set PHY address */
+	smi_cmd |= ((phy << SMI_CMD_PHY_ADR_SHIFT) & SMI_CMD_PHY_ADR_MASK);
+	/* Set PHY register offset */
+	smi_cmd |= ((reg << SMI_CMD_PHY_REG_ADR_SHIFT) &
+	    SMI_CMD_PHY_REG_ADR_MASK);
+
+	mdio_reg_write(sc, SMI_CMD, smi_cmd);
+	for (timeout = 1000; timeout > 0; timeout--) {
+		smi_rd_dat = mdio_reg_read(sc, SMI_RD_DAT);
+		if (smi_rd_dat & SMI_RD_DAT_PENDING)
+			DELAY(1000);
+		else
+			break;
+	}
+
+	if (smi_rd_dat & SMI_RD_DAT_VAL)
+		return (smi_rd_dat & SMI_RD_DAT_DAT_MASK);
+	else {
+		/* Return 0 on error */
+		return (0);
+	}
+}
+
+static int
+thunder_mdio_write(device_t dev, int phy, int reg, int data)
+{
+	struct thunder_mdio_softc *sc;
+	uint64_t smi_cmd, smi_wr_dat;
+	ssize_t timeout;
+
+	sc = device_get_softc(dev);
+
+	/* XXX Always C22 - for <= 1Gbps only */
+	thunder_mdio_set_mode(sc, MODE_IEEE_C22);
+
+	/* Prepare data for transmission */
+	mdio_reg_write(sc, SMI_WR_DAT, data & SMI_WR_DAT_DAT_MASK);
+	/*
+	 * Assemble command
+	 */
+	smi_cmd = 0;
+	/* Set opcode */
+	if (sc->mode == MODE_IEEE_C22)
+		smi_cmd |= SMI_CMD_PHY_OP_C22_WRITE;
+	else
+		smi_cmd |= SMI_CMD_PHY_OP_C45_WRITE;
+
+	/* Set PHY address */
+	smi_cmd |= ((phy << SMI_CMD_PHY_ADR_SHIFT) & SMI_CMD_PHY_ADR_MASK);
+	/* Set PHY register offset */
+	smi_cmd |= ((reg << SMI_CMD_PHY_REG_ADR_SHIFT) &
+	    SMI_CMD_PHY_REG_ADR_MASK);
+
+	mdio_reg_write(sc, SMI_CMD, smi_cmd);
+	for (timeout = 1000; timeout > 0; timeout--) {
+		smi_wr_dat = mdio_reg_read(sc, SMI_WR_DAT);
+		if (smi_wr_dat & SMI_WR_DAT_PENDING)
+			DELAY(1000);
+		else
+			break;
+	}
+
+	if (timeout <= 0)
+		return (EIO);
+	else {
+		/* Return 0 on success */
+		return (0);
+	}
+}
+
+static int
+thunder_ifmedia_change_stub(struct ifnet *ifp __unused)
+{
+	/* Will never be called by if_media */
+	return (0);
+}
+
+static void
+thunder_ifmedia_status_stub(struct ifnet *ifp __unused, struct ifmediareq
+    *ifmr __unused)
+{
+	/* Will never be called by if_media */
+}
+
+static __inline struct phy_desc *
+get_phy_desc(struct thunder_mdio_softc *sc, int lmacid)
+{
+	struct phy_desc *pd = NULL;
+
+	MDIO_LOCK_ASSERT(sc);
+	TAILQ_FOREACH(pd, &sc->phy_desc_head, phy_desc_list) {
+		if (pd->lmacid == lmacid)
+			break;
+	}
+
+	return (pd);
+}
+static int
+thunder_mdio_media_status(device_t dev, int lmacid, int *link, int *duplex,
+    int *speed)
+{
+	struct thunder_mdio_softc *sc;
+	struct mii_data *mii_sc;
+	struct phy_desc *pd;
+
+	sc = device_get_softc(dev);
+
+	MDIO_LOCK(sc);
+	pd = get_phy_desc(sc, lmacid);
+	if (pd == NULL) {
+		/* Panic when invariants are enabled, fail otherwise. */
+		KASSERT(0, ("%s: no PHY descriptor for LMAC%d",
+		    __func__, lmacid));
+		MDIO_UNLOCK(sc);
+		return (ENXIO);
+	}
+	mii_sc = device_get_softc(pd->miibus);
+
+	mii_tick(mii_sc);
+	if ((mii_sc->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
+	    (IFM_ACTIVE | IFM_AVALID)) {
+		/* Link is up */
+		*link = 1;
+	} else
+		*link = 0;
+
+	switch (IFM_SUBTYPE(mii_sc->mii_media_active)) {
+	case IFM_10_T:
+		*speed = 10;
+		break;
+	case IFM_100_TX:
+		*speed = 100;
+		break;
+	case IFM_1000_T:
+		*speed = 1000;
+		break;
+	default:
+		/* IFM_NONE */
+		*speed = 0;
+	}
+
+	if ((IFM_OPTIONS(mii_sc->mii_media_active) & IFM_FDX) != 0)
+		*duplex = 1;
+	else
+		*duplex = 0;
+
+	MDIO_UNLOCK(sc);
+
+	return (0);
+}
+
+static int
+thunder_mdio_media_change(device_t dev, int lmacid, int link, int duplex,
+    int speed)
+{
+
+	return (EIO);
+}
+
+static int
+thunder_mdio_phy_connect(device_t dev, int lmacid, int phy)
+{
+	struct thunder_mdio_softc *sc;
+	struct phy_desc *pd;
+	int err;
+
+	sc = device_get_softc(dev);
+
+	MDIO_LOCK(sc);
+	pd = get_phy_desc(sc, lmacid);
+	MDIO_UNLOCK(sc);
+	if (pd == NULL) {
+		pd = malloc(sizeof(*pd), M_THUNDER_MDIO, (M_NOWAIT | M_ZERO));
+		if (pd == NULL)
+			return (ENOMEM);
+		pd->ifp = if_alloc(IFT_ETHER);
+		if (pd->ifp == NULL) {
+			free(pd, M_THUNDER_MDIO);
+			return (ENOMEM);
+		}
+		pd->lmacid = lmacid;
+	}
+
+	err = mii_attach(dev, &pd->miibus, pd->ifp,
+	    thunder_ifmedia_change_stub, thunder_ifmedia_status_stub,
+	    BMSR_DEFCAPMASK, phy, MII_OFFSET_ANY, 0);
+
+	if (err != 0) {
+		device_printf(dev, "Could not attach PHY%d\n", phy);
+		if_free(pd->ifp);
+		free(pd, M_THUNDER_MDIO);
+		return (ENXIO);
+	}
+
+	MDIO_LOCK(sc);
+	TAILQ_INSERT_TAIL(&sc->phy_desc_head, pd, phy_desc_list);
+	MDIO_UNLOCK(sc);
+
+	return (0);
+}
+
+static int
+thunder_mdio_phy_disconnect(device_t dev, int lmacid, int phy)
+{
+	struct thunder_mdio_softc *sc;
+	struct phy_desc *pd;
+
+	sc = device_get_softc(dev);
+	MDIO_LOCK(sc);
+
+	pd = get_phy_desc(sc, lmacid);
+	if (pd == NULL) {
+		MDIO_UNLOCK(sc);
+		return (EINVAL);
+	}
+
+	/* Remove this PHY descriptor from the list */
+	TAILQ_REMOVE(&sc->phy_desc_head, pd, phy_desc_list);
+
+	/* Detach miibus */
+	bus_generic_detach(dev);
+	device_delete_child(dev, pd->miibus);
+	/* Free fake ifnet */
+	if_free(pd->ifp);
+	/* Free memory under phy descriptor */
+	free(pd, M_THUNDER_MDIO);
+	MDIO_UNLOCK(sc);
+
+	return (0);
+}
diff --git a/sys/dev/vnic/thunder_mdio_fdt.c b/sys/dev/vnic/thunder_mdio_fdt.c
new file mode 100644
index 000000000000..cd2d1cff9f09
--- /dev/null
+++ b/sys/dev/vnic/thunder_mdio_fdt.c
@@ -0,0 +1,95 @@
+/*-
+ * Copyright (c) 2015 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by Semihalf under
+ * the sponsorship of the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include "thunder_mdio_var.h"
+
+static int thunder_mdio_fdt_probe(device_t);
+static int thunder_mdio_fdt_attach(device_t);
+
+static device_method_t thunder_mdio_fdt_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		thunder_mdio_fdt_probe),
+	DEVMETHOD(device_attach,	thunder_mdio_fdt_attach),
+
+	/* End */
+	DEVMETHOD_END
+};
+
+DEFINE_CLASS_1(thunder_mdio, thunder_mdio_fdt_driver, thunder_mdio_fdt_methods,
+    sizeof(struct thunder_mdio_softc), thunder_mdio_driver);
+
+static devclass_t thunder_mdio_fdt_devclass;
+
+DRIVER_MODULE(thunder_mdio, ofwbus, thunder_mdio_fdt_driver,
+    thunder_mdio_fdt_devclass, 0, 0);
+
+static int
+thunder_mdio_fdt_probe(device_t dev)
+{
+
+	if (ofw_bus_is_compatible(dev, "cavium,octeon-3860-mdio")) {
+		device_set_desc(dev, THUNDER_MDIO_DEVSTR);
+		return (BUS_PROBE_DEFAULT);
+	}
+
+	return (ENXIO);
+}
+
+static int
+thunder_mdio_fdt_attach(device_t dev)
+{
+	phandle_t node;
+	int ret;
+
+	/* Call core attach */
+	ret = thunder_mdio_attach(dev);
+	if (ret != 0)
+		return (ret);
+	/*
+	 * Register device to this node/xref.
+	 * Thanks to that we will be able to retrieve device_t structure
+	 * while holding only node reference acquired from FDT.
+	 */
+	node = ofw_bus_get_node(dev);
+	OF_device_register_xref(OF_xref_from_node(node), dev);
+
+	return (0);
+}
diff --git a/sys/dev/vnic/thunder_mdio_var.h b/sys/dev/vnic/thunder_mdio_var.h
new file mode 100644
index 000000000000..01c4ca7f637c
--- /dev/null
+++ b/sys/dev/vnic/thunder_mdio_var.h
@@ -0,0 +1,63 @@
+/*-
+ * Copyright (c) 2015 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by Semihalf under
+ * the sponsorship of the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ *
+ */
+
+#ifndef __THUNDER_MDIO_VAR_H__
+#define	__THUNDER_MDIO_VAR_H__
+
+#define	THUNDER_MDIO_DEVSTR	"Cavium ThunderX SMI/MDIO driver"
+DECLARE_CLASS(thunder_mdio_driver);
+
+enum thunder_mdio_mode {
+	MODE_NONE = 0,
+	MODE_IEEE_C22,
+	MODE_IEEE_C45
+};
+
+struct phy_desc {
+	device_t		miibus; /* One miibus per LMAC */
+	struct ifnet *		ifp;	/* Fake ifp to satisfy miibus */
+	int			lmacid;	/* ID number of LMAC connected */
+	TAILQ_ENTRY(phy_desc)	phy_desc_list;
+};
+
+struct thunder_mdio_softc {
+	device_t		dev;
+	struct mtx		mtx;
+	struct resource *	reg_base;
+
+	enum thunder_mdio_mode	mode;
+
+	TAILQ_HEAD(,phy_desc)	phy_desc_head;
+};
+
+int thunder_mdio_attach(device_t);
+#endif

From fb5f64ee8b9ca967c59c0b2d89c00f0cb6d1c7c2 Mon Sep 17 00:00:00 2001
From: zbb <zbb@FreeBSD.org>
Date: Sun, 18 Oct 2015 22:13:21 +0000
Subject: [PATCH 366/380] Add ThunderX VNIC to arm64/GENERIC kernel

Add vnic to enabled networking cards and enable SR-IOV by the way.

Obtained from: Semihalf
Sponsored by:  The FreeBSD Foundation
---
 sys/arm64/conf/GENERIC | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/sys/arm64/conf/GENERIC b/sys/arm64/conf/GENERIC
index 1486f0793686..d671fa870c27 100644
--- a/sys/arm64/conf/GENERIC
+++ b/sys/arm64/conf/GENERIC
@@ -94,9 +94,11 @@ device		virtio_blk
 device		vtnet
 
 # Bus drivers
+options 	PCI_IOV
 device		pci
 
 # Ethernet NICs
+device		vnic		# Cavium ThunderX NIC
 device		em		# Intel PRO/1000 Gigabit Ethernet Family
 device		mii
 device		miibus		# MII bus support

From aa6bcff20c514c7c1fe9e239b22d8c59ec6e608b Mon Sep 17 00:00:00 2001
From: gonzo <gonzo@FreeBSD.org>
Date: Sun, 18 Oct 2015 23:58:05 +0000
Subject: [PATCH 367/380] Enable gpiobacklight in BEAGLEBONE config to support
 LCD capes by 4DSYSTEMS out of box

---
 sys/arm/conf/BEAGLEBONE | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sys/arm/conf/BEAGLEBONE b/sys/arm/conf/BEAGLEBONE
index 48d793406317..59c72792bde3 100644
--- a/sys/arm/conf/BEAGLEBONE
+++ b/sys/arm/conf/BEAGLEBONE
@@ -85,6 +85,7 @@ device		random			# Entropy device
 # GPIO
 device		gpio
 device		gpioled
+device		gpiobacklight
 
 # ADC support
 device		ti_adc

From 7177ad0e480aa29b6bea83bb31e31bb533e1f8f3 Mon Sep 17 00:00:00 2001
From: adrian <adrian@FreeBSD.org>
Date: Mon, 19 Oct 2015 01:14:26 +0000
Subject: [PATCH 368/380] otus(4) - add initial monitor mode; use lowest rate
 for EAPOL

The monitor mode stuff is from the openbsd driver, but it doesn't
100% work.  It doesn't seem to get all frames for all BSSes.
However, it's enough to at start debugging things.  That 0xffffffff
write is /I think/ the RX filter, but I am still not 100% sure about
it all.

Then, whilst here, use the lowest rate for EAPOL frames.  This is just
generally a good thing to do.
---
 sys/dev/otus/if_otus.c | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/sys/dev/otus/if_otus.c b/sys/dev/otus/if_otus.c
index 1808c950c496..1f0df91b82d3 100644
--- a/sys/dev/otus/if_otus.c
+++ b/sys/dev/otus/if_otus.c
@@ -759,6 +759,7 @@ otus_attachhook(struct otus_softc *sc)
 	    IEEE80211_C_WME |		/* WME/QoS */
 	    IEEE80211_C_SHSLOT |	/* Short slot time supported. */
 	    IEEE80211_C_FF |		/* Atheros fast-frames supported. */
+	    IEEE80211_C_MONITOR |
 	    IEEE80211_C_WPA;		/* WPA/RSN. */
 
 	/* XXX TODO: 11n */
@@ -2229,6 +2230,9 @@ otus_tx(struct otus_softc *sc, struct ieee80211_node *ni, struct mbuf *m,
 	    (wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK) != IEEE80211_FC0_TYPE_DATA) {
 		/* Get lowest rate */
 		rate = otus_rate_to_hw_rate(sc, 0);
+	} else if (m->m_flags & M_EAPOL) {
+		/* Get lowest rate */
+		rate = otus_rate_to_hw_rate(sc, 0);
 	} else {
 		(void) ieee80211_ratectl_rate(ni, NULL, 0);
 		rate = otus_rate_to_hw_rate(sc, ni->ni_txrate);
@@ -3094,13 +3098,22 @@ otus_init(struct otus_softc *sc)
 	}
 #endif
 
-	/* Expect STA operation */
-	otus_write(sc, 0x1c3700, 0x0f000002);
-	otus_write(sc, 0x1c3c40, 0x1);
+	switch (ic->ic_opmode) {
+	case IEEE80211_M_STA:
+		otus_write(sc, 0x1c3700, 0x0f000002);
+		otus_write(sc, 0x1c3c40, 0x1);
+		break;
+	case IEEE80211_M_MONITOR:
+		otus_write(sc, 0x1c368c, 0xffffffff);
+		break;
+	default:
+		break;
+	}
 
 	/* XXX ic_opmode? */
 	otus_write(sc, AR_MAC_REG_SNIFFER,
 	    (ic->ic_opmode == IEEE80211_M_MONITOR) ? 0x2000001 : 0x2000000);
+
 	(void)otus_write_barrier(sc);
 
 	sc->bb_reset = 1;	/* Force cold reset. */

From 12c5864844c0841a8c86a09401e1c89451777bc5 Mon Sep 17 00:00:00 2001
From: adrian <adrian@FreeBSD.org>
Date: Mon, 19 Oct 2015 01:21:29 +0000
Subject: [PATCH 369/380] otus(4) - use the local node alloc function so
 there's space for statistics.

* Use the correct malloc type for node allocation - M_80211_NODE - so
  the default node free method in net80211 will work correctly.
* Fix otus_node_alloc() to suit FreeBSD's net80211.
* .. and actually call otus_node_alloc() so there's space for the
  per-node tx statistics.  Otherwise, well, it will be scribbling over
  random memory.

Tested:

* AR9170, STA mode
---
 sys/dev/otus/if_otus.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/sys/dev/otus/if_otus.c b/sys/dev/otus/if_otus.c
index 1f0df91b82d3..8756419747e8 100644
--- a/sys/dev/otus/if_otus.c
+++ b/sys/dev/otus/if_otus.c
@@ -164,7 +164,8 @@ int		otus_cmd(struct otus_softc *, uint8_t, const void *, int,
 		    void *, int);
 void		otus_write(struct otus_softc *, uint32_t, uint32_t);
 int		otus_write_barrier(struct otus_softc *);
-struct		ieee80211_node *otus_node_alloc(struct ieee80211com *);
+static struct	ieee80211_node *otus_node_alloc(struct ieee80211vap *vap,
+		    const uint8_t mac[IEEE80211_ADDR_LEN]);
 int		otus_media_change(struct ifnet *);
 int		otus_read_eeprom(struct otus_softc *);
 void		otus_newassoc(struct ieee80211_node *, int);
@@ -814,6 +815,7 @@ otus_attachhook(struct otus_softc *sc)
 	ic->ic_ampdu_enable = otus_ampdu_enable;
 	ic->ic_wme.wme_update = otus_wme_update;
 	ic->ic_newassoc = otus_newassoc;
+	ic->ic_node_alloc = otus_node_alloc;
 
 #ifdef notyet
 	ic->ic_set_key = otus_set_key;
@@ -1387,10 +1389,12 @@ otus_write_barrier(struct otus_softc *sc)
 	return error;
 }
 
-struct ieee80211_node *
-otus_node_alloc(struct ieee80211com *ic)
+static struct ieee80211_node *
+otus_node_alloc(struct ieee80211vap *vap, const uint8_t mac[IEEE80211_ADDR_LEN])
 {
-	return malloc(sizeof (struct otus_node), M_DEVBUF, M_NOWAIT | M_ZERO);
+
+	return malloc(sizeof (struct otus_node), M_80211_NODE,
+	    M_NOWAIT | M_ZERO);
 }
 
 #if 0

From fe5ffcc51d3f24b527e81b42efa1b9413cbbbb9a Mon Sep 17 00:00:00 2001
From: hselasky <hselasky@FreeBSD.org>
Date: Mon, 19 Oct 2015 07:21:57 +0000
Subject: [PATCH 370/380] Add quirk for USB 3.0 PCI device.

Submitted by:	philipp.maechler@mamo.li
PR:		203650
MFC after:	1 week
---
 sys/dev/usb/controller/xhci_pci.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/sys/dev/usb/controller/xhci_pci.c b/sys/dev/usb/controller/xhci_pci.c
index 492fd3cfdaca..e6f937f892c6 100644
--- a/sys/dev/usb/controller/xhci_pci.c
+++ b/sys/dev/usb/controller/xhci_pci.c
@@ -115,6 +115,8 @@ xhci_pci_match(device_t self)
 		return ("Intel Lynx Point USB 3.0 controller");
 	case 0x8cb18086:
 		return ("Intel Wildcat Point USB 3.0 controller");
+	case 0x9cb18086:
+		return ("Broadwell Integrated PCH-LP chipset USB 3.0 controller");
 
 	case 0xa01b177d:
 		return ("Cavium ThunderX USB 3.0 controller");
@@ -216,6 +218,7 @@ xhci_pci_attach(device_t self)
 	case 0x1e318086:	/* Panther Point */
 	case 0x8c318086:	/* Lynx Point */
 	case 0x8cb18086:	/* Wildcat Point */
+	case 0x9cb18086:	/* Broadwell Mobile Integrated */
 		/*
 		 * On Intel chipsets, reroute ports from EHCI to XHCI
 		 * controller and use a different IMOD value.

From f204be76de3ab32ce7e295d0767252f547d6075f Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Mon, 19 Oct 2015 08:25:37 +0000
Subject: [PATCH 371/380] MFV r289561: 6328 Fix cstyle errors in zfs codebase

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Alex Reece <alex@delphix.com>
Reviewed by: Richard Elling <Richard.Elling@RichardElling.com>
Reviewed by: Jorgen Lundman <lundman@lundman.net>
Approved by: Robert Mustacchi <rm@joyent.com>
Author: Paul Dagnelie <pcd@delphix.com>

illumos/illumos-gate@9a686fbc186e8e2a64e9a5094d44c7d6fa0ea167
---
 cddl/contrib/opensolaris/cmd/zpool/zpool_main.c     |  3 ++-
 .../opensolaris/lib/libzfs/common/libzfs_iter.c     |  5 +++--
 .../opensolaris/lib/libzfs/common/libzfs_pool.c     |  5 +++--
 .../opensolaris/lib/libzfs/common/libzfs_util.c     |  2 +-
 .../opensolaris/lib/libzpool/common/kernel.c        |  4 ++--
 .../opensolaris/common/zfs/zfeature_common.c        |  3 ++-
 .../contrib/opensolaris/uts/common/fs/zfs/dmu.c     |  9 +++++----
 .../opensolaris/uts/common/fs/zfs/dsl_dataset.c     |  2 +-
 .../opensolaris/uts/common/fs/zfs/space_reftree.c   |  4 ++--
 .../opensolaris/uts/common/fs/zfs/sys/zrlock.h      |  9 +++------
 .../opensolaris/uts/common/fs/zfs/vdev_label.c      |  6 +++---
 .../opensolaris/uts/common/fs/zfs/zap_leaf.c        |  4 ++--
 .../opensolaris/uts/common/fs/zfs/zfeature.c        |  6 ++++--
 .../contrib/opensolaris/uts/common/fs/zfs/zfs_dir.c |  4 ++--
 .../opensolaris/uts/common/fs/zfs/zfs_ioctl.c       |  2 +-
 .../contrib/opensolaris/uts/common/fs/zfs/zfs_log.c | 13 +++++++------
 .../opensolaris/uts/common/fs/zfs/zfs_replay.c      |  4 ++--
 .../opensolaris/uts/common/fs/zfs/zfs_vnops.c       |  6 +++---
 .../contrib/opensolaris/uts/common/fs/zfs/zio.c     |  6 +++---
 .../opensolaris/uts/common/fs/zfs/zio_checksum.c    |  4 ++--
 .../contrib/opensolaris/uts/common/fs/zfs/zrlock.c  |  8 ++------
 21 files changed, 55 insertions(+), 54 deletions(-)

diff --git a/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c b/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c
index 52ad81d05988..6e1670e8a392 100644
--- a/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c
+++ b/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c
@@ -199,7 +199,8 @@ static boolean_t log_history = B_TRUE;
 static uint_t timestamp_fmt = NODATE;
 
 static const char *
-get_usage(zpool_help_t idx) {
+get_usage(zpool_help_t idx)
+{
 	switch (idx) {
 	case HELP_ADD:
 		return (gettext("\tadd [-fn] <pool> <vdev> ...\n"));
diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_iter.c b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_iter.c
index aadc1f96deaa..b6f6e733e8e3 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_iter.c
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_iter.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2013, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2012 Pawel Jakub Dawidek <pawel@dawidek.net>.
  * All rights reserved.
  * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
@@ -315,7 +315,8 @@ typedef struct {
 } snapspec_arg_t;
 
 static int
-snapspec_cb(zfs_handle_t *zhp, void *arg) {
+snapspec_cb(zfs_handle_t *zhp, void *arg)
+{
 	snapspec_arg_t *ssa = arg;
 	char *shortsnapname;
 	int err = 0;
diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_pool.c b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_pool.c
index bbab81b6cd78..7cdcfc90feca 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_pool.c
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_pool.c
@@ -22,7 +22,7 @@
 /*
  * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  */
 
@@ -1885,7 +1885,8 @@ zpool_scan(zpool_handle_t *zhp, pool_scan_func_t func)
  * and the like.
  */
 static int
-ctd_check_path(char *str) {
+ctd_check_path(char *str)
+{
 	/*
 	 * If it starts with a slash, check the last component.
 	 */
diff --git a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_util.c b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_util.c
index 79738882bdb1..1316aec7c680 100644
--- a/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_util.c
+++ b/cddl/contrib/opensolaris/lib/libzfs/common/libzfs_util.c
@@ -22,7 +22,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
- * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  */
 
 /*
diff --git a/cddl/contrib/opensolaris/lib/libzpool/common/kernel.c b/cddl/contrib/opensolaris/lib/libzpool/common/kernel.c
index f2515d5ab4a6..126a7c1347a3 100644
--- a/cddl/contrib/opensolaris/lib/libzpool/common/kernel.c
+++ b/cddl/contrib/opensolaris/lib/libzpool/common/kernel.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc.  All rights reserved.
  */
 
@@ -523,7 +523,7 @@ vn_openat(char *path, int x1, int flags, int mode, vnode_t **vpp, int x2,
 /*ARGSUSED*/
 int
 vn_rdwr(int uio, vnode_t *vp, void *addr, ssize_t len, offset_t offset,
-	int x1, int x2, rlim64_t x3, void *x4, ssize_t *residp)
+    int x1, int x2, rlim64_t x3, void *x4, ssize_t *residp)
 {
 	ssize_t iolen, split;
 
diff --git a/sys/cddl/contrib/opensolaris/common/zfs/zfeature_common.c b/sys/cddl/contrib/opensolaris/common/zfs/zfeature_common.c
index aa46ae201b95..6eade2a3a4a8 100644
--- a/sys/cddl/contrib/opensolaris/common/zfs/zfeature_common.c
+++ b/sys/cddl/contrib/opensolaris/common/zfs/zfeature_common.c
@@ -117,7 +117,8 @@ zfeature_lookup_name(const char *name, spa_feature_t *res)
 }
 
 boolean_t
-zfeature_depends_on(spa_feature_t fid, spa_feature_t check) {
+zfeature_depends_on(spa_feature_t fid, spa_feature_t check)
+{
 	zfeature_info_t *feature = &spa_feature_table[fid];
 
 	for (int i = 0; feature->fi_depends[i] != SPA_FEATURE_NONE; i++) {
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c
index f663cb91b33d..ed3f19cac856 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c
@@ -1739,7 +1739,7 @@ dmu_sync(zio_t *pio, uint64_t txg, dmu_sync_cb_t *done, zgd_t *zgd)
 
 int
 dmu_object_set_blocksize(objset_t *os, uint64_t object, uint64_t size, int ibs,
-	dmu_tx_t *tx)
+    dmu_tx_t *tx)
 {
 	dnode_t *dn;
 	int err;
@@ -1754,7 +1754,7 @@ dmu_object_set_blocksize(objset_t *os, uint64_t object, uint64_t size, int ibs,
 
 void
 dmu_object_set_checksum(objset_t *os, uint64_t object, uint8_t checksum,
-	dmu_tx_t *tx)
+    dmu_tx_t *tx)
 {
 	dnode_t *dn;
 
@@ -1774,7 +1774,7 @@ dmu_object_set_checksum(objset_t *os, uint64_t object, uint8_t checksum,
 
 void
 dmu_object_set_compress(objset_t *os, uint64_t object, uint8_t compress,
-	dmu_tx_t *tx)
+    dmu_tx_t *tx)
 {
 	dnode_t *dn;
 
@@ -1944,7 +1944,8 @@ dmu_offset_next(objset_t *os, uint64_t object, boolean_t hole, uint64_t *off)
  * ID and wait for that to be synced.
  */
 int
-dmu_object_wait_synced(objset_t *os, uint64_t object) {
+dmu_object_wait_synced(objset_t *os, uint64_t object)
+{
 	dnode_t *dn;
 	int error, i;
 
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c
index 0e3e8557a9f1..95e53923896a 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c
@@ -3558,7 +3558,7 @@ dsl_dataset_space_wouldfree(dsl_dataset_t *firstsnap,
  */
 boolean_t
 dsl_dataset_is_before(dsl_dataset_t *later, dsl_dataset_t *earlier,
-	uint64_t earlier_txg)
+    uint64_t earlier_txg)
 {
 	dsl_pool_t *dp = later->ds_dir->dd_pool;
 	int error;
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/space_reftree.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/space_reftree.c
index a508092c530e..3d990596f766 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/space_reftree.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/space_reftree.c
@@ -23,7 +23,7 @@
  * Use is subject to license terms.
  */
 /*
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2013, 2015 by Delphix. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -103,7 +103,7 @@ space_reftree_add_node(avl_tree_t *t, uint64_t offset, int64_t refcnt)
 
 void
 space_reftree_add_seg(avl_tree_t *t, uint64_t start, uint64_t end,
-	int64_t refcnt)
+    int64_t refcnt)
 {
 	space_reftree_add_node(t, start, refcnt);
 	space_reftree_add_node(t, end, -refcnt);
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zrlock.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zrlock.h
index dcd63f7b5b91..b6eba1a18ff4 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zrlock.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zrlock.h
@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015 by Delphix. All rights reserved.
  */
 
 #ifndef	_SYS_ZRLOCK_H
@@ -44,12 +45,8 @@ typedef struct zrlock {
 
 extern void zrl_init(zrlock_t *);
 extern void zrl_destroy(zrlock_t *);
-#ifdef	ZFS_DEBUG
-#define	zrl_add(_z)	zrl_add_debug((_z), __func__)
-extern void zrl_add_debug(zrlock_t *, const char *);
-#else
-extern void zrl_add(zrlock_t *);
-#endif
+#define	zrl_add(_z)	zrl_add_impl((_z), __func__)
+extern void zrl_add_impl(zrlock_t *, const char *);
 extern void zrl_remove(zrlock_t *);
 extern int zrl_tryenter(zrlock_t *);
 extern void zrl_exit(zrlock_t *);
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_label.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_label.c
index 966f2fa364b1..9befa75a5ee5 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_label.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_label.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
  */
 
 /*
@@ -185,7 +185,7 @@ vdev_label_number(uint64_t psize, uint64_t offset)
 
 static void
 vdev_label_read(zio_t *zio, vdev_t *vd, int l, void *buf, uint64_t offset,
-	uint64_t size, zio_done_func_t *done, void *private, int flags)
+    uint64_t size, zio_done_func_t *done, void *private, int flags)
 {
 	ASSERT(spa_config_held(zio->io_spa, SCL_STATE_ALL, RW_WRITER) ==
 	    SCL_STATE_ALL);
@@ -199,7 +199,7 @@ vdev_label_read(zio_t *zio, vdev_t *vd, int l, void *buf, uint64_t offset,
 
 static void
 vdev_label_write(zio_t *zio, vdev_t *vd, int l, void *buf, uint64_t offset,
-	uint64_t size, zio_done_func_t *done, void *private, int flags)
+    uint64_t size, zio_done_func_t *done, void *private, int flags)
 {
 	ASSERT(spa_config_held(zio->io_spa, SCL_ALL, RW_WRITER) == SCL_ALL ||
 	    (spa_config_held(zio->io_spa, SCL_CONFIG | SCL_STATE, RW_READER) ==
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zap_leaf.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zap_leaf.c
index 96358f7bd80f..c8c366075642 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zap_leaf.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zap_leaf.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2013, 2015 by Delphix. All rights reserved.
  */
 
 /*
@@ -535,7 +535,7 @@ zap_entry_read_name(zap_t *zap, const zap_entry_handle_t *zeh, uint16_t buflen,
 
 int
 zap_entry_update(zap_entry_handle_t *zeh,
-	uint8_t integer_size, uint64_t num_integers, const void *buf)
+    uint8_t integer_size, uint64_t num_integers, const void *buf)
 {
 	int delta_chunks;
 	zap_leaf_t *l = zeh->zeh_leaf;
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfeature.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfeature.c
index 80a3f0be3009..78b2912df1d6 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfeature.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfeature.c
@@ -269,7 +269,8 @@ feature_get_refcount_from_disk(spa_t *spa, zfeature_info_t *feature,
 
 
 static int
-feature_get_enabled_txg(spa_t *spa, zfeature_info_t *feature, uint64_t *res) {
+feature_get_enabled_txg(spa_t *spa, zfeature_info_t *feature, uint64_t *res)
+{
 	uint64_t enabled_txg_obj = spa->spa_feat_enabled_txg_obj;
 
 	ASSERT(zfeature_depends_on(feature->fi_feature,
@@ -493,7 +494,8 @@ spa_feature_is_active(spa_t *spa, spa_feature_t fid)
  * Returns B_FALSE otherwise (i.e. if the feature is not enabled).
  */
 boolean_t
-spa_feature_enabled_txg(spa_t *spa, spa_feature_t fid, uint64_t *txg) {
+spa_feature_enabled_txg(spa_t *spa, spa_feature_t fid, uint64_t *txg)
+{
 	int err;
 
 	ASSERT(VALID_FEATURE_FID(fid));
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_dir.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_dir.c
index f7c2b6a7e67d..e2a3bff64577 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_dir.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_dir.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2013, 2015 by Delphix. All rights reserved.
  */
 
 #include <sys/types.h>
@@ -806,7 +806,7 @@ zfs_dropname(zfs_dirlock_t *dl, znode_t *zp, znode_t *dzp, dmu_tx_t *tx,
  */
 int
 zfs_link_destroy(zfs_dirlock_t *dl, znode_t *zp, dmu_tx_t *tx, int flag,
-	boolean_t *unlinkedp)
+    boolean_t *unlinkedp)
 {
 	znode_t *dzp = dl->dl_dzp;
 	zfsvfs_t *zfsvfs = dzp->z_zfsvfs;
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c
index 2c5fc64bee0e..3e43bda846b0 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c
@@ -5708,7 +5708,7 @@ zfs_ioctl_register_dataset_read(zfs_ioc_t ioc, zfs_ioc_legacy_func_t *func)
 
 static void
 zfs_ioctl_register_dataset_modify(zfs_ioc_t ioc, zfs_ioc_legacy_func_t *func,
-	zfs_secpolicy_func_t *secpolicy)
+    zfs_secpolicy_func_t *secpolicy)
 {
 	zfs_ioctl_register_legacy(ioc, func, secpolicy,
 	    DATASET_NAME, B_TRUE, POOL_CHECK_SUSPENDED | POOL_CHECK_READONLY);
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_log.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_log.c
index 7432290d218c..30b3b5270abd 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_log.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_log.c
@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015 by Delphix. All rights reserved.
  */
 
 #include <sys/types.h>
@@ -348,7 +349,7 @@ zfs_log_create(zilog_t *zilog, dmu_tx_t *tx, uint64_t txtype,
  */
 void
 zfs_log_remove(zilog_t *zilog, dmu_tx_t *tx, uint64_t txtype,
-	znode_t *dzp, char *name, uint64_t foid)
+    znode_t *dzp, char *name, uint64_t foid)
 {
 	itx_t *itx;
 	lr_remove_t *lr;
@@ -372,7 +373,7 @@ zfs_log_remove(zilog_t *zilog, dmu_tx_t *tx, uint64_t txtype,
  */
 void
 zfs_log_link(zilog_t *zilog, dmu_tx_t *tx, uint64_t txtype,
-	znode_t *dzp, znode_t *zp, char *name)
+    znode_t *dzp, znode_t *zp, char *name)
 {
 	itx_t *itx;
 	lr_link_t *lr;
@@ -427,7 +428,7 @@ zfs_log_symlink(zilog_t *zilog, dmu_tx_t *tx, uint64_t txtype,
  */
 void
 zfs_log_rename(zilog_t *zilog, dmu_tx_t *tx, uint64_t txtype,
-	znode_t *sdzp, char *sname, znode_t *tdzp, char *dname, znode_t *szp)
+    znode_t *sdzp, char *sname, znode_t *tdzp, char *dname, znode_t *szp)
 {
 	itx_t *itx;
 	lr_rename_t *lr;
@@ -455,7 +456,7 @@ ssize_t zfs_immediate_write_sz = 32768;
 
 void
 zfs_log_write(zilog_t *zilog, dmu_tx_t *tx, int txtype,
-	znode_t *zp, offset_t off, ssize_t resid, int ioflag)
+    znode_t *zp, offset_t off, ssize_t resid, int ioflag)
 {
 	itx_wr_state_t write_state;
 	boolean_t slogging;
@@ -532,7 +533,7 @@ zfs_log_write(zilog_t *zilog, dmu_tx_t *tx, int txtype,
  */
 void
 zfs_log_truncate(zilog_t *zilog, dmu_tx_t *tx, int txtype,
-	znode_t *zp, uint64_t off, uint64_t len)
+    znode_t *zp, uint64_t off, uint64_t len)
 {
 	itx_t *itx;
 	lr_truncate_t *lr;
@@ -555,7 +556,7 @@ zfs_log_truncate(zilog_t *zilog, dmu_tx_t *tx, int txtype,
  */
 void
 zfs_log_setattr(zilog_t *zilog, dmu_tx_t *tx, int txtype,
-	znode_t *zp, vattr_t *vap, uint_t mask_applied, zfs_fuid_info_t *fuidp)
+    znode_t *zp, vattr_t *vap, uint_t mask_applied, zfs_fuid_info_t *fuidp)
 {
 	itx_t		*itx;
 	lr_setattr_t	*lr;
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_replay.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_replay.c
index 96f777a95f3a..6a76513fa1ac 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_replay.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_replay.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2013, 2015 by Delphix. All rights reserved.
  */
 
 #include <sys/types.h>
@@ -54,7 +54,7 @@
 
 static void
 zfs_init_vattr(vattr_t *vap, uint64_t mask, uint64_t mode,
-	uint64_t uid, uint64_t gid, uint64_t rdev, uint64_t nodeid)
+    uint64_t uid, uint64_t gid, uint64_t rdev, uint64_t nodeid)
 {
 	VATTR_NULL(vap);
 	vap->va_mask = (uint_t)mask;
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
index d5874ea3e943..d1ed9dac23f3 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
  * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
  */
 
@@ -4520,7 +4520,7 @@ top:
 /* ARGSUSED */
 static int
 zfs_null_putapage(vnode_t *vp, page_t *pp, u_offset_t *offp,
-		size_t *lenp, int flags, cred_t *cr)
+    size_t *lenp, int flags, cred_t *cr)
 {
 	pvn_write_done(pp, B_INVAL|B_FORCE|B_ERROR);
 	return (0);
@@ -4546,7 +4546,7 @@ zfs_null_putapage(vnode_t *vp, page_t *pp, u_offset_t *offp,
 /* ARGSUSED */
 static int
 zfs_putapage(vnode_t *vp, page_t *pp, u_offset_t *offp,
-		size_t *lenp, int flags, cred_t *cr)
+    size_t *lenp, int flags, cred_t *cr)
 {
 	znode_t		*zp = VTOZ(vp);
 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c
index 8ff8aabaed10..896c56e04328 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c
@@ -328,7 +328,7 @@ zio_data_buf_free(void *buf, size_t size)
  */
 static void
 zio_push_transform(zio_t *zio, void *data, uint64_t size, uint64_t bufsize,
-	zio_transform_func_t *transform)
+    zio_transform_func_t *transform)
 {
 	zio_transform_t *zt = kmem_alloc(sizeof (zio_transform_t), KM_SLEEP);
 
@@ -1066,8 +1066,8 @@ zio_vdev_child_io(zio_t *pio, blkptr_t *bp, vdev_t *vd, uint64_t offset,
 
 zio_t *
 zio_vdev_delegated_io(vdev_t *vd, uint64_t offset, void *data, uint64_t size,
-	int type, zio_priority_t priority, enum zio_flag flags,
-	zio_done_func_t *done, void *private)
+    int type, zio_priority_t priority, enum zio_flag flags,
+    zio_done_func_t *done, void *private)
 {
 	zio_t *zio;
 
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio_checksum.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio_checksum.c
index 0e11189b88a8..6ba64e085b40 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio_checksum.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio_checksum.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2013, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  * Copyright 2013 Saso Kiselkov. All rights reserved.
  */
@@ -249,7 +249,7 @@ zio_checksum_template_init(enum zio_checksum checksum, spa_t *spa)
  */
 void
 zio_checksum_compute(zio_t *zio, enum zio_checksum checksum,
-	void *data, uint64_t size)
+    void *data, uint64_t size)
 {
 	blkptr_t *bp = zio->io_bp;
 	uint64_t offset = zio->io_offset;
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zrlock.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zrlock.c
index 22151843e030..7f6beeed6149 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zrlock.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zrlock.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2014, 2015 by Delphix. All rights reserved.
  */
 
 /*
@@ -69,11 +69,7 @@ zrl_destroy(zrlock_t *zrl)
 }
 
 void
-#ifdef	ZFS_DEBUG
-zrl_add_debug(zrlock_t *zrl, const char *zc)
-#else
-zrl_add(zrlock_t *zrl)
-#endif
+zrl_add_impl(zrlock_t *zrl, const char *zc)
 {
 	uint32_t n = (uint32_t)zrl->zr_refcount;
 

From 5bb43cdb3135beef85654ab56a46c76f3879d641 Mon Sep 17 00:00:00 2001
From: hselasky <hselasky@FreeBSD.org>
Date: Mon, 19 Oct 2015 10:49:15 +0000
Subject: [PATCH 372/380] Merge LinuxKPI changes from DragonflyBSD: - Some
 minor whitespace fixes. - Added support for two new Linux functions.

Sponsored by:	Mellanox Technologies
---
 sys/ofed/include/linux/workqueue.h | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/sys/ofed/include/linux/workqueue.h b/sys/ofed/include/linux/workqueue.h
index 075016ad5a35..3f2c7fe55693 100644
--- a/sys/ofed/include/linux/workqueue.h
+++ b/sys/ofed/include/linux/workqueue.h
@@ -91,11 +91,11 @@ do {									\
 
 #define	flush_scheduled_work()	flush_taskqueue(taskqueue_thread)
 
-static inline int queue_work (struct workqueue_struct *q, struct work_struct *work)
+static inline int queue_work(struct workqueue_struct *q, struct work_struct *work)
 {
 	(work)->taskqueue = (q)->taskqueue;
 	/* Return opposite val to align with Linux logic */
-        return !taskqueue_enqueue((q)->taskqueue, &(work)->work_task);
+	return !taskqueue_enqueue((q)->taskqueue, &(work)->work_task);
 }
 
 static inline void
@@ -151,6 +151,12 @@ _create_workqueue_common(char *name, int cpus)
 #define	create_workqueue(name)						\
 	_create_workqueue_common(name, MAXCPU)
 
+#define	alloc_ordered_workqueue(name, flags)				\
+	_create_workqueue_common(name, 1)
+
+#define	alloc_workqueue(name, flags, max_active)			\
+	_create_workqueue_common(name, max_active)
+
 static inline void
 destroy_workqueue(struct workqueue_struct *wq)
 {

From 74fb822e8a392e1828b818713e4a1df3006c25d4 Mon Sep 17 00:00:00 2001
From: hselasky <hselasky@FreeBSD.org>
Date: Mon, 19 Oct 2015 10:54:24 +0000
Subject: [PATCH 373/380] Merge LinuxKPI changes from DragonflyBSD: - Added
 support for multiple new Linux functions. - Properly implement DEFINE_WAIT()
 and init_waitqueue_head() macros. - Removed FreeBSD specific
 __wait_queue_head structure definition.

Sponsored by:	Mellanox Technologies
---
 sys/ofed/include/linux/wait.h | 34 +++++++++++++++++++++++++++-------
 1 file changed, 27 insertions(+), 7 deletions(-)

diff --git a/sys/ofed/include/linux/wait.h b/sys/ofed/include/linux/wait.h
index 80047f2e5d8b..dccd0f3d5ad1 100644
--- a/sys/ofed/include/linux/wait.h
+++ b/sys/ofed/include/linux/wait.h
@@ -39,15 +39,18 @@
 #include <sys/kernel.h>
 #include <sys/proc.h>
 
-struct __wait_queue_head {
-	unsigned int	wchan;
-};
-typedef struct __wait_queue_head wait_queue_head_t;
+typedef struct {
+} wait_queue_t;
 
-#define	init_waitqueue_head(x)
+typedef struct {
+	unsigned int	wchan;
+} wait_queue_head_t;
+
+#define	init_waitqueue_head(x) \
+    do { } while (0)
 
 static inline void
-__wake_up(struct __wait_queue_head *q, int all)
+__wake_up(wait_queue_head_t *q, int all)
 {
 	int wakeup_swapper;
 	void *c;
@@ -108,6 +111,23 @@ do {									\
 	-_error;							\
 })
 
-#define	DEFINE_WAIT(x)
+static inline int
+waitqueue_active(wait_queue_head_t *q)
+{
+	return 0;	/* XXX: not really implemented */
+}
+
+#define DEFINE_WAIT(name)	\
+	wait_queue_t name = {}
+
+static inline void
+prepare_to_wait(wait_queue_head_t *q, wait_queue_t *wait, int state)
+{
+}
+
+static inline void
+finish_wait(wait_queue_head_t *q, wait_queue_t *wait)
+{
+}
 
 #endif	/* _LINUX_WAIT_H_ */

From 676076548047bc0a4249b444ca2ac6ef9763ddfe Mon Sep 17 00:00:00 2001
From: hselasky <hselasky@FreeBSD.org>
Date: Mon, 19 Oct 2015 10:56:32 +0000
Subject: [PATCH 374/380] Merge LinuxKPI changes from DragonflyBSD: - Implement
 pagefault_disable() and pagefault_enable().

Sponsored by:	Mellanox Technologies
---
 sys/ofed/include/linux/uaccess.h | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/sys/ofed/include/linux/uaccess.h b/sys/ofed/include/linux/uaccess.h
index 6ba34f7025b0..bab848c6ea09 100644
--- a/sys/ofed/include/linux/uaccess.h
+++ b/sys/ofed/include/linux/uaccess.h
@@ -2,7 +2,8 @@
  * Copyright (c) 2010 Isilon Systems, Inc.
  * Copyright (c) 2010 iX Systems, Inc.
  * Copyright (c) 2010 Panasas, Inc.
- * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd.
+ * Copyright (c) 2013-2015 Mellanox Technologies, Ltd.
+ * Copyright (c) 2015 François Tigeot
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -32,4 +33,14 @@
 #define	get_user(_x, _p)	-copyin((_p), &(_x), sizeof(*(_p)))
 #define	put_user(_x, _p)	-copyout(&(_x), (_p), sizeof(*(_p)))
 
+static inline void pagefault_disable(void)
+{
+	curthread_pflags_set(TDP_NOFAULTING | TDP_RESETSPUR);
+}
+
+static inline void pagefault_enable(void)
+{
+	curthread_pflags_restore(~(TDP_NOFAULTING | TDP_RESETSPUR));
+}
+
 #endif	/* _LINUX_UACCESS_H_ */

From c58bf9145bf400f6f98e6f221d2f329099acb120 Mon Sep 17 00:00:00 2001
From: hselasky <hselasky@FreeBSD.org>
Date: Mon, 19 Oct 2015 10:57:56 +0000
Subject: [PATCH 375/380] Merge LinuxKPI changes from DragonflyBSD: - Implement
 schedule_timeout().

Sponsored by:	Mellanox Technologies
---
 sys/ofed/include/linux/sched.h | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/sys/ofed/include/linux/sched.h b/sys/ofed/include/linux/sched.h
index da25359456fe..c2fb8dfd1418 100644
--- a/sys/ofed/include/linux/sched.h
+++ b/sys/ofed/include/linux/sched.h
@@ -107,4 +107,15 @@ do {									\
 
 #define	sched_yield()	sched_relinquish(curthread)
 
+static inline long
+schedule_timeout(signed long timeout)
+{
+	if (timeout < 0)
+		return 0;
+
+	pause("lstim", timeout);
+
+	return 0;
+}
+
 #endif	/* _LINUX_SCHED_H_ */

From bed28ab60d0d2d7172650d8d00dd31519885955c Mon Sep 17 00:00:00 2001
From: hselasky <hselasky@FreeBSD.org>
Date: Mon, 19 Oct 2015 11:09:51 +0000
Subject: [PATCH 376/380] Merge LinuxKPI changes from DragonflyBSD: - Avoid
 using PAGE_MASK, because Linux defines it differently.   Use (PAGE_SIZE - 1)
 instead. - Add support for for_each_sg_page() and sg_page_iter_dma_address().

Sponsored by:	Mellanox Technologies
---
 sys/ofed/include/linux/scatterlist.h | 69 +++++++++++++++++++++++++++-
 1 file changed, 68 insertions(+), 1 deletion(-)

diff --git a/sys/ofed/include/linux/scatterlist.h b/sys/ofed/include/linux/scatterlist.h
index 595e2e9c8d8b..b8e7c2955e6d 100644
--- a/sys/ofed/include/linux/scatterlist.h
+++ b/sys/ofed/include/linux/scatterlist.h
@@ -3,6 +3,7 @@
  * Copyright (c) 2010 iX Systems, Inc.
  * Copyright (c) 2010 Panasas, Inc.
  * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd.
+ * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -64,6 +65,12 @@ struct sg_table {
 	unsigned int orig_nents;        /* original size of list */
 };
 
+struct sg_page_iter {
+	struct scatterlist	*sg;
+	unsigned int		sg_pgoffset;	/* page index */
+	unsigned int		maxents;
+};
+
 /*
  * Maximum number of entries that will be allocated in one piece, if
  * a list larger than this is required then chaining will be utilized.
@@ -93,7 +100,7 @@ static inline void
 sg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen)
 {
 	sg_set_page(sg, virt_to_page(buf), buflen,
-	    ((uintptr_t)buf) & ~PAGE_MASK);
+	    ((uintptr_t)buf) & (PAGE_SIZE - 1));
 }
 
 static inline void
@@ -326,6 +333,66 @@ sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask)
 	return ret;
 }
 
+/*
+ * Iterate pages in sg list.
+ */
+static inline void
+_sg_iter_next(struct sg_page_iter *iter)
+{
+	struct scatterlist *sg;
+	unsigned int pgcount;
+
+	sg = iter->sg;
+	pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT;
+
+	++iter->sg_pgoffset;
+	while (iter->sg_pgoffset >= pgcount) {
+		iter->sg_pgoffset -= pgcount;
+		sg = sg_next(sg);
+		--iter->maxents;
+		if (sg == NULL || iter->maxents == 0)
+			break;
+		pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	}
+	iter->sg = sg;
+}
+
+/*
+ * NOTE: pgoffset is really a page index, not a byte offset.
+ */
+static inline void
+_sg_iter_init(struct scatterlist *sgl, struct sg_page_iter *iter,
+	      unsigned int nents, unsigned long pgoffset)
+{
+	if (nents) {
+		/*
+		 * Nominal case.  Note subtract 1 from starting page index
+		 * for initial _sg_iter_next() call.
+		 */
+		iter->sg = sgl;
+		iter->sg_pgoffset = pgoffset - 1;
+		iter->maxents = nents;
+		_sg_iter_next(iter);
+	} else {
+		/*
+		 * Degenerate case
+		 */
+		iter->sg = NULL;
+		iter->sg_pgoffset = 0;
+		iter->maxents = 0;
+	}
+}
+
+static inline dma_addr_t
+sg_page_iter_dma_address(struct sg_page_iter *spi)
+{
+	return spi->sg->address + (spi->sg_pgoffset << PAGE_SHIFT);
+}
+
+#define	for_each_sg_page(sgl, iter, nents, pgoffset)			\
+	for (_sg_iter_init(sgl, iter, nents, pgoffset);			\
+	     (iter)->sg; _sg_iter_next(iter))
+
 #define	for_each_sg(sglist, sg, sgmax, _itr)				\
 	for (_itr = 0, sg = (sglist); _itr < (sgmax); _itr++, sg = sg_next(sg))
 

From 109f18ddeb5495f39bc5b80cac6dce07a762327c Mon Sep 17 00:00:00 2001
From: hselasky <hselasky@FreeBSD.org>
Date: Mon, 19 Oct 2015 11:11:15 +0000
Subject: [PATCH 377/380] Merge LinuxKPI changes from DragonflyBSD: -
 Whitespace fixes.

Sponsored by:	Mellanox Technologies
---
 sys/ofed/include/linux/moduleparam.h | 36 ++++++++++++++--------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/sys/ofed/include/linux/moduleparam.h b/sys/ofed/include/linux/moduleparam.h
index 439237d8630a..c1817ea796a8 100644
--- a/sys/ofed/include/linux/moduleparam.h
+++ b/sys/ofed/include/linux/moduleparam.h
@@ -48,7 +48,7 @@ struct kernel_param {
 	u16		flags;
 	param_set_fn	set;
 	param_get_fn	get;
-	union { 
+	union {
 		void	*arg;
 		struct kparam_string	*str;
 		struct kparam_array	*arr;
@@ -83,8 +83,8 @@ param_sysinit(struct kernel_param *param)
 	SYSINIT(name##_param_sysinit, SI_SUB_DRIVERS, SI_ORDER_FIRST,	\
 	    param_sysinit, &__param_##name);
 
-#define module_param_string(name, string, len, perm)                    
-         
+#define	module_param_string(name, string, len, perm)
+
 #define	module_param_named(name, var, type, mode)			\
 	module_param_call(name, param_set_##type, param_get_##type, &var, mode)
 
@@ -118,7 +118,7 @@ param_set_short(const char *val, struct kernel_param *kp)
 	return 0;
 }
 
-static inline int 
+static inline int
 param_get_short(char *buffer, struct kernel_param *kp)
 {
 
@@ -126,14 +126,14 @@ param_get_short(char *buffer, struct kernel_param *kp)
 }
 
 
-static inline int 
+static inline int
 param_set_ushort(const char *val, struct kernel_param *kp)
 {
 
 	return 0;
 }
 
-static inline int 
+static inline int
 param_get_ushort(char *buffer, struct kernel_param *kp)
 {
 
@@ -141,14 +141,14 @@ param_get_ushort(char *buffer, struct kernel_param *kp)
 }
 
 
-static inline int 
+static inline int
 param_set_int(const char *val, struct kernel_param *kp)
 {
 
 	return 0;
 }
 
-static inline int 
+static inline int
 param_get_int(char *buffer, struct kernel_param *kp)
 {
 
@@ -156,14 +156,14 @@ param_get_int(char *buffer, struct kernel_param *kp)
 }
 
 
-static inline int 
+static inline int
 param_set_uint(const char *val, struct kernel_param *kp)
 {
 
 	return 0;
 }
 
-static inline int 
+static inline int
 param_get_uint(char *buffer, struct kernel_param *kp)
 {
 
@@ -171,14 +171,14 @@ param_get_uint(char *buffer, struct kernel_param *kp)
 }
 
 
-static inline int 
+static inline int
 param_set_long(const char *val, struct kernel_param *kp)
 {
 
 	return 0;
 }
 
-static inline int 
+static inline int
 param_get_long(char *buffer, struct kernel_param *kp)
 {
 
@@ -186,14 +186,14 @@ param_get_long(char *buffer, struct kernel_param *kp)
 }
 
 
-static inline int 
+static inline int
 param_set_ulong(const char *val, struct kernel_param *kp)
 {
 
 	return 0;
 }
 
-static inline int 
+static inline int
 param_get_ulong(char *buffer, struct kernel_param *kp)
 {
 
@@ -201,14 +201,14 @@ param_get_ulong(char *buffer, struct kernel_param *kp)
 }
 
 
-static inline int 
+static inline int
 param_set_charp(const char *val, struct kernel_param *kp)
 {
 
 	return 0;
 }
 
-static inline int 
+static inline int
 param_get_charp(char *buffer, struct kernel_param *kp)
 {
 
@@ -216,14 +216,14 @@ param_get_charp(char *buffer, struct kernel_param *kp)
 }
 
 
-static inline int 
+static inline int
 param_set_bool(const char *val, struct kernel_param *kp)
 {
 
 	return 0;
 }
 
-static inline int 
+static inline int
 param_get_bool(char *buffer, struct kernel_param *kp)
 {
 

From c2db9fa30b800aabd3c8367ddf2977cbdfbec8d0 Mon Sep 17 00:00:00 2001
From: hselasky <hselasky@FreeBSD.org>
Date: Mon, 19 Oct 2015 11:16:38 +0000
Subject: [PATCH 378/380] Merge LinuxKPI changes from DragonflyBSD: -
 Reimplement math64 header file to distinguish more from Linux.

Sponsored by:	Mellanox Technologies
---
 sys/ofed/include/linux/math64.h | 113 +++++---------------------------
 1 file changed, 17 insertions(+), 96 deletions(-)

diff --git a/sys/ofed/include/linux/math64.h b/sys/ofed/include/linux/math64.h
index cc3d946deff9..a898f37e2d85 100644
--- a/sys/ofed/include/linux/math64.h
+++ b/sys/ofed/include/linux/math64.h
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2007 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2014 Mellanox Technologies, Ltd. All rights reserved.
+ * Copyright (c) 2014-2015 Mellanox Technologies, Ltd. All rights reserved.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -26,108 +26,29 @@
  */
 
 #ifndef _LINUX_MATH64_H
-#define _LINUX_MATH64_H
+#define	_LINUX_MATH64_H
 
-#include <linux/types.h>
-#include <linux/bitops.h>
+#include <sys/stdint.h>
 
-#if BITS_PER_LONG == 64
- 
-# define do_div(n, base) ({                                    \
-	uint32_t __base = (base);                               \
-	uint32_t __rem;                                         \
-	__rem = ((uint64_t)(n)) % __base;                       \
-	(n) = ((uint64_t)(n)) / __base;                         \
-	__rem;                                                  \
+#define	do_div(n, base) ({			\
+	uint32_t __base = (base);		\
+	uint32_t __rem;				\
+	__rem = ((uint64_t)(n)) % __base;	\
+	(n) = ((uint64_t)(n)) / __base;		\
+	__rem;					\
 })
 
-/**
-* div_u64_rem - unsigned 64bit divide with 32bit divisor with remainder
-*
-* This is commonly provided by 32bit archs to provide an optimized 64bit
-* divide.
-*/
-static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder)
+static inline uint64_t
+div_u64_rem(uint64_t dividend, uint32_t divisor, uint32_t *remainder)
 {
-        *remainder = dividend % divisor;
-        return dividend / divisor;
+	*remainder = dividend % divisor;
+	return (dividend / divisor);
 }
 
-
-#elif BITS_PER_LONG == 32
-
-static uint32_t __div64_32(uint64_t *n, uint32_t base)
+static inline uint64_t
+div_u64(uint64_t dividend, uint32_t divisor)
 {
-	uint64_t rem = *n;
-	uint64_t b = base;
-	uint64_t res, d = 1;
-	uint32_t high = rem >> 32;
-
-	/* Reduce the thing a bit first */
-	res = 0;
-	if (high >= base) {
-		high /= base;
-		res = (uint64_t) high << 32;
-		rem -= (uint64_t) (high*base) << 32;
-	}
-
-	while ((int64_t)b > 0 && b < rem) {
-		b = b+b;
-		d = d+d;
-	}
-
-	do {
-		if (rem >= b) {
-			rem -= b;
-			res += d;
-		}
-		b >>= 1;
-		d >>= 1;
-	} while (d);
-
-	*n = res;
-	return rem;
+	return (dividend / divisor);
 }
- 
-# define do_div(n, base) ({                            \
-	uint32_t __base = (base);                       \
-	uint32_t __rem;                                 \
-	(void)(((typeof((n)) *)0) == ((uint64_t *)0));  \
-	if (likely(((n) >> 32) == 0)) {                 \
-		__rem = (uint32_t)(n) % __base;         \
-		(n) = (uint32_t)(n) / __base;           \
-	} else                                          \
-		__rem = __div64_32(&(n), __base);       \
-	__rem;                                          \
-})
 
-#ifndef div_u64_rem
-static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder)
-{
-        *remainder = do_div(dividend, divisor);
-        return dividend;
-}
-#endif
- 
-
-#endif /* BITS_PER_LONG */
-
-
-
-/**
- ** div_u64 - unsigned 64bit divide with 32bit divisor
- **
- ** This is the most common 64bit divide and should be used if possible,
- ** as many 32bit archs can optimize this variant better than a full 64bit
- ** divide.
- *  */
-#ifndef div_u64
-
-static inline u64 div_u64(u64 dividend, u32 divisor)
-{
-        u32 remainder;
-        return div_u64_rem(dividend, divisor, &remainder);
-}
-#endif
-
-#endif	/* _LINUX_MATH64_H */
+#endif					/* _LINUX_MATH64_H */

From d8710b53229e05873b6a3ac38688b591aba31542 Mon Sep 17 00:00:00 2001
From: tuexen <tuexen@FreeBSD.org>
Date: Mon, 19 Oct 2015 11:17:54 +0000
Subject: [PATCH 379/380] Use __func__ instead of __FUNCTION__.

This allows to compile the userland stack without errors using gcc5.
Thanks to saghul for makeing me aware and providing the patch.

MFC after: 1 week
---
 sys/netinet/sctp_auth.c   |  6 +++---
 sys/netinet/sctp_input.c  | 10 +++++-----
 sys/netinet/sctp_output.c | 10 +++++-----
 sys/netinet/sctp_pcb.c    | 16 ++++++++--------
 sys/netinet/sctp_usrreq.c |  2 +-
 sys/netinet/sctputil.c    |  8 ++++----
 6 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/sys/netinet/sctp_auth.c b/sys/netinet/sctp_auth.c
index 13454df92f15..07dbf8ba440d 100644
--- a/sys/netinet/sctp_auth.c
+++ b/sys/netinet/sctp_auth.c
@@ -558,7 +558,7 @@ sctp_auth_key_acquire(struct sctp_tcb *stcb, uint16_t key_id)
 		atomic_add_int(&skey->refcount, 1);
 		SCTPDBG(SCTP_DEBUG_AUTH2,
 		    "%s: stcb %p key %u refcount acquire to %d\n",
-		    __FUNCTION__, (void *)stcb, key_id, skey->refcount);
+		    __func__, (void *)stcb, key_id, skey->refcount);
 	}
 }
 
@@ -578,7 +578,7 @@ sctp_auth_key_release(struct sctp_tcb *stcb, uint16_t key_id, int so_locked
 	if (skey) {
 		SCTPDBG(SCTP_DEBUG_AUTH2,
 		    "%s: stcb %p key %u refcount release to %d\n",
-		    __FUNCTION__, (void *)stcb, key_id, skey->refcount);
+		    __func__, (void *)stcb, key_id, skey->refcount);
 
 		/* see if a notification should be generated */
 		if ((skey->refcount <= 2) && (skey->deactivated)) {
@@ -587,7 +587,7 @@ sctp_auth_key_release(struct sctp_tcb *stcb, uint16_t key_id, int so_locked
 			    key_id, 0, so_locked);
 			SCTPDBG(SCTP_DEBUG_AUTH2,
 			    "%s: stcb %p key %u no longer used, %d\n",
-			    __FUNCTION__, (void *)stcb, key_id, skey->refcount);
+			    __func__, (void *)stcb, key_id, skey->refcount);
 		}
 		sctp_free_sharedkey(skey);
 	}
diff --git a/sys/netinet/sctp_input.c b/sys/netinet/sctp_input.c
index 37d9e6051ea5..8ed155104a59 100644
--- a/sys/netinet/sctp_input.c
+++ b/sys/netinet/sctp_input.c
@@ -4625,7 +4625,7 @@ __attribute__((noinline))
 			}
 		}
 		if (stcb == NULL) {
-			snprintf(msg, sizeof(msg), "OOTB, %s:%d at %s", __FILE__, __LINE__, __FUNCTION__);
+			snprintf(msg, sizeof(msg), "OOTB, %s:%d at %s", __FILE__, __LINE__, __func__);
 			op_err = sctp_generate_cause(SCTP_BASE_SYSCTL(sctp_diag_info_code),
 			    msg);
 			/* no association, so it's out of the blue... */
@@ -4669,7 +4669,7 @@ __attribute__((noinline))
 				if (locked_tcb) {
 					SCTP_TCB_UNLOCK(locked_tcb);
 				}
-				snprintf(msg, sizeof(msg), "OOTB, %s:%d at %s", __FILE__, __LINE__, __FUNCTION__);
+				snprintf(msg, sizeof(msg), "OOTB, %s:%d at %s", __FILE__, __LINE__, __func__);
 				op_err = sctp_generate_cause(SCTP_BASE_SYSCTL(sctp_diag_info_code),
 				    msg);
 				sctp_handle_ootb(m, iphlen, *offset, src, dst,
@@ -5821,7 +5821,7 @@ sctp_common_input_processing(struct mbuf **mm, int iphlen, int offset, int lengt
 			 */
 			SCTP_TCB_UNLOCK(stcb);
 			stcb = NULL;
-			snprintf(msg, sizeof(msg), "OOTB, %s:%d at %s", __FILE__, __LINE__, __FUNCTION__);
+			snprintf(msg, sizeof(msg), "OOTB, %s:%d at %s", __FILE__, __LINE__, __func__);
 			op_err = sctp_generate_cause(SCTP_BASE_SYSCTL(sctp_diag_info_code),
 			    msg);
 			sctp_handle_ootb(m, iphlen, offset, src, dst, sh, inp, op_err,
@@ -5873,7 +5873,7 @@ sctp_common_input_processing(struct mbuf **mm, int iphlen, int offset, int lengt
 		}
 		if (stcb == NULL) {
 			/* out of the blue DATA chunk */
-			snprintf(msg, sizeof(msg), "OOTB, %s:%d at %s", __FILE__, __LINE__, __FUNCTION__);
+			snprintf(msg, sizeof(msg), "OOTB, %s:%d at %s", __FILE__, __LINE__, __func__);
 			op_err = sctp_generate_cause(SCTP_BASE_SYSCTL(sctp_diag_info_code),
 			    msg);
 			sctp_handle_ootb(m, iphlen, offset, src, dst, sh, inp, op_err,
@@ -5945,7 +5945,7 @@ sctp_common_input_processing(struct mbuf **mm, int iphlen, int offset, int lengt
 			/*
 			 * We consider OOTB any data sent during asoc setup.
 			 */
-			snprintf(msg, sizeof(msg), "OOTB, %s:%d at %s", __FILE__, __LINE__, __FUNCTION__);
+			snprintf(msg, sizeof(msg), "OOTB, %s:%d at %s", __FILE__, __LINE__, __func__);
 			op_err = sctp_generate_cause(SCTP_BASE_SYSCTL(sctp_diag_info_code),
 			    msg);
 			sctp_handle_ootb(m, iphlen, offset, src, dst, sh, inp, op_err,
diff --git a/sys/netinet/sctp_output.c b/sys/netinet/sctp_output.c
index f515d6a6cd2b..7d13aa22e952 100644
--- a/sys/netinet/sctp_output.c
+++ b/sys/netinet/sctp_output.c
@@ -2417,7 +2417,7 @@ sctp_is_addr_restricted(struct sctp_tcb *stcb, struct sctp_ifa *ifa)
 	LIST_FOREACH(laddr, &stcb->asoc.sctp_restricted_addrs, sctp_nxt_addr) {
 		if (laddr->ifa == NULL) {
 			SCTPDBG(SCTP_DEBUG_OUTPUT1, "%s: NULL ifa\n",
-			    __FUNCTION__);
+			    __func__);
 			continue;
 		}
 		if (laddr->ifa == ifa) {
@@ -2439,7 +2439,7 @@ sctp_is_addr_in_ep(struct sctp_inpcb *inp, struct sctp_ifa *ifa)
 	LIST_FOREACH(laddr, &inp->sctp_addr_list, sctp_nxt_addr) {
 		if (laddr->ifa == NULL) {
 			SCTPDBG(SCTP_DEBUG_OUTPUT1, "%s: NULL ifa\n",
-			    __FUNCTION__);
+			    __func__);
 			continue;
 		}
 		if ((laddr->ifa == ifa) && laddr->action == 0)
@@ -5524,7 +5524,7 @@ do_a_abort:
 		if (op_err == NULL) {
 			char msg[SCTP_DIAG_INFO_LEN];
 
-			snprintf(msg, sizeof(msg), "%s:%d at %s", __FILE__, __LINE__, __FUNCTION__);
+			snprintf(msg, sizeof(msg), "%s:%d at %s", __FILE__, __LINE__, __func__);
 			op_err = sctp_generate_cause(SCTP_BASE_SYSCTL(sctp_diag_info_code),
 			    msg);
 		}
@@ -6687,7 +6687,7 @@ sctp_sendall_iterator(struct sctp_inpcb *inp, struct sctp_tcb *stcb, void *ptr,
 
 				abort_anyway:
 						snprintf(msg, sizeof(msg),
-						    "%s:%d at %s", __FILE__, __LINE__, __FUNCTION__);
+						    "%s:%d at %s", __FILE__, __LINE__, __func__);
 						op_err = sctp_generate_cause(SCTP_BASE_SYSCTL(sctp_diag_info_code),
 						    msg);
 						atomic_add_int(&stcb->asoc.refcnt, 1);
@@ -13387,7 +13387,7 @@ dataless_eof:
 						free_cnt_applied = 0;
 					}
 					snprintf(msg, sizeof(msg),
-					    "%s:%d at %s", __FILE__, __LINE__, __FUNCTION__);
+					    "%s:%d at %s", __FILE__, __LINE__, __func__);
 					op_err = sctp_generate_cause(SCTP_BASE_SYSCTL(sctp_diag_info_code),
 					    msg);
 					sctp_abort_an_association(stcb->sctp_ep, stcb,
diff --git a/sys/netinet/sctp_pcb.c b/sys/netinet/sctp_pcb.c
index 07119f15d9ff..92a84d6a2999 100644
--- a/sys/netinet/sctp_pcb.c
+++ b/sys/netinet/sctp_pcb.c
@@ -1115,7 +1115,7 @@ sctp_tcb_special_locate(struct sctp_inpcb **inp_p, struct sockaddr *from,
 			LIST_FOREACH(laddr, &inp->sctp_addr_list, sctp_nxt_addr) {
 
 				if (laddr->ifa == NULL) {
-					SCTPDBG(SCTP_DEBUG_PCB1, "%s: NULL ifa\n", __FUNCTION__);
+					SCTPDBG(SCTP_DEBUG_PCB1, "%s: NULL ifa\n", __func__);
 					continue;
 				}
 				if (laddr->ifa->localifa_flags & SCTP_BEING_DELETED) {
@@ -1773,7 +1773,7 @@ sctp_endpoint_probe(struct sockaddr *nam, struct sctppcbhead *head,
 		LIST_FOREACH(laddr, &inp->sctp_addr_list, sctp_nxt_addr) {
 			if (laddr->ifa == NULL) {
 				SCTPDBG(SCTP_DEBUG_PCB1, "%s: NULL ifa\n",
-				    __FUNCTION__);
+				    __func__);
 				continue;
 			}
 			SCTPDBG(SCTP_DEBUG_PCB1, "Ok laddr->ifa:%p is possible, ",
@@ -2343,7 +2343,7 @@ sctp_findassociation_ep_asconf(struct mbuf *m, int offset,
 	    &parm_buf, sizeof(struct sctp_paramhdr));
 	if (phdr == NULL) {
 		SCTPDBG(SCTP_DEBUG_INPUT3, "%s: failed to get asconf lookup addr\n",
-		    __FUNCTION__);
+		    __func__);
 		return NULL;
 	}
 	ptype = (int)((uint32_t) ntohs(phdr->param_type));
@@ -2363,7 +2363,7 @@ sctp_findassociation_ep_asconf(struct mbuf *m, int offset,
 			    &p6_buf.ph, sizeof(*p6));
 			if (p6 == NULL) {
 				SCTPDBG(SCTP_DEBUG_INPUT3, "%s: failed to get asconf v6 lookup addr\n",
-				    __FUNCTION__);
+				    __func__);
 				return (NULL);
 			}
 			sin6 = &remote_store.sin6;
@@ -2390,7 +2390,7 @@ sctp_findassociation_ep_asconf(struct mbuf *m, int offset,
 			    &p4_buf.ph, sizeof(*p4));
 			if (p4 == NULL) {
 				SCTPDBG(SCTP_DEBUG_INPUT3, "%s: failed to get asconf v4 lookup addr\n",
-				    __FUNCTION__);
+				    __func__);
 				return (NULL);
 			}
 			sin = &remote_store.sin;
@@ -5300,7 +5300,7 @@ sctp_update_ep_vflag(struct sctp_inpcb *inp)
 	LIST_FOREACH(laddr, &inp->sctp_addr_list, sctp_nxt_addr) {
 		if (laddr->ifa == NULL) {
 			SCTPDBG(SCTP_DEBUG_PCB1, "%s: NULL ifa\n",
-			    __FUNCTION__);
+			    __func__);
 			continue;
 		}
 		if (laddr->ifa->localifa_flags & SCTP_BEING_DELETED) {
@@ -6258,7 +6258,7 @@ sctp_load_addresses_from_init(struct sctp_tcb *stcb, struct mbuf *m,
 							 * abort this guy
 							 */
 							snprintf(msg, sizeof(msg),
-							    "%s:%d at %s", __FILE__, __LINE__, __FUNCTION__);
+							    "%s:%d at %s", __FILE__, __LINE__, __func__);
 							op_err = sctp_generate_cause(SCTP_BASE_SYSCTL(sctp_diag_info_code),
 							    msg);
 							sctp_abort_an_association(stcb_tmp->sctp_ep,
@@ -6357,7 +6357,7 @@ sctp_load_addresses_from_init(struct sctp_tcb *stcb, struct mbuf *m,
 							 * abort this guy
 							 */
 							snprintf(msg, sizeof(msg),
-							    "%s:%d at %s", __FILE__, __LINE__, __FUNCTION__);
+							    "%s:%d at %s", __FILE__, __LINE__, __func__);
 							op_err = sctp_generate_cause(SCTP_BASE_SYSCTL(sctp_diag_info_code),
 							    msg);
 							sctp_abort_an_association(stcb_tmp->sctp_ep,
diff --git a/sys/netinet/sctp_usrreq.c b/sys/netinet/sctp_usrreq.c
index 78f4508c5015..266861ecb9c6 100644
--- a/sys/netinet/sctp_usrreq.c
+++ b/sys/netinet/sctp_usrreq.c
@@ -5815,7 +5815,7 @@ sctp_setopt(struct socket *so, int optname, void *optval, size_t optsize,
 					LIST_FOREACH(laddr, &inp->sctp_addr_list, sctp_nxt_addr) {
 						if (laddr->ifa == NULL) {
 							SCTPDBG(SCTP_DEBUG_OUTPUT1, "%s: NULL ifa\n",
-							    __FUNCTION__);
+							    __func__);
 							continue;
 						}
 						if (laddr->ifa == ifa) {
diff --git a/sys/netinet/sctputil.c b/sys/netinet/sctputil.c
index 8a4a65e48b58..e451cb18d382 100644
--- a/sys/netinet/sctputil.c
+++ b/sys/netinet/sctputil.c
@@ -2183,13 +2183,13 @@ sctp_timer_start(int t_type, struct sctp_inpcb *inp, struct sctp_tcb *stcb,
 		break;
 	default:
 		SCTPDBG(SCTP_DEBUG_TIMER1, "%s: Unknown timer type %d\n",
-		    __FUNCTION__, t_type);
+		    __func__, t_type);
 		return;
 		break;
 	}
 	if ((to_ticks <= 0) || (tmr == NULL)) {
 		SCTPDBG(SCTP_DEBUG_TIMER1, "%s: %d:software error to_ticks:%d tmr:%p not set ??\n",
-		    __FUNCTION__, t_type, to_ticks, (void *)tmr);
+		    __func__, t_type, to_ticks, (void *)tmr);
 		return;
 	}
 	if (SCTP_OS_TIMER_PENDING(&tmr->timer)) {
@@ -2345,7 +2345,7 @@ sctp_timer_stop(int t_type, struct sctp_inpcb *inp, struct sctp_tcb *stcb,
 		break;
 	default:
 		SCTPDBG(SCTP_DEBUG_TIMER1, "%s: Unknown timer type %d\n",
-		    __FUNCTION__, t_type);
+		    __func__, t_type);
 		break;
 	}
 	if (tmr == NULL) {
@@ -3777,7 +3777,7 @@ sctp_ulp_notify(uint32_t notification, struct sctp_tcb *stcb,
 		break;
 	default:
 		SCTPDBG(SCTP_DEBUG_UTIL1, "%s: unknown notification %xh (%u)\n",
-		    __FUNCTION__, notification, notification);
+		    __func__, notification, notification);
 		break;
 	}			/* end switch */
 }

From 5855be3ec6724e6451361f6ef8bd165bf9f949df Mon Sep 17 00:00:00 2001
From: hselasky <hselasky@FreeBSD.org>
Date: Mon, 19 Oct 2015 11:29:50 +0000
Subject: [PATCH 380/380] Fix compile warning.

Sponsored by:	Mellanox Technologies
---
 sys/ofed/include/linux/dma-mapping.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/ofed/include/linux/dma-mapping.h b/sys/ofed/include/linux/dma-mapping.h
index f9fc3cb095b1..3af3e227be01 100644
--- a/sys/ofed/include/linux/dma-mapping.h
+++ b/sys/ofed/include/linux/dma-mapping.h
@@ -87,7 +87,7 @@ struct dma_map_ops {
 	int is_phys;
 };
 
-#define	DMA_BIT_MASK(n)	(((n) == 64) ? ~0ULL : ((1ULL << (n)) - 1))
+#define	DMA_BIT_MASK(n)	((2ULL << ((n) - 1)) - 1ULL)
 
 static inline int
 dma_supported(struct device *dev, u64 mask)