From b8d06fca089fae4680c3a552fc55c512bfb02202 Mon Sep 17 00:00:00 2001 From: Richard Yao Date: Mon, 7 May 2012 13:49:51 -0400 Subject: [PATCH] Switch KM_SLEEP to KM_PUSHPAGE Differences between how paging is done on Solaris and Linux can cause deadlocks if KM_SLEEP is used in any the following contexts. * The txg_sync thread * The zvol write/discard threads * The zpl_putpage() VFS callback This is because KM_SLEEP will allow for direct reclaim which may result in the VM calling back in to the filesystem or block layer to write out pages. If a lock is held over this operation the potential exists to deadlock the system. To ensure forward progress all memory allocations in these contexts must us KM_PUSHPAGE which disables performing any I/O to accomplish the memory allocation. Previously, this behavior was acheived by setting PF_MEMALLOC on the thread. However, that resulted in unexpected side effects such as the exhaustion of pages in ZONE_DMA. This approach touchs more of the zfs code, but it is more consistent with the right way to handle these cases under Linux. This is patch lays the ground work for being able to safely revert the following commits which used PF_MEMALLOC: 21ade34 Disable direct reclaim for z_wr_* threads cfc9a5c Fix zpl_writepage() deadlock eec8164 Fix ASSERTION(!dsl_pool_sync_context(tx->tx_pool)) Signed-off-by: Richard Yao Signed-off-by: Brian Behlendorf Issue #726 --- include/sys/dbuf.h | 14 +++---- include/sys/dsl_dataset.h | 2 +- include/sys/dsl_dir.h | 2 +- include/sys/spa.h | 12 +++--- module/zcommon/zprop_common.c | 2 +- module/zfs/arc.c | 2 +- module/zfs/bplist.c | 2 +- module/zfs/dbuf.c | 6 +-- module/zfs/ddt.c | 2 +- module/zfs/dmu.c | 12 +++--- module/zfs/dmu_objset.c | 2 +- module/zfs/dmu_traverse.c | 6 +-- module/zfs/dmu_tx.c | 6 +-- module/zfs/dmu_zfetch.c | 2 +- module/zfs/dnode.c | 6 +-- module/zfs/dsl_dataset.c | 2 +- module/zfs/dsl_deadlist.c | 6 ++- module/zfs/dsl_dir.c | 10 ++--- module/zfs/dsl_prop.c | 6 +-- module/zfs/lzjb.c | 2 +- module/zfs/metaslab.c | 10 ++--- module/zfs/refcount.c | 4 +- module/zfs/sa.c | 26 ++++++------ module/zfs/spa.c | 74 +++++++++++++++++------------------ module/zfs/spa_config.c | 24 ++++++------ module/zfs/spa_history.c | 10 ++--- module/zfs/spa_misc.c | 10 ++--- module/zfs/space_map.c | 6 +-- module/zfs/txg.c | 2 +- module/zfs/unique.c | 2 +- module/zfs/vdev.c | 10 ++--- module/zfs/vdev_cache.c | 4 +- module/zfs/vdev_disk.c | 6 +-- module/zfs/vdev_file.c | 2 +- module/zfs/vdev_label.c | 16 ++++---- module/zfs/vdev_mirror.c | 4 +- module/zfs/vdev_raidz.c | 6 +-- module/zfs/zap.c | 8 ++-- module/zfs/zap_micro.c | 6 +-- module/zfs/zfs_acl.c | 6 +-- module/zfs/zfs_fm.c | 6 +-- module/zfs/zfs_rlock.c | 2 +- module/zfs/zfs_znode.c | 8 ++-- module/zfs/zil.c | 14 +++---- module/zfs/zvol.c | 2 +- 45 files changed, 187 insertions(+), 185 deletions(-) diff --git a/include/sys/dbuf.h b/include/sys/dbuf.h index f3e14aad3669..ef93eb54ca86 100644 --- a/include/sys/dbuf.h +++ b/include/sys/dbuf.h @@ -345,13 +345,13 @@ boolean_t dbuf_is_metadata(dmu_buf_impl_t *db); } \ _NOTE(CONSTCOND) } while (0) -#define dprintf_dbuf_bp(db, bp, fmt, ...) do { \ - if (zfs_flags & ZFS_DEBUG_DPRINTF) { \ - char *__blkbuf = kmem_alloc(BP_SPRINTF_LEN, KM_SLEEP); \ - sprintf_blkptr(__blkbuf, bp); \ - dprintf_dbuf(db, fmt " %s\n", __VA_ARGS__, __blkbuf); \ - kmem_free(__blkbuf, BP_SPRINTF_LEN); \ - } \ +#define dprintf_dbuf_bp(db, bp, fmt, ...) do { \ + if (zfs_flags & ZFS_DEBUG_DPRINTF) { \ + char *__blkbuf = kmem_alloc(BP_SPRINTF_LEN, KM_PUSHPAGE); \ + sprintf_blkptr(__blkbuf, bp); \ + dprintf_dbuf(db, fmt " %s\n", __VA_ARGS__, __blkbuf); \ + kmem_free(__blkbuf, BP_SPRINTF_LEN); \ + } \ _NOTE(CONSTCOND) } while (0) #define DBUF_VERIFY(db) dbuf_verify(db) diff --git a/include/sys/dsl_dataset.h b/include/sys/dsl_dataset.h index c4530a8f0ae7..948abb020d3f 100644 --- a/include/sys/dsl_dataset.h +++ b/include/sys/dsl_dataset.h @@ -271,7 +271,7 @@ int dsl_destroy_inconsistent(const char *dsname, void *arg); #ifdef ZFS_DEBUG #define dprintf_ds(ds, fmt, ...) do { \ if (zfs_flags & ZFS_DEBUG_DPRINTF) { \ - char *__ds_name = kmem_alloc(MAXNAMELEN, KM_SLEEP); \ + char *__ds_name = kmem_alloc(MAXNAMELEN, KM_PUSHPAGE); \ dsl_dataset_name(ds, __ds_name); \ dprintf("ds=%s " fmt, __ds_name, __VA_ARGS__); \ kmem_free(__ds_name, MAXNAMELEN); \ diff --git a/include/sys/dsl_dir.h b/include/sys/dsl_dir.h index 2191635dd813..65ad202bba2d 100644 --- a/include/sys/dsl_dir.h +++ b/include/sys/dsl_dir.h @@ -150,7 +150,7 @@ timestruc_t dsl_dir_snap_cmtime(dsl_dir_t *dd); #define dprintf_dd(dd, fmt, ...) do { \ if (zfs_flags & ZFS_DEBUG_DPRINTF) { \ char *__ds_name = kmem_alloc(MAXNAMELEN + strlen(MOS_DIR_NAME) + 1, \ - KM_SLEEP); \ + KM_PUSHPAGE); \ dsl_dir_name(dd, __ds_name); \ dprintf("dd=%s " fmt, __ds_name, __VA_ARGS__); \ kmem_free(__ds_name, MAXNAMELEN + strlen(MOS_DIR_NAME) + 1); \ diff --git a/include/sys/spa.h b/include/sys/spa.h index 9de5736edb15..28bb4e1de150 100644 --- a/include/sys/spa.h +++ b/include/sys/spa.h @@ -690,12 +690,12 @@ extern void spa_configfile_set(spa_t *, nvlist_t *, boolean_t); extern void spa_event_notify(spa_t *spa, vdev_t *vdev, const char *name); #ifdef ZFS_DEBUG -#define dprintf_bp(bp, fmt, ...) do { \ - if (zfs_flags & ZFS_DEBUG_DPRINTF) { \ - char *__blkbuf = kmem_alloc(BP_SPRINTF_LEN, KM_SLEEP); \ - sprintf_blkptr(__blkbuf, (bp)); \ - dprintf(fmt " %s\n", __VA_ARGS__, __blkbuf); \ - kmem_free(__blkbuf, BP_SPRINTF_LEN); \ +#define dprintf_bp(bp, fmt, ...) do { \ + if (zfs_flags & ZFS_DEBUG_DPRINTF) { \ + char *__blkbuf = kmem_alloc(BP_SPRINTF_LEN, KM_PUSHPAGE); \ + sprintf_blkptr(__blkbuf, (bp)); \ + dprintf(fmt " %s\n", __VA_ARGS__, __blkbuf); \ + kmem_free(__blkbuf, BP_SPRINTF_LEN); \ } \ _NOTE(CONSTCOND) } while (0) #else diff --git a/module/zcommon/zprop_common.c b/module/zcommon/zprop_common.c index ab5b4662b79c..0a0af2334a08 100644 --- a/module/zcommon/zprop_common.c +++ b/module/zcommon/zprop_common.c @@ -171,7 +171,7 @@ zprop_iter_common(zprop_func func, void *cb, boolean_t show_all, size = num_props * sizeof (zprop_desc_t *); #if defined(_KERNEL) - order = kmem_alloc(size, KM_SLEEP); + order = kmem_alloc(size, KM_PUSHPAGE); #else if ((order = malloc(size)) == NULL) return (ZPROP_CONT); diff --git a/module/zfs/arc.c b/module/zfs/arc.c index aad37a39bafd..e75a3f59a0b5 100644 --- a/module/zfs/arc.c +++ b/module/zfs/arc.c @@ -3547,7 +3547,7 @@ arc_write(zio_t *pio, spa_t *spa, uint64_t txg, ASSERT(hdr->b_acb == NULL); if (l2arc) hdr->b_flags |= ARC_L2CACHE; - callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP); + callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_PUSHPAGE); callback->awcb_ready = ready; callback->awcb_done = done; callback->awcb_private = private; diff --git a/module/zfs/bplist.c b/module/zfs/bplist.c index 5d1cf7e76312..d196351dcf96 100644 --- a/module/zfs/bplist.c +++ b/module/zfs/bplist.c @@ -44,7 +44,7 @@ bplist_destroy(bplist_t *bpl) void bplist_append(bplist_t *bpl, const blkptr_t *bp) { - bplist_entry_t *bpe = kmem_alloc(sizeof (*bpe), KM_SLEEP); + bplist_entry_t *bpe = kmem_alloc(sizeof (*bpe), KM_PUSHPAGE); mutex_enter(&bpl->bpl_lock); bpe->bpe_blk = *bp; diff --git a/module/zfs/dbuf.c b/module/zfs/dbuf.c index 42d82bbfb8f4..1f6fa93406c9 100644 --- a/module/zfs/dbuf.c +++ b/module/zfs/dbuf.c @@ -298,7 +298,7 @@ dbuf_init(void) #if defined(_KERNEL) && defined(HAVE_SPL) /* Large allocations which do not require contiguous pages * should be using vmem_alloc() in the linux kernel */ - h->hash_table = vmem_zalloc(hsize * sizeof (void *), KM_SLEEP); + h->hash_table = vmem_zalloc(hsize * sizeof (void *), KM_PUSHPAGE); #else h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); #endif @@ -1719,7 +1719,7 @@ dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); ASSERT(dn->dn_type != DMU_OT_NONE); - db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); + db = kmem_cache_alloc(dbuf_cache, KM_PUSHPAGE); db->db_objset = os; db->db.db_object = dn->dn_object; @@ -2019,7 +2019,7 @@ dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse, int error; dh = kmem_zalloc(sizeof(struct dbuf_hold_impl_data) * - DBUF_HOLD_IMPL_MAX_DEPTH, KM_SLEEP); + DBUF_HOLD_IMPL_MAX_DEPTH, KM_PUSHPAGE); __dbuf_hold_impl_init(dh, dn, level, blkid, fail_sparse, tag, dbp, 0); error = __dbuf_hold_impl(dh); diff --git a/module/zfs/ddt.c b/module/zfs/ddt.c index 18c6cef5a84f..f93024d8f988 100644 --- a/module/zfs/ddt.c +++ b/module/zfs/ddt.c @@ -504,7 +504,7 @@ ddt_get_dedup_stats(spa_t *spa, ddt_stat_t *dds_total) ddt_histogram_t *ddh_total; /* XXX: Move to a slab */ - ddh_total = kmem_zalloc(sizeof (ddt_histogram_t), KM_SLEEP); + ddh_total = kmem_zalloc(sizeof (ddt_histogram_t), KM_PUSHPAGE); ddt_get_dedup_histogram(spa, ddh_total); ddt_histogram_stat(dds_total, ddh_total); kmem_free(ddh_total, sizeof (ddt_histogram_t)); diff --git a/module/zfs/dmu.c b/module/zfs/dmu.c index cda4f8428483..1d4d1257d54e 100644 --- a/module/zfs/dmu.c +++ b/module/zfs/dmu.c @@ -381,7 +381,7 @@ dmu_buf_hold_array_by_dnode(dnode_t *dn, uint64_t offset, uint64_t length, } nblks = 1; } - dbp = kmem_zalloc(sizeof (dmu_buf_t *) * nblks, KM_SLEEP | KM_NODEBUG); + dbp = kmem_zalloc(sizeof (dmu_buf_t *) * nblks, KM_PUSHPAGE | KM_NODEBUG); if (dn->dn_objset->os_dsl_dataset) dp = dn->dn_objset->os_dsl_dataset->ds_dir->dd_pool; @@ -863,11 +863,11 @@ dmu_xuio_init(xuio_t *xuio, int nblk) uio_t *uio = &xuio->xu_uio; uio->uio_iovcnt = nblk; - uio->uio_iov = kmem_zalloc(nblk * sizeof (iovec_t), KM_SLEEP); + uio->uio_iov = kmem_zalloc(nblk * sizeof (iovec_t), KM_PUSHPAGE); - priv = kmem_zalloc(sizeof (dmu_xuio_t), KM_SLEEP); + priv = kmem_zalloc(sizeof (dmu_xuio_t), KM_PUSHPAGE); priv->cnt = nblk; - priv->bufs = kmem_zalloc(nblk * sizeof (arc_buf_t *), KM_SLEEP); + priv->bufs = kmem_zalloc(nblk * sizeof (arc_buf_t *), KM_PUSHPAGE); priv->iovp = uio->uio_iov; XUIO_XUZC_PRIV(xuio) = priv; @@ -1431,7 +1431,7 @@ dmu_sync_late_arrival(zio_t *pio, objset_t *os, dmu_sync_cb_t *done, zgd_t *zgd, return (EIO); /* Make zl_get_data do txg_waited_synced() */ } - dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP); + dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_PUSHPAGE); dsa->dsa_dr = NULL; dsa->dsa_done = done; dsa->dsa_zgd = zgd; @@ -1555,7 +1555,7 @@ dmu_sync(zio_t *pio, uint64_t txg, dmu_sync_cb_t *done, zgd_t *zgd) dr->dt.dl.dr_override_state = DR_IN_DMU_SYNC; mutex_exit(&db->db_mtx); - dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP); + dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_PUSHPAGE); dsa->dsa_dr = dr; dsa->dsa_done = done; dsa->dsa_zgd = zgd; diff --git a/module/zfs/dmu_objset.c b/module/zfs/dmu_objset.c index 4e043243c185..a34584ebfe76 100644 --- a/module/zfs/dmu_objset.c +++ b/module/zfs/dmu_objset.c @@ -262,7 +262,7 @@ dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp, ASSERT(ds == NULL || MUTEX_HELD(&ds->ds_opening_lock)); - os = kmem_zalloc(sizeof (objset_t), KM_SLEEP); + os = kmem_zalloc(sizeof (objset_t), KM_PUSHPAGE); os->os_dsl_dataset = ds; os->os_spa = spa; os->os_rootbp = bp; diff --git a/module/zfs/dmu_traverse.c b/module/zfs/dmu_traverse.c index 998cb4b6aee4..376f60f828b6 100644 --- a/module/zfs/dmu_traverse.c +++ b/module/zfs/dmu_traverse.c @@ -361,9 +361,9 @@ traverse_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *rootbp, zbookmark_t *czb; int err; - td = kmem_alloc(sizeof(traverse_data_t), KM_SLEEP); - pd = kmem_zalloc(sizeof(prefetch_data_t), KM_SLEEP); - czb = kmem_alloc(sizeof(zbookmark_t), KM_SLEEP); + td = kmem_alloc(sizeof(traverse_data_t), KM_PUSHPAGE); + pd = kmem_zalloc(sizeof(prefetch_data_t), KM_PUSHPAGE); + czb = kmem_alloc(sizeof(zbookmark_t), KM_PUSHPAGE); td->td_spa = spa; td->td_objset = ds ? ds->ds_object : 0; diff --git a/module/zfs/dmu_tx.c b/module/zfs/dmu_tx.c index ead0f3e2a140..81c6dfea2eab 100644 --- a/module/zfs/dmu_tx.c +++ b/module/zfs/dmu_tx.c @@ -63,7 +63,7 @@ static kstat_t *dmu_tx_ksp; dmu_tx_t * dmu_tx_create_dd(dsl_dir_t *dd) { - dmu_tx_t *tx = kmem_zalloc(sizeof (dmu_tx_t), KM_SLEEP); + dmu_tx_t *tx = kmem_zalloc(sizeof (dmu_tx_t), KM_PUSHPAGE); tx->tx_dir = dd; if (dd) tx->tx_pool = dd->dd_pool; @@ -141,7 +141,7 @@ dmu_tx_hold_object_impl(dmu_tx_t *tx, objset_t *os, uint64_t object, } } - txh = kmem_zalloc(sizeof (dmu_tx_hold_t), KM_SLEEP); + txh = kmem_zalloc(sizeof (dmu_tx_hold_t), KM_PUSHPAGE); txh->txh_tx = tx; txh->txh_dnode = dn; #ifdef DEBUG_DMU_TX @@ -1241,7 +1241,7 @@ dmu_tx_callback_register(dmu_tx_t *tx, dmu_tx_callback_func_t *func, void *data) { dmu_tx_callback_t *dcb; - dcb = kmem_alloc(sizeof (dmu_tx_callback_t), KM_SLEEP); + dcb = kmem_alloc(sizeof (dmu_tx_callback_t), KM_PUSHPAGE); dcb->dcb_func = func; dcb->dcb_data = data; diff --git a/module/zfs/dmu_zfetch.c b/module/zfs/dmu_zfetch.c index 897ea8adbcb1..1763bae5184a 100644 --- a/module/zfs/dmu_zfetch.c +++ b/module/zfs/dmu_zfetch.c @@ -699,7 +699,7 @@ dmu_zfetch(zfetch_t *zf, uint64_t offset, uint64_t size, int prefetched) if (cur_streams >= max_streams) { return; } - newstream = kmem_zalloc(sizeof (zstream_t), KM_SLEEP); + newstream = kmem_zalloc(sizeof (zstream_t), KM_PUSHPAGE); } newstream->zst_offset = zst.zst_offset; diff --git a/module/zfs/dnode.c b/module/zfs/dnode.c index 5438f60d0003..99ac6256561d 100644 --- a/module/zfs/dnode.c +++ b/module/zfs/dnode.c @@ -372,7 +372,7 @@ static dnode_t * dnode_create(objset_t *os, dnode_phys_t *dnp, dmu_buf_impl_t *db, uint64_t object, dnode_handle_t *dnh) { - dnode_t *dn = kmem_cache_alloc(dnode_cache, KM_SLEEP); + dnode_t *dn = kmem_cache_alloc(dnode_cache, KM_PUSHPAGE); ASSERT(!POINTER_IS_VALID(dn->dn_objset)); dn->dn_moved = 0; @@ -1491,7 +1491,7 @@ dnode_clear_range(dnode_t *dn, uint64_t blkid, uint64_t nblks, dmu_tx_t *tx) } else if (blkid > rp->fr_blkid && endblk < fr_endblk) { /* clear a chunk out of this range */ free_range_t *new_rp = - kmem_alloc(sizeof (free_range_t), KM_SLEEP); + kmem_alloc(sizeof (free_range_t), KM_PUSHPAGE); new_rp->fr_blkid = endblk; new_rp->fr_nblks = fr_endblk - endblk; @@ -1669,7 +1669,7 @@ dnode_free_range(dnode_t *dn, uint64_t off, uint64_t len, dmu_tx_t *tx) avl_tree_t *tree = &dn->dn_ranges[tx->tx_txg&TXG_MASK]; /* Add new range to dn_ranges */ - rp = kmem_alloc(sizeof (free_range_t), KM_SLEEP); + rp = kmem_alloc(sizeof (free_range_t), KM_PUSHPAGE); rp->fr_blkid = blkid; rp->fr_nblks = nblks; found = avl_find(tree, rp, &where); diff --git a/module/zfs/dsl_dataset.c b/module/zfs/dsl_dataset.c index fce6d3c1a969..e7cbc513800b 100644 --- a/module/zfs/dsl_dataset.c +++ b/module/zfs/dsl_dataset.c @@ -390,7 +390,7 @@ dsl_dataset_get_ref(dsl_pool_t *dp, uint64_t dsobj, void *tag, if (ds == NULL) { dsl_dataset_t *winner = NULL; - ds = kmem_zalloc(sizeof (dsl_dataset_t), KM_SLEEP); + ds = kmem_zalloc(sizeof (dsl_dataset_t), KM_PUSHPAGE); ds->ds_dbuf = dbuf; ds->ds_object = dsobj; ds->ds_phys = dbuf->db_data; diff --git a/module/zfs/dsl_deadlist.c b/module/zfs/dsl_deadlist.c index dd6db2120b31..1e89a68d7708 100644 --- a/module/zfs/dsl_deadlist.c +++ b/module/zfs/dsl_deadlist.c @@ -80,7 +80,9 @@ dsl_deadlist_load_tree(dsl_deadlist_t *dl) for (zap_cursor_init(&zc, dl->dl_os, dl->dl_object); zap_cursor_retrieve(&zc, &za) == 0; zap_cursor_advance(&zc)) { - dsl_deadlist_entry_t *dle = kmem_alloc(sizeof (*dle), KM_SLEEP); + dsl_deadlist_entry_t *dle; + + dle = kmem_alloc(sizeof (*dle), KM_PUSHPAGE); dle->dle_mintxg = strtonum(za.za_name, NULL); VERIFY3U(0, ==, bpobj_open(&dle->dle_bpobj, dl->dl_os, za.za_first_integer)); @@ -215,7 +217,7 @@ dsl_deadlist_add_key(dsl_deadlist_t *dl, uint64_t mintxg, dmu_tx_t *tx) dsl_deadlist_load_tree(dl); - dle = kmem_alloc(sizeof (*dle), KM_SLEEP); + dle = kmem_alloc(sizeof (*dle), KM_PUSHPAGE); dle->dle_mintxg = mintxg; obj = bpobj_alloc(dl->dl_os, SPA_MAXBLOCKSIZE, tx); VERIFY3U(0, ==, bpobj_open(&dle->dle_bpobj, dl->dl_os, obj)); diff --git a/module/zfs/dsl_dir.c b/module/zfs/dsl_dir.c index 224cc0a3c423..d615832c56c2 100644 --- a/module/zfs/dsl_dir.c +++ b/module/zfs/dsl_dir.c @@ -96,7 +96,7 @@ dsl_dir_open_obj(dsl_pool_t *dp, uint64_t ddobj, if (dd == NULL) { dsl_dir_t *winner; - dd = kmem_zalloc(sizeof (dsl_dir_t), KM_SLEEP); + dd = kmem_zalloc(sizeof (dsl_dir_t), KM_PUSHPAGE); dd->dd_object = ddobj; dd->dd_dbuf = dbuf; dd->dd_pool = dp; @@ -791,7 +791,7 @@ dsl_dir_tempreserve_impl(dsl_dir_t *dd, uint64_t asize, boolean_t netfree, asize - ref_rsrv); mutex_exit(&dd->dd_lock); - tr = kmem_zalloc(sizeof (struct tempreserve), KM_SLEEP); + tr = kmem_zalloc(sizeof (struct tempreserve), KM_PUSHPAGE); tr->tr_ds = dd; tr->tr_size = asize; list_insert_tail(tr_list, tr); @@ -825,7 +825,7 @@ dsl_dir_tempreserve_space(dsl_dir_t *dd, uint64_t lsize, uint64_t asize, return (0); } - tr_list = kmem_alloc(sizeof (list_t), KM_SLEEP); + tr_list = kmem_alloc(sizeof (list_t), KM_PUSHPAGE); list_create(tr_list, sizeof (struct tempreserve), offsetof(struct tempreserve, tr_node)); ASSERT3S(asize, >, 0); @@ -835,7 +835,7 @@ dsl_dir_tempreserve_space(dsl_dir_t *dd, uint64_t lsize, uint64_t asize, if (err == 0) { struct tempreserve *tr; - tr = kmem_zalloc(sizeof (struct tempreserve), KM_SLEEP); + tr = kmem_zalloc(sizeof (struct tempreserve), KM_PUSHPAGE); tr->tr_size = lsize; list_insert_tail(tr_list, tr); @@ -851,7 +851,7 @@ dsl_dir_tempreserve_space(dsl_dir_t *dd, uint64_t lsize, uint64_t asize, if (err == 0) { struct tempreserve *tr; - tr = kmem_zalloc(sizeof (struct tempreserve), KM_SLEEP); + tr = kmem_zalloc(sizeof (struct tempreserve), KM_PUSHPAGE); tr->tr_dp = dd->dd_pool; tr->tr_size = asize; list_insert_tail(tr_list, tr); diff --git a/module/zfs/dsl_prop.c b/module/zfs/dsl_prop.c index 4b340b63a121..64c44b991dfd 100644 --- a/module/zfs/dsl_prop.c +++ b/module/zfs/dsl_prop.c @@ -247,9 +247,9 @@ dsl_prop_register(dsl_dataset_t *ds, const char *propname, return (err); } - cbr = kmem_alloc(sizeof (dsl_prop_cb_record_t), KM_SLEEP); + cbr = kmem_alloc(sizeof (dsl_prop_cb_record_t), KM_PUSHPAGE); cbr->cbr_ds = ds; - cbr->cbr_propname = kmem_alloc(strlen(propname)+1, KM_SLEEP); + cbr->cbr_propname = kmem_alloc(strlen(propname)+1, KM_PUSHPAGE); (void) strcpy((char *)cbr->cbr_propname, propname); cbr->cbr_func = callback; cbr->cbr_arg = cbarg; @@ -534,7 +534,7 @@ dsl_prop_changed_notify(dsl_pool_t *dp, uint64_t ddobj, } mutex_exit(&dd->dd_lock); - za = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP); + za = kmem_alloc(sizeof (zap_attribute_t), KM_PUSHPAGE); for (zap_cursor_init(&zc, mos, dd->dd_phys->dd_child_dir_zapobj); zap_cursor_retrieve(&zc, za) == 0; diff --git a/module/zfs/lzjb.c b/module/zfs/lzjb.c index 4da30cf17d98..43d0df055d8e 100644 --- a/module/zfs/lzjb.c +++ b/module/zfs/lzjb.c @@ -56,7 +56,7 @@ lzjb_compress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) uint16_t *hp; uint16_t *lempel; - lempel = kmem_zalloc(LEMPEL_SIZE * sizeof (uint16_t), KM_SLEEP); + lempel = kmem_zalloc(LEMPEL_SIZE * sizeof (uint16_t), KM_PUSHPAGE); while (src < (uchar_t *)s_start + s_len) { if ((copymask <<= 1) == (1 << NBBY)) { if (dst >= (uchar_t *)d_start + d_len - 1 - 2 * NBBY) { diff --git a/module/zfs/metaslab.c b/module/zfs/metaslab.c index c33c5e8c7a30..d06012ffb980 100644 --- a/module/zfs/metaslab.c +++ b/module/zfs/metaslab.c @@ -102,7 +102,7 @@ metaslab_class_create(spa_t *spa, space_map_ops_t *ops) { metaslab_class_t *mc; - mc = kmem_zalloc(sizeof (metaslab_class_t), KM_SLEEP); + mc = kmem_zalloc(sizeof (metaslab_class_t), KM_PUSHPAGE); mc->mc_spa = spa; mc->mc_rotor = NULL; @@ -217,7 +217,7 @@ metaslab_group_create(metaslab_class_t *mc, vdev_t *vd) { metaslab_group_t *mg; - mg = kmem_zalloc(sizeof (metaslab_group_t), KM_SLEEP); + mg = kmem_zalloc(sizeof (metaslab_group_t), KM_PUSHPAGE); mutex_init(&mg->mg_lock, NULL, MUTEX_DEFAULT, NULL); avl_create(&mg->mg_metaslab_tree, metaslab_compare, sizeof (metaslab_t), offsetof(struct metaslab, ms_group_node)); @@ -422,9 +422,9 @@ metaslab_pp_load(space_map_t *sm) space_seg_t *ss; ASSERT(sm->sm_ppd == NULL); - sm->sm_ppd = kmem_zalloc(64 * sizeof (uint64_t), KM_SLEEP); + sm->sm_ppd = kmem_zalloc(64 * sizeof (uint64_t), KM_PUSHPAGE); - sm->sm_pp_root = kmem_alloc(sizeof (avl_tree_t), KM_SLEEP); + sm->sm_pp_root = kmem_alloc(sizeof (avl_tree_t), KM_PUSHPAGE); avl_create(sm->sm_pp_root, metaslab_segsize_compare, sizeof (space_seg_t), offsetof(struct space_seg, ss_pp_node)); @@ -725,7 +725,7 @@ metaslab_init(metaslab_group_t *mg, space_map_obj_t *smo, vdev_t *vd = mg->mg_vd; metaslab_t *msp; - msp = kmem_zalloc(sizeof (metaslab_t), KM_SLEEP); + msp = kmem_zalloc(sizeof (metaslab_t), KM_PUSHPAGE); mutex_init(&msp->ms_lock, NULL, MUTEX_DEFAULT, NULL); msp->ms_smo_syncing = *smo; diff --git a/module/zfs/refcount.c b/module/zfs/refcount.c index a6155460625a..e43807c8e3f4 100644 --- a/module/zfs/refcount.c +++ b/module/zfs/refcount.c @@ -114,7 +114,7 @@ refcount_add_many(refcount_t *rc, uint64_t number, void *holder) int64_t count; if (reference_tracking_enable) { - ref = kmem_cache_alloc(reference_cache, KM_SLEEP); + ref = kmem_cache_alloc(reference_cache, KM_PUSHPAGE); ref->ref_holder = holder; ref->ref_number = number; } @@ -158,7 +158,7 @@ refcount_remove_many(refcount_t *rc, uint64_t number, void *holder) if (reference_history > 0) { ref->ref_removed = kmem_cache_alloc(reference_history_cache, - KM_SLEEP); + KM_PUSHPAGE); list_insert_head(&rc->rc_removed, ref); rc->rc_removed_count++; if (rc->rc_removed_count >= reference_history) { diff --git a/module/zfs/sa.c b/module/zfs/sa.c index a26e5ebdd6a5..d4b28cc90ddd 100644 --- a/module/zfs/sa.c +++ b/module/zfs/sa.c @@ -433,10 +433,10 @@ sa_add_layout_entry(objset_t *os, sa_attr_type_t *attrs, int attr_count, avl_index_t loc; ASSERT(MUTEX_HELD(&sa->sa_lock)); - tb = kmem_zalloc(sizeof (sa_lot_t), KM_SLEEP); + tb = kmem_zalloc(sizeof (sa_lot_t), KM_PUSHPAGE); tb->lot_attr_count = attr_count; tb->lot_attrs = kmem_alloc(sizeof (sa_attr_type_t) * attr_count, - KM_SLEEP); + KM_PUSHPAGE); bcopy(attrs, tb->lot_attrs, sizeof (sa_attr_type_t) * attr_count); tb->lot_num = lot_num; tb->lot_hash = hash; @@ -721,7 +721,7 @@ sa_build_layouts(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc, int attr_count, buf_space = hdl->sa_bonus->db_size - hdrsize; attrs_start = attrs = kmem_alloc(sizeof (sa_attr_type_t) * attr_count, - KM_SLEEP); + KM_PUSHPAGE); lot_count = 0; for (i = 0, len_idx = 0, hash = -1ULL; i != attr_count; i++) { @@ -842,7 +842,7 @@ sa_attr_table_setup(objset_t *os, sa_attr_reg_t *reg_attrs, int count) dmu_objset_type_t ostype = dmu_objset_type(os); sa->sa_user_table = - kmem_zalloc(count * sizeof (sa_attr_type_t), KM_SLEEP); + kmem_zalloc(count * sizeof (sa_attr_type_t), KM_PUSHPAGE); sa->sa_user_table_sz = count * sizeof (sa_attr_type_t); if (sa->sa_reg_attr_obj != 0) { @@ -901,7 +901,7 @@ sa_attr_table_setup(objset_t *os, sa_attr_reg_t *reg_attrs, int count) sa->sa_num_attrs = sa_attr_count; tb = sa->sa_attr_table = - kmem_zalloc(sizeof (sa_attr_table_t) * sa_attr_count, KM_SLEEP); + kmem_zalloc(sizeof (sa_attr_table_t) * sa_attr_count, KM_PUSHPAGE); /* * Attribute table is constructed from requested attribute list, @@ -926,7 +926,7 @@ sa_attr_table_setup(objset_t *os, sa_attr_reg_t *reg_attrs, int count) continue; } tb[ATTR_NUM(value)].sa_name = - kmem_zalloc(strlen(za.za_name) +1, KM_SLEEP); + kmem_zalloc(strlen(za.za_name) +1, KM_PUSHPAGE); (void) strlcpy(tb[ATTR_NUM(value)].sa_name, za.za_name, strlen(za.za_name) +1); } @@ -952,7 +952,7 @@ sa_attr_table_setup(objset_t *os, sa_attr_reg_t *reg_attrs, int count) tb[i].sa_registered = B_FALSE; tb[i].sa_name = kmem_zalloc(strlen(sa_legacy_attrs[i].sa_name) +1, - KM_SLEEP); + KM_PUSHPAGE); (void) strlcpy(tb[i].sa_name, sa_legacy_attrs[i].sa_name, strlen(sa_legacy_attrs[i].sa_name) + 1); @@ -970,7 +970,7 @@ sa_attr_table_setup(objset_t *os, sa_attr_reg_t *reg_attrs, int count) tb[attr_id].sa_byteswap = reg_attrs[i].sa_byteswap; tb[attr_id].sa_attr = attr_id; tb[attr_id].sa_name = - kmem_zalloc(strlen(reg_attrs[i].sa_name) + 1, KM_SLEEP); + kmem_zalloc(strlen(reg_attrs[i].sa_name) + 1, KM_PUSHPAGE); (void) strlcpy(tb[attr_id].sa_name, reg_attrs[i].sa_name, strlen(reg_attrs[i].sa_name) + 1); } @@ -1007,7 +1007,7 @@ sa_setup(objset_t *os, uint64_t sa_obj, sa_attr_reg_t *reg_attrs, int count, return (0); } - sa = kmem_zalloc(sizeof (sa_os_t), KM_SLEEP); + sa = kmem_zalloc(sizeof (sa_os_t), KM_PUSHPAGE); mutex_init(&sa->sa_lock, NULL, MUTEX_DEFAULT, NULL); sa->sa_master_obj = sa_obj; @@ -1055,7 +1055,7 @@ sa_setup(objset_t *os, uint64_t sa_obj, sa_attr_reg_t *reg_attrs, int count, uint64_t lot_num; lot_attrs = kmem_zalloc(sizeof (sa_attr_type_t) * - za.za_num_integers, KM_SLEEP); + za.za_num_integers, KM_PUSHPAGE); if ((error = (zap_lookup(os, sa->sa_layout_attr_obj, za.za_name, 2, za.za_num_integers, @@ -1540,14 +1540,14 @@ sa_find_idx_tab(objset_t *os, dmu_object_type_t bonustype, void *data) } /* No such luck, create a new entry */ - idx_tab = kmem_zalloc(sizeof (sa_idx_tab_t), KM_SLEEP); + idx_tab = kmem_zalloc(sizeof (sa_idx_tab_t), KM_PUSHPAGE); idx_tab->sa_idx_tab = - kmem_zalloc(sizeof (uint32_t) * sa->sa_num_attrs, KM_SLEEP); + kmem_zalloc(sizeof (uint32_t) * sa->sa_num_attrs, KM_PUSHPAGE); idx_tab->sa_layout = tb; refcount_create(&idx_tab->sa_refcount); if (tb->lot_var_sizes) idx_tab->sa_variable_lengths = kmem_alloc(sizeof (uint16_t) * - tb->lot_var_sizes, KM_SLEEP); + tb->lot_var_sizes, KM_PUSHPAGE); sa_attr_iter(os, hdr, bonustype, sa_build_idx_tab, tb, idx_tab); diff --git a/module/zfs/spa.c b/module/zfs/spa.c index 7abe69902b6c..a3d52c8b1c70 100644 --- a/module/zfs/spa.c +++ b/module/zfs/spa.c @@ -150,7 +150,7 @@ spa_prop_add_list(nvlist_t *nvl, zpool_prop_t prop, char *strval, const char *propname = zpool_prop_to_name(prop); nvlist_t *propval; - VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP) == 0); + VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_uint64(propval, ZPROP_SOURCE, src) == 0); if (strval != NULL) @@ -237,7 +237,7 @@ spa_prop_get(spa_t *spa, nvlist_t **nvp) zap_attribute_t za; int err; - err = nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP); + err = nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_PUSHPAGE); if (err) return err; @@ -289,7 +289,7 @@ spa_prop_get(spa_t *spa, nvlist_t **nvp) strval = kmem_alloc( MAXNAMELEN + strlen(MOS_DIR_NAME) + 1, - KM_SLEEP); + KM_PUSHPAGE); dsl_dataset_name(ds, strval); dsl_dataset_rele(ds, FTAG); rw_exit(&dp->dp_config_rwlock); @@ -308,7 +308,7 @@ spa_prop_get(spa_t *spa, nvlist_t **nvp) case 1: /* string property */ - strval = kmem_alloc(za.za_num_integers, KM_SLEEP); + strval = kmem_alloc(za.za_num_integers, KM_PUSHPAGE); err = zap_lookup(mos, spa->spa_pool_props_object, za.za_name, 1, za.za_num_integers, strval); if (err) { @@ -528,7 +528,7 @@ spa_configfile_set(spa_t *spa, nvlist_t *nvp, boolean_t need_sync) return; dp = kmem_alloc(sizeof (spa_config_dirent_t), - KM_SLEEP); + KM_PUSHPAGE); if (cachefile[0] == '\0') dp->scd_path = spa_strdup(spa_config_path); @@ -1140,7 +1140,7 @@ spa_load_spares(spa_t *spa) * active configuration, then we also mark this vdev as an active spare. */ spa->spa_spares.sav_vdevs = kmem_alloc(nspares * sizeof (void *), - KM_SLEEP); + KM_PUSHPAGE); for (i = 0; i < spa->spa_spares.sav_count; i++) { VERIFY(spa_config_parse(spa, &vd, spares[i], NULL, 0, VDEV_ALLOC_SPARE) == 0); @@ -1188,7 +1188,7 @@ spa_load_spares(spa_t *spa) DATA_TYPE_NVLIST_ARRAY) == 0); spares = kmem_alloc(spa->spa_spares.sav_count * sizeof (void *), - KM_SLEEP); + KM_PUSHPAGE); for (i = 0; i < spa->spa_spares.sav_count; i++) spares[i] = vdev_config_generate(spa, spa->spa_spares.sav_vdevs[i], B_TRUE, VDEV_CONFIG_SPARE); @@ -1222,7 +1222,7 @@ spa_load_l2cache(spa_t *spa) if (sav->sav_config != NULL) { VERIFY(nvlist_lookup_nvlist_array(sav->sav_config, ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0); - newvdevs = kmem_alloc(nl2cache * sizeof (void *), KM_SLEEP); + newvdevs = kmem_alloc(nl2cache * sizeof (void *), KM_PUSHPAGE); } else { nl2cache = 0; } @@ -1316,7 +1316,7 @@ spa_load_l2cache(spa_t *spa) VERIFY(nvlist_remove(sav->sav_config, ZPOOL_CONFIG_L2CACHE, DATA_TYPE_NVLIST_ARRAY) == 0); - l2cache = kmem_alloc(sav->sav_count * sizeof (void *), KM_SLEEP); + l2cache = kmem_alloc(sav->sav_count * sizeof (void *), KM_PUSHPAGE); for (i = 0; i < sav->sav_count; i++) l2cache[i] = vdev_config_generate(spa, sav->sav_vdevs[i], B_TRUE, VDEV_CONFIG_L2CACHE); @@ -1342,7 +1342,7 @@ load_nvlist(spa_t *spa, uint64_t obj, nvlist_t **value) nvsize = *(uint64_t *)db->db_data; dmu_buf_rele(db, FTAG); - packed = kmem_alloc(nvsize, KM_SLEEP | KM_NODEBUG); + packed = kmem_alloc(nvsize, KM_PUSHPAGE | KM_NODEBUG); error = dmu_read(spa->spa_meta_objset, obj, 0, nvsize, packed, DMU_READ_PREFETCH); if (error == 0) @@ -1398,8 +1398,8 @@ spa_config_valid(spa_t *spa, nvlist_t *config) uint64_t idx = 0; child = kmem_alloc(rvd->vdev_children * sizeof (nvlist_t **), - KM_SLEEP); - VERIFY(nvlist_alloc(&nv, NV_UNIQUE_NAME, KM_SLEEP) == 0); + KM_PUSHPAGE); + VERIFY(nvlist_alloc(&nv, NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); for (c = 0; c < rvd->vdev_children; c++) { vdev_t *tvd = rvd->vdev_child[c]; @@ -1754,7 +1754,7 @@ spa_try_repair(spa_t *spa, nvlist_t *config) &glist, &gcount) != 0) return; - vd = kmem_zalloc(gcount * sizeof (vdev_t *), KM_SLEEP); + vd = kmem_zalloc(gcount * sizeof (vdev_t *), KM_PUSHPAGE); /* attempt to online all the vdevs & validate */ attempt_reopen = B_TRUE; @@ -1840,7 +1840,7 @@ spa_load(spa_t *spa, spa_load_state_t state, spa_import_type_t type, if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) == 0) { VERIFY(nvlist_dup(nvl, &spa->spa_config_splitting, - KM_SLEEP) == 0); + KM_PUSHPAGE) == 0); } gethrestime(&spa->spa_loaded_ts); @@ -2497,7 +2497,7 @@ spa_open_common(const char *pool, spa_t **spapp, void *tag, nvlist_t *nvpolicy, */ if (config != NULL && spa->spa_config) { VERIFY(nvlist_dup(spa->spa_config, config, - KM_SLEEP) == 0); + KM_PUSHPAGE) == 0); VERIFY(nvlist_add_nvlist(*config, ZPOOL_CONFIG_LOAD_INFO, spa->spa_load_info) == 0); @@ -2873,13 +2873,13 @@ spa_set_aux_vdevs(spa_aux_vdev_t *sav, nvlist_t **devs, int ndevs, &olddevs, &oldndevs) == 0); newdevs = kmem_alloc(sizeof (void *) * - (ndevs + oldndevs), KM_SLEEP); + (ndevs + oldndevs), KM_PUSHPAGE); for (i = 0; i < oldndevs; i++) VERIFY(nvlist_dup(olddevs[i], &newdevs[i], - KM_SLEEP) == 0); + KM_PUSHPAGE) == 0); for (i = 0; i < ndevs; i++) VERIFY(nvlist_dup(devs[i], &newdevs[i + oldndevs], - KM_SLEEP) == 0); + KM_PUSHPAGE) == 0); VERIFY(nvlist_remove(sav->sav_config, config, DATA_TYPE_NVLIST_ARRAY) == 0); @@ -2894,7 +2894,7 @@ spa_set_aux_vdevs(spa_aux_vdev_t *sav, nvlist_t **devs, int ndevs, * Generate a new dev list. */ VERIFY(nvlist_alloc(&sav->sav_config, NV_UNIQUE_NAME, - KM_SLEEP) == 0); + KM_PUSHPAGE) == 0); VERIFY(nvlist_add_nvlist_array(sav->sav_config, config, devs, ndevs) == 0); } @@ -3020,7 +3020,7 @@ spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props, if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0) { VERIFY(nvlist_alloc(&spa->spa_spares.sav_config, NV_UNIQUE_NAME, - KM_SLEEP) == 0); + KM_PUSHPAGE) == 0); VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES, spares, nspares) == 0); spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); @@ -3035,7 +3035,7 @@ spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props, if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0) { VERIFY(nvlist_alloc(&spa->spa_l2cache.sav_config, - NV_UNIQUE_NAME, KM_SLEEP) == 0); + NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_nvlist_array(spa->spa_l2cache.sav_config, ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0); spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); @@ -3173,7 +3173,7 @@ spa_generate_rootconf(char *devpath, char *devid, uint64_t *guid) /* * Put this pool's top-level vdevs into a root vdev. */ - VERIFY(nvlist_alloc(&nvroot, NV_UNIQUE_NAME, KM_SLEEP) == 0); + VERIFY(nvlist_alloc(&nvroot, NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_string(nvroot, ZPOOL_CONFIG_TYPE, VDEV_TYPE_ROOT) == 0); VERIFY(nvlist_add_uint64(nvroot, ZPOOL_CONFIG_ID, 0ULL) == 0); @@ -3484,7 +3484,7 @@ spa_import(const char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags) ZPOOL_CONFIG_SPARES, DATA_TYPE_NVLIST_ARRAY) == 0); else VERIFY(nvlist_alloc(&spa->spa_spares.sav_config, - NV_UNIQUE_NAME, KM_SLEEP) == 0); + NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES, spares, nspares) == 0); spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); @@ -3499,7 +3499,7 @@ spa_import(const char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags) ZPOOL_CONFIG_L2CACHE, DATA_TYPE_NVLIST_ARRAY) == 0); else VERIFY(nvlist_alloc(&spa->spa_l2cache.sav_config, - NV_UNIQUE_NAME, KM_SLEEP) == 0); + NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_nvlist_array(spa->spa_l2cache.sav_config, ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0); spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); @@ -3582,7 +3582,7 @@ spa_tryimport(nvlist_t *tryconfig) * pools are bootable. */ if ((!error || error == EEXIST) && spa->spa_bootfs) { - char *tmpname = kmem_alloc(MAXPATHLEN, KM_SLEEP); + char *tmpname = kmem_alloc(MAXPATHLEN, KM_PUSHPAGE); /* * We have to play games with the name since the @@ -3591,7 +3591,7 @@ spa_tryimport(nvlist_t *tryconfig) if (dsl_dsobj_to_dsname(spa_name(spa), spa->spa_bootfs, tmpname) == 0) { char *cp; - char *dsname = kmem_alloc(MAXPATHLEN, KM_SLEEP); + char *dsname = kmem_alloc(MAXPATHLEN, KM_PUSHPAGE); cp = strchr(tmpname, '/'); if (cp == NULL) { @@ -3996,7 +3996,7 @@ spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing) if (strcmp(oldvd->vdev_path, newvd->vdev_path) == 0) { spa_strfree(oldvd->vdev_path); oldvd->vdev_path = kmem_alloc(strlen(newvd->vdev_path) + 5, - KM_SLEEP); + KM_PUSHPAGE); (void) sprintf(oldvd->vdev_path, "%s/%s", newvd->vdev_path, "old"); if (oldvd->vdev_devid != NULL) { @@ -4391,8 +4391,8 @@ spa_vdev_split_mirror(spa_t *spa, char *newname, nvlist_t *config, nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_L2CACHE, &tmp) == 0) return (spa_vdev_exit(spa, NULL, txg, EINVAL)); - vml = kmem_zalloc(children * sizeof (vdev_t *), KM_SLEEP); - glist = kmem_zalloc(children * sizeof (uint64_t), KM_SLEEP); + vml = kmem_zalloc(children * sizeof (vdev_t *), KM_PUSHPAGE); + glist = kmem_zalloc(children * sizeof (uint64_t), KM_PUSHPAGE); /* then, loop over each vdev and validate it */ for (c = 0; c < children; c++) { @@ -4472,7 +4472,7 @@ spa_vdev_split_mirror(spa_t *spa, char *newname, nvlist_t *config, * Temporarily record the splitting vdevs in the spa config. This * will disappear once the config is regenerated. */ - VERIFY(nvlist_alloc(&nvl, NV_UNIQUE_NAME, KM_SLEEP) == 0); + VERIFY(nvlist_alloc(&nvl, NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, glist, children) == 0); kmem_free(glist, children * sizeof (uint64_t)); @@ -4519,7 +4519,7 @@ spa_vdev_split_mirror(spa_t *spa, char *newname, nvlist_t *config, /* if that worked, generate a real config for the new pool */ if (newspa->spa_root_vdev != NULL) { VERIFY(nvlist_alloc(&newspa->spa_config_splitting, - NV_UNIQUE_NAME, KM_SLEEP) == 0); + NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_uint64(newspa->spa_config_splitting, ZPOOL_CONFIG_SPLIT_GUID, spa_guid(spa)) == 0); spa_config_set(newspa, spa_config_generate(newspa, NULL, -1ULL, @@ -4631,12 +4631,12 @@ spa_vdev_remove_aux(nvlist_t *config, char *name, nvlist_t **dev, int count, int i, j; if (count > 1) - newdev = kmem_alloc((count - 1) * sizeof (void *), KM_SLEEP); + newdev = kmem_alloc((count - 1) * sizeof (void *), KM_PUSHPAGE); for (i = 0, j = 0; i < count; i++) { if (dev[i] == dev_to_remove) continue; - VERIFY(nvlist_dup(dev[i], &newdev[j++], KM_SLEEP) == 0); + VERIFY(nvlist_dup(dev[i], &newdev[j++], KM_PUSHPAGE) == 0); } VERIFY(nvlist_remove(config, name, DATA_TYPE_NVLIST_ARRAY) == 0); @@ -5291,10 +5291,10 @@ spa_sync_nvlist(spa_t *spa, uint64_t obj, nvlist_t *nv, dmu_tx_t *tx) * saves us a pre-read to get data we don't actually care about. */ bufsize = P2ROUNDUP(nvsize, SPA_CONFIG_BLOCKSIZE); - packed = vmem_alloc(bufsize, KM_SLEEP); + packed = vmem_alloc(bufsize, KM_PUSHPAGE); VERIFY(nvlist_pack(nv, &packed, &nvsize, NV_ENCODE_XDR, - KM_SLEEP) == 0); + KM_PUSHPAGE) == 0); bzero(packed + nvsize, bufsize - nvsize); dmu_write(spa->spa_meta_objset, obj, 0, bufsize, packed, tx); @@ -5332,11 +5332,11 @@ spa_sync_aux_dev(spa_t *spa, spa_aux_vdev_t *sav, dmu_tx_t *tx, &sav->sav_object, tx) == 0); } - VERIFY(nvlist_alloc(&nvroot, NV_UNIQUE_NAME, KM_SLEEP) == 0); + VERIFY(nvlist_alloc(&nvroot, NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); if (sav->sav_count == 0) { VERIFY(nvlist_add_nvlist_array(nvroot, config, NULL, 0) == 0); } else { - list = kmem_alloc(sav->sav_count * sizeof (void *), KM_SLEEP); + list = kmem_alloc(sav->sav_count * sizeof (void *), KM_PUSHPAGE); for (i = 0; i < sav->sav_count; i++) list[i] = vdev_config_generate(spa, sav->sav_vdevs[i], B_FALSE, VDEV_CONFIG_L2CACHE); diff --git a/module/zfs/spa_config.c b/module/zfs/spa_config.c index d814ae21762a..c86884148494 100644 --- a/module/zfs/spa_config.c +++ b/module/zfs/spa_config.c @@ -83,7 +83,7 @@ spa_config_load(void) /* * Open the configuration file. */ - pathname = kmem_alloc(MAXPATHLEN, KM_SLEEP); + pathname = kmem_alloc(MAXPATHLEN, KM_PUSHPAGE); (void) snprintf(pathname, MAXPATHLEN, "%s%s", (rootdir != NULL) ? "./" : "", spa_config_path); @@ -98,7 +98,7 @@ spa_config_load(void) if (kobj_get_filesize(file, &fsize) != 0) goto out; - buf = kmem_alloc(fsize, KM_SLEEP | KM_NODEBUG); + buf = kmem_alloc(fsize, KM_PUSHPAGE | KM_NODEBUG); /* * Read the nvlist from the file. @@ -109,7 +109,7 @@ spa_config_load(void) /* * Unpack the nvlist. */ - if (nvlist_unpack(buf, fsize, &nvlist, KM_SLEEP) != 0) + if (nvlist_unpack(buf, fsize, &nvlist, KM_PUSHPAGE) != 0) goto out; /* @@ -161,11 +161,11 @@ spa_config_write(spa_config_dirent_t *dp, nvlist_t *nvl) */ VERIFY(nvlist_size(nvl, &buflen, NV_ENCODE_XDR) == 0); - buf = kmem_alloc(buflen, KM_SLEEP | KM_NODEBUG); - temp = kmem_zalloc(MAXPATHLEN, KM_SLEEP); + buf = kmem_alloc(buflen, KM_PUSHPAGE | KM_NODEBUG); + temp = kmem_zalloc(MAXPATHLEN, KM_PUSHPAGE); VERIFY(nvlist_pack(nvl, &buf, &buflen, NV_ENCODE_XDR, - KM_SLEEP) == 0); + KM_PUSHPAGE) == 0); /* * Write the configuration to disk. We need to do the traditional @@ -234,7 +234,7 @@ spa_config_sync(spa_t *target, boolean_t removing, boolean_t postsysevent) if (nvl == NULL) VERIFY(nvlist_alloc(&nvl, NV_UNIQUE_NAME, - KM_SLEEP) == 0); + KM_PUSHPAGE) == 0); VERIFY(nvlist_add_nvlist(nvl, spa->spa_name, spa->spa_config) == 0); @@ -277,7 +277,7 @@ spa_all_configs(uint64_t *generation) if (*generation == spa_config_generation) return (NULL); - VERIFY(nvlist_alloc(&pools, NV_UNIQUE_NAME, KM_SLEEP) == 0); + VERIFY(nvlist_alloc(&pools, NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); mutex_enter(&spa_namespace_lock); while ((spa = spa_next(spa)) != NULL) { @@ -334,7 +334,7 @@ spa_config_generate(spa_t *spa, vdev_t *vd, uint64_t txg, int getstats) if (txg == -1ULL) txg = spa->spa_config_txg; - VERIFY(nvlist_alloc(&config, NV_UNIQUE_NAME, KM_SLEEP) == 0); + VERIFY(nvlist_alloc(&config, NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, spa_version(spa)) == 0); @@ -413,21 +413,21 @@ spa_config_generate(spa_t *spa, vdev_t *vd, uint64_t txg, int getstats) ddt_stat_t *dds; ddt_object_t *ddo; - ddh = kmem_zalloc(sizeof (ddt_histogram_t), KM_SLEEP); + ddh = kmem_zalloc(sizeof (ddt_histogram_t), KM_PUSHPAGE); ddt_get_dedup_histogram(spa, ddh); VERIFY(nvlist_add_uint64_array(config, ZPOOL_CONFIG_DDT_HISTOGRAM, (uint64_t *)ddh, sizeof (*ddh) / sizeof (uint64_t)) == 0); kmem_free(ddh, sizeof (ddt_histogram_t)); - ddo = kmem_zalloc(sizeof (ddt_object_t), KM_SLEEP); + ddo = kmem_zalloc(sizeof (ddt_object_t), KM_PUSHPAGE); ddt_get_dedup_object_stats(spa, ddo); VERIFY(nvlist_add_uint64_array(config, ZPOOL_CONFIG_DDT_OBJ_STATS, (uint64_t *)ddo, sizeof (*ddo) / sizeof (uint64_t)) == 0); kmem_free(ddo, sizeof (ddt_object_t)); - dds = kmem_zalloc(sizeof (ddt_stat_t), KM_SLEEP); + dds = kmem_zalloc(sizeof (ddt_stat_t), KM_PUSHPAGE); ddt_get_dedup_stats(spa, dds); VERIFY(nvlist_add_uint64_array(config, ZPOOL_CONFIG_DDT_STATS, diff --git a/module/zfs/spa_history.c b/module/zfs/spa_history.c index 7a25378752f8..9fb75f391b9a 100644 --- a/module/zfs/spa_history.c +++ b/module/zfs/spa_history.c @@ -234,7 +234,7 @@ spa_history_log_sync(void *arg1, void *arg2, dmu_tx_t *tx) } #endif - VERIFY(nvlist_alloc(&nvrecord, NV_UNIQUE_NAME, KM_SLEEP) == 0); + VERIFY(nvlist_alloc(&nvrecord, NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_uint64(nvrecord, ZPOOL_HIST_TIME, gethrestime_sec()) == 0); VERIFY(nvlist_add_uint64(nvrecord, ZPOOL_HIST_WHO, hap->ha_uid) == 0); @@ -266,10 +266,10 @@ spa_history_log_sync(void *arg1, void *arg2, dmu_tx_t *tx) } VERIFY(nvlist_size(nvrecord, &reclen, NV_ENCODE_XDR) == 0); - record_packed = kmem_alloc(reclen, KM_SLEEP); + record_packed = kmem_alloc(reclen, KM_PUSHPAGE); VERIFY(nvlist_pack(nvrecord, &record_packed, &reclen, - NV_ENCODE_XDR, KM_SLEEP) == 0); + NV_ENCODE_XDR, KM_PUSHPAGE) == 0); mutex_enter(&spa->spa_history_lock); if (hap->ha_log_type == LOG_CMD_POOL_CREATE) @@ -316,7 +316,7 @@ spa_history_log(spa_t *spa, const char *history_str, history_log_type_t what) return (err); } - ha = kmem_alloc(sizeof (history_arg_t), KM_SLEEP); + ha = kmem_alloc(sizeof (history_arg_t), KM_PUSHPAGE); ha->ha_history_str = strdup(history_str); ha->ha_zone = strdup(spa_history_zone()); ha->ha_log_type = what; @@ -442,7 +442,7 @@ log_internal(history_internal_events_t event, spa_t *spa, if (tx->tx_txg == TXG_INITIAL) return; - ha = kmem_alloc(sizeof (history_arg_t), KM_SLEEP); + ha = kmem_alloc(sizeof (history_arg_t), KM_PUSHPAGE); va_copy(adx_copy, adx); ha->ha_history_str = kmem_vasprintf(fmt, adx_copy); va_end(adx_copy); diff --git a/module/zfs/spa_misc.c b/module/zfs/spa_misc.c index c82dca6c5693..4a8e6adfd83c 100644 --- a/module/zfs/spa_misc.c +++ b/module/zfs/spa_misc.c @@ -425,7 +425,7 @@ spa_add(const char *name, nvlist_t *config, const char *altroot) ASSERT(MUTEX_HELD(&spa_namespace_lock)); - spa = kmem_zalloc(sizeof (spa_t), KM_SLEEP | KM_NODEBUG); + spa = kmem_zalloc(sizeof (spa_t), KM_PUSHPAGE | KM_NODEBUG); mutex_init(&spa->spa_async_lock, NULL, MUTEX_DEFAULT, NULL); mutex_init(&spa->spa_errlist_lock, NULL, MUTEX_DEFAULT, NULL); @@ -472,12 +472,12 @@ spa_add(const char *name, nvlist_t *config, const char *altroot) list_create(&spa->spa_config_list, sizeof (spa_config_dirent_t), offsetof(spa_config_dirent_t, scd_link)); - dp = kmem_zalloc(sizeof (spa_config_dirent_t), KM_SLEEP); + dp = kmem_zalloc(sizeof (spa_config_dirent_t), KM_PUSHPAGE); dp->scd_path = altroot ? NULL : spa_strdup(spa_config_path); list_insert_head(&spa->spa_config_list, dp); VERIFY(nvlist_alloc(&spa->spa_load_info, NV_UNIQUE_NAME, - KM_SLEEP) == 0); + KM_PUSHPAGE) == 0); if (config != NULL) VERIFY(nvlist_dup(config, &spa->spa_config, 0) == 0); @@ -647,7 +647,7 @@ spa_aux_add(vdev_t *vd, avl_tree_t *avl) if ((aux = avl_find(avl, &search, &where)) != NULL) { aux->aux_count++; } else { - aux = kmem_zalloc(sizeof (spa_aux_t), KM_SLEEP); + aux = kmem_zalloc(sizeof (spa_aux_t), KM_PUSHPAGE); aux->aux_guid = vd->vdev_guid; aux->aux_count = 1; avl_insert(avl, aux, where); @@ -1131,7 +1131,7 @@ spa_strdup(const char *s) char *new; len = strlen(s); - new = kmem_alloc(len + 1, KM_SLEEP); + new = kmem_alloc(len + 1, KM_PUSHPAGE); bcopy(s, new, len); new[len] = '\0'; diff --git a/module/zfs/space_map.c b/module/zfs/space_map.c index 1ce7b2a3d466..9c0cdb6beecc 100644 --- a/module/zfs/space_map.c +++ b/module/zfs/space_map.c @@ -134,7 +134,7 @@ space_map_add(space_map_t *sm, uint64_t start, uint64_t size) avl_remove(sm->sm_pp_root, ss_after); ss = ss_after; } else { - ss = kmem_alloc(sizeof (*ss), KM_SLEEP); + ss = kmem_alloc(sizeof (*ss), KM_PUSHPAGE); ss->ss_start = start; ss->ss_end = end; avl_insert(&sm->sm_root, ss, where); @@ -181,7 +181,7 @@ space_map_remove(space_map_t *sm, uint64_t start, uint64_t size) avl_remove(sm->sm_pp_root, ss); if (left_over && right_over) { - newseg = kmem_alloc(sizeof (*newseg), KM_SLEEP); + newseg = kmem_alloc(sizeof (*newseg), KM_PUSHPAGE); newseg->ss_start = end; newseg->ss_end = ss->ss_end; ss->ss_end = start; @@ -551,7 +551,7 @@ space_map_ref_add_node(avl_tree_t *t, uint64_t offset, int64_t refcnt) { space_ref_t *sr; - sr = kmem_alloc(sizeof (*sr), KM_SLEEP); + sr = kmem_alloc(sizeof (*sr), KM_PUSHPAGE); sr->sr_offset = offset; sr->sr_refcnt = refcnt; diff --git a/module/zfs/txg.c b/module/zfs/txg.c index aefda6f69149..e14bff269b3e 100644 --- a/module/zfs/txg.c +++ b/module/zfs/txg.c @@ -349,7 +349,7 @@ txg_dispatch_callbacks(dsl_pool_t *dp, uint64_t txg) TASKQ_THREADS_CPU_PCT | TASKQ_PREPOPULATE); } - cb_list = kmem_alloc(sizeof (list_t), KM_SLEEP); + cb_list = kmem_alloc(sizeof (list_t), KM_PUSHPAGE); list_create(cb_list, sizeof (dmu_tx_callback_t), offsetof(dmu_tx_callback_t, dcb_node)); diff --git a/module/zfs/unique.c b/module/zfs/unique.c index 8c1d2e2f9856..5c7ca48759b0 100644 --- a/module/zfs/unique.c +++ b/module/zfs/unique.c @@ -79,7 +79,7 @@ uint64_t unique_insert(uint64_t value) { avl_index_t idx; - unique_t *un = kmem_alloc(sizeof (unique_t), KM_SLEEP); + unique_t *un = kmem_alloc(sizeof (unique_t), KM_PUSHPAGE); un->un_value = value; diff --git a/module/zfs/vdev.c b/module/zfs/vdev.c index 0c8ce1b1c064..06c7d0c8937b 100644 --- a/module/zfs/vdev.c +++ b/module/zfs/vdev.c @@ -195,7 +195,7 @@ vdev_add_child(vdev_t *pvd, vdev_t *cvd) pvd->vdev_children = MAX(pvd->vdev_children, id + 1); newsize = pvd->vdev_children * sizeof (vdev_t *); - newchild = kmem_zalloc(newsize, KM_SLEEP); + newchild = kmem_zalloc(newsize, KM_PUSHPAGE); if (pvd->vdev_child != NULL) { bcopy(pvd->vdev_child, newchild, oldsize); kmem_free(pvd->vdev_child, oldsize); @@ -265,7 +265,7 @@ vdev_compact_children(vdev_t *pvd) if (pvd->vdev_child[c]) newc++; - newchild = kmem_alloc(newc * sizeof (vdev_t *), KM_SLEEP); + newchild = kmem_alloc(newc * sizeof (vdev_t *), KM_PUSHPAGE); for (c = newc = 0; c < oldc; c++) { if ((cvd = pvd->vdev_child[c]) != NULL) { @@ -288,7 +288,7 @@ vdev_alloc_common(spa_t *spa, uint_t id, uint64_t guid, vdev_ops_t *ops) vdev_t *vd; int t; - vd = kmem_zalloc(sizeof (vdev_t), KM_SLEEP); + vd = kmem_zalloc(sizeof (vdev_t), KM_PUSHPAGE); if (spa->spa_root_vdev == NULL) { ASSERT(ops == &vdev_root_ops); @@ -838,7 +838,7 @@ vdev_metaslab_init(vdev_t *vd, uint64_t txg) ASSERT(oldc <= newc); - mspp = kmem_zalloc(newc * sizeof (*mspp), KM_SLEEP | KM_NODEBUG); + mspp = kmem_zalloc(newc * sizeof (*mspp), KM_PUSHPAGE | KM_NODEBUG); if (oldc != 0) { bcopy(vd->vdev_ms, mspp, oldc * sizeof (*mspp)); @@ -993,7 +993,7 @@ vdev_probe(vdev_t *vd, zio_t *zio) mutex_enter(&vd->vdev_probe_lock); if ((pio = vd->vdev_probe_zio) == NULL) { - vps = kmem_zalloc(sizeof (*vps), KM_SLEEP); + vps = kmem_zalloc(sizeof (*vps), KM_PUSHPAGE); vps->vps_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_PROBE | ZIO_FLAG_DONT_CACHE | ZIO_FLAG_DONT_AGGREGATE | diff --git a/module/zfs/vdev_cache.c b/module/zfs/vdev_cache.c index e2f8040d1319..bf4ae7b2bc33 100644 --- a/module/zfs/vdev_cache.c +++ b/module/zfs/vdev_cache.c @@ -177,7 +177,7 @@ vdev_cache_allocate(zio_t *zio) vdev_cache_evict(vc, ve); } - ve = kmem_zalloc(sizeof (vdev_cache_entry_t), KM_SLEEP); + ve = kmem_zalloc(sizeof (vdev_cache_entry_t), KM_PUSHPAGE); ve->ve_offset = offset; ve->ve_lastused = ddi_get_lbolt(); ve->ve_data = zio_buf_alloc(VCBS); @@ -274,7 +274,7 @@ vdev_cache_read(zio_t *zio) mutex_enter(&vc->vc_lock); - ve_search = kmem_alloc(sizeof(vdev_cache_entry_t), KM_SLEEP); + ve_search = kmem_alloc(sizeof(vdev_cache_entry_t), KM_PUSHPAGE); ve_search->ve_offset = cache_offset; ve = avl_find(&vc->vc_offset_tree, ve_search, NULL); kmem_free(ve_search, sizeof(vdev_cache_entry_t)); diff --git a/module/zfs/vdev_disk.c b/module/zfs/vdev_disk.c index eee03d08055d..fd40b10055fb 100644 --- a/module/zfs/vdev_disk.c +++ b/module/zfs/vdev_disk.c @@ -236,7 +236,7 @@ vdev_disk_open(vdev_t *v, uint64_t *psize, uint64_t *ashift) return EINVAL; } - vd = kmem_zalloc(sizeof(vdev_disk_t), KM_SLEEP); + vd = kmem_zalloc(sizeof(vdev_disk_t), KM_PUSHPAGE); if (vd == NULL) return ENOMEM; @@ -320,7 +320,7 @@ vdev_disk_dio_alloc(int bio_count) int i; dr = kmem_zalloc(sizeof(dio_request_t) + - sizeof(struct bio *) * bio_count, KM_SLEEP); + sizeof(struct bio *) * bio_count, KM_PUSHPAGE); if (dr) { init_completion(&dr->dr_comp); atomic_set(&dr->dr_ref, 0); @@ -789,7 +789,7 @@ vdev_disk_read_rootlabel(char *devpath, char *devid, nvlist_t **config) } size = P2ALIGN_TYPED(s, sizeof(vdev_label_t), uint64_t); - label = vmem_alloc(sizeof(vdev_label_t), KM_SLEEP); + label = vmem_alloc(sizeof(vdev_label_t), KM_PUSHPAGE); for (i = 0; i < VDEV_LABELS; i++) { uint64_t offset, state, txg = 0; diff --git a/module/zfs/vdev_file.c b/module/zfs/vdev_file.c index ce49fe08fb71..25d0bad3fa4c 100644 --- a/module/zfs/vdev_file.c +++ b/module/zfs/vdev_file.c @@ -72,7 +72,7 @@ vdev_file_open(vdev_t *vd, uint64_t *psize, uint64_t *ashift) goto skip_open; } - vf = vd->vdev_tsd = kmem_zalloc(sizeof (vdev_file_t), KM_SLEEP); + vf = vd->vdev_tsd = kmem_zalloc(sizeof (vdev_file_t), KM_PUSHPAGE); /* * We always open the files from the root of the global zone, even if diff --git a/module/zfs/vdev_label.c b/module/zfs/vdev_label.c index 3774d71528ee..7ac23500f7f9 100644 --- a/module/zfs/vdev_label.c +++ b/module/zfs/vdev_label.c @@ -212,7 +212,7 @@ vdev_config_generate(spa_t *spa, vdev_t *vd, boolean_t getstats, { nvlist_t *nv = NULL; - VERIFY(nvlist_alloc(&nv, NV_UNIQUE_NAME, KM_SLEEP) == 0); + VERIFY(nvlist_alloc(&nv, NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_string(nv, ZPOOL_CONFIG_TYPE, vd->vdev_ops->vdev_op_type) == 0); @@ -319,7 +319,7 @@ vdev_config_generate(spa_t *spa, vdev_t *vd, boolean_t getstats, ASSERT(!vd->vdev_ishole); child = kmem_alloc(vd->vdev_children * sizeof (nvlist_t *), - KM_SLEEP); + KM_PUSHPAGE); for (c = 0, idx = 0; c < vd->vdev_children; c++) { vdev_t *cvd = vd->vdev_child[c]; @@ -408,7 +408,7 @@ vdev_top_config_generate(spa_t *spa, nvlist_t *config) uint64_t *array; uint_t c, idx; - array = kmem_alloc(rvd->vdev_children * sizeof (uint64_t), KM_SLEEP); + array = kmem_alloc(rvd->vdev_children * sizeof (uint64_t), KM_PUSHPAGE); for (c = 0, idx = 0; c < rvd->vdev_children; c++) { vdev_t *tvd = rvd->vdev_child[c]; @@ -709,7 +709,7 @@ vdev_label_init(vdev_t *vd, uint64_t crtxg, vdev_labeltype_t reason) * active hot spare (in which case we want to revert the * labels). */ - VERIFY(nvlist_alloc(&label, NV_UNIQUE_NAME, KM_SLEEP) == 0); + VERIFY(nvlist_alloc(&label, NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_uint64(label, ZPOOL_CONFIG_VERSION, spa_version(spa)) == 0); @@ -722,7 +722,7 @@ vdev_label_init(vdev_t *vd, uint64_t crtxg, vdev_labeltype_t reason) /* * For level 2 ARC devices, add a special label. */ - VERIFY(nvlist_alloc(&label, NV_UNIQUE_NAME, KM_SLEEP) == 0); + VERIFY(nvlist_alloc(&label, NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_uint64(label, ZPOOL_CONFIG_VERSION, spa_version(spa)) == 0); @@ -749,7 +749,7 @@ vdev_label_init(vdev_t *vd, uint64_t crtxg, vdev_labeltype_t reason) buf = vp->vp_nvlist; buflen = sizeof (vp->vp_nvlist); - error = nvlist_pack(label, &buf, &buflen, NV_ENCODE_XDR, KM_SLEEP); + error = nvlist_pack(label, &buf, &buflen, NV_ENCODE_XDR, KM_PUSHPAGE); if (error != 0) { nvlist_free(label); zio_buf_free(vp, sizeof (vdev_phys_t)); @@ -1061,7 +1061,7 @@ vdev_label_sync(zio_t *zio, vdev_t *vd, int l, uint64_t txg, int flags) buf = vp->vp_nvlist; buflen = sizeof (vp->vp_nvlist); - if (nvlist_pack(label, &buf, &buflen, NV_ENCODE_XDR, KM_SLEEP) == 0) { + if (nvlist_pack(label, &buf, &buflen, NV_ENCODE_XDR, KM_PUSHPAGE) == 0) { for (; l < VDEV_LABELS; l += 2) { vdev_label_write(zio, vd, l, vp, offsetof(vdev_label_t, vl_vdev_phys), @@ -1094,7 +1094,7 @@ vdev_label_sync_list(spa_t *spa, int l, uint64_t txg, int flags) ASSERT(!vd->vdev_ishole); - good_writes = kmem_zalloc(sizeof (uint64_t), KM_SLEEP); + good_writes = kmem_zalloc(sizeof (uint64_t), KM_PUSHPAGE); vio = zio_null(zio, spa, NULL, (vd->vdev_islog || vd->vdev_aux != NULL) ? vdev_label_sync_ignore_done : vdev_label_sync_top_done, diff --git a/module/zfs/vdev_mirror.c b/module/zfs/vdev_mirror.c index 47181d439e28..96623d2cf5e2 100644 --- a/module/zfs/vdev_mirror.c +++ b/module/zfs/vdev_mirror.c @@ -79,7 +79,7 @@ vdev_mirror_map_alloc(zio_t *zio) c = BP_GET_NDVAS(zio->io_bp); - mm = kmem_zalloc(offsetof(mirror_map_t, mm_child[c]), KM_SLEEP); + mm = kmem_zalloc(offsetof(mirror_map_t, mm_child[c]), KM_PUSHPAGE); mm->mm_children = c; mm->mm_replacing = B_FALSE; mm->mm_preferred = spa_get_random(c); @@ -106,7 +106,7 @@ vdev_mirror_map_alloc(zio_t *zio) } else { c = vd->vdev_children; - mm = kmem_zalloc(offsetof(mirror_map_t, mm_child[c]), KM_SLEEP); + mm = kmem_zalloc(offsetof(mirror_map_t, mm_child[c]), KM_PUSHPAGE); mm->mm_children = c; mm->mm_replacing = (vd->vdev_ops == &vdev_replacing_ops || vd->vdev_ops == &vdev_spare_ops); diff --git a/module/zfs/vdev_raidz.c b/module/zfs/vdev_raidz.c index b987ac40b136..21c6e032bc19 100644 --- a/module/zfs/vdev_raidz.c +++ b/module/zfs/vdev_raidz.c @@ -456,7 +456,7 @@ vdev_raidz_map_alloc(zio_t *zio, uint64_t unit_shift, uint64_t dcols, ASSERT3U(acols, <=, scols); - rm = kmem_alloc(offsetof(raidz_map_t, rm_col[scols]), KM_SLEEP); + rm = kmem_alloc(offsetof(raidz_map_t, rm_col[scols]), KM_PUSHPAGE); rm->rm_cols = acols; rm->rm_scols = scols; @@ -1196,7 +1196,7 @@ vdev_raidz_matrix_reconstruct(raidz_map_t *rm, int n, int nmissing, size_t psize; psize = sizeof (invlog[0][0]) * n * nmissing; - p = kmem_alloc(psize, KM_SLEEP); + p = kmem_alloc(psize, KM_PUSHPAGE); for (pp = p, i = 0; i < nmissing; i++) { invlog[i] = pp; @@ -1313,7 +1313,7 @@ vdev_raidz_reconstruct_general(raidz_map_t *rm, int *tgts, int ntgts) psize = (sizeof (rows[0][0]) + sizeof (invrows[0][0])) * nmissing_rows * n + sizeof (used[0]) * n; - p = kmem_alloc(psize, KM_SLEEP); + p = kmem_alloc(psize, KM_PUSHPAGE); for (pp = p, i = 0; i < nmissing_rows; i++) { rows[i] = pp; diff --git a/module/zfs/zap.c b/module/zfs/zap.c index fa7e61711af0..fac54eab082b 100644 --- a/module/zfs/zap.c +++ b/module/zfs/zap.c @@ -114,7 +114,7 @@ fzap_upgrade(zap_t *zap, dmu_tx_t *tx, zap_flags_t flags) 1<l_dbuf = db; l->l_phys = db->db_data; @@ -390,7 +390,7 @@ static zap_leaf_t * zap_create_leaf(zap_t *zap, dmu_tx_t *tx) { void *winner; - zap_leaf_t *l = kmem_alloc(sizeof (zap_leaf_t), KM_SLEEP); + zap_leaf_t *l = kmem_alloc(sizeof (zap_leaf_t), KM_PUSHPAGE); ASSERT(RW_WRITE_HELD(&zap->zap_rwlock)); @@ -452,7 +452,7 @@ zap_open_leaf(uint64_t blkid, dmu_buf_t *db) ASSERT(blkid != 0); - l = kmem_alloc(sizeof (zap_leaf_t), KM_SLEEP); + l = kmem_alloc(sizeof (zap_leaf_t), KM_PUSHPAGE); rw_init(&l->l_rwlock, NULL, RW_DEFAULT, NULL); rw_enter(&l->l_rwlock, RW_WRITER); l->l_blkid = blkid; @@ -957,7 +957,7 @@ zap_value_search(objset_t *os, uint64_t zapobj, uint64_t value, uint64_t mask, if (mask == 0) mask = -1ULL; - za = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP); + za = kmem_alloc(sizeof (zap_attribute_t), KM_PUSHPAGE); for (zap_cursor_init(&zc, os, zapobj); (err = zap_cursor_retrieve(&zc, za)) == 0; zap_cursor_advance(&zc)) { diff --git a/module/zfs/zap_micro.c b/module/zfs/zap_micro.c index 3072475f2740..178ab02719a5 100644 --- a/module/zfs/zap_micro.c +++ b/module/zfs/zap_micro.c @@ -172,7 +172,7 @@ zap_name_free(zap_name_t *zn) zap_name_t * zap_name_alloc(zap_t *zap, const char *key, matchtype_t mt) { - zap_name_t *zn = kmem_alloc(sizeof (zap_name_t), KM_SLEEP); + zap_name_t *zn = kmem_alloc(sizeof (zap_name_t), KM_PUSHPAGE); zn->zn_zap = zap; zn->zn_key_intlen = sizeof (*key); @@ -271,7 +271,7 @@ mze_insert(zap_t *zap, int chunkid, uint64_t hash) ASSERT(zap->zap_ismicro); ASSERT(RW_WRITE_HELD(&zap->zap_rwlock)); - mze = kmem_alloc(sizeof (mzap_ent_t), KM_SLEEP); + mze = kmem_alloc(sizeof (mzap_ent_t), KM_PUSHPAGE); mze->mze_chunkid = chunkid; mze->mze_hash = hash; mze->mze_cd = MZE_PHYS(zap, mze)->mze_cd; @@ -365,7 +365,7 @@ mzap_open(objset_t *os, uint64_t obj, dmu_buf_t *db) ASSERT3U(MZAP_ENT_LEN, ==, sizeof (mzap_ent_phys_t)); - zap = kmem_zalloc(sizeof (zap_t), KM_SLEEP); + zap = kmem_zalloc(sizeof (zap_t), KM_PUSHPAGE); rw_init(&zap->zap_rwlock, NULL, RW_DEFAULT, NULL); rw_enter(&zap->zap_rwlock, RW_WRITER); zap->zap_objset = os; diff --git a/module/zfs/zfs_acl.c b/module/zfs/zfs_acl.c index df690b364d06..98c0019503cd 100644 --- a/module/zfs/zfs_acl.c +++ b/module/zfs/zfs_acl.c @@ -452,7 +452,7 @@ zfs_acl_alloc(int vers) { zfs_acl_t *aclp; - aclp = kmem_zalloc(sizeof (zfs_acl_t), KM_SLEEP); + aclp = kmem_zalloc(sizeof (zfs_acl_t), KM_PUSHPAGE); list_create(&aclp->z_acl, sizeof (zfs_acl_node_t), offsetof(zfs_acl_node_t, z_next)); aclp->z_version = vers; @@ -468,9 +468,9 @@ zfs_acl_node_alloc(size_t bytes) { zfs_acl_node_t *aclnode; - aclnode = kmem_zalloc(sizeof (zfs_acl_node_t), KM_SLEEP); + aclnode = kmem_zalloc(sizeof (zfs_acl_node_t), KM_PUSHPAGE); if (bytes) { - aclnode->z_acldata = kmem_alloc(bytes, KM_SLEEP); + aclnode->z_acldata = kmem_alloc(bytes, KM_PUSHPAGE); aclnode->z_allocdata = aclnode->z_acldata; aclnode->z_allocsize = bytes; aclnode->z_size = bytes; diff --git a/module/zfs/zfs_fm.c b/module/zfs/zfs_fm.c index 7801837f104b..0b98231ec0f9 100644 --- a/module/zfs/zfs_fm.c +++ b/module/zfs/zfs_fm.c @@ -519,7 +519,7 @@ annotate_ecksum(nvlist_t *ereport, zio_bad_cksum_t *info, size_t offset = 0; ssize_t start = -1; - zfs_ecksum_info_t *eip = kmem_zalloc(sizeof (*eip), KM_SLEEP); + zfs_ecksum_info_t *eip = kmem_zalloc(sizeof (*eip), KM_PUSHPAGE); /* don't do any annotation for injected checksum errors */ if (info != NULL && info->zbc_injected) @@ -688,7 +688,7 @@ zfs_ereport_start_checksum(spa_t *spa, vdev_t *vd, struct zio *zio, uint64_t offset, uint64_t length, void *arg, zio_bad_cksum_t *info) { - zio_cksum_report_t *report = kmem_zalloc(sizeof (*report), KM_SLEEP); + zio_cksum_report_t *report = kmem_zalloc(sizeof (*report), KM_PUSHPAGE); if (zio->io_vsd != NULL) zio->io_vsd_ops->vsd_cksum_report(zio, report, arg); @@ -697,7 +697,7 @@ zfs_ereport_start_checksum(spa_t *spa, vdev_t *vd, /* copy the checksum failure information if it was provided */ if (info != NULL) { - report->zcr_ckinfo = kmem_zalloc(sizeof (*info), KM_SLEEP); + report->zcr_ckinfo = kmem_zalloc(sizeof (*info), KM_PUSHPAGE); bcopy(info, report->zcr_ckinfo, sizeof (*info)); } diff --git a/module/zfs/zfs_rlock.c b/module/zfs/zfs_rlock.c index f3ada1706419..4f34b3758fa0 100644 --- a/module/zfs/zfs_rlock.c +++ b/module/zfs/zfs_rlock.c @@ -426,7 +426,7 @@ zfs_range_lock(znode_t *zp, uint64_t off, uint64_t len, rl_type_t type) ASSERT(type == RL_READER || type == RL_WRITER || type == RL_APPEND); - new = kmem_alloc(sizeof (rl_t), KM_SLEEP); + new = kmem_alloc(sizeof (rl_t), KM_PUSHPAGE); new->r_zp = zp; new->r_off = off; if (len + off < off) /* overflow */ diff --git a/module/zfs/zfs_znode.c b/module/zfs/zfs_znode.c index 3a6872f3e45e..33fb0083b8ba 100644 --- a/module/zfs/zfs_znode.c +++ b/module/zfs/zfs_znode.c @@ -627,7 +627,7 @@ zfs_mknode(znode_t *dzp, vattr_t *vap, dmu_tx_t *tx, cred_t *cr, * order for DMU_OT_ZNODE is critical since it needs to be constructed * in the old znode_phys_t format. Don't change this ordering */ - sa_attrs = kmem_alloc(sizeof(sa_bulk_attr_t) * ZPL_END, KM_SLEEP); + sa_attrs = kmem_alloc(sizeof(sa_bulk_attr_t) * ZPL_END, KM_PUSHPAGE); if (obj_type == DMU_OT_ZNODE) { SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_ATIME(zsb), @@ -1502,13 +1502,13 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx) vattr.va_uid = crgetuid(cr); vattr.va_gid = crgetgid(cr); - rootzp = kmem_cache_alloc(znode_cache, KM_SLEEP); + rootzp = kmem_cache_alloc(znode_cache, KM_PUSHPAGE); rootzp->z_moved = 0; rootzp->z_unlinked = 0; rootzp->z_atime_dirty = 0; rootzp->z_is_sa = USE_SA(version, os); - zsb = kmem_zalloc(sizeof (zfs_sb_t), KM_SLEEP); + zsb = kmem_zalloc(sizeof (zfs_sb_t), KM_PUSHPAGE); zsb->z_os = os; zsb->z_parent = zsb; zsb->z_version = version; @@ -1516,7 +1516,7 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx) zsb->z_use_sa = USE_SA(version, os); zsb->z_norm = norm; - sb = kmem_zalloc(sizeof (struct super_block), KM_SLEEP); + sb = kmem_zalloc(sizeof (struct super_block), KM_PUSHPAGE); sb->s_fs_info = zsb; ZTOI(rootzp)->i_sb = sb; diff --git a/module/zfs/zil.c b/module/zfs/zil.c index 9ab02d70c095..292aea27d219 100644 --- a/module/zfs/zil.c +++ b/module/zfs/zil.c @@ -165,7 +165,7 @@ zil_bp_tree_add(zilog_t *zilog, const blkptr_t *bp) if (avl_find(t, dva, &where) != NULL) return (EEXIST); - zn = kmem_alloc(sizeof (zil_bp_node_t), KM_SLEEP); + zn = kmem_alloc(sizeof (zil_bp_node_t), KM_PUSHPAGE); zn->zn_dva = *dva; avl_insert(t, zn, where); @@ -455,7 +455,7 @@ zil_alloc_lwb(zilog_t *zilog, blkptr_t *bp, uint64_t txg) { lwb_t *lwb; - lwb = kmem_cache_alloc(zil_lwb_cache, KM_SLEEP); + lwb = kmem_cache_alloc(zil_lwb_cache, KM_PUSHPAGE); lwb->lwb_zilog = zilog; lwb->lwb_blk = *bp; lwb->lwb_buf = zio_buf_alloc(BP_GET_LSIZE(bp)); @@ -752,7 +752,7 @@ zil_add_block(zilog_t *zilog, const blkptr_t *bp) for (i = 0; i < ndvas; i++) { zvsearch.zv_vdev = DVA_GET_VDEV(&bp->blk_dva[i]); if (avl_find(t, &zvsearch, &where) == NULL) { - zv = kmem_alloc(sizeof (*zv), KM_SLEEP); + zv = kmem_alloc(sizeof (*zv), KM_PUSHPAGE); zv->zv_vdev = zvsearch.zv_vdev; avl_insert(t, zv, where); } @@ -1277,7 +1277,7 @@ zil_itx_assign(zilog_t *zilog, itx_t *itx, dmu_tx_t *tx) } ASSERT(itxg->itxg_sod == 0); itxg->itxg_txg = txg; - itxs = itxg->itxg_itxs = kmem_zalloc(sizeof (itxs_t), KM_SLEEP); + itxs = itxg->itxg_itxs = kmem_zalloc(sizeof (itxs_t), KM_PUSHPAGE); list_create(&itxs->i_sync_list, sizeof (itx_t), offsetof(itx_t, itx_node)); @@ -1297,7 +1297,7 @@ zil_itx_assign(zilog_t *zilog, itx_t *itx, dmu_tx_t *tx) ian = avl_find(t, &foid, &where); if (ian == NULL) { - ian = kmem_alloc(sizeof (itx_async_node_t), KM_SLEEP); + ian = kmem_alloc(sizeof (itx_async_node_t), KM_PUSHPAGE); list_create(&ian->ia_list, sizeof (itx_t), offsetof(itx_t, itx_node)); ian->ia_foid = foid; @@ -1685,7 +1685,7 @@ zil_alloc(objset_t *os, zil_header_t *zh_phys) zilog_t *zilog; int i; - zilog = kmem_zalloc(sizeof (zilog_t), KM_SLEEP); + zilog = kmem_zalloc(sizeof (zilog_t), KM_PUSHPAGE); zilog->zl_header = zh_phys; zilog->zl_os = os; @@ -2007,7 +2007,7 @@ zil_replay(objset_t *os, void *arg, zil_replay_func_t *replay_func[TX_MAX_TYPE]) zr.zr_replay = replay_func; zr.zr_arg = arg; zr.zr_byteswap = BP_SHOULD_BYTESWAP(&zh->zh_log); - zr.zr_lr = vmem_alloc(2 * SPA_MAXBLOCKSIZE, KM_SLEEP); + zr.zr_lr = vmem_alloc(2 * SPA_MAXBLOCKSIZE, KM_PUSHPAGE); /* * Wait for in-progress removes to sync before starting replay. diff --git a/module/zfs/zvol.c b/module/zfs/zvol.c index 60ff64ebe7b0..22f14a0066a5 100644 --- a/module/zfs/zvol.c +++ b/module/zfs/zvol.c @@ -785,7 +785,7 @@ zvol_get_data(void *arg, lr_write_t *lr, char *buf, zio_t *zio) ASSERT(zio != NULL); ASSERT(size != 0); - zgd = (zgd_t *)kmem_zalloc(sizeof (zgd_t), KM_SLEEP); + zgd = (zgd_t *)kmem_zalloc(sizeof (zgd_t), KM_PUSHPAGE); zgd->zgd_zilog = zv->zv_zilog; zgd->zgd_rl = zfs_range_lock(&zv->zv_znode, offset, size, RL_READER);