3c0177b887
Without the 'car limit' enabled (before this), running sequential ZFS scrub on HDD without command queuing support, I've measured latency on concurrent random reads reaching 4 seconds (surprised that not more). Enabling this reduced the latency to 65 milliseconds, while scrub still doing ~180MB/s. For disks with command queuing this does not make much difference (if any), since most time all the requests are queued down to the disk or HBA, leaving nothing in the queue to sort. And even if something does not fit, staying on the queue, it is likely not for long. To not limit sorting in such bursty scenarios I've added batched counter zeroing when the queue is getting empty. The internal scheduler of the SAS HDD I was testing seems to be even more loyal to random I/O, reducing the scrub speed to ~120MB/s. So in case somebody worried this is limit is too strict -- it actually looks relaxed. MFC after: 2 weeks Sponsored by: iXsystems, Inc.
302 lines
8.5 KiB
C
302 lines
8.5 KiB
C
/*-
|
|
* SPDX-License-Identifier: Beerware
|
|
*
|
|
* ----------------------------------------------------------------------------
|
|
* "THE BEER-WARE LICENSE" (Revision 42):
|
|
* <phk@FreeBSD.ORG> wrote this file. As long as you retain this notice you
|
|
* can do whatever you want with this stuff. If we meet some day, and you think
|
|
* this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp
|
|
* ----------------------------------------------------------------------------
|
|
*
|
|
* The bioq_disksort() (and the specification of the bioq API)
|
|
* have been written by Luigi Rizzo and Fabio Checconi under the same
|
|
* license as above.
|
|
*/
|
|
|
|
#include <sys/cdefs.h>
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
#include "opt_geom.h"
|
|
|
|
#include <sys/param.h>
|
|
#include <sys/systm.h>
|
|
#include <sys/bio.h>
|
|
#include <sys/conf.h>
|
|
#include <sys/disk.h>
|
|
#include <sys/sysctl.h>
|
|
#include <geom/geom_disk.h>
|
|
|
|
static int bioq_batchsize = 128;
|
|
SYSCTL_INT(_debug, OID_AUTO, bioq_batchsize, CTLFLAG_RW,
|
|
&bioq_batchsize, 0, "BIOQ batch size");
|
|
|
|
/*-
|
|
* Disk error is the preface to plaintive error messages
|
|
* about failing disk transfers. It prints messages of the form
|
|
* "hp0g: BLABLABLA cmd=read fsbn 12345 of 12344-12347"
|
|
* blkdone should be -1 if the position of the error is unknown.
|
|
* The message is printed with printf.
|
|
*/
|
|
void
|
|
disk_err(struct bio *bp, const char *what, int blkdone, int nl)
|
|
{
|
|
daddr_t sn;
|
|
|
|
if (bp->bio_dev != NULL)
|
|
printf("%s: %s ", devtoname(bp->bio_dev), what);
|
|
else if (bp->bio_disk != NULL)
|
|
printf("%s%d: %s ",
|
|
bp->bio_disk->d_name, bp->bio_disk->d_unit, what);
|
|
else
|
|
printf("disk??: %s ", what);
|
|
switch(bp->bio_cmd) {
|
|
case BIO_READ: printf("cmd=read "); break;
|
|
case BIO_WRITE: printf("cmd=write "); break;
|
|
case BIO_DELETE: printf("cmd=delete "); break;
|
|
case BIO_GETATTR: printf("cmd=getattr "); break;
|
|
case BIO_FLUSH: printf("cmd=flush "); break;
|
|
default: printf("cmd=%x ", bp->bio_cmd); break;
|
|
}
|
|
sn = bp->bio_pblkno;
|
|
if (bp->bio_bcount <= DEV_BSIZE) {
|
|
printf("fsbn %jd%s", (intmax_t)sn, nl ? "\n" : "");
|
|
return;
|
|
}
|
|
if (blkdone >= 0) {
|
|
sn += blkdone;
|
|
printf("fsbn %jd of ", (intmax_t)sn);
|
|
}
|
|
printf("%jd-%jd", (intmax_t)bp->bio_pblkno,
|
|
(intmax_t)(bp->bio_pblkno + (bp->bio_bcount - 1) / DEV_BSIZE));
|
|
if (nl)
|
|
printf("\n");
|
|
}
|
|
|
|
/*
|
|
* BIO queue implementation
|
|
*
|
|
* Please read carefully the description below before making any change
|
|
* to the code, or you might change the behaviour of the data structure
|
|
* in undesirable ways.
|
|
*
|
|
* A bioq stores disk I/O request (bio), normally sorted according to
|
|
* the distance of the requested position (bio->bio_offset) from the
|
|
* current head position (bioq->last_offset) in the scan direction, i.e.
|
|
*
|
|
* (uoff_t)(bio_offset - last_offset)
|
|
*
|
|
* Note that the cast to unsigned (uoff_t) is fundamental to insure
|
|
* that the distance is computed in the scan direction.
|
|
*
|
|
* The main methods for manipulating the bioq are:
|
|
*
|
|
* bioq_disksort() performs an ordered insertion;
|
|
*
|
|
* bioq_first() return the head of the queue, without removing;
|
|
*
|
|
* bioq_takefirst() return and remove the head of the queue,
|
|
* updating the 'current head position' as
|
|
* bioq->last_offset = bio->bio_offset + bio->bio_length;
|
|
*
|
|
* When updating the 'current head position', we assume that the result of
|
|
* bioq_takefirst() is dispatched to the device, so bioq->last_offset
|
|
* represents the head position once the request is complete.
|
|
*
|
|
* If the bioq is manipulated using only the above calls, it starts
|
|
* with a sorted sequence of requests with bio_offset >= last_offset,
|
|
* possibly followed by another sorted sequence of requests with
|
|
* 0 <= bio_offset < bioq->last_offset
|
|
*
|
|
* NOTE: historical behaviour was to ignore bio->bio_length in the
|
|
* update, but its use tracks the head position in a better way.
|
|
* Historical behaviour was also to update the head position when
|
|
* the request under service is complete, rather than when the
|
|
* request is extracted from the queue. However, the current API
|
|
* has no method to update the head position; secondly, once
|
|
* a request has been submitted to the disk, we have no idea of
|
|
* the actual head position, so the final one is our best guess.
|
|
*
|
|
* --- Direct queue manipulation ---
|
|
*
|
|
* A bioq uses an underlying TAILQ to store requests, so we also
|
|
* export methods to manipulate the TAILQ, in particular:
|
|
*
|
|
* bioq_insert_tail() insert an entry at the end.
|
|
* It also creates a 'barrier' so all subsequent
|
|
* insertions through bioq_disksort() will end up
|
|
* after this entry;
|
|
*
|
|
* bioq_insert_head() insert an entry at the head, update
|
|
* bioq->last_offset = bio->bio_offset so that
|
|
* all subsequent insertions through bioq_disksort()
|
|
* will end up after this entry;
|
|
*
|
|
* bioq_remove() remove a generic element from the queue, act as
|
|
* bioq_takefirst() if invoked on the head of the queue.
|
|
*
|
|
* The semantic of these methods is the same as the operations
|
|
* on the underlying TAILQ, but with additional guarantees on
|
|
* subsequent bioq_disksort() calls. E.g. bioq_insert_tail()
|
|
* can be useful for making sure that all previous ops are flushed
|
|
* to disk before continuing.
|
|
*
|
|
* Updating bioq->last_offset on a bioq_insert_head() guarantees
|
|
* that the bio inserted with the last bioq_insert_head() will stay
|
|
* at the head of the queue even after subsequent bioq_disksort().
|
|
*
|
|
* Note that when the direct queue manipulation functions are used,
|
|
* the queue may contain multiple inversion points (i.e. more than
|
|
* two sorted sequences of requests).
|
|
*
|
|
*/
|
|
|
|
void
|
|
bioq_init(struct bio_queue_head *head)
|
|
{
|
|
|
|
TAILQ_INIT(&head->queue);
|
|
head->last_offset = 0;
|
|
head->insert_point = NULL;
|
|
head->total = 0;
|
|
head->batched = 0;
|
|
}
|
|
|
|
void
|
|
bioq_remove(struct bio_queue_head *head, struct bio *bp)
|
|
{
|
|
|
|
if (head->insert_point == NULL) {
|
|
if (bp == TAILQ_FIRST(&head->queue))
|
|
head->last_offset = bp->bio_offset + bp->bio_length;
|
|
} else if (bp == head->insert_point)
|
|
head->insert_point = NULL;
|
|
|
|
TAILQ_REMOVE(&head->queue, bp, bio_queue);
|
|
if (TAILQ_EMPTY(&head->queue))
|
|
head->batched = 0;
|
|
head->total--;
|
|
}
|
|
|
|
void
|
|
bioq_flush(struct bio_queue_head *head, struct devstat *stp, int error)
|
|
{
|
|
struct bio *bp;
|
|
|
|
while ((bp = bioq_takefirst(head)) != NULL)
|
|
biofinish(bp, stp, error);
|
|
}
|
|
|
|
void
|
|
bioq_insert_head(struct bio_queue_head *head, struct bio *bp)
|
|
{
|
|
|
|
if (head->insert_point == NULL)
|
|
head->last_offset = bp->bio_offset;
|
|
TAILQ_INSERT_HEAD(&head->queue, bp, bio_queue);
|
|
head->total++;
|
|
head->batched = 0;
|
|
}
|
|
|
|
void
|
|
bioq_insert_tail(struct bio_queue_head *head, struct bio *bp)
|
|
{
|
|
|
|
TAILQ_INSERT_TAIL(&head->queue, bp, bio_queue);
|
|
head->total++;
|
|
head->batched = 0;
|
|
head->insert_point = bp;
|
|
head->last_offset = bp->bio_offset;
|
|
}
|
|
|
|
struct bio *
|
|
bioq_first(struct bio_queue_head *head)
|
|
{
|
|
|
|
return (TAILQ_FIRST(&head->queue));
|
|
}
|
|
|
|
struct bio *
|
|
bioq_takefirst(struct bio_queue_head *head)
|
|
{
|
|
struct bio *bp;
|
|
|
|
bp = TAILQ_FIRST(&head->queue);
|
|
if (bp != NULL)
|
|
bioq_remove(head, bp);
|
|
return (bp);
|
|
}
|
|
|
|
/*
|
|
* Compute the sorting key. The cast to unsigned is
|
|
* fundamental for correctness, see the description
|
|
* near the beginning of the file.
|
|
*/
|
|
static inline uoff_t
|
|
bioq_bio_key(struct bio_queue_head *head, struct bio *bp)
|
|
{
|
|
|
|
return ((uoff_t)(bp->bio_offset - head->last_offset));
|
|
}
|
|
|
|
/*
|
|
* Seek sort for disks.
|
|
*
|
|
* Sort all requests in a single queue while keeping
|
|
* track of the current position of the disk with last_offset.
|
|
* See above for details.
|
|
*/
|
|
void
|
|
bioq_disksort(struct bio_queue_head *head, struct bio *bp)
|
|
{
|
|
struct bio *cur, *prev;
|
|
uoff_t key;
|
|
|
|
if ((bp->bio_flags & BIO_ORDERED) != 0) {
|
|
/*
|
|
* Ordered transactions can only be dispatched
|
|
* after any currently queued transactions. They
|
|
* also have barrier semantics - no transactions
|
|
* queued in the future can pass them.
|
|
*/
|
|
bioq_insert_tail(head, bp);
|
|
return;
|
|
}
|
|
|
|
/*
|
|
* We should only sort requests of types that have concept of offset.
|
|
* Other types, such as BIO_FLUSH or BIO_ZONE, may imply some degree
|
|
* of ordering even if strict ordering is not requested explicitly.
|
|
*/
|
|
if (bp->bio_cmd != BIO_READ && bp->bio_cmd != BIO_WRITE &&
|
|
bp->bio_cmd != BIO_DELETE) {
|
|
bioq_insert_tail(head, bp);
|
|
return;
|
|
}
|
|
|
|
if (bioq_batchsize > 0 && head->batched > bioq_batchsize) {
|
|
bioq_insert_tail(head, bp);
|
|
return;
|
|
}
|
|
|
|
prev = NULL;
|
|
key = bioq_bio_key(head, bp);
|
|
cur = TAILQ_FIRST(&head->queue);
|
|
|
|
if (head->insert_point) {
|
|
prev = head->insert_point;
|
|
cur = TAILQ_NEXT(head->insert_point, bio_queue);
|
|
}
|
|
|
|
while (cur != NULL && key >= bioq_bio_key(head, cur)) {
|
|
prev = cur;
|
|
cur = TAILQ_NEXT(cur, bio_queue);
|
|
}
|
|
|
|
if (prev == NULL)
|
|
TAILQ_INSERT_HEAD(&head->queue, bp, bio_queue);
|
|
else
|
|
TAILQ_INSERT_AFTER(&head->queue, prev, bp, bio_queue);
|
|
head->total++;
|
|
head->batched++;
|
|
}
|