diff --git a/block/block-backend.c b/block/block-backend.c index 0056b526b8010503c22fc530b795c577cda764a0..2d812e22547e8c350a9f46aa6ef993f81ba9de50 100644 --- a/block/block-backend.c +++ b/block/block-backend.c @@ -88,6 +88,15 @@ struct BlockBackend { * Accessed with atomic ops. */ unsigned int in_flight; + + /* Timer for retry on errors. */ + QEMUTimer *retry_timer; + /* Interval in ms to trigger next retry. */ + int64_t retry_interval; + /* Start time of the first error. Used to check timeout. */ + int64_t retry_start_time; + /* Retry timeout. 0 represents infinite retry. */ + int64_t retry_timeout; }; typedef struct BlockBackendAIOCB { @@ -337,6 +346,11 @@ BlockBackend *blk_new(AioContext *ctx, uint64_t perm, uint64_t shared_perm) blk->on_read_error = BLOCKDEV_ON_ERROR_REPORT; blk->on_write_error = BLOCKDEV_ON_ERROR_ENOSPC; + blk->retry_timer = NULL; + blk->retry_interval = BLOCK_BACKEND_DEFAULT_RETRY_INTERVAL; + blk->retry_start_time = 0; + blk->retry_timeout = 0; + block_acct_init(&blk->stats); notifier_list_init(&blk->remove_bs_notifiers); @@ -423,6 +437,10 @@ static void blk_delete(BlockBackend *blk) QTAILQ_REMOVE(&block_backends, blk, link); drive_info_del(blk->legacy_dinfo); block_acct_cleanup(&blk->stats); + if (blk->retry_timer) { + timer_del(blk->retry_timer); + timer_free(blk->retry_timer); + } g_free(blk); } @@ -937,6 +955,14 @@ void blk_set_dev_ops(BlockBackend *blk, const BlockDevOps *ops, blk->dev_ops = ops; blk->dev_opaque = opaque; + if ((blk->on_read_error == BLOCKDEV_ON_ERROR_RETRY || + blk->on_write_error == BLOCKDEV_ON_ERROR_RETRY) && + ops->retry_request_cb) { + blk->retry_timer = aio_timer_new(blk->ctx, QEMU_CLOCK_REALTIME, + SCALE_MS, ops->retry_request_cb, + opaque); + } + /* Are we currently quiesced? Should we enforce this right now? */ if (blk->quiesce_counter && ops->drained_begin) { ops->drained_begin(opaque); @@ -1604,6 +1630,39 @@ void blk_drain_all(void) bdrv_drain_all_end(); } +void blk_set_on_error_retry_interval(BlockBackend *blk, int64_t interval) +{ + blk->retry_interval = interval; +} + +void blk_set_on_error_retry_timeout(BlockBackend *blk, int64_t timeout) +{ + blk->retry_timeout = timeout; +} + +static bool blk_error_retry_timeout(BlockBackend *blk) +{ + /* No timeout set, infinite retries. */ + if (!blk->retry_timeout) { + return false; + } + + /* The first time an error occurs. */ + if (!blk->retry_start_time) { + blk->retry_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + return false; + } + + return qemu_clock_get_ms(QEMU_CLOCK_REALTIME) > (blk->retry_start_time + + blk->retry_timeout); +} + +void blk_error_retry_reset_timeout(BlockBackend *blk) +{ + if (blk->retry_timer && blk->retry_start_time) + blk->retry_start_time = 0; +} + void blk_set_on_error(BlockBackend *blk, BlockdevOnError on_read_error, BlockdevOnError on_write_error) { @@ -1631,6 +1690,9 @@ BlockErrorAction blk_get_error_action(BlockBackend *blk, bool is_read, return BLOCK_ERROR_ACTION_REPORT; case BLOCKDEV_ON_ERROR_IGNORE: return BLOCK_ERROR_ACTION_IGNORE; + case BLOCKDEV_ON_ERROR_RETRY: + return (blk->retry_timer && !blk_error_retry_timeout(blk)) ? + BLOCK_ERROR_ACTION_RETRY : BLOCK_ERROR_ACTION_REPORT; case BLOCKDEV_ON_ERROR_AUTO: default: abort(); @@ -1678,6 +1740,10 @@ void blk_error_action(BlockBackend *blk, BlockErrorAction action, qemu_system_vmstop_request_prepare(); send_qmp_error_event(blk, action, is_read, error); qemu_system_vmstop_request(RUN_STATE_IO_ERROR); + } else if (action == BLOCK_ERROR_ACTION_RETRY) { + timer_mod(blk->retry_timer, qemu_clock_get_ms(QEMU_CLOCK_REALTIME) + + blk->retry_interval); + send_qmp_error_event(blk, action, is_read, error); } else { send_qmp_error_event(blk, action, is_read, error); } diff --git a/blockdev.c b/blockdev.c index 4d141e9a1fba68a40a5ad69f43ef270d279d025f..99c92b96d25831cebd8d90ad566bda94c8035220 100644 --- a/blockdev.c +++ b/blockdev.c @@ -319,6 +319,8 @@ static int parse_block_error_action(const char *buf, bool is_read, Error **errp) return BLOCKDEV_ON_ERROR_STOP; } else if (!strcmp(buf, "report")) { return BLOCKDEV_ON_ERROR_REPORT; + } else if (!strcmp(buf, "retry")) { + return BLOCKDEV_ON_ERROR_RETRY; } else { error_setg(errp, "'%s' invalid %s error action", buf, is_read ? "read" : "write"); @@ -468,6 +470,7 @@ static BlockBackend *blockdev_init(const char *file, QDict *bs_opts, const char *buf; int bdrv_flags = 0; int on_read_error, on_write_error; + int64_t retry_interval, retry_timeout; bool account_invalid, account_failed; bool writethrough, read_only; BlockBackend *blk; @@ -563,6 +566,10 @@ static BlockBackend *blockdev_init(const char *file, QDict *bs_opts, } } + retry_interval = qemu_opt_get_number(opts, "retry_interval", + BLOCK_BACKEND_DEFAULT_RETRY_INTERVAL); + retry_timeout = qemu_opt_get_number(opts, "retry_timeout", 0); + if (snapshot) { bdrv_flags |= BDRV_O_SNAPSHOT; } @@ -627,6 +634,11 @@ static BlockBackend *blockdev_init(const char *file, QDict *bs_opts, blk_set_enable_write_cache(blk, !writethrough); blk_set_on_error(blk, on_read_error, on_write_error); + if (on_read_error == BLOCKDEV_ON_ERROR_RETRY || + on_write_error == BLOCKDEV_ON_ERROR_RETRY) { + blk_set_on_error_retry_interval(blk, retry_interval); + blk_set_on_error_retry_timeout(blk, retry_timeout); + } if (!monitor_add_blk(blk, id, errp)) { blk_unref(blk); @@ -752,6 +764,14 @@ QemuOptsList qemu_legacy_drive_opts = { .name = "werror", .type = QEMU_OPT_STRING, .help = "write error action", + },{ + .name = "retry_interval", + .type = QEMU_OPT_NUMBER, + .help = "interval for retry action in millisecond", + },{ + .name = "retry_timeout", + .type = QEMU_OPT_NUMBER, + .help = "timeout for retry action in millisecond", },{ .name = "copy-on-read", .type = QEMU_OPT_BOOL, @@ -774,6 +794,7 @@ DriveInfo *drive_new(QemuOpts *all_opts, BlockInterfaceType block_default_type, BlockInterfaceType type; int max_devs, bus_id, unit_id, index; const char *werror, *rerror; + int64_t retry_interval, retry_timeout; bool read_only = false; bool copy_on_read; const char *filename; @@ -990,6 +1011,29 @@ DriveInfo *drive_new(QemuOpts *all_opts, BlockInterfaceType block_default_type, qdict_put_str(bs_opts, "rerror", rerror); } + if (qemu_opt_find(legacy_opts, "retry_interval")) { + if ((werror == NULL || strcmp(werror, "retry")) && + (rerror == NULL || strcmp(rerror, "retry"))) { + error_setg(errp, "retry_interval is only supported " + "by werror/rerror=retry"); + goto fail; + } + retry_interval = qemu_opt_get_number(legacy_opts, "retry_interval", + BLOCK_BACKEND_DEFAULT_RETRY_INTERVAL); + qdict_put_int(bs_opts, "retry_interval", retry_interval); + } + + if (qemu_opt_find(legacy_opts, "retry_timeout")) { + if ((werror == NULL || strcmp(werror, "retry")) && + (rerror == NULL || strcmp(rerror, "retry"))) { + error_setg(errp, "retry_timeout is only supported " + "by werror/rerror=retry"); + goto fail; + } + retry_timeout = qemu_opt_get_number(legacy_opts, "retry_timeout", 0); + qdict_put_int(bs_opts, "retry_timeout", retry_timeout); + } + /* Actual block device init: Functionality shared with blockdev-add */ blk = blockdev_init(filename, bs_opts, &local_err); bs_opts = NULL; @@ -4591,6 +4635,14 @@ QemuOptsList qemu_common_drive_opts = { .name = "werror", .type = QEMU_OPT_STRING, .help = "write error action", + },{ + .name = "retry_interval", + .type = QEMU_OPT_NUMBER, + .help = "interval for retry action in millisecond", + },{ + .name = "retry_timeout", + .type = QEMU_OPT_NUMBER, + .help = "timeout for retry action in millisecond", },{ .name = BDRV_OPT_READ_ONLY, .type = QEMU_OPT_BOOL, diff --git a/hw/block/block.c b/hw/block/block.c index bf56c7612bc0ee33b57275218169992536a7749e..56141940ca8daa18d5de543ec5c2e089dd60489c 100644 --- a/hw/block/block.c +++ b/hw/block/block.c @@ -134,6 +134,16 @@ bool blkconf_apply_backend_options(BlockConf *conf, bool readonly, blk_set_enable_write_cache(blk, wce); blk_set_on_error(blk, rerror, werror); + if (rerror == BLOCKDEV_ON_ERROR_RETRY || + werror == BLOCKDEV_ON_ERROR_RETRY) { + if (conf->retry_interval >= 0) { + blk_set_on_error_retry_interval(blk, conf->retry_interval); + } + if (conf->retry_timeout >= 0) { + blk_set_on_error_retry_timeout(blk, conf->retry_timeout); + } + } + return true; } diff --git a/hw/block/dataplane/virtio-blk.c b/hw/block/dataplane/virtio-blk.c index 5fea76df8556099917d1f2ed0fe19416e8695d4b..4476f97960371fe7b5a8f0ac11a1dc30f37f932b 100644 --- a/hw/block/dataplane/virtio-blk.c +++ b/hw/block/dataplane/virtio-blk.c @@ -219,6 +219,9 @@ int virtio_blk_data_plane_start(VirtIODevice *vdev) goto fail_guest_notifiers; } + /* Process queued requests before the ones in vring */ + virtio_blk_process_queued_requests(vblk, false); + /* Kick right away to begin processing requests already in vring */ for (i = 0; i < nvqs; i++) { VirtQueue *vq = virtio_get_queue(s->vdev, i); @@ -238,6 +241,11 @@ int virtio_blk_data_plane_start(VirtIODevice *vdev) return 0; fail_guest_notifiers: + /* + * If we failed to set up the guest notifiers queued requests will be + * processed on the main context. + */ + virtio_blk_process_queued_requests(vblk, false); vblk->dataplane_disabled = true; s->starting = false; vblk->dataplane_started = true; diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c index 703ed4c93bff863e2b6fa4a2f8092abaec6f83c3..2db9804cfef19d82e72ff36baa30e1afcec18ac4 100644 --- a/hw/block/virtio-blk.c +++ b/hw/block/virtio-blk.c @@ -101,6 +101,10 @@ static int virtio_blk_handle_rw_error(VirtIOBlockReq *req, int error, block_acct_failed(blk_get_stats(s->blk), &req->acct); } virtio_blk_free_request(req); + } else if (action == BLOCK_ERROR_ACTION_RETRY) { + req->mr_next = NULL; + req->next = s->rq; + s->rq = req; } blk_error_action(s->blk, action, is_read, error); @@ -142,6 +146,7 @@ static void virtio_blk_rw_complete(void *opaque, int ret) } } + blk_error_retry_reset_timeout(s->blk); virtio_blk_req_complete(req, VIRTIO_BLK_S_OK); block_acct_done(blk_get_stats(s->blk), &req->acct); virtio_blk_free_request(req); @@ -161,6 +166,7 @@ static void virtio_blk_flush_complete(void *opaque, int ret) } } + blk_error_retry_reset_timeout(s->blk); virtio_blk_req_complete(req, VIRTIO_BLK_S_OK); block_acct_done(blk_get_stats(s->blk), &req->acct); virtio_blk_free_request(req); @@ -183,6 +189,7 @@ static void virtio_blk_discard_write_zeroes_complete(void *opaque, int ret) } } + blk_error_retry_reset_timeout(s->blk); virtio_blk_req_complete(req, VIRTIO_BLK_S_OK); if (is_write_zeroes) { block_acct_done(blk_get_stats(s->blk), &req->acct); @@ -809,18 +816,14 @@ static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq) virtio_blk_handle_output_do(s, vq); } -static void virtio_blk_dma_restart_bh(void *opaque) +void virtio_blk_process_queued_requests(VirtIOBlock *s, bool is_bh) { - VirtIOBlock *s = opaque; - VirtIOBlockReq *req = s->rq; + VirtIOBlockReq *req; MultiReqBuffer mrb = {}; - qemu_bh_delete(s->bh); - s->bh = NULL; - - s->rq = NULL; - aio_context_acquire(blk_get_aio_context(s->conf.conf.blk)); + req = s->rq; + s->rq = NULL; while (req) { VirtIOBlockReq *next = req->next; if (virtio_blk_handle_request(req, &mrb)) { @@ -841,22 +844,38 @@ static void virtio_blk_dma_restart_bh(void *opaque) if (mrb.num_reqs) { virtio_blk_submit_multireq(s->blk, &mrb); } - blk_dec_in_flight(s->conf.conf.blk); + if (is_bh) { + blk_dec_in_flight(s->conf.conf.blk); + } aio_context_release(blk_get_aio_context(s->conf.conf.blk)); } +static void virtio_blk_dma_restart_bh(void *opaque) +{ + VirtIOBlock *s = opaque; + + qemu_bh_delete(s->bh); + s->bh = NULL; + + virtio_blk_process_queued_requests(s, true); +} + static void virtio_blk_dma_restart_cb(void *opaque, int running, RunState state) { VirtIOBlock *s = opaque; + BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s))); + VirtioBusState *bus = VIRTIO_BUS(qbus); if (!running) { return; } - if (!s->bh) { - /* FIXME The data plane is not started yet, so these requests are - * processed in the main thread. */ + /* + * If ioeventfd is enabled, don't schedule the BH here as queued + * requests will be processed while starting the data plane. + */ + if (!s->bh && !virtio_bus_ioeventfd_enabled(bus)) { s->bh = aio_bh_new(blk_get_aio_context(s->conf.conf.blk), virtio_blk_dma_restart_bh, s); blk_inc_in_flight(s->conf.conf.blk); @@ -1089,8 +1108,16 @@ static void virtio_blk_resize(void *opaque) virtio_notify_config(vdev); } +static void virtio_blk_retry_request(void *opaque) +{ + VirtIOBlock *s = VIRTIO_BLK(opaque); + + virtio_blk_process_queued_requests(s, false); +} + static const BlockDevOps virtio_block_ops = { .resize_cb = virtio_blk_resize, + .retry_request_cb = virtio_blk_retry_request, }; static void virtio_blk_device_realize(DeviceState *dev, Error **errp) diff --git a/hw/scsi/scsi-bus.c b/hw/scsi/scsi-bus.c index fdc3a0e4e03732f4b72cb45f1b771a2b1e2f1945..9dc09b5f3ed02ea360eb2e185039a516718b358c 100644 --- a/hw/scsi/scsi-bus.c +++ b/hw/scsi/scsi-bus.c @@ -99,14 +99,10 @@ void scsi_bus_new(SCSIBus *bus, size_t bus_size, DeviceState *host, qbus_set_bus_hotplug_handler(BUS(bus), &error_abort); } -static void scsi_dma_restart_bh(void *opaque) +void scsi_retry_requests(SCSIDevice *s) { - SCSIDevice *s = opaque; SCSIRequest *req, *next; - qemu_bh_delete(s->bh); - s->bh = NULL; - aio_context_acquire(blk_get_aio_context(s->conf.blk)); QTAILQ_FOREACH_SAFE(req, &s->requests, next, next) { scsi_req_ref(req); @@ -128,6 +124,16 @@ static void scsi_dma_restart_bh(void *opaque) aio_context_release(blk_get_aio_context(s->conf.blk)); } +static void scsi_dma_restart_bh(void *opaque) +{ + SCSIDevice *s = opaque; + + qemu_bh_delete(s->bh); + s->bh = NULL; + + scsi_retry_requests(s); +} + void scsi_req_retry(SCSIRequest *req) { /* No need to save a reference, because scsi_dma_restart_bh just diff --git a/hw/scsi/scsi-disk.c b/hw/scsi/scsi-disk.c index cd90cd780e3c49942eda0813a15d430427f664a4..93fdd913fea40db498ded73af8895a9249a5653d 100644 --- a/hw/scsi/scsi-disk.c +++ b/hw/scsi/scsi-disk.c @@ -184,6 +184,8 @@ static void scsi_disk_load_request(QEMUFile *f, SCSIRequest *req) static bool scsi_disk_req_check_error(SCSIDiskReq *r, int ret, bool acct_failed) { + SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev); + if (r->req.io_canceled) { scsi_req_cancel_complete(&r->req); return true; @@ -193,6 +195,7 @@ static bool scsi_disk_req_check_error(SCSIDiskReq *r, int ret, bool acct_failed) return scsi_handle_rw_error(r, -ret, acct_failed); } + blk_error_retry_reset_timeout(s->qdev.conf.blk); return false; } @@ -480,6 +483,10 @@ static bool scsi_handle_rw_error(SCSIDiskReq *r, int error, bool acct_failed) } } + if (action == BLOCK_ERROR_ACTION_RETRY) { + scsi_req_retry(&r->req); + } + blk_error_action(s->qdev.conf.blk, action, is_read, error); if (action == BLOCK_ERROR_ACTION_IGNORE) { scsi_req_complete(&r->req, 0); @@ -2252,6 +2259,13 @@ static void scsi_disk_resize_cb(void *opaque) } } +static void scsi_disk_retry_request(void *opaque) +{ + SCSIDiskState *s = opaque; + + scsi_retry_requests(&s->qdev); +} + static void scsi_cd_change_media_cb(void *opaque, bool load, Error **errp) { SCSIDiskState *s = opaque; @@ -2300,10 +2314,12 @@ static const BlockDevOps scsi_disk_removable_block_ops = { .is_medium_locked = scsi_cd_is_medium_locked, .resize_cb = scsi_disk_resize_cb, + .retry_request_cb = scsi_disk_retry_request, }; static const BlockDevOps scsi_disk_block_ops = { .resize_cb = scsi_disk_resize_cb, + .retry_request_cb = scsi_disk_retry_request, }; static void scsi_disk_unit_attention_reported(SCSIDevice *dev) diff --git a/include/hw/block/block.h b/include/hw/block/block.h index 607539057a6b5b87286b3368abcc277cda710389..d12603aabd7ca395adf4dab11a47018b8661fcba 100644 --- a/include/hw/block/block.h +++ b/include/hw/block/block.h @@ -30,6 +30,8 @@ typedef struct BlockConf { bool share_rw; BlockdevOnError rerror; BlockdevOnError werror; + int64_t retry_interval; + int64_t retry_timeout; } BlockConf; static inline unsigned int get_physical_block_exp(BlockConf *conf) @@ -71,7 +73,10 @@ static inline unsigned int get_physical_block_exp(BlockConf *conf) DEFINE_PROP_BLOCKDEV_ON_ERROR("rerror", _state, _conf.rerror, \ BLOCKDEV_ON_ERROR_AUTO), \ DEFINE_PROP_BLOCKDEV_ON_ERROR("werror", _state, _conf.werror, \ - BLOCKDEV_ON_ERROR_AUTO) + BLOCKDEV_ON_ERROR_AUTO), \ + DEFINE_PROP_INT64("retry_interval", _state, _conf.retry_interval, \ + -1), \ + DEFINE_PROP_INT64("retry_timeout", _state, _conf.retry_timeout, -1) /* Backend access helpers */ diff --git a/include/hw/scsi/scsi.h b/include/hw/scsi/scsi.h index 426566a5c66f7fd5ec1a53bc2b75abe165d3ce3b..1231d30b35018ce9fe7990eac862281a063e3b0d 100644 --- a/include/hw/scsi/scsi.h +++ b/include/hw/scsi/scsi.h @@ -184,6 +184,7 @@ void scsi_req_cancel_complete(SCSIRequest *req); void scsi_req_cancel(SCSIRequest *req); void scsi_req_cancel_async(SCSIRequest *req, Notifier *notifier); void scsi_req_retry(SCSIRequest *req); +void scsi_retry_requests(SCSIDevice *s); void scsi_device_purge_requests(SCSIDevice *sdev, SCSISense sense); void scsi_device_set_ua(SCSIDevice *sdev, SCSISense sense); void scsi_device_report_change(SCSIDevice *dev, SCSISense sense); diff --git a/include/hw/virtio/virtio-blk.h b/include/hw/virtio/virtio-blk.h index cddcfbebe913f52b951b60d5f2de1189d56d844c..e77f0db3b05dca693476da893e664060fa40454f 100644 --- a/include/hw/virtio/virtio-blk.h +++ b/include/hw/virtio/virtio-blk.h @@ -84,5 +84,6 @@ typedef struct MultiReqBuffer { } MultiReqBuffer; bool virtio_blk_handle_vq(VirtIOBlock *s, VirtQueue *vq); +void virtio_blk_process_queued_requests(VirtIOBlock *s, bool is_bh); #endif diff --git a/include/sysemu/block-backend.h b/include/sysemu/block-backend.h index 733c4957eb29fc445847ff744f1aba1c539c7614..dc10e507ae2a834ac67e0b8e941f0de5934671ab 100644 --- a/include/sysemu/block-backend.h +++ b/include/sysemu/block-backend.h @@ -25,6 +25,9 @@ */ #include "block/block.h" +/* block backend default retry interval */ +#define BLOCK_BACKEND_DEFAULT_RETRY_INTERVAL 1000 + /* Callbacks for block device models */ typedef struct BlockDevOps { /* @@ -66,6 +69,10 @@ typedef struct BlockDevOps { * Runs when the backend's last drain request ends. */ void (*drained_end)(void *opaque); + /* + * Runs when retrying failed requests. + */ + void (*retry_request_cb)(void *opaque); } BlockDevOps; /* This struct is embedded in (the private) BlockBackend struct and contains @@ -180,6 +187,9 @@ void blk_inc_in_flight(BlockBackend *blk); void blk_dec_in_flight(BlockBackend *blk); void blk_drain(BlockBackend *blk); void blk_drain_all(void); +void blk_set_on_error_retry_interval(BlockBackend *blk, int64_t interval); +void blk_set_on_error_retry_timeout(BlockBackend *blk, int64_t timeout); +void blk_error_retry_reset_timeout(BlockBackend *blk); void blk_set_on_error(BlockBackend *blk, BlockdevOnError on_read_error, BlockdevOnError on_write_error); BlockdevOnError blk_get_on_error(BlockBackend *blk, bool is_read); diff --git a/qapi/block-core.json b/qapi/block-core.json index 0d43d4f37c1afd0e71ed8aba75ecdf8a6a92a9aa..db24f0dfe586bc21b8a914b0692f28798ba96b95 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -1113,7 +1113,7 @@ # Since: 1.3 ## { 'enum': 'BlockdevOnError', - 'data': ['report', 'ignore', 'enospc', 'stop', 'auto'] } + 'data': ['report', 'ignore', 'enospc', 'stop', 'auto', 'retry'] } ## # @MirrorSyncMode: @@ -4894,7 +4894,7 @@ # Since: 2.1 ## { 'enum': 'BlockErrorAction', - 'data': [ 'ignore', 'report', 'stop' ] } + 'data': [ 'ignore', 'report', 'stop', 'retry' ] } ##