lkml.org 
[lkml]   [2019]   [Dec]   [17]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
Patch in this message
/
From
Subject[PATCH v2 3/3] io_uring: move *queue_link_head() from common path
Date
Move io_queue_link_head() to links handling code in io_submit_sqe(),
so it wouldn't need extra checks and would have better data locality.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
fs/io_uring.c | 32 ++++++++++++++------------------
1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/fs/io_uring.c b/fs/io_uring.c
index ee461bcd3121..8ab96ca0ad28 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -3373,13 +3373,15 @@ static bool io_submit_sqe(struct io_kiocb *req, struct io_submit_state *state,
struct io_kiocb **link)
{
struct io_ring_ctx *ctx = req->ctx;
+ unsigned int sqe_flags;
int ret;

+ sqe_flags = READ_ONCE(req->sqe->flags);
req->user_data = READ_ONCE(req->sqe->user_data);
trace_io_uring_submit_sqe(ctx, req->user_data, true, req->in_async);

/* enforce forwards compatibility on users */
- if (unlikely(req->sqe->flags & ~SQE_VALID_FLAGS)) {
+ if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) {
ret = -EINVAL;
goto err_req;
}
@@ -3402,10 +3404,10 @@ static bool io_submit_sqe(struct io_kiocb *req, struct io_submit_state *state,
if (*link) {
struct io_kiocb *head = *link;

- if (req->sqe->flags & IOSQE_IO_DRAIN)
+ if (sqe_flags & IOSQE_IO_DRAIN)
head->flags |= REQ_F_DRAIN_LINK | REQ_F_IO_DRAIN;

- if (req->sqe->flags & IOSQE_IO_HARDLINK)
+ if (sqe_flags & IOSQE_IO_HARDLINK)
req->flags |= REQ_F_HARDLINK;

if (io_alloc_async_ctx(req)) {
@@ -3421,9 +3423,15 @@ static bool io_submit_sqe(struct io_kiocb *req, struct io_submit_state *state,
}
trace_io_uring_link(ctx, req, head);
list_add_tail(&req->link_list, &head->link_list);
- } else if (req->sqe->flags & (IOSQE_IO_LINK|IOSQE_IO_HARDLINK)) {
+
+ /* last request of a link, enqueue the link */
+ if (!(sqe_flags & (IOSQE_IO_LINK|IOSQE_IO_HARDLINK))) {
+ io_queue_link_head(head);
+ *link = NULL;
+ }
+ } else if (sqe_flags & (IOSQE_IO_LINK|IOSQE_IO_HARDLINK)) {
req->flags |= REQ_F_LINK;
- if (req->sqe->flags & IOSQE_IO_HARDLINK)
+ if (sqe_flags & IOSQE_IO_HARDLINK)
req->flags |= REQ_F_HARDLINK;

INIT_LIST_HEAD(&req->link_list);
@@ -3540,10 +3548,8 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
}

for (i = 0; i < nr; i++) {
- struct io_kiocb *req;
- unsigned int sqe_flags;
+ struct io_kiocb *req = io_get_req(ctx, statep);

- req = io_get_req(ctx, statep);
if (unlikely(!req)) {
if (!submitted)
submitted = -EAGAIN;
@@ -3563,8 +3569,6 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
}

submitted++;
- sqe_flags = req->sqe->flags;
-
req->ring_file = ring_file;
req->ring_fd = ring_fd;
req->has_user = *mm != NULL;
@@ -3572,14 +3576,6 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
req->needs_fixed_file = async;
if (!io_submit_sqe(req, statep, &link))
break;
- /*
- * If previous wasn't linked and we have a linked command,
- * that's the end of the chain. Submit the previous link.
- */
- if (!(sqe_flags & (IOSQE_IO_LINK|IOSQE_IO_HARDLINK)) && link) {
- io_queue_link_head(link);
- link = NULL;
- }
}

if (link)
--
2.24.0
\
 
 \ /
  Last update: 2019-12-17 20:27    [W:0.069 / U:0.396 seconds]
©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site