From f87b543af45ea217fa1d000483f9a40944c1ff73 Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Tue, 12 Mar 2019 12:06:35 -0400 Subject: [PATCH 1/8] fix null pointer deref in tracepoints in back channel Backchannel doesn't have the rq_task->tk_clientid pointer set. Otherwise can lead to the following oops: ocalhost login: [ 111.385319] BUG: unable to handle kernel NULL pointer dereference at 0000000000000004 [ 111.388073] #PF error: [normal kernel read fault] [ 111.389452] PGD 80000000290d8067 P4D 80000000290d8067 PUD 75f25067 PMD 0 [ 111.391224] Oops: 0000 [#1] SMP PTI [ 111.392151] CPU: 0 PID: 3533 Comm: NFSv4 callback Not tainted 5.0.0-rc7+ #1 [ 111.393787] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015 [ 111.396340] RIP: 0010:trace_event_raw_event_xprt_enq_xmit+0x6f/0xf0 [sunrpc] [ 111.397974] Code: 00 00 00 48 89 ee 48 89 e7 e8 bd 0a 85 d7 48 85 c0 74 4a 41 0f b7 94 24 e0 00 00 00 48 89 e7 89 50 08 49 8b 94 24 a8 00 00 00 <8b> 52 04 89 50 0c 49 8b 94 24 c0 00 00 00 8b 92 a8 00 00 00 0f ca [ 111.402215] RSP: 0018:ffffb98743263cf8 EFLAGS: 00010286 [ 111.403406] RAX: ffffa0890fc3bc88 RBX: 0000000000000003 RCX: 0000000000000000 [ 111.405057] RDX: 0000000000000000 RSI: 0000000000000000 RDI: ffffb98743263cf8 [ 111.406656] RBP: ffffa0896f5368f0 R08: 0000000000000246 R09: 0000000000000000 [ 111.408437] R10: ffffe19b01c01500 R11: 0000000000000000 R12: ffffa08977d28a00 [ 111.410210] R13: 0000000000000004 R14: ffffa089315303f0 R15: ffffa08931530000 [ 111.411856] FS: 0000000000000000(0000) GS:ffffa0897bc00000(0000) knlGS:0000000000000000 [ 111.413699] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 111.415068] CR2: 0000000000000004 CR3: 000000002ac90004 CR4: 00000000001606f0 [ 111.416745] Call Trace: [ 111.417339] xprt_request_enqueue_transmit+0x2b6/0x4a0 [sunrpc] [ 111.418709] ? rpc_task_need_encode+0x40/0x40 [sunrpc] [ 111.419957] call_bc_transmit+0xd5/0x170 [sunrpc] [ 111.421067] __rpc_execute+0x7e/0x3f0 [sunrpc] [ 111.422177] rpc_run_bc_task+0x78/0xd0 [sunrpc] [ 111.423212] bc_svc_process+0x281/0x340 [sunrpc] [ 111.424325] nfs41_callback_svc+0x130/0x1c0 [nfsv4] [ 111.425430] ? remove_wait_queue+0x60/0x60 [ 111.426398] kthread+0xf5/0x130 [ 111.427155] ? nfs_callback_authenticate+0x50/0x50 [nfsv4] [ 111.428388] ? kthread_bind+0x10/0x10 [ 111.429270] ret_from_fork+0x1f/0x30 localhost login: [ 467.462259] BUG: unable to handle kernel NULL pointer dereference at 0000000000000004 [ 467.464411] #PF error: [normal kernel read fault] [ 467.465445] PGD 80000000728c1067 P4D 80000000728c1067 PUD 728c0067 PMD 0 [ 467.466980] Oops: 0000 [#1] SMP PTI [ 467.467759] CPU: 0 PID: 3517 Comm: NFSv4 callback Not tainted 5.0.0-rc7+ #1 [ 467.469393] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015 [ 467.471840] RIP: 0010:trace_event_raw_event_xprt_transmit+0x7c/0xf0 [sunrpc] [ 467.473392] Code: f6 48 85 c0 74 4b 49 8b 94 24 98 00 00 00 48 89 e7 0f b7 92 e0 00 00 00 89 50 08 49 8b 94 24 98 00 00 00 48 8b 92 a8 00 00 00 <8b> 52 04 89 50 0c 41 8b 94 24 a8 00 00 00 0f ca 89 50 10 41 8b 94 [ 467.477605] RSP: 0018:ffffabe7434fbcd0 EFLAGS: 00010282 [ 467.478793] RAX: ffff99720fc3bce0 RBX: 0000000000000003 RCX: 0000000000000000 [ 467.480409] RDX: 0000000000000000 RSI: 0000000000000000 RDI: ffffabe7434fbcd0 [ 467.482011] RBP: ffff99726f631948 R08: 0000000000000246 R09: 0000000000000000 [ 467.483591] R10: 0000000070000000 R11: 0000000000000000 R12: ffff997277dfcc00 [ 467.485226] R13: 0000000000000000 R14: 0000000000000000 R15: ffff99722fecdca8 [ 467.486830] FS: 0000000000000000(0000) GS:ffff99727bc00000(0000) knlGS:0000000000000000 [ 467.488596] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 467.489931] CR2: 0000000000000004 CR3: 00000000270e6006 CR4: 00000000001606f0 [ 467.491559] Call Trace: [ 467.492128] xprt_transmit+0x303/0x3f0 [sunrpc] [ 467.493143] ? rpc_task_need_encode+0x40/0x40 [sunrpc] [ 467.494328] call_bc_transmit+0x49/0x170 [sunrpc] [ 467.495379] __rpc_execute+0x7e/0x3f0 [sunrpc] [ 467.496451] rpc_run_bc_task+0x78/0xd0 [sunrpc] [ 467.497467] bc_svc_process+0x281/0x340 [sunrpc] [ 467.498507] nfs41_callback_svc+0x130/0x1c0 [nfsv4] [ 467.499751] ? remove_wait_queue+0x60/0x60 [ 467.500686] kthread+0xf5/0x130 [ 467.501438] ? nfs_callback_authenticate+0x50/0x50 [nfsv4] [ 467.502640] ? kthread_bind+0x10/0x10 [ 467.503454] ret_from_fork+0x1f/0x30 Signed-off-by: Olga Kornievskaia Signed-off-by: Trond Myklebust --- include/trace/events/sunrpc.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/include/trace/events/sunrpc.h b/include/trace/events/sunrpc.h index 8451f30c6a0f..7e899e635d33 100644 --- a/include/trace/events/sunrpc.h +++ b/include/trace/events/sunrpc.h @@ -712,7 +712,8 @@ TRACE_EVENT(xprt_transmit, TP_fast_assign( __entry->task_id = rqst->rq_task->tk_pid; - __entry->client_id = rqst->rq_task->tk_client->cl_clid; + __entry->client_id = rqst->rq_task->tk_client ? + rqst->rq_task->tk_client->cl_clid : -1; __entry->xid = be32_to_cpu(rqst->rq_xid); __entry->seqno = rqst->rq_seqno; __entry->status = status; @@ -742,7 +743,8 @@ TRACE_EVENT(xprt_enq_xmit, TP_fast_assign( __entry->task_id = task->tk_pid; - __entry->client_id = task->tk_client->cl_clid; + __entry->client_id = task->tk_client ? + task->tk_client->cl_clid : -1; __entry->xid = be32_to_cpu(task->tk_rqstp->rq_xid); __entry->seqno = task->tk_rqstp->rq_seqno; __entry->stage = stage; From 400417b05f3ec0531544ca5f94e64d838d8b8849 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Tue, 12 Mar 2019 16:04:51 -0400 Subject: [PATCH 2/8] pNFS: Fix a typo in pnfs_update_layout We're supposed to wait for the outstanding layout count to go to zero, but that got lost somehow. Fixes: d03360aaf5cca ("pNFS: Ensure we return the error if someone...") Reported-by: Anna Schumaker Signed-off-by: Trond Myklebust --- fs/nfs/pnfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index 8247bd1634cb..7066cd7c7aff 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -1889,7 +1889,7 @@ lookup_again: atomic_read(&lo->plh_outstanding) != 0) { spin_unlock(&ino->i_lock); lseg = ERR_PTR(wait_var_event_killable(&lo->plh_outstanding, - atomic_read(&lo->plh_outstanding))); + !atomic_read(&lo->plh_outstanding))); if (IS_ERR(lseg) || !list_empty(&lo->plh_segs)) goto out_put_layout_hdr; pnfs_put_layout_hdr(lo); From 9734ad57b0f1a367fd3a00d717f97f8c00d9edb7 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 15 Mar 2019 12:47:34 -0400 Subject: [PATCH 3/8] SUNRPC: Fix a client regression when handling oversized replies If the server sends a reply that is larger than the pre-allocated buffer, then the current code may fail to register how much of the stream that it has finished reading. This again can lead to hangs. Fixes: e92053a52e68 ("SUNRPC: Handle zero length fragments correctly") Signed-off-by: Trond Myklebust --- net/sunrpc/xprtsock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c index 42f45d33dc56..9359539907ba 100644 --- a/net/sunrpc/xprtsock.c +++ b/net/sunrpc/xprtsock.c @@ -453,7 +453,7 @@ xs_read_xdr_buf(struct socket *sock, struct msghdr *msg, int flags, goto out; if (ret != want) goto out; - } else + } else if (offset < seek_init) offset = seek_init; ret = -EMSGSIZE; out: From 513149607d19bc3821386fb5ac75f8b99fd4b115 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 15 Mar 2019 12:55:59 -0400 Subject: [PATCH 4/8] SUNRPC: Fix the minimal size for reply buffer allocation We must at minimum allocate enough memory to be able to see any auth errors in the reply from the server. Fixes: 2c94b8eca1a26 ("SUNRPC: Use au_rslack when computing reply...") Signed-off-by: Trond Myklebust --- net/sunrpc/clnt.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index 4216fe33204a..310873895578 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -1730,7 +1730,12 @@ call_allocate(struct rpc_task *task) req->rq_callsize = RPC_CALLHDRSIZE + (auth->au_cslack << 1) + proc->p_arglen; req->rq_callsize <<= 2; - req->rq_rcvsize = RPC_REPHDRSIZE + auth->au_rslack + proc->p_replen; + /* + * Note: the reply buffer must at minimum allocate enough space + * for the 'struct accepted_reply' from RFC5531. + */ + req->rq_rcvsize = RPC_REPHDRSIZE + auth->au_rslack + \ + max_t(size_t, proc->p_replen, 2); req->rq_rcvsize <<= 2; status = xprt->ops->buf_alloc(task); From 27adc785928ae6b34cdda96f472735b77c91e247 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 15 Mar 2019 08:01:16 -0400 Subject: [PATCH 5/8] SUNRPC: Use the ENOTCONN error on socket disconnect When the socket is closed, we currently send an EAGAIN error to all pending requests in order to ask them to retransmit. Use ENOTCONN instead, to ensure that they try to reconnect before attempting to transmit. This also helps SOFTCONN tasks to behave correctly in this situation. Signed-off-by: Trond Myklebust --- net/sunrpc/xprt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c index e096c5a725df..d7117d241460 100644 --- a/net/sunrpc/xprt.c +++ b/net/sunrpc/xprt.c @@ -664,7 +664,7 @@ void xprt_disconnect_done(struct rpc_xprt *xprt) spin_lock_bh(&xprt->transport_lock); xprt_clear_connected(xprt); xprt_clear_write_space_locked(xprt); - xprt_wake_pending_tasks(xprt, -EAGAIN); + xprt_wake_pending_tasks(xprt, -ENOTCONN); spin_unlock_bh(&xprt->transport_lock); } EXPORT_SYMBOL_GPL(xprt_disconnect_done); From eb90a16e9087063943859ae99bbdddd1fbfcf477 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 15 Mar 2019 09:29:00 -0400 Subject: [PATCH 6/8] SUNRPC: rpc_decode_header() must always return a non-zero value on error Ensure that when the "garbage args" case falls through, we do set an error of EIO. Fixes: a0584ee9aed8 ("SUNRPC: Use struct xdr_stream when decoding...") Signed-off-by: Trond Myklebust --- net/sunrpc/clnt.c | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index 310873895578..1f47801e0002 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -2454,7 +2454,7 @@ static noinline int rpc_decode_header(struct rpc_task *task, struct xdr_stream *xdr) { struct rpc_clnt *clnt = task->tk_client; - int error = -EACCES; + int error; __be32 *p; /* RFC-1014 says that the representation of XDR data must be a @@ -2463,7 +2463,7 @@ rpc_decode_header(struct rpc_task *task, struct xdr_stream *xdr) * undefined results */ if (task->tk_rqstp->rq_rcv_buf.len & 3) - goto out_badlen; + goto out_unparsable; p = xdr_inline_decode(xdr, 3 * sizeof(*p)); if (!p) @@ -2498,9 +2498,10 @@ rpc_decode_header(struct rpc_task *task, struct xdr_stream *xdr) goto out_err; case rpc_garbage_args: trace_rpc__garbage_args(task); + error = -EIO; break; default: - trace_rpc__unparsable(task); + goto out_unparsable; } out_garbage: @@ -2514,11 +2515,6 @@ out_err: rpc_exit(task, error); return error; -out_badlen: - trace_rpc__unparsable(task); - error = -EIO; - goto out_err; - out_unparsable: trace_rpc__unparsable(task); error = -EIO; @@ -2529,6 +2525,7 @@ out_verifier: goto out_garbage; out_msg_denied: + error = -EACCES; p = xdr_inline_decode(xdr, sizeof(*p)); if (!p) goto out_unparsable; @@ -2540,9 +2537,7 @@ out_msg_denied: error = -EPROTONOSUPPORT; goto out_err; default: - trace_rpc__unparsable(task); - error = -EIO; - goto out_err; + goto out_unparsable; } p = xdr_inline_decode(xdr, sizeof(*p)); @@ -2577,8 +2572,7 @@ out_msg_denied: task->tk_xprt->servername); break; default: - trace_rpc__unparsable(task); - error = -EIO; + goto out_unparsable; } goto out_err; } From 928d42f7d8737e1d6327e09668525f59725dabf9 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 15 Mar 2019 10:12:30 -0400 Subject: [PATCH 7/8] SUNRPC: Handle the SYSTEM_ERR rpc error Handle the SYSTEM_ERR rpc error by retrying the RPC call as if it were a garbage argument. Signed-off-by: Trond Myklebust --- net/sunrpc/clnt.c | 1 + 1 file changed, 1 insertion(+) diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index 1f47801e0002..cb73d6c25857 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -2497,6 +2497,7 @@ rpc_decode_header(struct rpc_task *task, struct xdr_stream *xdr) error = -EOPNOTSUPP; goto out_err; case rpc_garbage_args: + case rpc_system_err: trace_rpc__garbage_args(task); error = -EIO; break; From 5e3863fd597eba8c6679de805681631b1aad9bdb Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 15 Mar 2019 13:11:36 -0400 Subject: [PATCH 8/8] SUNRPC: Remove redundant check for the reply length in call_decode() Now that we're using the xdr_stream functions to decode the header, the test for the minimum reply length is redundant. Signed-off-by: Trond Myklebust --- net/sunrpc/clnt.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index cb73d6c25857..228970e6e52b 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -2392,9 +2392,6 @@ call_decode(struct rpc_task *task) WARN_ON(memcmp(&req->rq_rcv_buf, &req->rq_private_buf, sizeof(req->rq_rcv_buf)) != 0); - if (req->rq_rcv_buf.len < 12) - goto out_retry; - xdr_init_decode(&xdr, &req->rq_rcv_buf, req->rq_rcv_buf.head[0].iov_base, req); switch (rpc_decode_header(task, &xdr)) { @@ -2405,7 +2402,6 @@ call_decode(struct rpc_task *task) task->tk_pid, __func__, task->tk_status); return; case -EAGAIN: -out_retry: task->tk_status = 0; /* Note: rpc_decode_header() may have freed the RPC slot */ if (task->tk_rqstp == req) {