From simon@comsys.ntu-kpi.kiev.ua  Fri Feb  4 09:55:41 2011
Return-Path: <simon@comsys.ntu-kpi.kiev.ua>
Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34])
	by hub.freebsd.org (Postfix) with ESMTP id 0838A1065675
	for <FreeBSD-gnats-submit@freebsd.org>; Fri,  4 Feb 2011 09:55:41 +0000 (UTC)
	(envelope-from simon@comsys.ntu-kpi.kiev.ua)
Received: from comsys.kpi.ua (comsys.kpi.ua [77.47.192.42])
	by mx1.freebsd.org (Postfix) with ESMTP id 69B7B8FC25
	for <FreeBSD-gnats-submit@freebsd.org>; Fri,  4 Feb 2011 09:55:40 +0000 (UTC)
Received: from pm513-1.comsys.kpi.ua ([10.18.52.101] helo=pm513-1.comsys.ntu-kpi.kiev.ua)
	by comsys.kpi.ua with esmtpsa (TLSv1:AES256-SHA:256)
	(Exim 4.63)
	(envelope-from <simon@comsys.ntu-kpi.kiev.ua>)
	id 1PlINU-0005vE-P2
	for FreeBSD-gnats-submit@freebsd.org; Fri, 04 Feb 2011 11:54:56 +0200
Received: by pm513-1.comsys.ntu-kpi.kiev.ua (Postfix, from userid 1001)
	id 399D11CC1E; Fri,  4 Feb 2011 11:55:37 +0200 (EET)
Message-Id: <20110204095537.GA47583@pm513-1.comsys.ntu-kpi.kiev.ua>
Date: Fri, 4 Feb 2011 11:55:37 +0200
From: Andrey Simonenko <simon@comsys.ntu-kpi.kiev.ua>
To: FreeBSD-gnats-submit@freebsd.org
Subject: PF_LOCAL stream connection is stuck in sbwait when recv(MSG_WAITALL)
 is used

>Number:         154504
>Category:       kern
>Synopsis:       [libc] [patch] recv(2): PF_LOCAL stream connection is stuck in sbwait when recv(MSG_WAITALL) is used
>Confidential:   no
>Severity:       serious
>Priority:       medium
>Responsible:    freebsd-bugs
>State:          closed
>Quarter:        
>Keywords:       
>Date-Required:  
>Class:          sw-bug
>Submitter-Id:   current-users
>Arrival-Date:   Fri Feb 04 10:00:23 UTC 2011
>Closed-Date:    Thu Jun 16 05:57:40 UTC 2011
>Last-Modified:  Thu Jun 16 05:57:40 UTC 2011
>Originator:     Andrey Simonenko
>Release:        FreeBSD 9.0-CURRENT and 8-STABLE
>Organization:
>Environment:

FreeBSD 9.0-CURRENT and 8-STABLE.

>Description:

Let's there are a client and a server that use stream sockets in the PF_LOCAL
communication domain.  A client sends data to a server, then having received
data from a client a server sends data to a client and a client receives it.
Both of them use recv(2) with the MSG_WAITALL flag.

At some moment of such communication a client and sometimes a server
stuck in sbwait.  I could not reproduce this situation on FreeBSD 7.1-STABLE,
but can reproduce it on 8.2-PRERELEASE (real amd64 hardware) and on just
updated 9.0-CURRENT (qemu for amd64).

>How-To-Repeat:

I wrote a test program.  Run "msg_waitall -s" in one terminal and run
"msg_waitall_client.sh" in another terminal.  At some moment a client will
stuck in sbwait, sometimes a server also is blocked in sbwait:

 2297 sbwait ./msg_waitall -s
 2298 wait   /bin/sh ./msg_waitall_client.sh
 3189 sbwait ./msg_waitall -c

All test were made with the default net.local.stream.recvspace and
net.local.stream.sendspace values.  Also if these variable are smaller,
then the effect of blocking should happen faster.

The effect of blocking also exists when TCP_SORECEIVE_STREAM is defined
on 9.0-CURRENT.

diff -ruNp msg_waitall.orig/Makefile msg_waitall/Makefile
--- msg_waitall.orig/Makefile	1970-01-01 03:00:00.000000000 +0300
+++ msg_waitall/Makefile	2011-02-02 13:30:25.000000000 +0200
@@ -0,0 +1,7 @@
+PROG=msg_waitall
+
+NO_MAN=
+
+WARNS=6
+
+.include <bsd.prog.mk>
diff -ruNp msg_waitall.orig/msg_waitall.c msg_waitall/msg_waitall.c
--- msg_waitall.orig/msg_waitall.c	1970-01-01 03:00:00.000000000 +0300
+++ msg_waitall/msg_waitall.c	2011-02-03 17:39:50.000000000 +0200
@@ -0,0 +1,186 @@
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+
+#include <err.h>
+#include <errno.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#define SOCKET_PATH	"./socket"
+#define BUFFER_SIZE	(30 * 1024)
+
+static void
+client(int use_read, int use_write)
+{
+	char buf[BUFFER_SIZE];
+	struct sockaddr_un un;
+	ssize_t nread, nsent;
+	int fd;
+
+	printf("CLIENT: using %s() and %s()\n",
+	    use_read ? "read" : "recv", use_write ? "write" : "send");
+
+	if (sizeof(SOCKET_PATH) > sizeof(un.sun_path))
+		errx(EXIT_FAILURE, "path %s is too long", SOCKET_PATH);
+
+	memset(&un, 0, sizeof(un));
+	un.sun_family = PF_LOCAL;
+	strncpy(un.sun_path, SOCKET_PATH, sizeof(un.sun_path) - 1);
+
+	fd = socket(PF_LOCAL, SOCK_STREAM, 0);
+	if (fd < 0)
+		err(EXIT_FAILURE, "socket");
+
+	if (connect(fd, (struct sockaddr *)&un, SUN_LEN(&un)) < 0)
+		err(EXIT_FAILURE, "connect");
+
+	if (use_write) {
+		nsent = write(fd, buf, sizeof(buf));
+		if (nsent < 0)
+			err(EXIT_FAILURE, "write");
+	} else {
+		nsent = send(fd, buf, sizeof(buf), 0);
+		if (nsent < 0)
+			err(EXIT_FAILURE, "send");
+	}
+			
+	if (nsent != -1 && nsent != sizeof(buf))
+		errx(EXIT_FAILURE, "short send: %zd of %zu",
+		    nsent, sizeof(buf));
+
+	fprintf(stderr, "sent");
+
+	if (use_read) {
+		nread = read(fd, buf, sizeof(buf));
+		if (nread < 0)
+			err(EXIT_FAILURE, "read");
+	} else {
+		nread = recv(fd, buf, sizeof(buf), MSG_WAITALL);
+		if (nread < 0)
+			err(EXIT_FAILURE, "recv");
+	}
+
+	if (nread != -1 && nread != sizeof(buf))
+		errx(EXIT_FAILURE, "short read: %zd of %zu",
+		    nread, sizeof(buf));
+
+	fprintf(stderr, ", received\n");
+
+	if (close(fd) < 0)
+		err(EXIT_FAILURE, "close");
+}
+
+static void
+server(int use_read, int use_write)
+{
+	char buf[BUFFER_SIZE];
+	struct sockaddr_un un;
+	ssize_t nread, nsent;
+	int fdl, fd;
+
+	printf("SERVER: using %s() and %s()\n",
+	    use_read ? "read" : "recv", use_write ? "write" : "send");
+
+	signal(SIGPIPE, SIG_IGN);
+
+	if (sizeof(SOCKET_PATH) > sizeof(un.sun_path))
+		errx(EXIT_FAILURE, "path %s is too long", SOCKET_PATH);
+
+	fdl = socket(PF_LOCAL, SOCK_STREAM, 0);
+	if (fdl < 0)
+		err(EXIT_FAILURE, "socket");
+
+	if (unlink(SOCKET_PATH) < 0)
+		if (errno != ENOENT)
+			err(EXIT_FAILURE, "unlink");
+
+	memset(&un, 0, sizeof(un));
+	un.sun_family = PF_LOCAL;
+	strncpy(un.sun_path, SOCKET_PATH, sizeof(un.sun_path) - 1);
+	if (bind(fdl, (struct sockaddr *)&un, SUN_LEN(&un)) < 0)
+		err(EXIT_FAILURE, "bind");
+
+	if (listen(fdl, 10) < 0)
+		err(EXIT_FAILURE, "listen");
+
+	for (;;) {
+		fd = accept(fdl, (struct sockaddr *)NULL, (socklen_t *)NULL);
+		if (fd < 0)
+			err(EXIT_FAILURE, "accept");
+
+		if (use_read) {
+			nread = read(fd, buf, sizeof(buf));
+			if (nread < 0)
+				warn("read");
+		} else {
+			nread = recv(fd, buf, sizeof(buf), MSG_WAITALL);
+			if (nread < 0)
+				warn("recv");
+		}
+
+		if (nread != -1 && nread != sizeof(buf))
+			warnx("short read: %zd of %zu",
+			    nread, sizeof(buf));
+
+		fprintf(stderr, "received");
+
+		if (use_write) {
+			nsent = write(fd, buf, sizeof(buf));
+			if (nsent < 0)
+				warn("write");
+		} else {
+			nsent = send(fd, buf, sizeof(buf), 0);
+			if (nsent < 0)
+				warn("send");
+		}
+
+		if (nsent != -1 && nsent != sizeof(buf))
+			warnx("short send: %zd of %zu",
+			    nsent, sizeof(buf));
+
+		fprintf(stderr, ", sent\n");
+
+		if (close(fd) < 0)
+			warn("close");
+	}
+}
+
+int
+main(int argc, char *argv[])
+{
+	int opt, opt_s, opt_c, opt_r, opt_w;
+
+	opt_s = opt_c = opt_r = opt_w = 0;
+	while ((opt = getopt(argc, argv, "scrw")) != -1)
+		switch (opt) {
+		case 's':
+			opt_s = 1;
+			break;
+		case 'c':
+			opt_c = 1;
+			break;
+		case 'r':
+			opt_r = 1;
+			break;
+		case 'w':
+			opt_w = 1;
+			break;
+		default:
+			errx(EXIT_FAILURE, "Usage: %s: [-rw] -s|c",
+			    getprogname());
+		}
+
+	if (opt_s == 0 && opt_c == 0)
+		errx(EXIT_FAILURE, "specify -s or -c");
+
+	if (opt_c)
+		client(opt_r, opt_w);
+	else
+		server(opt_r, opt_w);
+
+	return (EXIT_SUCCESS);
+}
diff -ruNp msg_waitall.orig/msg_waitall_client.sh msg_waitall/msg_waitall_client.sh
--- msg_waitall.orig/msg_waitall_client.sh	1970-01-01 03:00:00.000000000 +0300
+++ msg_waitall/msg_waitall_client.sh	2011-02-03 13:20:03.000000000 +0200
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+while [ true ]; do
+	./msg_waitall -c || exit 1
+done
+


>Fix:


>Release-Note:
>Audit-Trail:
Responsible-Changed-From-To: freebsd-bugs->freebsd-pf 
Responsible-Changed-By: linimon 
Responsible-Changed-When: Fri Feb 4 19:35:14 UTC 2011 
Responsible-Changed-Why:  
Over to maintainer(s). 

http://www.freebsd.org/cgi/query-pr.cgi?pr=154504 
Responsible-Changed-From-To: freebsd-pf->freebsd-bugs 
Responsible-Changed-By: linimon 
Responsible-Changed-When: Fri Feb 4 20:58:46 UTC 2011 
Responsible-Changed-Why:  
Fix misclassification. 

Pointed out by: Gary Palmer 

http://www.freebsd.org/cgi/query-pr.cgi?pr=154504 

From: Mikolaj Golub <trociny@freebsd.org>
To: Andrey Simonenko <simon@comsys.ntu-kpi.kiev.ua>
Cc: bug-followup@FreeBSD.org, freebsd-net@freebsd.org,
    Pawel Jakub Dawidek <pjd@FreeBSD.org>, Kostik Belousov <kib@FreeBSD.org>
Subject: Re: kern/154504: [libc] recv(2): PF_LOCAL stream connection is stuck in sbwait when recv(MSG_WAITALL) is used
Date: Sun, 10 Apr 2011 16:49:25 +0300

 --=-=-=
 
 Hi,
 
 Does the attached patch fix the problem for you?
 
 -- 
 Mikolaj Golub
 
 --=-=-=
 Content-Type: text/x-patch
 Content-Disposition: attachment; filename=uipc_socket.c.soreceive.patch
 
 Index: sys/kern/uipc_socket.c
 ===================================================================
 --- sys/kern/uipc_socket.c	(revision 220485)
 +++ sys/kern/uipc_socket.c	(working copy)
 @@ -1845,10 +1845,16 @@ dontblock:
  			}
  			SBLASTRECORDCHK(&so->so_rcv);
  			SBLASTMBUFCHK(&so->so_rcv);
 -			error = sbwait(&so->so_rcv);
 -			if (error) {
 -				SOCKBUF_UNLOCK(&so->so_rcv);
 -				goto release;
 +			/*
 +			 * We could receive some data while was notifying the
 +			 * the protocol. Skip blocking in this case.
 +			 */
 +			if (so->so_rcv.sb_mb == NULL) {
 +				error = sbwait(&so->so_rcv);
 +				if (error) {
 +					SOCKBUF_UNLOCK(&so->so_rcv);
 +					goto release;
 +				}
  			}
  			m = so->so_rcv.sb_mb;
  			if (m != NULL)
 
 --=-=-=--

From: dfilter@FreeBSD.ORG (dfilter service)
To: bug-followup@FreeBSD.org
Cc:  
Subject: Re: kern/154504: commit references a PR
Date: Sun, 29 May 2011 18:01:06 +0000 (UTC)

 Author: trociny
 Date: Sun May 29 18:00:50 2011
 New Revision: 222454
 URL: http://svn.freebsd.org/changeset/base/222454
 
 Log:
   In soreceive_generic(), if MSG_WAITALL is set but the request is
   larger than the receive buffer, we have to receive in sections.
   When notifying the protocol that some data has been drained the
   lock is released for a moment. Returning we block waiting for the
   rest of data. There is a race, when data could arrive while the
   lock was released and then the connection stalls in sbwait.
   
   Fix this by checking for data before blocking and skip blocking
   if there are some.
   
   PR:		kern/154504
   Reported by:	Andrey Simonenko <simon@comsys.ntu-kpi.kiev.ua>
   Tested by:	Andrey Simonenko <simon@comsys.ntu-kpi.kiev.ua>
   Reviewed by:	rwatson
   Approved by:	kib (co-mentor)
   MFC after:	2 weeks
 
 Modified:
   head/sys/kern/uipc_socket.c
 
 Modified: head/sys/kern/uipc_socket.c
 ==============================================================================
 --- head/sys/kern/uipc_socket.c	Sun May 29 15:10:12 2011	(r222453)
 +++ head/sys/kern/uipc_socket.c	Sun May 29 18:00:50 2011	(r222454)
 @@ -1845,10 +1845,16 @@ dontblock:
  			}
  			SBLASTRECORDCHK(&so->so_rcv);
  			SBLASTMBUFCHK(&so->so_rcv);
 -			error = sbwait(&so->so_rcv);
 -			if (error) {
 -				SOCKBUF_UNLOCK(&so->so_rcv);
 -				goto release;
 +			/*
 +			 * We could receive some data while was notifying
 +			 * the protocol. Skip blocking in this case.
 +			 */
 +			if (so->so_rcv.sb_mb == NULL) {
 +				error = sbwait(&so->so_rcv);
 +				if (error) {
 +					SOCKBUF_UNLOCK(&so->so_rcv);
 +					goto release;
 +				}
  			}
  			m = so->so_rcv.sb_mb;
  			if (m != NULL)
 _______________________________________________
 svn-src-all@freebsd.org mailing list
 http://lists.freebsd.org/mailman/listinfo/svn-src-all
 To unsubscribe, send any mail to "svn-src-all-unsubscribe@freebsd.org"
 

From: dfilter@FreeBSD.ORG (dfilter service)
To: bug-followup@FreeBSD.org
Cc:  
Subject: Re: kern/154504: commit references a PR
Date: Wed, 15 Jun 2011 20:34:51 +0000 (UTC)

 Author: trociny
 Date: Wed Jun 15 20:34:40 2011
 New Revision: 223119
 URL: http://svn.freebsd.org/changeset/base/223119
 
 Log:
   MFC r222454:
   
   In soreceive_generic(), if MSG_WAITALL is set but the request is
   larger than the receive buffer, we have to receive in sections.
   When notifying the protocol that some data has been drained the
   lock is released for a moment. Returning we block waiting for the
   rest of data. There is a race, when data could arrive while the
   lock was released and then the connection stalls in sbwait.
   
   Fix this by checking for data before blocking and skip blocking
   if there are some.
   
   PR:		kern/154504
   Reported by:	Andrey Simonenko <simon@comsys.ntu-kpi.kiev.ua>
   Tested by:	Andrey Simonenko <simon@comsys.ntu-kpi.kiev.ua>
   Reviewed by:	rwatson
   
   Approved by:	pjd (mentor)
 
 Modified:
   stable/8/sys/kern/uipc_socket.c
 Directory Properties:
   stable/8/sys/   (props changed)
   stable/8/sys/amd64/include/xen/   (props changed)
   stable/8/sys/cddl/contrib/opensolaris/   (props changed)
   stable/8/sys/contrib/dev/acpica/   (props changed)
   stable/8/sys/contrib/pf/   (props changed)
 
 Modified: stable/8/sys/kern/uipc_socket.c
 ==============================================================================
 --- stable/8/sys/kern/uipc_socket.c	Wed Jun 15 19:53:08 2011	(r223118)
 +++ stable/8/sys/kern/uipc_socket.c	Wed Jun 15 20:34:40 2011	(r223119)
 @@ -1836,10 +1836,16 @@ dontblock:
  			}
  			SBLASTRECORDCHK(&so->so_rcv);
  			SBLASTMBUFCHK(&so->so_rcv);
 -			error = sbwait(&so->so_rcv);
 -			if (error) {
 -				SOCKBUF_UNLOCK(&so->so_rcv);
 -				goto release;
 +			/*
 +			 * We could receive some data while was notifying
 +			 * the protocol. Skip blocking in this case.
 +			 */
 +			if (so->so_rcv.sb_mb == NULL) {
 +				error = sbwait(&so->so_rcv);
 +				if (error) {
 +					SOCKBUF_UNLOCK(&so->so_rcv);
 +					goto release;
 +				}
  			}
  			m = so->so_rcv.sb_mb;
  			if (m != NULL)
 _______________________________________________
 svn-src-all@freebsd.org mailing list
 http://lists.freebsd.org/mailman/listinfo/svn-src-all
 To unsubscribe, send any mail to "svn-src-all-unsubscribe@freebsd.org"
 
State-Changed-From-To: open->closed 
State-Changed-By: trociny 
State-Changed-When: Thu Jun 16 05:55:47 UTC 2011 
State-Changed-Why:  
Fixed. 

http://www.freebsd.org/cgi/query-pr.cgi?pr=154504 
>Unformatted:
