From thomas@cuivre.fr.eu.org  Thu May  1 09:14:46 2014
Return-Path: <thomas@cuivre.fr.eu.org>
Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1])
	(using TLSv1 with cipher ADH-AES256-SHA (256/256 bits))
	(No client certificate requested)
	by hub.freebsd.org (Postfix) with ESMTPS id 2FA70CED
	for <FreeBSD-gnats-submit@freebsd.org>; Thu,  1 May 2014 09:14:46 +0000 (UTC)
Received: from melamine.cuivre.fr.eu.org (houdart.cuivre.fr.eu.org [81.57.40.110])
	(using TLSv1 with cipher DHE-RSA-CAMELLIA256-SHA (256/256 bits))
	(Client did not present a certificate)
	by mx1.freebsd.org (Postfix) with ESMTPS id E43AD12BD
	for <FreeBSD-gnats-submit@freebsd.org>; Thu,  1 May 2014 09:14:45 +0000 (UTC)
Received: by melamine.cuivre.fr.eu.org (Postfix, from userid 1000)
	id 11CCB110AC; Thu,  1 May 2014 11:06:29 +0200 (CEST)
Message-Id: <20140501090629.11CCB110AC@melamine.cuivre.fr.eu.org>
Date: Thu,  1 May 2014 11:06:28 +0200 (CEST)
From: Thomas Quinot <thomas@cuivre.fr.eu.org>
Reply-To: Thomas Quinot <thomas@cuivre.fr.eu.org>
To: FreeBSD-gnats-submit@freebsd.org
Cc:
Subject: dd conv=sparse bs=64k EINVAL on sparse last block
X-Send-Pr-Version: 3.114
X-GNATS-Notify:

>Number:         189174
>Category:       bin
>Synopsis:       [patch] dd(1): dd conv=sparse bs=64k EINVAL on sparse last block
>Confidential:   no
>Severity:       serious
>Priority:       medium
>Responsible:    freebsd-bugs
>State:          closed
>Quarter:        
>Keywords:       
>Date-Required:  
>Class:          sw-bug
>Submitter-Id:   current-users
>Arrival-Date:   Thu May 01 09:20:00 UTC 2014
>Closed-Date:    Wed May 07 19:55:07 UTC 2014
>Last-Modified:  Wed May 21 07:50:00 UTC 2014
>Originator:     Thomas Quinot
>Release:        FreeBSD 9.2-STABLE amd64
>Organization:
>Environment:
System: FreeBSD melamine.cuivre.fr.eu.org 9.2-STABLE FreeBSD 9.2-STABLE #0 r256080: Sun Oct 6 15:33:30 CEST 2013 thomas@melamine.cuivre.fr.eu.org:/usr/obj/users/thomas/projects/FreeBSD/base/stable/9/sys/GENERIC amd64


	
>Description:
	When copying to an md device (or a slice thereof) with conv=sparse
	and an explicit block size, dd will produce an error if the last
	block is a sparse and partial one.

	(This breaks NanoBSD builds in the case where two system images
	are created, and the slice size computed from the media size
	is not an intregral multiple of 64k).

	This is with dd(1) from stable/9 as of rev 265172.

>How-To-Repeat:
# dd if=/dev/zero of=f1 count=129
129+0 records in
129+0 records out
66048 bytes transferred in 0.003451 secs (19139518 bytes/sec)
# dd if=/dev/zero of=f2 count=129
129+0 records in
129+0 records out
66048 bytes transferred in 0.003485 secs (18952274 bytes/sec)
# mdconfig -f f1
md0
# mdconfig -f f2
md1
# dd if=/dev/md0 of=/dev/md1 conv=sparse bs=64k
dd: /dev/md1: Invalid argument
1+1 records in
1+0 records out
65536 bytes transferred in 0.000338 secs (193849018 bytes/sec)

>Fix:

	


>Release-Note:
>Audit-Trail:

From: Thomas Quinot <thomas@quinot.org>
To: bug-followup@freebsd.org
Cc:  
Subject: Re: bin/189174: dd conv=sparse bs=64k EINVAL on sparse last block
Date: Thu, 1 May 2014 11:28:25 +0200

 * FreeBSD-gnats-submit@FreeBSD.org, 2014-05-01 :
 
 > >Category:       bin
 > >Responsible:    freebsd-bugs
 > >Synopsis:       dd conv=sparse bs=64k EINVAL on sparse last block
 > >Arrival-Date:   Thu May 01 09:20:00 UTC 2014
 
 The following fix appears to fix the problem here:
 
 Index: dd.c
 ===================================================================
 --- dd.c	(rvision 265172)
 +++ dd.c	(copie de travail)
 @@ -450,13 +450,16 @@
  				nw = cnt;
  			} else {
  				if (pending != 0) {
 -					if (force)
 +					/* If forced to write, and we have no data left, we need
 +                                         * to write an explicit 0.
 +                                         */
 +					if (force && cnt == 0)
  						pending--;
  					if (lseek(out.fd, pending, SEEK_CUR) ==
  					    -1)
  						err(2, "%s: seek error creating sparse file",
  						    out.name);
 -					if (force)
 +					if (force && cnt == 0)
  						write(out.fd, outp, 1);
  					pending = 0;
  				}

From: dfilter@FreeBSD.ORG (dfilter service)
To: bug-followup@FreeBSD.org
Cc:  
Subject: Re: bin/189174: commit references a PR
Date: Wed,  7 May 2014 19:33:34 +0000 (UTC)

 Author: thomas
 Date: Wed May  7 19:33:29 2014
 New Revision: 265593
 URL: http://svnweb.freebsd.org/changeset/base/265593
 
 Log:
   (dd_out): Fix handling of all-zeroes block at end of input with
   conv=sparse.
   
   This change fixes two separate issues observed when the last output
   block is all zeroes, and conv=sparse is in use. In this case, care
   must be taken to roll back the last seek and write the entire last zero
   block at the original offset where it should have occurred: when the
   destination file is a block device, it is not possible to roll back
   by just one character as the write would then not be properly aligned.
   
   Furthermore, the buffer used to write this last all-zeroes block
   needs to be properly zeroed-out. This was not the case previously,
   resulting in a junk data byte appearing instead of a zero in the
   output stream.
   
   PR:		bin/189174
   PR:		bin/189284
   Reviewed by:	kib
   MFC after:	2 weeks
 
 Modified:
   head/bin/dd/dd.c
 
 Modified: head/bin/dd/dd.c
 ==============================================================================
 --- head/bin/dd/dd.c	Wed May  7 19:30:28 2014	(r265592)
 +++ head/bin/dd/dd.c	Wed May  7 19:33:29 2014	(r265593)
 @@ -51,6 +51,7 @@ __FBSDID("$FreeBSD$");
  #include <sys/disklabel.h>
  #include <sys/filio.h>
  
 +#include <assert.h>
  #include <ctype.h>
  #include <err.h>
  #include <errno.h>
 @@ -77,6 +78,7 @@ STAT	st;			/* statistics */
  void	(*cfunc)(void);		/* conversion function */
  uintmax_t cpy_cnt;		/* # of blocks to copy */
  static off_t	pending = 0;	/* pending seek if sparse */
 +static off_t	last_sp = 0;	/* size of last added sparse block */
  u_int	ddflags = 0;		/* conversion options */
  size_t	cbsz;			/* conversion block size */
  uintmax_t files_cnt = 1;	/* # of files to copy */
 @@ -174,6 +176,8 @@ setup(void)
  	} else if ((in.db = malloc(MAX(in.dbsz, cbsz) + cbsz)) == NULL ||
  	    (out.db = malloc(out.dbsz + cbsz)) == NULL)
  		err(1, "output buffer");
 +
 +	/* dbp is the first free position in each buffer. */
  	in.dbp = in.db;
  	out.dbp = out.db;
  
 @@ -436,8 +440,15 @@ dd_out(int force)
  	 * we play games with the buffer size, and it's usually a partial write.
  	 */
  	outp = out.db;
 +
 +	/*
 +	 * If force, first try to write all pending data, else try to write
 +	 * just one block. Subsequently always write data one full block at
 +	 * a time at most.
 +	 */
  	for (n = force ? out.dbcnt : out.dbsz;; n = out.dbsz) {
 -		for (cnt = n;; cnt -= nw) {
 +		cnt = n;
 +		do {
  			sparse = 0;
  			if (ddflags & C_SPARSE) {
  				sparse = 1;	/* Is buffer sparse? */
 @@ -449,18 +460,24 @@ dd_out(int force)
  			}
  			if (sparse && !force) {
  				pending += cnt;
 +				last_sp = cnt;
  				nw = cnt;
  			} else {
  				if (pending != 0) {
 -					if (force)
 -						pending--;
 +					/* If forced to write, and we have no
 +					 * data left, we need to write the last
 +					 * sparse block explicitly.
 +					 */
 +					if (force && cnt == 0) {
 +						pending -= last_sp;
 +						assert(outp == out.db);
 +						memset(outp, 0, cnt);
 +					}
  					if (lseek(out.fd, pending, SEEK_CUR) ==
  					    -1)
  						err(2, "%s: seek error creating sparse file",
  						    out.name);
 -					if (force)
 -						write(out.fd, outp, 1);
 -					pending = 0;
 +					pending = last_sp = 0;
  				}
  				if (cnt)
  					nw = write(out.fd, outp, cnt);
 @@ -475,27 +492,29 @@ dd_out(int force)
  					err(1, "%s", out.name);
  				nw = 0;
  			}
 +
  			outp += nw;
  			st.bytes += nw;
 -			if ((size_t)nw == n) {
 -				if (n != out.dbsz)
 -					++st.out_part;
 -				else
 -					++st.out_full;
 -				break;
 -			}
 -			++st.out_part;
 -			if ((size_t)nw == cnt)
 -				break;
 -			if (out.flags & ISTAPE)
 -				errx(1, "%s: short write on tape device",
 -				    out.name);
 -			if (out.flags & ISCHR && !warned) {
 -				warned = 1;
 -				warnx("%s: short write on character device",
 -				    out.name);
 +
 +			if ((size_t)nw == n && n == out.dbsz)
 +				++st.out_full;
 +			else
 +				++st.out_part;
 +
 +			if ((size_t) nw != cnt) {
 +				if (out.flags & ISTAPE)
 +					errx(1, "%s: short write on tape device",
 +				    	out.name);
 +				if (out.flags & ISCHR && !warned) {
 +					warned = 1;
 +					warnx("%s: short write on character device",
 +				    	out.name);
 +				}
  			}
 -		}
 +
 +			cnt -= nw;
 +		} while (cnt != 0);
 +
  		if ((out.dbcnt -= n) < out.dbsz)
  			break;
  	}
 _______________________________________________
 svn-src-all@freebsd.org mailing list
 http://lists.freebsd.org/mailman/listinfo/svn-src-all
 To unsubscribe, send any mail to "svn-src-all-unsubscribe@freebsd.org"
 
State-Changed-From-To: open->closed 
State-Changed-By: thomas 
State-Changed-When: Wed May 7 19:54:36 UTC 2014 
State-Changed-Why:  
Fixed in head. 

http://www.freebsd.org/cgi/query-pr.cgi?pr=189174 

From: dfilter@FreeBSD.ORG (dfilter service)
To: bug-followup@FreeBSD.org
Cc:  
Subject: Re: bin/189174: commit references a PR
Date: Wed, 21 May 2014 07:21:40 +0000 (UTC)

 Author: thomas
 Date: Wed May 21 07:21:36 2014
 New Revision: 266488
 URL: http://svnweb.freebsd.org/changeset/base/266488
 
 Log:
   MFC rev. 265593:
   (dd_out): Fix handling of all-zeroes block at end of input with
   conv=sparse.
   
   PR:		bin/189174
   PR:		bin/189284
   Reviewed by:	kib
 
 Modified:
   stable/10/bin/dd/dd.c
 Directory Properties:
   stable/10/   (props changed)
 
 Modified: stable/10/bin/dd/dd.c
 ==============================================================================
 --- stable/10/bin/dd/dd.c	Wed May 21 06:33:21 2014	(r266487)
 +++ stable/10/bin/dd/dd.c	Wed May 21 07:21:36 2014	(r266488)
 @@ -52,6 +52,7 @@ __FBSDID("$FreeBSD$");
  #include <sys/filio.h>
  #include <sys/time.h>
  
 +#include <assert.h>
  #include <ctype.h>
  #include <err.h>
  #include <errno.h>
 @@ -76,6 +77,7 @@ STAT	st;			/* statistics */
  void	(*cfunc)(void);		/* conversion function */
  uintmax_t cpy_cnt;		/* # of blocks to copy */
  static off_t	pending = 0;	/* pending seek if sparse */
 +static off_t	last_sp = 0;	/* size of last added sparse block */
  u_int	ddflags = 0;		/* conversion options */
  size_t	cbsz;			/* conversion block size */
  uintmax_t files_cnt = 1;	/* # of files to copy */
 @@ -173,6 +175,8 @@ setup(void)
  	} else if ((in.db = malloc(MAX(in.dbsz, cbsz) + cbsz)) == NULL ||
  	    (out.db = malloc(out.dbsz + cbsz)) == NULL)
  		err(1, "output buffer");
 +
 +	/* dbp is the first free position in each buffer. */
  	in.dbp = in.db;
  	out.dbp = out.db;
  
 @@ -434,8 +438,15 @@ dd_out(int force)
  	 * we play games with the buffer size, and it's usually a partial write.
  	 */
  	outp = out.db;
 +
 +	/*
 +	 * If force, first try to write all pending data, else try to write
 +	 * just one block. Subsequently always write data one full block at
 +	 * a time at most.
 +	 */
  	for (n = force ? out.dbcnt : out.dbsz;; n = out.dbsz) {
 -		for (cnt = n;; cnt -= nw) {
 +		cnt = n;
 +		do {
  			sparse = 0;
  			if (ddflags & C_SPARSE) {
  				sparse = 1;	/* Is buffer sparse? */
 @@ -447,18 +458,24 @@ dd_out(int force)
  			}
  			if (sparse && !force) {
  				pending += cnt;
 +				last_sp = cnt;
  				nw = cnt;
  			} else {
  				if (pending != 0) {
 -					if (force)
 -						pending--;
 +					/* If forced to write, and we have no
 +					 * data left, we need to write the last
 +					 * sparse block explicitly.
 +					 */
 +					if (force && cnt == 0) {
 +						pending -= last_sp;
 +						assert(outp == out.db);
 +						memset(outp, 0, cnt);
 +					}
  					if (lseek(out.fd, pending, SEEK_CUR) ==
  					    -1)
  						err(2, "%s: seek error creating sparse file",
  						    out.name);
 -					if (force)
 -						write(out.fd, outp, 1);
 -					pending = 0;
 +					pending = last_sp = 0;
  				}
  				if (cnt)
  					nw = write(out.fd, outp, cnt);
 @@ -473,27 +490,29 @@ dd_out(int force)
  					err(1, "%s", out.name);
  				nw = 0;
  			}
 +
  			outp += nw;
  			st.bytes += nw;
 -			if ((size_t)nw == n) {
 -				if (n != out.dbsz)
 -					++st.out_part;
 -				else
 -					++st.out_full;
 -				break;
 -			}
 -			++st.out_part;
 -			if ((size_t)nw == cnt)
 -				break;
 -			if (out.flags & ISTAPE)
 -				errx(1, "%s: short write on tape device",
 -				    out.name);
 -			if (out.flags & ISCHR && !warned) {
 -				warned = 1;
 -				warnx("%s: short write on character device",
 -				    out.name);
 +
 +			if ((size_t)nw == n && n == out.dbsz)
 +				++st.out_full;
 +			else
 +				++st.out_part;
 +
 +			if ((size_t) nw != cnt) {
 +				if (out.flags & ISTAPE)
 +					errx(1, "%s: short write on tape device",
 +				    	out.name);
 +				if (out.flags & ISCHR && !warned) {
 +					warned = 1;
 +					warnx("%s: short write on character device",
 +				    	out.name);
 +				}
  			}
 -		}
 +
 +			cnt -= nw;
 +		} while (cnt != 0);
 +
  		if ((out.dbcnt -= n) < out.dbsz)
  			break;
  	}
 _______________________________________________
 svn-src-all@freebsd.org mailing list
 http://lists.freebsd.org/mailman/listinfo/svn-src-all
 To unsubscribe, send any mail to "svn-src-all-unsubscribe@freebsd.org"
 

From: dfilter@FreeBSD.ORG (dfilter service)
To: bug-followup@FreeBSD.org
Cc:  
Subject: Re: bin/189174: commit references a PR
Date: Wed, 21 May 2014 07:42:48 +0000 (UTC)

 Author: thomas
 Date: Wed May 21 07:42:42 2014
 New Revision: 266489
 URL: http://svnweb.freebsd.org/changeset/base/266489
 
 Log:
   MFC rev. 265593:
   (dd_out): Fix handling of all-zeroes block at end of input with
   conv=sparse.
   
   PR:		bin/189174
   PR:		bin/189284
   Reviewed by:	kib
 
 Modified:
   stable/9/bin/dd/dd.c
 Directory Properties:
   stable/9/   (props changed)
   stable/9/bin/   (props changed)
   stable/9/bin/dd/   (props changed)
 
 Modified: stable/9/bin/dd/dd.c
 ==============================================================================
 --- stable/9/bin/dd/dd.c	Wed May 21 07:21:36 2014	(r266488)
 +++ stable/9/bin/dd/dd.c	Wed May 21 07:42:42 2014	(r266489)
 @@ -52,6 +52,7 @@ __FBSDID("$FreeBSD$");
  #include <sys/filio.h>
  #include <sys/time.h>
  
 +#include <assert.h>
  #include <ctype.h>
  #include <err.h>
  #include <errno.h>
 @@ -76,6 +77,7 @@ STAT	st;			/* statistics */
  void	(*cfunc)(void);		/* conversion function */
  uintmax_t cpy_cnt;		/* # of blocks to copy */
  static off_t	pending = 0;	/* pending seek if sparse */
 +static off_t	last_sp = 0;	/* size of last added sparse block */
  u_int	ddflags = 0;		/* conversion options */
  size_t	cbsz;			/* conversion block size */
  uintmax_t files_cnt = 1;	/* # of files to copy */
 @@ -173,6 +175,8 @@ setup(void)
  	} else if ((in.db = malloc(MAX(in.dbsz, cbsz) + cbsz)) == NULL ||
  	    (out.db = malloc(out.dbsz + cbsz)) == NULL)
  		err(1, "output buffer");
 +
 +	/* dbp is the first free position in each buffer. */
  	in.dbp = in.db;
  	out.dbp = out.db;
  
 @@ -434,8 +438,15 @@ dd_out(int force)
  	 * we play games with the buffer size, and it's usually a partial write.
  	 */
  	outp = out.db;
 +
 +	/*
 +	 * If force, first try to write all pending data, else try to write
 +	 * just one block. Subsequently always write data one full block at
 +	 * a time at most.
 +	 */
  	for (n = force ? out.dbcnt : out.dbsz;; n = out.dbsz) {
 -		for (cnt = n;; cnt -= nw) {
 +		cnt = n;
 +		do {
  			sparse = 0;
  			if (ddflags & C_SPARSE) {
  				sparse = 1;	/* Is buffer sparse? */
 @@ -447,18 +458,24 @@ dd_out(int force)
  			}
  			if (sparse && !force) {
  				pending += cnt;
 +				last_sp = cnt;
  				nw = cnt;
  			} else {
  				if (pending != 0) {
 -					if (force)
 -						pending--;
 +					/* If forced to write, and we have no
 +					 * data left, we need to write the last
 +					 * sparse block explicitly.
 +					 */
 +					if (force && cnt == 0) {
 +						pending -= last_sp;
 +						assert(outp == out.db);
 +						memset(outp, 0, cnt);
 +					}
  					if (lseek(out.fd, pending, SEEK_CUR) ==
  					    -1)
  						err(2, "%s: seek error creating sparse file",
  						    out.name);
 -					if (force)
 -						write(out.fd, outp, 1);
 -					pending = 0;
 +					pending = last_sp = 0;
  				}
  				if (cnt)
  					nw = write(out.fd, outp, cnt);
 @@ -473,27 +490,29 @@ dd_out(int force)
  					err(1, "%s", out.name);
  				nw = 0;
  			}
 +
  			outp += nw;
  			st.bytes += nw;
 -			if ((size_t)nw == n) {
 -				if (n != out.dbsz)
 -					++st.out_part;
 -				else
 -					++st.out_full;
 -				break;
 -			}
 -			++st.out_part;
 -			if ((size_t)nw == cnt)
 -				break;
 -			if (out.flags & ISTAPE)
 -				errx(1, "%s: short write on tape device",
 -				    out.name);
 -			if (out.flags & ISCHR && !warned) {
 -				warned = 1;
 -				warnx("%s: short write on character device",
 -				    out.name);
 +
 +			if ((size_t)nw == n && n == out.dbsz)
 +				++st.out_full;
 +			else
 +				++st.out_part;
 +
 +			if ((size_t) nw != cnt) {
 +				if (out.flags & ISTAPE)
 +					errx(1, "%s: short write on tape device",
 +				    	out.name);
 +				if (out.flags & ISCHR && !warned) {
 +					warned = 1;
 +					warnx("%s: short write on character device",
 +				    	out.name);
 +				}
  			}
 -		}
 +
 +			cnt -= nw;
 +		} while (cnt != 0);
 +
  		if ((out.dbcnt -= n) < out.dbsz)
  			break;
  	}
 _______________________________________________
 svn-src-all@freebsd.org mailing list
 http://lists.freebsd.org/mailman/listinfo/svn-src-all
 To unsubscribe, send any mail to "svn-src-all-unsubscribe@freebsd.org"
 
>Unformatted:
