From imura@ryu16.org  Fri Nov 28 09:42:20 2003
Return-Path: <imura@ryu16.org>
Received: from mx1.FreeBSD.org (mx1.freebsd.org [216.136.204.125])
	by hub.freebsd.org (Postfix) with ESMTP
	id 37DE016A4CE; Fri, 28 Nov 2003 09:42:20 -0800 (PST)
Received: from mail.ryu16.org (YahooBB219005044050.bbtec.net [219.5.44.50])
	by mx1.FreeBSD.org (Postfix) with ESMTP
	id A170A43F85; Fri, 28 Nov 2003 09:42:17 -0800 (PST)
	(envelope-from imura@ryu16.org)
Received: from redeye.xt.ryu16.org (localhost [127.0.0.1])
	by mail.ryu16.org (8.12.9p1/8.12.9) with ESMTP id hASHgGmh053767;
	Sat, 29 Nov 2003 02:42:16 +0900 (JST)
	(envelope-from imura@redeye.xt.ryu16.org)
Received: (from imura@localhost)
	by redeye.xt.ryu16.org (8.12.9p1/8.12.9/Submit) id hASHgFIX053766;
	Sat, 29 Nov 2003 02:42:15 +0900 (JST)
	(envelope-from imura)
Message-Id: <200311281742.hASHgFIX053766@redeye.xt.ryu16.org>
Date: Sat, 29 Nov 2003 02:42:15 +0900 (JST)
From: Ryuichiro Imura <imura@ryu16.org>
Reply-To: Ryuichiro Imura <imura@ryu16.org>
To: FreeBSD-gnats-submit@freebsd.org
Cc: Max Khon <fjoe@freebsd.org>
Subject: msdosfs long file name matching should be case insensitve
X-Send-Pr-Version: 3.113
X-GNATS-Notify:

>Number:         59765
>Category:       kern
>Synopsis:       msdosfs long file name matching should be case insensitve
>Confidential:   no
>Severity:       serious
>Priority:       high
>Responsible:    fjoe
>State:          closed
>Quarter:        
>Keywords:       
>Date-Required:  
>Class:          sw-bug
>Submitter-Id:   current-users
>Arrival-Date:   Fri Nov 28 09:50:06 PST 2003
>Closed-Date:    Mon Dec 08 00:34:15 PST 2003
>Last-Modified:  Mon Dec 08 00:34:15 PST 2003
>Originator:     Ryuichiro Imura
>Release:        FreeBSD 5.2-BETA i186
>Organization:
>Environment:
System: FreeBSD bluewind.xt.ryu16.org 5.2-BETA FreeBSD 5.2-BETA #22: Wed Nov 26 02:09:08 JST 2003     root@bluewind.xt.ryu16.org:/usr/obj/usr/src/sys/BLUE  i386

>Description:

Although msdosfs's long file name matching should be case insensitive,
current code has been changed to do in case sensitive after kiconv(3) changes.
Sorry for that.
Just FYI, the first problem was reported at FreeBSD-users-jp@jp.freebsd.org 
list.

>How-To-Repeat:
>Fix:

Fixing this problem should be very easy if nobody using kiconv feature, but
sad to say that iconv_xlat16.c has a bug when using KICONV_FROM_XX with UCS-2,
This patch fixes this problem and iconv_xlat16.c's bug.

Index: sbin/mount_msdosfs/mount_msdosfs.c
===================================================================
RCS file: /home/ncvs/src/sbin/mount_msdosfs/mount_msdosfs.c,v
retrieving revision 1.31
diff -u -r1.31 mount_msdosfs.c
--- sbin/mount_msdosfs/mount_msdosfs.c	23 Oct 2003 16:09:20 -0000	1.31
+++ sbin/mount_msdosfs/mount_msdosfs.c	24 Nov 2003 06:34:07 -0000
@@ -339,7 +339,7 @@
 	if ((args->cs_win = malloc(ICONV_CSNMAXLEN)) == NULL)
 		return (-1);
 	strncpy(args->cs_win, ENCODING_UNICODE, ICONV_CSNMAXLEN);
-	error = kiconv_add_xlat16_cspair(args->cs_win, args->cs_local, 0);
+	error = kiconv_add_xlat16_cspair(args->cs_win, args->cs_local, KICONV_FROM_LOWER);
 	if (error)
 		return (-1);
 	error = kiconv_add_xlat16_cspair(args->cs_local, args->cs_win, 0);
Index: sys/fs/msdosfs/msdosfs_conv.c
===================================================================
RCS file: /home/ncvs/src/sys/fs/msdosfs/msdosfs_conv.c,v
retrieving revision 1.34
diff -u -r1.34 msdosfs_conv.c
--- sys/fs/msdosfs/msdosfs_conv.c	26 Sep 2003 20:26:22 -0000	1.34
+++ sys/fs/msdosfs/msdosfs_conv.c	28 Nov 2003 17:37:35 -0000
@@ -800,10 +800,12 @@
 
 	for (np = dirbuf.d_name; unlen > 0 && len > 0;) {
 		/*
-		 * Should comparison be case insensitive?
+		 * Comparison must be case insensitive, because FAT disallows
+		 * to look up or create files case sensitive even when
+		 * it's a long file name.
 		 */
-		c1 = unix2winchr((const u_char **)&np, (size_t *)&len, 0, pmp);
-		c2 = unix2winchr(&un, (size_t *)&unlen, 0, pmp);
+		c1 = unix2winchr((const u_char **)&np, (size_t *)&len, LCASE_BASE, pmp);
+		c2 = unix2winchr(&un, (size_t *)&unlen, LCASE_BASE, pmp);
 		if (c1 != c2)
 			return -2;
 	}
Index: sys/libkern/iconv_xlat16.c
===================================================================
RCS file: /home/ncvs/src/sys/libkern/iconv_xlat16.c,v
retrieving revision 1.1
diff -u -r1.1 iconv_xlat16.c
--- sys/libkern/iconv_xlat16.c	26 Sep 2003 20:26:24 -0000	1.1
+++ sys/libkern/iconv_xlat16.c	24 Nov 2003 17:32:09 -0000
@@ -96,11 +96,11 @@
 	struct iconv_xlat16 *dp = (struct iconv_xlat16*)d2p;
 	const char *src;
 	char *dst;
-	int ret = 0;
+	int nullin, ret = 0;
 	size_t in, on, ir, or, inlen;
 	uint32_t code;
 	u_char u, l;
-	u_int16_t c1, c2;
+	uint16_t c1, c2;
 
 	if (inbuf == NULL || *inbuf == NULL || outbuf == NULL || *outbuf == NULL)
 		return (0);
@@ -146,7 +146,8 @@
 			}
 		}
 
-		if ((inlen == 1) && (code & XLAT16_ACCEPT_NULL_IN)) {
+		nullin = (code & XLAT16_ACCEPT_NULL_IN) ? 1 : 0;
+		if (inlen == 1 && nullin) {
 			/*
 			 * XLAT16_ACCEPT_NULL_IN requires inbuf has 2byte
 			 */
@@ -157,6 +158,14 @@
 		/*
 		 * now start translation
 		 */
+		if ((casetype == KICONV_FROM_LOWER && code & XLAT16_HAS_FROM_LOWER_CASE) ||
+		    (casetype == KICONV_FROM_UPPER && code & XLAT16_HAS_FROM_UPPER_CASE)) {
+			c2 = (u_char)(code >> 16);
+			c1 = c2 & 0x80 ? 0x100 : 0;
+			c2 = c2 & 0x80 ? c2 & 0x7f : c2;
+			code = dp->d_table[c1][c2];
+		}
+
 		u = (u_char)(code >> 8);
 		l = (u_char)code;
 
@@ -184,9 +193,6 @@
 			if ((casetype == KICONV_LOWER && code & XLAT16_HAS_LOWER_CASE) ||
 			    (casetype == KICONV_UPPER && code & XLAT16_HAS_UPPER_CASE))
 				*dst++ = (u_char)(code >> 16);
-			else if ((casetype == KICONV_FROM_LOWER && code & XLAT16_HAS_FROM_LOWER_CASE) ||
-				 (casetype == KICONV_FROM_UPPER && code & XLAT16_HAS_FROM_UPPER_CASE))
-				*dst++ = dp->d_table[0][(u_char)(code >> 16)];
 			else
 				*dst++ = l;
 			or--;
@@ -197,8 +203,7 @@
 			 * there is a case that inbuf char is a single
 			 * byte char while inlen == 2
 			 */
-			if ((u_char)*(src+1) == 0 &&
-			    (code & XLAT16_ACCEPT_NULL_IN) == 0 ) {
+			if ((u_char)*(src+1) == 0 && !nullin ) {
 				src++;
 				ir--;
 			} else {
>Release-Note:
>Audit-Trail:

From: "R. Imura" <imura@ryu16.org>
To: FreeBSD-gnats-submit@FreeBSD.org
Cc: Max Khon <fjoe@FreeBSD.org>
Subject: Re: kern/59765: msdosfs long file name matching should be case insensitve
Date: Mon, 1 Dec 2003 23:43:13 +0900

 On Sat, Nov 29, 2003 at 02:42:15AM +0900, Ryuichiro Imura wrote:
 > Although msdosfs's long file name matching should be case insensitive,
 > current code has been changed to do in case sensitive after kiconv(3) changes.
 > Sorry for that.
 
 I noticed the previous patch is insufficient.
 Since convertion tables are shared by other file systems, if you mount
 msdosfs after mount cd9660, there aren't any upper/lower tables.
 We need to create upper/lower table for every file systems and need to
 reserve it for msdosfs.  Now I'd like to add a new function which wraps
 upper/lower operations, and replace existent kiconv functions with it.
 
 - R. Imura
 
 Index: lib/libkiconv/Makefile
 ===================================================================
 RCS file: /home/ncvs/src/lib/libkiconv/Makefile,v
 retrieving revision 1.1
 diff -u -r1.1 Makefile
 --- lib/libkiconv/Makefile	26 Sep 2003 20:26:20 -0000	1.1
 +++ lib/libkiconv/Makefile	1 Dec 2003 14:41:21 -0000
 @@ -10,6 +10,7 @@
  MAN=		kiconv.3
  
  MLINKS+=	kiconv.3 kiconv_add_xlat16_cspair.3 \
 +		kiconv.3 kiconv_add_xlat16_cspairs.3 \
  		kiconv.3 kiconv_add_xlat16_table.3
  
  CFLAGS+=	-I${.CURDIR}/../../sys
 Index: lib/libkiconv/kiconv.3
 ===================================================================
 RCS file: /home/ncvs/src/lib/libkiconv/kiconv.3,v
 retrieving revision 1.2
 diff -u -r1.2 kiconv.3
 --- lib/libkiconv/kiconv.3	5 Oct 2003 13:39:28 -0000	1.2
 +++ lib/libkiconv/kiconv.3	30 Nov 2003 23:28:05 -0000
 @@ -30,6 +30,7 @@
  .Os
  .Sh NAME
  .Nm kiconv_add_xlat16_cspair ,
 +.Nm kiconv_add_xlat16_cspairs ,
  .Nm kiconv_add_xlat16_table
  .Nd Kernel side iconv library
  .Sh LIBRARY
 @@ -43,6 +44,11 @@
  .Fa "int flag"
  .Fc
  .Ft int
 +.Fo kiconv_add_xlat16_cspairs
 +.Fa "const char *foreigncode"
 +.Fa "const char *localcode"
 +.Fc
 +.Ft int
  .Fo kiconv_add_xlat16_table
  .Fa "const char *tocode"
  .Fa "const char *fromcode"
 @@ -92,6 +98,17 @@
  .Pp
  A tolower/toupper conversion is limited to single-byte characters.
  .Pp 
 +.Fn kiconv_add_xlat16_cspairs
 +defines two conversion tables which are from
 +.Ar localcode
 +to
 +.Ar foreigncode
 +and from
 +.Ar foreigncode
 +to
 +.Ar localcode .
 +This conversion tables also contain both of tolower and toupper tables.
 +.Pp
  .Fn kiconv_add_xlat16_table
  defines a conversion table directly pointed by
  .Ar data
 Index: lib/libkiconv/xlat16_iconv.c
 ===================================================================
 RCS file: /home/ncvs/src/lib/libkiconv/xlat16_iconv.c,v
 retrieving revision 1.1
 diff -u -r1.1 xlat16_iconv.c
 --- lib/libkiconv/xlat16_iconv.c	26 Sep 2003 20:26:20 -0000	1.1
 +++ lib/libkiconv/xlat16_iconv.c	30 Nov 2003 23:37:10 -0000
 @@ -113,6 +113,23 @@
  	return (-1);
  }
  
 +int
 +kiconv_add_xlat16_cspairs(const char *foreigncode, const char *localcode)
 +{
 +	int error;
 +
 +	error = kiconv_add_xlat16_cspair(foreigncode, localcode,
 +	    KICONV_FROM_LOWER | KICONV_FROM_UPPER);
 +	if (error)
 +		return (error);
 +	error = kiconv_add_xlat16_cspair(localcode, foreigncode,
 +	    KICONV_LOWER | KICONV_UPPER);
 +	if (error)
 +		return (error);
 +	
 +	return (0);
 +}
 +
  static struct xlat16_table
  kiconv_xlat16_open(const char *tocode, const char *fromcode, int lcase)
  {
 Index: sbin/mount_cd9660/mount_cd9660.c
 ===================================================================
 RCS file: /home/ncvs/src/sbin/mount_cd9660/mount_cd9660.c,v
 retrieving revision 1.25
 diff -u -r1.25 mount_cd9660.c
 --- sbin/mount_cd9660/mount_cd9660.c	4 Nov 2003 21:04:14 -0000	1.25
 +++ sbin/mount_cd9660/mount_cd9660.c	30 Nov 2003 23:28:06 -0000
 @@ -255,10 +255,7 @@
  	strncpy(args->cs_disk, ENCODING_UNICODE, ICONV_CSNMAXLEN);
  	strncpy(args->cs_local, kiconv_quirkcs(localcs, KICONV_VENDOR_MICSFT),
  	    ICONV_CSNMAXLEN);
 -	error = kiconv_add_xlat16_cspair(args->cs_local, args->cs_disk, 0);
 -	if (error)
 -		return (-1);
 -	error = kiconv_add_xlat16_cspair(args->cs_disk, args->cs_local, 0);
 +	error = kiconv_add_xlat16_cspairs(args->cs_disk, args->cs_local);
  	if (error)
  		return (-1);
  
 Index: sbin/mount_msdosfs/mount_msdosfs.c
 ===================================================================
 RCS file: /home/ncvs/src/sbin/mount_msdosfs/mount_msdosfs.c,v
 retrieving revision 1.31
 diff -u -r1.31 mount_msdosfs.c
 --- sbin/mount_msdosfs/mount_msdosfs.c	23 Oct 2003 16:09:20 -0000	1.31
 +++ sbin/mount_msdosfs/mount_msdosfs.c	30 Nov 2003 23:28:06 -0000
 @@ -339,17 +339,11 @@
  	if ((args->cs_win = malloc(ICONV_CSNMAXLEN)) == NULL)
  		return (-1);
  	strncpy(args->cs_win, ENCODING_UNICODE, ICONV_CSNMAXLEN);
 -	error = kiconv_add_xlat16_cspair(args->cs_win, args->cs_local, 0);
 -	if (error)
 -		return (-1);
 -	error = kiconv_add_xlat16_cspair(args->cs_local, args->cs_win, 0);
 +	error = kiconv_add_xlat16_cspairs(args->cs_win, args->cs_local);
  	if (error)
  		return (-1);
  	if (args->cs_dos) {
 -		error = kiconv_add_xlat16_cspair(args->cs_dos, args->cs_local, KICONV_FROM_UPPER);
 -		if (error)
 -			return (-1);
 -		error = kiconv_add_xlat16_cspair(args->cs_local, args->cs_dos, KICONV_LOWER);
 +		error = kiconv_add_xlat16_cspairs(args->cs_dos, args->cs_local);
  		if (error)
  			return (-1);
  	} else {
 Index: sbin/mount_ntfs/mount_ntfs.c
 ===================================================================
 RCS file: /home/ncvs/src/sbin/mount_ntfs/mount_ntfs.c,v
 retrieving revision 1.9
 diff -u -r1.9 mount_ntfs.c
 --- sbin/mount_ntfs/mount_ntfs.c	26 Sep 2003 20:26:21 -0000	1.9
 +++ sbin/mount_ntfs/mount_ntfs.c	30 Nov 2003 23:28:06 -0000
 @@ -275,10 +275,7 @@
  	if ((pargs->cs_ntfs = malloc(ICONV_CSNMAXLEN)) == NULL)
  		return (-1);
  	strncpy(pargs->cs_ntfs, ENCODING_UNICODE, ICONV_CSNMAXLEN);
 -	error = kiconv_add_xlat16_cspair(pargs->cs_local, pargs->cs_ntfs, 0);
 -	if (error)
 -		return (-1);
 -	error = kiconv_add_xlat16_cspair(pargs->cs_ntfs, pargs->cs_local, 0);
 +	error = kiconv_add_xlat16_cspairs(pargs->cs_ntfs, pargs->cs_local);
  	if (error)
  		return (-1);
  
 Index: sbin/mount_udf/mount_udf.c
 ===================================================================
 RCS file: /home/ncvs/src/sbin/mount_udf/mount_udf.c,v
 retrieving revision 1.8
 diff -u -r1.8 mount_udf.c
 --- sbin/mount_udf/mount_udf.c	24 Nov 2003 16:14:32 -0000	1.8
 +++ sbin/mount_udf/mount_udf.c	30 Nov 2003 23:28:07 -0000
 @@ -172,14 +172,9 @@
  		return (-1);
  	strncpy(*cs_disk, ENCODING_UNICODE, ICONV_CSNMAXLEN);
  	strncpy(*cs_local, localcs, ICONV_CSNMAXLEN);
 -	error = kiconv_add_xlat16_cspair(*cs_local, *cs_disk, 0);
 +	error = kiconv_add_xlat16_cspairs(*cs_disk, *cs_local);
  	if (error)
  		return (-1);
 -#if 0
 -	error = kiconv_add_xlat16_cspair(*cs_disk, *cs_local, 0);
 -	if (error)
 -		return (-1);
 -#endif
  
  	return (0);
  }
 Index: sys/fs/msdosfs/msdosfs_conv.c
 ===================================================================
 RCS file: /home/ncvs/src/sys/fs/msdosfs/msdosfs_conv.c,v
 retrieving revision 1.34
 diff -u -r1.34 msdosfs_conv.c
 --- sys/fs/msdosfs/msdosfs_conv.c	26 Sep 2003 20:26:22 -0000	1.34
 +++ sys/fs/msdosfs/msdosfs_conv.c	30 Nov 2003 23:28:06 -0000
 @@ -800,10 +800,12 @@
  
  	for (np = dirbuf.d_name; unlen > 0 && len > 0;) {
  		/*
 -		 * Should comparison be case insensitive?
 +		 * Comparison must be case insensitive, because FAT disallows
 +		 * to look up or create files in case sensitive even when
 +		 * it's a long file name.
  		 */
 -		c1 = unix2winchr((const u_char **)&np, (size_t *)&len, 0, pmp);
 -		c2 = unix2winchr(&un, (size_t *)&unlen, 0, pmp);
 +		c1 = unix2winchr((const u_char **)&np, (size_t *)&len, LCASE_BASE, pmp);
 +		c2 = unix2winchr(&un, (size_t *)&unlen, LCASE_BASE, pmp);
  		if (c1 != c2)
  			return -2;
  	}
 Index: sys/libkern/iconv_xlat16.c
 ===================================================================
 RCS file: /home/ncvs/src/sys/libkern/iconv_xlat16.c,v
 retrieving revision 1.1
 diff -u -r1.1 iconv_xlat16.c
 --- sys/libkern/iconv_xlat16.c	26 Sep 2003 20:26:24 -0000	1.1
 +++ sys/libkern/iconv_xlat16.c	24 Nov 2003 17:32:09 -0000
 @@ -96,11 +96,11 @@
  	struct iconv_xlat16 *dp = (struct iconv_xlat16*)d2p;
  	const char *src;
  	char *dst;
 -	int ret = 0;
 +	int nullin, ret = 0;
  	size_t in, on, ir, or, inlen;
  	uint32_t code;
  	u_char u, l;
 -	u_int16_t c1, c2;
 +	uint16_t c1, c2;
  
  	if (inbuf == NULL || *inbuf == NULL || outbuf == NULL || *outbuf == NULL)
  		return (0);
 @@ -146,7 +146,8 @@
  			}
  		}
  
 -		if ((inlen == 1) && (code & XLAT16_ACCEPT_NULL_IN)) {
 +		nullin = (code & XLAT16_ACCEPT_NULL_IN) ? 1 : 0;
 +		if (inlen == 1 && nullin) {
  			/*
  			 * XLAT16_ACCEPT_NULL_IN requires inbuf has 2byte
  			 */
 @@ -157,6 +158,14 @@
  		/*
  		 * now start translation
  		 */
 +		if ((casetype == KICONV_FROM_LOWER && code & XLAT16_HAS_FROM_LOWER_CASE) ||
 +		    (casetype == KICONV_FROM_UPPER && code & XLAT16_HAS_FROM_UPPER_CASE)) {
 +			c2 = (u_char)(code >> 16);
 +			c1 = c2 & 0x80 ? 0x100 : 0;
 +			c2 = c2 & 0x80 ? c2 & 0x7f : c2;
 +			code = dp->d_table[c1][c2];
 +		}
 +
  		u = (u_char)(code >> 8);
  		l = (u_char)code;
  
 @@ -184,9 +193,6 @@
  			if ((casetype == KICONV_LOWER && code & XLAT16_HAS_LOWER_CASE) ||
  			    (casetype == KICONV_UPPER && code & XLAT16_HAS_UPPER_CASE))
  				*dst++ = (u_char)(code >> 16);
 -			else if ((casetype == KICONV_FROM_LOWER && code & XLAT16_HAS_FROM_LOWER_CASE) ||
 -				 (casetype == KICONV_FROM_UPPER && code & XLAT16_HAS_FROM_UPPER_CASE))
 -				*dst++ = dp->d_table[0][(u_char)(code >> 16)];
  			else
  				*dst++ = l;
  			or--;
 @@ -197,8 +203,7 @@
  			 * there is a case that inbuf char is a single
  			 * byte char while inlen == 2
  			 */
 -			if ((u_char)*(src+1) == 0 &&
 -			    (code & XLAT16_ACCEPT_NULL_IN) == 0 ) {
 +			if ((u_char)*(src+1) == 0 && !nullin ) {
  				src++;
  				ir--;
  			} else {
 Index: sys/sys/iconv.h
 ===================================================================
 RCS file: /home/ncvs/src/sys/sys/iconv.h,v
 retrieving revision 1.7
 diff -u -r1.7 iconv.h
 --- sys/sys/iconv.h	5 Nov 2003 06:27:40 -0000	1.7
 +++ sys/sys/iconv.h	30 Nov 2003 23:35:36 -0000
 @@ -92,6 +103,7 @@
  
  int   kiconv_add_xlat_table(const char *, const char *, const u_char *);
  int   kiconv_add_xlat16_cspair(const char *, const char *, int);
 +int   kiconv_add_xlat16_cspairs(const char *, const char *);
  int   kiconv_add_xlat16_table(const char *, const char *, const void *, int);
  const char *kiconv_quirkcs(const char *, int);
  
Responsible-Changed-From-To: freebsd-bugs->fjoe 
Responsible-Changed-By: fjoe 
Responsible-Changed-When: Tue Dec 2 17:54:44 PST 2003 
Responsible-Changed-Why:  
I'll take this. 

http://www.freebsd.org/cgi/query-pr.cgi?pr=59765 
State-Changed-From-To: open->closed 
State-Changed-By: fjoe 
State-Changed-When: Mon Dec 8 00:34:01 PST 2003 
State-Changed-Why:  
Committed, thanks! 

http://www.freebsd.org/cgi/query-pr.cgi?pr=59765 
>Unformatted:
