Update setmode module from NetBSD

Merge some interesting changes.

src/setmode.c

@@ -1,3 +1,5 @@
+/*	$NetBSD: setmode.c,v 1.34 2012/06/25 22:32:43 abs Exp $	*/
+
 /*
  * Copyright (c) 1989, 1993, 1994
  *	The Regents of the University of California.  All rights reserved.
@@ -13,7 +15,7 @@
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
+ * 3. Neither the name of the University nor the names of its contributors
  *    may be used to endorse or promote products derived from this software
  *    without specific prior written permission.
  *
@@ -30,19 +32,23 @@
  * SUCH DAMAGE.
  */
 
-#if defined(LIBC_SCCS) && !defined(lint)
-static char sccsid[] = "@(#)setmode.c	8.2 (Berkeley) 3/25/94";
-#endif /* LIBC_SCCS and not lint */
 #include <sys/cdefs.h>
-__FBSDID("$FreeBSD: src/lib/libc/gen/setmode.c,v 1.11 2007/01/09 00:27:55 imp Exp $");
+#if defined(LIBC_SCCS) && !defined(lint)
+#if 0
+static char sccsid[] = "@(#)setmode.c	8.2 (Berkeley) 3/25/94";
+#else
+__RCSID("$NetBSD: setmode.c,v 1.34 2012/06/25 22:32:43 abs Exp $");
+#endif
+#endif /* LIBC_SCCS and not lint */
 
 #include <sys/types.h>
 #include <sys/stat.h>
 
 #include <ctype.h>
+#include <errno.h>
 #include <signal.h>
-#include <stddef.h>
 #include <stdlib.h>
+#include <limits.h>
 #include <unistd.h>
 
 #ifdef SETMODE_DEBUG
@@ -64,7 +70,7 @@ typedef struct bitcmd {
 #define	CMD2_OBITS	0x08
 #define	CMD2_UBITS	0x10
 
-static BITCMD	*addcmd(BITCMD *, int, int, int, u_int);
+static BITCMD	*addcmd(BITCMD *, mode_t, mode_t, mode_t, mode_t);
 static void	 compress_mode(BITCMD *);
 #ifdef SETMODE_DEBUG
 static void	 dumpmode(BITCMD *);
@@ -144,38 +150,38 @@ common:			if (set->cmd2 & CMD2_CLR) {
 		}
 }
 
-#define	ADDCMD(a, b, c, d)						\
+#define	ADDCMD(a, b, c, d) do {						\
 	if (set >= endset) {						\
 		BITCMD *newset;						\
 		setlen += SET_LEN_INCR;					\
 		newset = realloc(saveset, sizeof(BITCMD) * setlen);	\
-		if (!newset) {						\
-			if (saveset)					\
-				free(saveset);				\
-			saveset = NULL;					\
-			return (NULL);					\
-		}							\
+		if (newset == NULL)					\
+			goto out;					\
 		set = newset + (set - saveset);				\
 		saveset = newset;					\
 		endset = newset + (setlen - 2);				\
 	}								\
-	set = addcmd(set, (a), (b), (c), (d))
+	set = addcmd(set, (mode_t)(a), (mode_t)(b), (mode_t)(c), (d));	\
+} while (/*CONSTCOND*/0)
 
 #define	STANDARD_BITS	(S_ISUID|S_ISGID|S_IRWXU|S_IRWXG|S_IRWXO)
 
 void *
 setmode(const char *p)
 {
-	int perm, who;
+	int serrno;
 	char op, *ep;
 	BITCMD *set, *saveset, *endset;
-	sigset_t sigset, sigoset;
-	mode_t mask;
-	int equalopdone=0, permXbits, setlen;
-	long perml;
+	sigset_t signset, sigoset;
+	mode_t mask, perm, permXbits, who;
+	long lval;
+	int equalopdone = 0;	/* pacify gcc */
+	int setlen;
 
-	if (!*p)
-		return (NULL);
+	if (!*p) {
+		errno = EINVAL;
+		return NULL;
+	}
 
 	/*
 	 * Get a copy of the mask for the permissions that are mask relative.
@@ -183,8 +189,8 @@ setmode(const char *p)
 	 * the caller is opening files inside a signal handler, protect them
 	 * as best we can.
 	 */
-	sigfillset(&sigset);
-	(void)sigprocmask(SIG_BLOCK, &sigset, &sigoset);
+	sigfillset(&signset);
+	(void)sigprocmask(SIG_BLOCK, &signset, &sigoset);
 	(void)umask(mask = umask(0));
 	mask = ~mask;
 	(void)sigprocmask(SIG_SETMASK, &sigoset, NULL);
@@ -201,12 +207,19 @@ setmode(const char *p)
 	 * or illegal bits.
 	 */
 	if (isdigit((unsigned char)*p)) {
-		perml = strtol(p, &ep, 8);
-		if (*ep || perml < 0 || perml & ~(STANDARD_BITS|S_ISTXT)) {
-			free(saveset);
-			return (NULL);
+		errno = 0;
+		lval = strtol(p, &ep, 8);
+		if (*ep) {
+			errno = EINVAL;
+			goto out;
 		}
-		perm = (mode_t)perml;
+		if (errno == ERANGE && (lval == LONG_MAX || lval == LONG_MIN))
+			goto out;
+		if (lval & ~(STANDARD_BITS|S_ISTXT)) {
+			errno = EINVAL;
+			goto out;
+		}
+		perm = (mode_t)lval;
 		ADDCMD('=', (STANDARD_BITS|S_ISTXT), perm, mask);
 		set->cmd = 0;
 		return (saveset);
@@ -238,8 +251,8 @@ setmode(const char *p)
 		}
 
 getop:		if ((op = *p++) != '+' && op != '-' && op != '=') {
-			free(saveset);
-			return (NULL);
+			errno = EINVAL;
+			goto out;
 		}
 		if (op == '=')
 			equalopdone = 0;
@@ -251,13 +264,19 @@ getop:		if ((op = *p++) != '+' && op != '-' && op != '=') {
 				perm |= S_IRUSR|S_IRGRP|S_IROTH;
 				break;
 			case 's':
-				/* If only "other" bits ignore set-id. */
-				if (!who || who & ~S_IRWXO)
+				/*
+				 * If specific bits where requested and
+				 * only "other" bits ignore set-id.
+				 */
+				if (who == 0 || (who & ~S_IRWXO))
 					perm |= S_ISUID|S_ISGID;
 				break;
 			case 't':
-				/* If only "other" bits ignore sticky. */
-				if (!who || who & ~S_IRWXO) {
+				/*
+				 * If specific bits where requested and
+				 * only "other" bits ignore set-id.
+				 */
+				if (who == 0 || (who & ~S_IRWXO)) {
 					who |= S_ISTXT;
 					perm |= S_ISTXT;
 				}
@@ -328,10 +347,15 @@ apply:		if (!*p)
 	dumpmode(saveset);
 #endif
 	return (saveset);
+out:
+	serrno = errno;
+	free(saveset);
+	errno = serrno;
+	return NULL;
 }
 
 static BITCMD *
-addcmd(BITCMD *set, int op, int who, int oparg, u_int mask)
+addcmd(BITCMD *set, mode_t op, mode_t who, mode_t oparg, mode_t mask)
 {
 	switch (op) {
 	case '=':