Patch for rlimit in 32bit binaries on x86-64

[Stepan Kasal]

Hello,
  when tracing i386 binary on x86-64, the parameters to getrlimit and
setrlimit are not displayed correctly.
Attached please find an updated version of the patch which fixes
that.
(The previous version of this patch was introducing a regression:
when tracing native binaries, RLIM64_INFINITY was displayed, instead
of RLIM_INFINITY.)

Is this suitable for checking in to the CVS?

Have a nice day,
	Stepan
-------------- next part --------------
2006-04-21  Stepan Kasal  <kasal at ucw.cz>

	Interpret struct rlimit according to current personality on x86-64.
	* linux/x86_64/syscallent.h (getrlimit, setrlimit): Use
	sys_getrlimit64 and sys_setrlimit64, respectively.
	* linux/syscall.h (sys_getrlimit64, sys_setrlimit64): Add.
	* resource.c (sys_getrlimit64, sys_setrlimit64): Compile these
	on Linux/x86-64, too.
	(sys_getrlimit, sys_setrlimit): Adapt these to handle the 32bit
	versions of the syscalls.
	Fixes RH#171394.

Index: resource.c
===================================================================
RCS file: /cvsroot/strace/strace/resource.c,v
retrieving revision 1.16
diff -u -r1.16 resource.c
--- resource.c	2 Dec 2005 04:08:27 -0000	1.16
+++ resource.c	23 May 2006 18:59:17 -0000
@@ -46,19 +46,6 @@
 #include <sys/time.h>
 #endif
 
-#if HAVE_LONG_LONG_RLIM_T
-/*
- * Hacks for systems that have a long long rlim_t
- */
-
-#define rlimit64 rlimit			/* Ugly hack */
-#define rlim64_t rlim_t			/* Ugly hack */
-#define RLIM64_INFINITY RLIM_INFINITY	/* You guessed it */
-
-#define sys_getrlimit64	sys_getrlimit
-#define sys_setrlimit64	sys_setrlimit
-#endif
-
 static const struct xlat resources[] = {
 #ifdef RLIMIT_AS
 	{ RLIMIT_AS,	"RLIMIT_AS"	},
@@ -111,7 +98,14 @@
 	{ 0,		NULL		},
 };
 
+
 #if !HAVE_LONG_LONG_RLIM_T
+
+/* On Linux/x86-64, this handler will be used only for the 32bit syscalls. */
+# if LINUX && X86_64
+#   define rlimit { int rlim_cur; int rlim_max; }
+# endif
+
 static char *
 sprintrlim(lim)
 long lim;
@@ -143,8 +137,10 @@
 		else if (umove(tcp, tcp->u_arg[1], &rlim) < 0)
 			tprintf("{...}");
 		else {
-			tprintf("{rlim_cur=%s,", sprintrlim(rlim.rlim_cur));
-			tprintf(" rlim_max=%s}", sprintrlim(rlim.rlim_max));
+			/* The cast is needed to preserve the sign
+			 * if rlim_t is smaller than long. */
+			tprintf("{rlim_cur=%s,", sprintrlim((long)rlim.rlim_cur));
+			tprintf(" rlim_max=%s}", sprintrlim((long)rlim.rlim_max));
 		}
 	}
 	return 0;
@@ -164,23 +160,42 @@
 		else if (umove(tcp, tcp->u_arg[1], &rlim) < 0)
 			tprintf("{...}");
 		else {
-			tprintf("{rlim_cur=%s,", sprintrlim(rlim.rlim_cur));
-			tprintf(" rlim_max=%s}", sprintrlim(rlim.rlim_max));
+			/* The cast is needed to preserve the sign
+			 * if rlim_t is smaller than long. */
+			tprintf("{rlim_cur=%s,", sprintrlim((long)rlim.rlim_cur));
+			tprintf(" rlim_max=%s}", sprintrlim((long)rlim.rlim_max));
 		}
 	}
 	return 0;
 }
+#   undef rlimit
 #endif /* !HAVE_LONG_LONG_RLIM_T */
 
-#if _LFS64_LARGEFILE || HAVE_LONG_LONG_RLIM_T
+
+#if _LFS64_LARGEFILE || HAVE_LONG_LONG_RLIM_T || (LINUX && X86_64)
+
+# if HAVE_LONG_LONG_RLIM_T || (LINUX && X86_64)
+#   define rlimit64 rlimit
+#   define rlim64_t rlim_t
+# endif
+# if HAVE_LONG_LONG_RLIM_T
+#   define sys_getrlimit64	sys_getrlimit
+#   define sys_setrlimit64	sys_setrlimit
+# endif
+
 static char *
 sprintrlim64(lim)
 rlim64_t lim;
 {
 	static char buf[64];
 
+# if HAVE_LONG_LONG_RLIM_T || (LINUX && X86_64)
+	if (lim == RLIM_INFINITY)
+		sprintf(buf, "RLIM_INFINITY");
+# else
 	if (lim == RLIM64_INFINITY)
 		sprintf(buf, "RLIM64_INFINITY");
+# endif
 	else if (lim > 1024 && lim%1024 == 0)
 		sprintf(buf, "%lld*1024", (long long) lim/1024);
 	else
@@ -231,7 +246,12 @@
 	}
 	return 0;
 }
-#endif /* _LFS64_LARGEFILES || HAVE_LONG_LONG_RLIM_T */
+#   undef rlimit64
+#   undef rlim64_t
+#   undef sys_getrlimit64
+#   undef sys_setrlimit64
+#endif /* _LFS64_LARGEFILE || HAVE_LONG_LONG_RLIM_T || (LINUX && X86_64) */
+
 
 #ifndef SVR4
 
Index: linux/syscall.h
===================================================================
RCS file: /cvsroot/strace/strace/linux/syscall.h,v
retrieving revision 1.46
diff -u -r1.46 syscall.h
--- linux/syscall.h	12 Jan 2006 22:34:50 -0000	1.46
+++ linux/syscall.h	23 May 2006 18:59:17 -0000
@@ -295,3 +295,7 @@
 #ifdef M68K
 int sys_cacheflush();
 #endif
+
+#if X86_64
+int sys_setrlimit64(), sys_getrlimit64();
+#endif
Index: linux/x86_64/syscallent.h
===================================================================
RCS file: /cvsroot/strace/strace/linux/x86_64/syscallent.h,v
retrieving revision 1.19
diff -u -r1.19 syscallent.h
--- linux/x86_64/syscallent.h	12 Jan 2006 22:34:50 -0000	1.19
+++ linux/x86_64/syscallent.h	23 May 2006 18:59:18 -0000
@@ -95,7 +95,7 @@
 	{ 3,	TF,	sys_chown,		"lchown"	},  /* 94 */
 	{ 1,	0,	sys_umask,		"umask"		},  /* 95 */
 	{ 2,	0,	sys_gettimeofday,	"gettimeofday"	},  /* 96 */
-	{ 2,	0,	sys_getrlimit,		"getrlimit"	},  /* 97 */
+	{ 2,	0,	sys_getrlimit64,	"getrlimit"	},  /* 97 */
 	{ 2,	0,	sys_getrusage,		"getrusage"	},  /* 98 */
 	{ 1,	0,	sys_sysinfo,		"sysinfo"	},  /* 99 */
 	{ 1,	0,	sys_times,		"times"		},  /* 100 */
@@ -158,7 +158,7 @@
 	{ 5,	0,	printargs,		"prctl"		},  /* 157 */
 	{ 2,	TP,	sys_arch_prctl,		"arch_prctl"	},  /* 158 */
 	{ 1,	0,	sys_adjtimex,		"adjtimex"	},  /* 159 */
-	{ 2,	0,	sys_setrlimit,		"setrlimit"	},  /* 160 */
+	{ 2,	0,	sys_setrlimit64,	"setrlimit"	},  /* 160 */
 	{ 1,	TF,	sys_chroot,		"chroot"	},  /* 161 */
 	{ 0,	0,	sys_sync,		"sync"		},  /* 162 */
 	{ 1,	TF,	sys_acct,		"acct"		},  /* 163 */