--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,14 @@
+2012-06-24  Bruno Haible  <bruno@clisp.org>
+
+	ptsname_r: Fix test failures on IRIX, Solaris.
+	* m4/ptsname_r.m4 (gl_PREREQ_PTSNAME_R): Test whether isatty sets
+	errno when it fails. Define ISATTY_FAILS_WITHOUT_SETTING_ERRNO
+	accordingly.
+	* lib/ptsname_r.c: Include <fcntl.h>.
+	(__ptsname_r): When isatty returned false, then on IRIX, Solaris
+	set errno if fd is invalid.
+	* tests/test-isatty.c (main): Update comments.
+
 2012-06-24  Bruno Haible  <bruno@clisp.org>
 
 	ptsname test: Extend test.

--- a/lib/ptsname_r.c
+++ b/lib/ptsname_r.c
@@ -19,6 +19,7 @@
 #include <stdlib.h>
 
 #include <errno.h>
+#include <fcntl.h>
 #include <string.h>
 #include <sys/stat.h>
 #include <unistd.h>
@@ -59,8 +60,16 @@
     }
 
   if (!__isatty (fd))
-    /* We rely on isatty to set errno properly (i.e. EBADF or ENOTTY).  */
-    return errno;
+    {
+#if ISATTY_FAILS_WITHOUT_SETTING_ERRNO && defined F_GETFL /* IRIX, Solaris */
+      /* Set errno.  */
+      if (fcntl (fd, F_GETFL) != -1)
+        errno = ENOTTY;
+#else
+      /* We rely on isatty to set errno properly (i.e. EBADF or ENOTTY).  */
+#endif
+      return errno;
+    }
 
   if (buflen < strlen (_PATH_TTY) + 3)
     {

--- a/m4/ptsname_r.m4
+++ b/m4/ptsname_r.m4
@@ -1,4 +1,4 @@
-# ptsname_r.m4 serial 2
+# ptsname_r.m4 serial 3
 dnl Copyright (C) 2010-2012 Free Software Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
@@ -46,5 +46,38 @@
 
 # Prerequisites of lib/ptsname.c.
 AC_DEFUN([gl_PREREQ_PTSNAME_R], [
-  :
+  AC_REQUIRE([AC_CANONICAL_HOST]) dnl for cross-compiles
+  AC_CACHE_CHECK([whether isatty sets errno when it fails],
+    [gl_cv_func_isatty_sets_errno],
+    [AC_RUN_IFELSE(
+       [AC_LANG_PROGRAM(
+          [[#include <errno.h>
+            #if (defined _WIN32 || defined __WIN32__) && ! defined __CYGWIN__
+            # include <io.h>
+            #else
+            # include <unistd.h>
+            #endif
+          ]],
+          [[errno = 0;
+            isatty (-1);
+            return errno == 0;
+          ]])
+       ],
+       [gl_cv_func_isatty_sets_errno=yes],
+       [gl_cv_func_isatty_sets_errno=no],
+       [case "$host_os" in
+          irix* | solaris* | mingw*)
+            gl_cv_func_isatty_sets_errno="guessing no";;
+          *)
+            gl_cv_func_isatty_sets_errno="guessing yes";;
+        esac
+       ])
+    ])
+  case "$gl_cv_func_isatty_sets_errno" in
+    *yes) ;;
+    *)
+      AC_DEFINE([ISATTY_FAILS_WITHOUT_SETTING_ERRNO], [1],
+        [Define to 1 if isatty() may fail without setting errno.])
+      ;;
+  esac
 ])

--- a/tests/test-isatty.c
+++ b/tests/test-isatty.c
@@ -45,14 +45,14 @@
     errno = 0;
     ASSERT (isatty (-1) == 0);
     ASSERT (errno == EBADF
-            || errno == 0 /* seen on Solaris 10 */
+            || errno == 0 /* seen on IRIX 6.5, Solaris 10 */
            );
   }
   {
     errno = 0;
     ASSERT (isatty (99) == 0);
     ASSERT (errno == EBADF
-            || errno == 0 /* seen on Solaris 10 */
+            || errno == 0 /* seen on IRIX 6.5, Solaris 10 */
            );
   }