--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,17 @@
+2011-07-08  Bruno Haible  <bruno@clisp.org>
+
+	pthread_sigmask: Work around bug in single-threaded implementation.
+	* m4/pthread_sigmask.m4 (gl_FUNC_PTHREAD_SIGMASK): Test for the
+	FreeBSD, HP-UX, Solaris bug.
+	(gl_PREREQ_PTHREAD_SIGMASK): New macro.
+	* lib/pthread_sigmask.c: Include <stddef.h>.
+	(pthread_sigmask): If HAVE_PTHREAD_SIGMASK, define as a wrapper around
+	the system's pthread_sigmask function.
+	* modules/pthread_sigmask (configure.ac): Invoke
+	gl_PREREQ_PTHREAD_SIGMASK.
+	* doc/posix-functions/pthread_sigmask.texi: Mention bug on FreeBSD,
+	HP-UX, Solaris.
+
 2011-07-08  Eric Blake  <eblake@redhat.com>
 
 	test-sigprocmask: avoid compiler warning

--- a/doc/posix-functions/pthread_sigmask.texi
+++ b/doc/posix-functions/pthread_sigmask.texi
@@ -15,6 +15,10 @@
 This function is declared in @code{<pthread.h>} instead of @code{<signal.h>}
 on some platforms:
 MacOS X 10.3, FreeBSD 6.4, OpenBSD 3.8, OSF/1 4.0, Solaris 2.6.
+@item
+This function does nothing and always returns 0 in programs that are not
+linked with @code{-lpthread} on some platforms:
+FreeBSD 6.4, HP-UX 11.31, Solaris 9.
 @end itemize
 
 Portability problems not fixed by Gnulib:

--- a/lib/pthread_sigmask.c
+++ b/lib/pthread_sigmask.c
@@ -20,10 +20,32 @@
 #include <signal.h>
 
 #include <errno.h>
+#include <stddef.h>
 
 int
 pthread_sigmask (int how, const sigset_t *new_mask, sigset_t *old_mask)
+#undef pthread_sigmask
 {
+#if HAVE_PTHREAD_SIGMASK
+  int ret = pthread_sigmask (how, new_mask, old_mask);
+# if PTHREAD_SIGMASK_INEFFECTIVE
+  if (ret == 0)
+    {
+      /* Detect whether pthread_sigmask is currently ineffective.
+         Don't cache the information: libpthread.so could be dynamically
+         loaded after the program started and after pthread_sigmask was
+         called for the first time.  */
+      if (pthread_sigmask (1729, NULL, NULL) == 0)
+        {
+          /* pthread_sigmask is currently ineffective.  The program is not
+             linked to -lpthread.  So use sigprocmask instead.  */
+          return (sigprocmask (how, new_mask, old_mask) < 0 ? errno : 0);
+        }
+    }
+# endif
+  return ret;
+#else
   int ret = sigprocmask (how, new_mask, old_mask);
   return (ret < 0 ? errno : 0);
+#endif
 }

--- a/m4/pthread_sigmask.m4
+++ b/m4/pthread_sigmask.m4
@@ -1,4 +1,4 @@
-# pthread_sigmask.m4 serial 7
+# pthread_sigmask.m4 serial 8
 dnl Copyright (C) 2011 Free Software Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
@@ -74,4 +74,59 @@
   dnl We don't need a variable LTLIB_PTHREAD_SIGMASK, because when
   dnl "$gl_threads_api" = posix, $LTLIBMULTITHREAD and $LIBMULTITHREAD are the
   dnl same: either both empty or both "-lpthread".
+
+  dnl Now test for some bugs in the system function.
+  if test $HAVE_PTHREAD_SIGMASK = 1; then
+    AC_REQUIRE([AC_PROG_CC])
+    AC_REQUIRE([AC_CANONICAL_HOST]) dnl for cross-compiles
+
+    dnl On FreeBSD 6.4, HP-UX 11.31, Solaris 9, in programs that are not linked
+    dnl with -lpthread, the pthread_sigmask() function always returns 0 and has
+    dnl no effect.
+    if test -z "$LIB_PTHREAD_SIGMASK"; then
+      AC_CACHE_CHECK([whether pthread_sigmask works without -lpthread],
+        [gl_cv_func_pthread_sigmask_in_libc_works],
+        [
+          AC_RUN_IFELSE(
+            [AC_LANG_SOURCE([[
+#include <pthread.h>
+#include <signal.h>
+#include <stddef.h>
+int main ()
+{
+  sigset_t set;
+  sigemptyset (&set);
+  return pthread_sigmask (1729, &set, NULL) != 0;
+}]])],
+            [gl_cv_func_pthread_sigmask_in_libc_works=no],
+            [gl_cv_func_pthread_sigmask_in_libc_works=yes],
+            [
+changequote(,)dnl
+             case "$host_os" in
+               freebsd* | hpux* | solaris | solaris2.[2-9]*)
+                 gl_cv_func_pthread_sigmask_in_libc_works="guessing no";;
+               *)
+                 gl_cv_func_pthread_sigmask_in_libc_works="guessing yes";;
+             esac
+changequote([,])dnl
+            ])
+        ])
+      case "$gl_cv_func_pthread_sigmask_in_libc_works" in
+        *no)
+          REPLACE_PTHREAD_SIGMASK=1
+          AC_DEFINE([PTHREAD_SIGMASK_INEFFECTIVE], [1],
+            [Define to 1 if pthread_mask() may returns 0 and have no effect.])
+          ;;
+      esac
+    fi
+  fi
 ])
+
+# Prerequisite of lib/pthread_sigmask.c.
+AC_DEFUN([gl_PREREQ_PTHREAD_SIGMASK],
+[
+  if test $HAVE_PTHREAD_SIGMASK = 1; then
+    AC_DEFINE([HAVE_PTHREAD_SIGMASK], [1],
+      [Define to 1 if the pthread_sigmask function can be used (despite bugs).])
+  fi
+])

--- a/modules/pthread_sigmask
+++ b/modules/pthread_sigmask
@@ -13,6 +13,7 @@
 gl_FUNC_PTHREAD_SIGMASK
 if test $HAVE_PTHREAD_SIGMASK = 0 || test $REPLACE_PTHREAD_SIGMASK = 1; then
   AC_LIBOBJ([pthread_sigmask])
+  gl_PREREQ_PTHREAD_SIGMASK
 fi
 gl_SIGNAL_MODULE_INDICATOR([pthread_sigmask])