From fbf38229857f3d1982aa305c20da5e1ea0195b3e Mon Sep 17 00:00:00 2001
From: Solar Designer <solar@openwall.com>
Date: Wed, 12 Jul 2023 18:17:30 +0200
Subject: [PATCH] Fix macOS and Solaris builds

Also bring us closer to having a working AIX build (but not yet, need more
Makefile changes).

Resolves: https://github.com/openwall/passwdqc/issues/31
---
 Makefile          | 22 +++++++++++-----------
 pam_passwdqc.c    |  8 +-------
 passwdqc_filter.h |  9 ++++++++-
 3 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/Makefile b/Makefile
index be75c1d..9283dfa 100644
--- a/Makefile
+++ b/Makefile
@@ -133,7 +133,7 @@ all locales pam utils install install_lib install_locales install_pam install_ut
 			LDLIBS_pam="$(LDLIBS_pam_LINUX)" \
 			$@_wrapped;; \
 	SunOS)	$(MAKE) -e CPPFLAGS_lib="$(CPPFLAGS_lib) -DHAVE_SHADOW" \
-			LD_lib=ld \
+			LD_lib=/usr/ccs/bin/ld \
 			LDFLAGS_lib="$(LDFLAGS_lib_SUN)" \
 			LDFLAGS_pam="$(LDFLAGS_pam_SUN)" \
 			LDLIBS_pam="$(LDLIBS_pam_SUN)" \
@@ -174,9 +174,9 @@ $(SHARED_LIB): $(OBJS_LIB) $(MAP_LIB)
 	$(LD_lib) $(LDFLAGS_lib) $(OBJS_LIB) $(LDLIBS_lib) -o $(SHARED_LIB)
 
 $(DEVEL_LIB): $(SHARED_LIB)
-ifndef CYGWIN
-	$(LN_s) $(SHARED_LIB) $(DEVEL_LIB)
-endif
+	if [ "$(CYGWIN)" != true ]; then \
+		$(LN_s) $(SHARED_LIB) $(DEVEL_LIB); \
+	fi
 
 $(SHARED_PAM): $(OBJS_PAM) $(MAP_PAM) $(DEVEL_LIB)
 	$(LD_lib) $(LDFLAGS_pam) $(OBJS_PAM) $(LDLIBS_pam) -L. -lpasswdqc -o $(SHARED_PAM)
@@ -203,7 +203,7 @@ pwqfilter.o: pwqfilter.c passwdqc_filter.h passwdqc.h
 	$(CC) $(CPPFLAGS_lib) $(CFLAGS_lib) -c $*.c
 
 $(PKGCONFIG): $(PKGCONFIG).in
-	sed -e "s|@VERSION@|$(VERSION)|g" $< > $@
+	sed -e "s|@VERSION@|$(VERSION)|g" $(PKGCONFIG).in > $@
 
 concat.o: concat.h
 pam_passwdqc.o: passwdqc.h pam_macros.h
@@ -225,12 +225,12 @@ install_lib_wrapped:
 	$(INSTALL) -m $(SHLIBMODE) $(SHARED_LIB) $(DESTDIR)$(SHARED_LIBDIR)/
 
 	$(MKDIR) $(DESTDIR)$(DEVEL_LIBDIR)
-ifndef CYGWIN
-	$(LN_s) $(SHARED_LIBDIR_REL)/$(SHARED_LIB) \
-		$(DESTDIR)$(DEVEL_LIBDIR)/$(DEVEL_LIB)
-else
-	$(INSTALL) -m $(SHLIBMODE) $(DEVEL_LIB) $(DESTDIR)$(DEVEL_LIBDIR)/
-endif
+	if [ "$(CYGWIN)" != true ]; then \
+		$(LN_s) $(SHARED_LIBDIR_REL)/$(SHARED_LIB) \
+			$(DESTDIR)$(DEVEL_LIBDIR)/$(DEVEL_LIB); \
+	else \
+		$(INSTALL) -m $(SHLIBMODE) $(DEVEL_LIB) $(DESTDIR)$(DEVEL_LIBDIR)/; \
+	fi
 
 	$(MKDIR) $(DESTDIR)$(INCLUDEDIR)
 	$(INSTALL) -m $(INCMODE) $(HEADER) $(DESTDIR)$(INCLUDEDIR)/
diff --git a/pam_passwdqc.c b/pam_passwdqc.c
index 6be3d9e..3d1b66a 100644
--- a/pam_passwdqc.c
+++ b/pam_passwdqc.c
@@ -5,15 +5,9 @@
  * See LICENSE
  */
 
-#ifdef __FreeBSD__
-/* For vsnprintf(3) */
 #define _XOPEN_SOURCE 600
-#else
-#define _XOPEN_SOURCE 500
 #define _XOPEN_SOURCE_EXTENDED
-#define _XOPEN_VERSION 500
 #define _DEFAULT_SOURCE
-#endif
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdarg.h>
@@ -45,7 +39,7 @@
 #define PAM_AUTHTOK_RECOVERY_ERR	PAM_AUTHTOK_RECOVER_ERR
 #endif
 
-#if (defined(__sun) || defined(__hpux)) && \
+#if (defined(__sun) || defined(__hpux) || defined(_AIX)) && \
     !defined(LINUX_PAM) && !defined(_OPENPAM)
 /* Sun's PAM doesn't use const here, while Linux-PAM and OpenPAM do */
 #define lo_const
diff --git a/passwdqc_filter.h b/passwdqc_filter.h
index f8315fc..40b72fe 100644
--- a/passwdqc_filter.h
+++ b/passwdqc_filter.h
@@ -34,8 +34,15 @@ extern int passwdqc_filter_close(passwdqc_filter_t *flt);
 /* Lower-level inlines for shared use by pwqfilter.c and passwdqc_filter.c */
 
 #include <string.h> /* for strcspn() */
-#if !defined(_MSC_VER) && !defined(__APPLE__)
+
+/* We assume little-endian if this leaves __BYTE_ORDER undefined */
+#if !defined(_MSC_VER) && !defined(__APPLE__) && !defined(__sun) && !defined(_AIX)
 #include <endian.h>
+#elif (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) || \
+    defined(__sparc) || defined(_POWER)
+#define __LITTLE_ENDIAN 1234
+#define __BIG_ENDIAN 4321
+#define __BYTE_ORDER __BIG_ENDIAN
 #endif
 
 #include "md4.h"