Add tests using nm to check so contents (#693)

We want to ensure that the symbols included in shared object files (.so, .dylib, .dll) are as expected.

This verifies that our Autoconf and CMake builds are using the `-fvisibility` flags and attributes correctly.

As it turns out, the CMake build on Solaris is not working, and was exposing internal symbols due to a CMake issue.

Author: NWilson

.github/workflows/build.yml

@@ -54,6 +54,7 @@ jobs:
         run: |
           make install "DESTDIR=`pwd`/install-dir"
           maint/RunManifestTest install-dir maint/manifest-makeinstall-linux
+          maint/RunSymbolTest install-dir/usr/local/lib/ maint/
 
   alpine:
     name: alpine
@@ -93,6 +94,7 @@ jobs:
         run: |
           make install "DESTDIR=`pwd`/install-dir"
           maint/RunManifestTest install-dir maint/manifest-makeinstall-linux
+          maint/RunSymbolTest install-dir/usr/local/lib/ maint/
 
   macos:
     name: macOS universal
@@ -117,6 +119,7 @@ jobs:
           cd build
           cmake --install . --prefix install-dir
           ../maint/RunManifestTest install-dir ../maint/manifest-cmakeinstall-macos
+          ../maint/RunSymbolTest install-dir/lib/ ../maint/
 
   windows:
     name: Windows
@@ -145,6 +148,7 @@ jobs:
           cd build
           cmake --install . --config Release --prefix install-dir
           ../maint/RunManifestTest.ps1 install-dir ../maint/manifest-cmakeinstall-windows
+          ../maint/RunSymbolTest.ps1 install-dir/bin ../maint/
 
   freebsd:
     name: FreeBSD
@@ -182,6 +186,7 @@ jobs:
 
             make install "DESTDIR=`pwd`/install-dir"
             maint/RunManifestTest install-dir maint/manifest-makeinstall-freebsd
+            maint/RunSymbolTest install-dir/usr/local/lib/ maint/
 
             echo "== CMake =="
             cd ../build-cmake
@@ -192,6 +197,7 @@ jobs:
             ctest -j3 --output-on-failure
             cmake --install . --prefix install-dir
             ../maint/RunManifestTest install-dir ../maint/manifest-cmakeinstall-freebsd
+            ../maint/RunSymbolTest install-dir/lib/ ../maint/
 
   solaris:
     name: Solaris
@@ -251,6 +257,7 @@ jobs:
 
             make install "DESTDIR=`pwd`/install-dir"
             maint/RunManifestTest install-dir maint/manifest-makeinstall-solaris
+            maint/RunSymbolTest install-dir/usr/local/lib/ maint/
 
             echo "== Autoconf, 64-bit =="
             cd ../build-autoconf-64
@@ -261,6 +268,7 @@ jobs:
 
             make install "DESTDIR=`pwd`/install-dir"
             maint/RunManifestTest install-dir maint/manifest-makeinstall-solaris
+            maint/RunSymbolTest install-dir/usr/local/lib/ maint/
 
             echo "== CMake, 64-bit =="
             cd ../build-cmake-64
@@ -271,6 +279,7 @@ jobs:
             ctest -j3 --output-on-failure
             cmake --install . --prefix install-dir
             ../maint/RunManifestTest install-dir ../maint/manifest-cmakeinstall-solaris
+            ../maint/RunSymbolTest install-dir/lib/ ../maint/
 
   distcheck:
     name: Build & verify distribution

.github/workflows/dev.yml

@@ -112,6 +112,7 @@ jobs:
           cd build
           cmake --install . --prefix install-dir
           ../maint/RunManifestTest install-dir ../maint/manifest-cmakeinstall-linux
+          ../maint/RunSymbolTest install-dir/lib/ ../maint/
 
   dodo:
     # Tests with: Autconf on oldest supported Ubuntu (in non-extended support)
@@ -139,6 +140,7 @@ jobs:
         run: |
           make install "DESTDIR=`pwd`/install-dir"
           maint/RunManifestTest install-dir maint/manifest-makeinstall-linux
+          maint/RunSymbolTest install-dir/usr/local/lib/ maint/
 
   wasp:
     # Tests with: French locale; oldest supported CMake; no JIT; -Os; libreadline
@@ -185,6 +187,7 @@ jobs:
           cd build
           cmake --install . --prefix install-dir
           ../maint/RunManifestTest install-dir ../maint/manifest-cmakeinstall-linux
+          ../maint/RunSymbolTest install-dir/lib/ ../maint/
 
   bat:
     # Tests with: MSVC 32-bit, and a variety of CMake options

CMakeLists.txt

@@ -119,6 +119,17 @@ set(CMAKE_C_STANDARD_REQUIRED TRUE)
 
 set(CMAKE_C_VISIBILITY_PRESET hidden)
 
+# Solaris-specific fix for "CMAKE_C_VISIBILITY_PRESET": this feature was only
+# recently added to CMake for the `cc` compiler (Oracle Developer Studio). The
+# CMake version from OpenCSW and Oracle's package repository is too old and
+# requires this fix.
+if (CMAKE_SYSTEM_NAME STREQUAL "SunOS" AND
+    CMAKE_VERSION VERSION_LESS 3.31 AND
+    CMAKE_C_COMPILER_ID STREQUAL "SunPro" AND
+    CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 5.15)
+  set(CMAKE_C_COMPILE_OPTIONS_VISIBILITY "-fvisibility=")
+endif()
+
 # The following policies have been set in the PCRE2 CMake file in the past.
 # Since we specify a minimum of CMake 3.15, these are no longer required.
 # cmake_policy(SET CMP0063 NEW)

maint/RunManifestTest

@@ -3,6 +3,8 @@
 # Script to test a directory listing. We use this to verify that the list of
 # files installed by "make install" or "cmake --install" matches what we expect.
 
+set -e
+
 LANG=C # Ensure stable ordering of `sort` output
 export LANG
 
@@ -17,14 +19,10 @@ expected_manifest="$2"
 base=`basename $expected_manifest`
 
 sed=sed
-grep=grep
-# Helpers for Solaris
+# Helper for Solaris
 if [ -f /usr/bin/gsed ] ; then
   sed=/usr/bin/gsed
 fi
-if [ -f /usr/bin/ggrep ] ; then
-  grep=/usr/bin/ggrep
-fi
 
 find "$input_dir" -print | \
   sort | \

maint/RunSymbolTest

@@ -0,0 +1,72 @@
+#! /bin/sh
+
+# Script to test that all the symbols of a shared object are as expected.
+
+set -e
+
+LANG=C # Ensure stable ordering of `sort` output
+export LANG
+
+if [ "$1" = "" -o "$2" = "" ] ; then
+    echo "Usage: $0 <so_dir> <manifest_dir>" >&2
+  exit 1
+fi
+
+input_dir="$1"
+manifest_dir="$2"
+
+sed=sed
+grep=grep
+# Helpers for Solaris
+if [ -f /usr/bin/gsed ] ; then
+  sed=/usr/bin/gsed
+fi
+if [ -f /usr/bin/ggrep ] ; then
+  grep=/usr/bin/ggrep
+fi
+
+nm="nm -B -D"
+if [ "`uname -s`" = "SunOS" ]; then
+  nm="nm -p -h -D -g"
+elif [ "`uname -s`" = "Darwin" ]; then
+  nm="nm -B -g"
+fi
+
+so_ext=so
+so_mangling() { cat; }
+if [ "`uname -s`" = "Darwin" ]; then
+  so_ext=dylib
+  so_mangling()
+    {
+    sed -E -e 's/_([_0-9a-zA-Z]+)$/\1/g'
+    }
+fi
+
+for so_name in "libpcre2-8" "libpcre2-16" "libpcre2-32" "libpcre2-posix"; do
+  expected_file="$manifest_dir/manifest-$so_name.so"
+  so_file="$input_dir/$so_name.$so_ext"
+  base=`basename $expected_file`
+
+  $nm "$so_file" | \
+    $sed -E -e 's/^[0-9a-fA-F]* *//g' | \
+    $grep -E -v '^[Uw] ' | \
+    $grep -E -v ' (_init|_fini)$' | \
+    $grep -E -v ' (_end|_DYNAMIC|_GLOBAL_OFFSET_TABLE_|_PROCEDURE_LINKAGE_TABLE_|_edata|_etext)$' | \
+    so_mangling | \
+    sort \
+    > "$base"
+
+  if ! diff -u "$expected_file" "$base"; then
+    echo "Shared object contents for $so_file differ from expected" >&2
+
+    echo "===Actual===" >&2
+    cat "$base" >&2
+    echo "===End===" >&2
+
+    exit 1
+  fi
+
+  echo "Shared object contents for $so_file match expected"
+  rm -f "$base"
+
+done

maint/RunSymbolTest.ps1

@@ -0,0 +1,54 @@
+# Script to test that all the symbols of a DLL are as expected.
+
+param (
+    [Parameter(Mandatory=$true)]
+    [string]$inputDir,
+
+    [Parameter(Mandatory=$true)]
+    [string]$manifestDir
+)
+
+if ((-not $inputDir) -or (-not $manifestDir)) {
+  throw "Usage: .\RunSymbolTest.ps1 <dll_dir> <manifest_dir>"
+}
+
+$dllNames = @("pcre2-8", "pcre2-16", "pcre2-32", "pcre2-posix")
+
+foreach ($dllName in $dllNames) {
+    $expectedFile = Join-Path $manifestDir ("manifest-lib$dllName.so")
+    $dllFile = Join-Path $inputDir ("$dllName.dll")
+    $base = [System.IO.Path]::GetFileName($expectedFile)
+
+    # Get path to dumpbin using vswhere
+    $vswhere = "C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"
+    $dumpbin = & $vswhere -latest -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -find VC\Tools\MSVC\*\bin\Hostx64\x64\dumpbin.exe | Select-Object -First 1
+
+    $actualSymbols = & $dumpbin /exports $dllFile |
+        ForEach-Object {
+            if ($_ -match '^\s*\d+\s+[0-9A-Fa-f]+\s+[0-9A-Fa-f]+\s+(\S+)') {
+                "T $($matches[1])"
+            }
+        } |
+        Where-Object {
+            $_ -match '^T '
+        } |
+        Sort-Object
+
+    $actualOutput = ($actualSymbols -join "`n") + "`n"
+    $null = New-Item -Force $base -Value $actualOutput
+
+    $expectedOutput = Get-Content -Path $expectedFile -Raw
+    $actualOutput = Get-Content -Path $base -Raw
+
+    if ($expectedOutput -ne $actualOutput) {
+        Write-Host "Shared object contents for $dllFile differ from expected"
+        Write-Host "===Actual==="
+        Write-Host $actualOutput
+        Write-Host "===End==="
+        throw "Symbol test failed"
+    } else {
+        Write-Host "Shared object contents for $dllFile match expected"
+    }
+
+    Remove-Item -Path $base -Force
+}

maint/manifest-libpcre2-16.so

@@ -0,0 +1,78 @@
+T pcre2_callout_enumerate_16
+T pcre2_code_copy_16
+T pcre2_code_copy_with_tables_16
+T pcre2_code_free_16
+T pcre2_compile_16
+T pcre2_compile_context_copy_16
+T pcre2_compile_context_create_16
+T pcre2_compile_context_free_16
+T pcre2_config_16
+T pcre2_convert_context_copy_16
+T pcre2_convert_context_create_16
+T pcre2_convert_context_free_16
+T pcre2_converted_pattern_free_16
+T pcre2_dfa_match_16
+T pcre2_general_context_copy_16
+T pcre2_general_context_create_16
+T pcre2_general_context_free_16
+T pcre2_get_error_message_16
+T pcre2_get_mark_16
+T pcre2_get_match_data_heapframes_size_16
+T pcre2_get_match_data_size_16
+T pcre2_get_ovector_count_16
+T pcre2_get_ovector_pointer_16
+T pcre2_get_startchar_16
+T pcre2_jit_compile_16
+T pcre2_jit_free_unused_memory_16
+T pcre2_jit_match_16
+T pcre2_jit_stack_assign_16
+T pcre2_jit_stack_create_16
+T pcre2_jit_stack_free_16
+T pcre2_maketables_16
+T pcre2_maketables_free_16
+T pcre2_match_16
+T pcre2_match_context_copy_16
+T pcre2_match_context_create_16
+T pcre2_match_context_free_16
+T pcre2_match_data_create_16
+T pcre2_match_data_create_from_pattern_16
+T pcre2_match_data_free_16
+T pcre2_pattern_convert_16
+T pcre2_pattern_info_16
+T pcre2_serialize_decode_16
+T pcre2_serialize_encode_16
+T pcre2_serialize_free_16
+T pcre2_serialize_get_number_of_codes_16
+T pcre2_set_bsr_16
+T pcre2_set_callout_16
+T pcre2_set_character_tables_16
+T pcre2_set_compile_extra_options_16
+T pcre2_set_compile_recursion_guard_16
+T pcre2_set_depth_limit_16
+T pcre2_set_glob_escape_16
+T pcre2_set_glob_separator_16
+T pcre2_set_heap_limit_16
+T pcre2_set_match_limit_16
+T pcre2_set_max_pattern_compiled_length_16
+T pcre2_set_max_pattern_length_16
+T pcre2_set_max_varlookbehind_16
+T pcre2_set_newline_16
+T pcre2_set_offset_limit_16
+T pcre2_set_optimize_16
+T pcre2_set_parens_nest_limit_16
+T pcre2_set_recursion_limit_16
+T pcre2_set_recursion_memory_management_16
+T pcre2_set_substitute_callout_16
+T pcre2_set_substitute_case_callout_16
+T pcre2_substitute_16
+T pcre2_substring_copy_byname_16
+T pcre2_substring_copy_bynumber_16
+T pcre2_substring_free_16
+T pcre2_substring_get_byname_16
+T pcre2_substring_get_bynumber_16
+T pcre2_substring_length_byname_16
+T pcre2_substring_length_bynumber_16
+T pcre2_substring_list_free_16
+T pcre2_substring_list_get_16
+T pcre2_substring_nametable_scan_16
+T pcre2_substring_number_from_name_16