For Clang -fsanitize=vptr use -fvisibility-ms-compat, not -fvisibility=hidden

As discussed in b4f6b26b5a1a78fecfa95ec2eb7ac8b80495d8aa "SAL_DLLPUBLIC_RTTI for
proper RTTI visibility for LLVM," RTTI-based -fsanitize= checks with Clang on
Linux need special precautions to make RTTI symbols visible across DSOs.  The
approach taken there, as well as in 598d8194b0ea1a64e0ebba28a86c128bafa57c7c
"Visible function type RTTI for Clang -fsanitize=function," was to add explicit
SAL_DLLPUBLIC_RTTI annontations to relevant type definitions.  However, for
-fsanitize=vptr that would have required many more of those, so it appears
easier to "misuse" -fsanitize-ms-compat in that case, which happens to give all
RTTI symbols default visibility (while otherwise still honoring our
SAL_DLLPUBLIC/PRIVATE annotations).

The SAL_DLLPUBLIC_RTTI annotations from 598d8194b0ea1a64e0ebba28a86c128bafa57c7c
"Visible function type RTTI for Clang -fsanitize=function" can likely be removed
again.

Change-Id: Ibeff7ab8c908111a7dc66ff0677204f112b24db8

19 files changed, 185 insertions, 2 deletions

diff --git a/external/libabw/UnpackedTarball_libabw.mk b/external/libabw/UnpackedTarball_libabw.mk
index 6288b0ef83aa..eb6b75cf470b 100644
--- a/external/libabw/UnpackedTarball_libabw.mk
+++ b/external/libabw/UnpackedTarball_libabw.mk
@@ -11,4 +11,14 @@ $(eval $(call gb_UnpackedTarball_UnpackedTarball,libabw))
 
 $(eval $(call gb_UnpackedTarball_set_tarball,libabw,$(ABW_TARBALL)))
 
+$(eval $(call gb_UnpackedTarball_set_patchlevel,libabw,0))
+
+ifeq ($(COM_GCC_IS_CLANG),TRUE)
+ifneq ($(filter -fsanitize=%,$(CC)),)
+$(eval $(call gb_UnpackedTarball_add_patches,libabw, \
+    external/libabw/ubsan-visibility.patch \
+))
+endif
+endif
+
 # vim: set noet sw=4 ts=4:
diff --git a/external/libabw/ubsan-visibility.patch b/external/libabw/ubsan-visibility.patch
new file mode 100644
index 000000000000..2285ef873e2c
--- /dev/null
+++ b/external/libabw/ubsan-visibility.patch
@@ -0,0 +1,11 @@
+--- configure
++++ configure
+@@ -16635,7 +16635,7 @@
+         { $as_echo "$as_me:${as_lineno-$LINENO}: checking for -fvisibility=hidden compiler flag" >&5
+ $as_echo_n "checking for -fvisibility=hidden compiler flag... " >&6; }
+         saved_CXXFLAGS="$CXXFLAGS"
+-        CXXFLAGS="$CXXFLAGS -fvisibility=hidden"
++        CXXFLAGS="$CXXFLAGS -fvisibility-ms-compat"
+         cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+ /* end confdefs.h.  */
+ 
diff --git a/external/libcdr/UnpackedTarball_libcdr.mk b/external/libcdr/UnpackedTarball_libcdr.mk
index a91bc8e907ca..f43bd1b53dfd 100644
--- a/external/libcdr/UnpackedTarball_libcdr.mk
+++ b/external/libcdr/UnpackedTarball_libcdr.mk
@@ -11,4 +11,14 @@ $(eval $(call gb_UnpackedTarball_UnpackedTarball,libcdr))
 
 $(eval $(call gb_UnpackedTarball_set_tarball,libcdr,$(CDR_TARBALL)))
 
+$(eval $(call gb_UnpackedTarball_set_patchlevel,libcdr,0))
+
+ifeq ($(COM_GCC_IS_CLANG),TRUE)
+ifneq ($(filter -fsanitize=%,$(CC)),)
+$(eval $(call gb_UnpackedTarball_add_patches,libcdr, \
+    external/libcdr/ubsan-visibility.patch \
+))
+endif
+endif
+
 # vim: set noet sw=4 ts=4:
diff --git a/external/libcdr/ubsan-visibility.patch b/external/libcdr/ubsan-visibility.patch
new file mode 100644
index 000000000000..2581786e47c3
--- /dev/null
+++ b/external/libcdr/ubsan-visibility.patch
@@ -0,0 +1,11 @@
+--- configure
++++ configure
+@@ -17651,7 +17641,7 @@
+         { $as_echo "$as_me:${as_lineno-$LINENO}: checking for -fvisibility=hidden compiler flag" >&5
+ $as_echo_n "checking for -fvisibility=hidden compiler flag... " >&6; }
+         saved_CXXFLAGS="$CXXFLAGS"
+-        CXXFLAGS="$CXXFLAGS -fvisibility=hidden"
++        CXXFLAGS="$CXXFLAGS -fvisibility-ms-compat"
+         cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+ /* end confdefs.h.  */
+ 
diff --git a/external/libebook/UnpackedTarball_libebook.mk b/external/libebook/UnpackedTarball_libebook.mk
index 02c17db4aa5f..4d8622efa8a7 100644
--- a/external/libebook/UnpackedTarball_libebook.mk
+++ b/external/libebook/UnpackedTarball_libebook.mk
@@ -18,4 +18,12 @@ $(eval $(call gb_UnpackedTarball_add_patches,libebook, \
     external/libebook/0001-lrf-compute-color-interpolation-coeff.-correctly.patch.1 \
 ))
 
+ifeq ($(COM_GCC_IS_CLANG),TRUE)
+ifneq ($(filter -fsanitize=%,$(CC)),)
+$(eval $(call gb_UnpackedTarball_add_patches,libebook, \
+    external/libebook/ubsan-visibility.patch \
+))
+endif
+endif
+
 # vim: set noet sw=4 ts=4:
diff --git a/external/libebook/ubsan-visibility.patch b/external/libebook/ubsan-visibility.patch
new file mode 100644
index 000000000000..b18f6a4312a5
--- /dev/null
+++ b/external/libebook/ubsan-visibility.patch
@@ -0,0 +1,11 @@
+--- configure
++++ configure
+@@ -17254,7 +17254,7 @@
+         { $as_echo "$as_me:${as_lineno-$LINENO}: checking for -fvisibility=hidden compiler flag" >&5
+ $as_echo_n "checking for -fvisibility=hidden compiler flag... " >&6; }
+         saved_CXXFLAGS="$CXXFLAGS"
+-        CXXFLAGS="$CXXFLAGS -fvisibility=hidden"
++        CXXFLAGS="$CXXFLAGS -fvisibility-ms-compat"
+         cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+ /* end confdefs.h.  */
+ 
diff --git a/external/libmspub/UnpackedTarball_libmspub.mk b/external/libmspub/UnpackedTarball_libmspub.mk
index cce7b5423ad0..a8c673c877f3 100644
--- a/external/libmspub/UnpackedTarball_libmspub.mk
+++ b/external/libmspub/UnpackedTarball_libmspub.mk
@@ -17,4 +17,12 @@ $(eval $(call gb_UnpackedTarball_add_patches,libmspub,\
     external/libmspub/ubsan.patch \
 ))
 
+ifeq ($(COM_GCC_IS_CLANG),TRUE)
+ifneq ($(filter -fsanitize=%,$(CC)),)
+$(eval $(call gb_UnpackedTarball_add_patches,libmspub, \
+    external/libmspub/ubsan-visibility.patch \
+))
+endif
+endif
+
 # vim: set noet sw=4 ts=4:
diff --git a/external/libmspub/ubsan-visibility.patch b/external/libmspub/ubsan-visibility.patch
new file mode 100644
index 000000000000..edaa1d0249ee
--- /dev/null
+++ b/external/libmspub/ubsan-visibility.patch
@@ -0,0 +1,11 @@
+--- configure
++++ configure
+@@ -17060,7 +17060,7 @@
+         { $as_echo "$as_me:${as_lineno-$LINENO}: checking for -fvisibility=hidden compiler flag" >&5
+ $as_echo_n "checking for -fvisibility=hidden compiler flag... " >&6; }
+         saved_CXXFLAGS="$CXXFLAGS"
+-        CXXFLAGS="$CXXFLAGS -fvisibility=hidden"
++        CXXFLAGS="$CXXFLAGS -fvisibility-ms-compat"
+         cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+ /* end confdefs.h.  */
+ 
diff --git a/external/libmwaw/UnpackedTarball_libmwaw.mk b/external/libmwaw/UnpackedTarball_libmwaw.mk
index 5338ce52c691..f88842a63c19 100644
--- a/external/libmwaw/UnpackedTarball_libmwaw.mk
+++ b/external/libmwaw/UnpackedTarball_libmwaw.mk
@@ -19,4 +19,12 @@ $(eval $(call gb_UnpackedTarball_add_patches,libmwaw,\
 	external/libmwaw/ubsan.patch.0 \
 ))
 
+ifeq ($(COM_GCC_IS_CLANG),TRUE)
+ifneq ($(filter -fsanitize=%,$(CC)),)
+$(eval $(call gb_UnpackedTarball_add_patches,libmwaw, \
+    external/libmwaw/ubsan-visibility.patch.0 \
+))
+endif
+endif
+
 # vim: set noet sw=4 ts=4:
diff --git a/external/libmwaw/ubsan-visibility.patch.0 b/external/libmwaw/ubsan-visibility.patch.0
new file mode 100644
index 000000000000..a14d201ea4c5
--- /dev/null
+++ b/external/libmwaw/ubsan-visibility.patch.0
@@ -0,0 +1,11 @@
+--- configure
++++ configure
+@@ -17225,7 +17225,7 @@
+         { $as_echo "$as_me:${as_lineno-$LINENO}: checking for -fvisibility=hidden compiler flag" >&5
+ $as_echo_n "checking for -fvisibility=hidden compiler flag... " >&6; }
+         saved_CXXFLAGS="$CXXFLAGS"
+-        CXXFLAGS="$CXXFLAGS -fvisibility=hidden"
++        CXXFLAGS="$CXXFLAGS -fvisibility-ms-compat"
+         cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+ /* end confdefs.h.  */
+ 
diff --git a/external/libodfgen/UnpackedTarball_libodfgen.mk b/external/libodfgen/UnpackedTarball_libodfgen.mk
index 965a278b8230..1aa716f244c5 100644
--- a/external/libodfgen/UnpackedTarball_libodfgen.mk
+++ b/external/libodfgen/UnpackedTarball_libodfgen.mk
@@ -18,4 +18,12 @@ $(eval $(call gb_UnpackedTarball_add_patches,libodfgen, \
 	external/libodfgen/rtti.patch \
 ))
 
+ifeq ($(COM_GCC_IS_CLANG),TRUE)
+ifneq ($(filter -fsanitize=%,$(CC)),)
+$(eval $(call gb_UnpackedTarball_add_patches,libodfgen, \
+    external/libodfgen/ubsan-visibility.patch \
+))
+endif
+endif
+
 # vim: set noet sw=4 ts=4:
diff --git a/external/libodfgen/ubsan-visibility.patch b/external/libodfgen/ubsan-visibility.patch
new file mode 100644
index 000000000000..5dcae64c3b14
--- /dev/null
+++ b/external/libodfgen/ubsan-visibility.patch
@@ -0,0 +1,11 @@
+--- configure
++++ configure
+@@ -15721,7 +15721,7 @@
+         { $as_echo "$as_me:${as_lineno-$LINENO}: checking for -fvisibility=hidden compiler flag" >&5
+ $as_echo_n "checking for -fvisibility=hidden compiler flag... " >&6; }
+         saved_CXXFLAGS="$CXXFLAGS"
+-        CXXFLAGS="$CXXFLAGS -fvisibility=hidden"
++        CXXFLAGS="$CXXFLAGS -fvisibility-ms-compat"
+         cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+ /* end confdefs.h.  */
+ 
diff --git a/external/libpagemaker/UnpackedTarball_libpagemaker.mk b/external/libpagemaker/UnpackedTarball_libpagemaker.mk
index fec1c03871dd..6322c2e7091a 100644
--- a/external/libpagemaker/UnpackedTarball_libpagemaker.mk
+++ b/external/libpagemaker/UnpackedTarball_libpagemaker.mk
@@ -11,4 +11,14 @@ $(eval $(call gb_UnpackedTarball_UnpackedTarball,libpagemaker))
 
 $(eval $(call gb_UnpackedTarball_set_tarball,libpagemaker,$(PAGEMAKER_TARBALL)))
 
+$(eval $(call gb_UnpackedTarball_set_patchlevel,libpagemaker,0))
+
+ifeq ($(COM_GCC_IS_CLANG),TRUE)
+ifneq ($(filter -fsanitize=%,$(CC)),)
+$(eval $(call gb_UnpackedTarball_add_patches,libpagemaker, \
+    external/libpagemaker/ubsan-visibility.patch \
+))
+endif
+endif
+
 # vim: set noet sw=4 ts=4:
diff --git a/external/libpagemaker/ubsan-visibility.patch b/external/libpagemaker/ubsan-visibility.patch
new file mode 100644
index 000000000000..90ead2b844b2
--- /dev/null
+++ b/external/libpagemaker/ubsan-visibility.patch
@@ -0,0 +1,11 @@
+--- configure
++++ configure
+@@ -16787,7 +16787,7 @@
+         { $as_echo "$as_me:${as_lineno-$LINENO}: checking for -fvisibility=hidden compiler flag" >&5
+ $as_echo_n "checking for -fvisibility=hidden compiler flag... " >&6; }
+         saved_CXXFLAGS="$CXXFLAGS"
+-        CXXFLAGS="$CXXFLAGS -fvisibility=hidden"
++        CXXFLAGS="$CXXFLAGS -fvisibility-ms-compat"
+         cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+ /* end confdefs.h.  */
+ 
diff --git a/external/librevenge/UnpackedTarball_librevenge.mk b/external/librevenge/UnpackedTarball_librevenge.mk
index 470f58f9d001..dabad6a8f14a 100644
--- a/external/librevenge/UnpackedTarball_librevenge.mk
+++ b/external/librevenge/UnpackedTarball_librevenge.mk
@@ -11,6 +11,14 @@ $(eval $(call gb_UnpackedTarball_UnpackedTarball,librevenge))
 
 $(eval $(call gb_UnpackedTarball_set_tarball,librevenge,$(REVENGE_TARBALL)))
 
-$(eval $(call gb_UnpackedTarball_set_patchlevel,librevenge,1))
+$(eval $(call gb_UnpackedTarball_set_patchlevel,librevenge,0))
+
+ifeq ($(COM_GCC_IS_CLANG),TRUE)
+ifneq ($(filter -fsanitize=%,$(CC)),)
+$(eval $(call gb_UnpackedTarball_add_patches,librevenge, \
+    external/librevenge/ubsan-visibility.patch \
+))
+endif
+endif
 
 # vim: set noet sw=4 ts=4:
diff --git a/external/librevenge/ubsan-visibility.patch b/external/librevenge/ubsan-visibility.patch
new file mode 100644
index 000000000000..ede44d4fee7d
--- /dev/null
+++ b/external/librevenge/ubsan-visibility.patch
@@ -0,0 +1,11 @@
+--- src/lib/Makefile.in
++++ src/lib/Makefile.in
+@@ -79,7 +79,7 @@
+ POST_UNINSTALL = :
+ build_triplet = @build@
+ host_triplet = @host@
+-@HAVE_VISIBILITY_TRUE@am__append_1 = -fvisibility=hidden
++@HAVE_VISIBILITY_TRUE@am__append_1 = -fvisibility-ms-compat
+ @HAVE_VISIBILITY_TRUE@am__append_2 = -DLIBREVENGE_VISIBILITY
+ @BUILD_GENERATORS_TRUE@am__append_3 = librevenge-generators-@RVNG_MAJOR_VERSION@.@RVNG_MINOR_VERSION@.la
+ @BUILD_GENERATORS_TRUE@@HAVE_VISIBILITY_TRUE@am__append_4 = -DLIBREVENGE_GENERATORS_VISIBILITY
diff --git a/external/libvisio/UnpackedTarball_libvisio.mk b/external/libvisio/UnpackedTarball_libvisio.mk
index caae90391ec5..4c55ecec1baf 100644
--- a/external/libvisio/UnpackedTarball_libvisio.mk
+++ b/external/libvisio/UnpackedTarball_libvisio.mk
@@ -18,4 +18,12 @@ $(eval $(call gb_UnpackedTarball_add_patches,libvisio,\
        external/libvisio/ubsan.patch \
 ))
 
+ifeq ($(COM_GCC_IS_CLANG),TRUE)
+ifneq ($(filter -fsanitize=%,$(CC)),)
+$(eval $(call gb_UnpackedTarball_add_patches,libvisio, \
+    external/libvisio/ubsan-visibility.patch \
+))
+endif
+endif
+
 # vim: set noet sw=4 ts=4:
diff --git a/external/libvisio/ubsan-visibility.patch b/external/libvisio/ubsan-visibility.patch
new file mode 100644
index 000000000000..1855c737bd79
--- /dev/null
+++ b/external/libvisio/ubsan-visibility.patch
@@ -0,0 +1,11 @@
+--- configure
++++ configure
+@@ -17847,7 +17847,7 @@
+         { $as_echo "$as_me:${as_lineno-$LINENO}: checking for -fvisibility=hidden compiler flag" >&5
+ $as_echo_n "checking for -fvisibility=hidden compiler flag... " >&6; }
+         saved_CXXFLAGS="$CXXFLAGS"
+-        CXXFLAGS="$CXXFLAGS -fvisibility=hidden"
++        CXXFLAGS="$CXXFLAGS -fvisibility-ms-compat"
+         cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+ /* end confdefs.h.  */
+ 
diff --git a/solenv/gbuild/platform/com_GCC_defs.mk b/solenv/gbuild/platform/com_GCC_defs.mk
index 7f8b617bfaba..319ea9fb0495 100644
--- a/solenv/gbuild/platform/com_GCC_defs.mk
+++ b/solenv/gbuild/platform/com_GCC_defs.mk
@@ -89,7 +89,13 @@ ifeq ($(HAVE_GCC_VISIBILITY_FEATURE),TRUE)
 gb_VISIBILITY_FLAGS := -DHAVE_GCC_VISIBILITY_FEATURE
 # If CC or CXX already include -fvisibility=hidden, don't duplicate it
 ifeq (,$(filter -fvisibility=hidden,$(CC)))
-gb_VISIBILITY_FLAGS += -fvisibility=hidden
+gb__visibility_hidden := -fvisibility=hidden
+ifeq ($(COM_GCC_IS_CLANG),TRUE)
+ifneq ($(filter -fsanitize=%,$(CC)),)
+gb__visibility_hidden := -fvisibility-ms-compat
+endif
+endif
+gb_VISIBILITY_FLAGS += $(gb__visibility_hidden)
 endif
 ifneq ($(HAVE_GCC_VISIBILITY_BROKEN),TRUE)
 gb_VISIBILITY_FLAGS_CXX := -fvisibility-inlines-hidden