external/firebird: Support Clang ASan

First of all, LD_LIBRARY_PATH must not affect the make executable called for the
firebird build.  That would cause problems when that make executable depends on
system versions of dynamic libraries that also happen to be available as LO-
built versions on LD_LIBRARY_PATH:

> make: symbol lookup error: /data/sbergman/lo-san/core/instdir/program/libfreebl3.so: undefined symbol: __asan_option_detect_stack_use_after_return
> make: *** [/data/sbergman/lo-san/core/external/firebird/ExternalProject_firebird.mk:41: /data/sbergman/lo-san/core/workdir/ExternalProject/firebird/build] Error 127

For that, gb_Helper_extend_ld_path is now tunneled to the relevant places in the
firebird build, where executables are called that use firebird's
UnicodeUtil::getConversionICU
(workdir/UnpackedTarball/firebird/src/common/unicode_util.cpp) to dynamically
load icu libraries.  (Similar to 3a2818280ad65c3d0aa9e720f62fec571713b2e7
"external/liblangtag: Tunnel LD_LIBRARY_PATH to where it's actually needed".)

Then, what appears to work is to disable firebird's global operators new/delete
(so that ASan's versions kick in).

MemPool::{alloc,release}Raw would still be called at start-up through
Firebird::GlobalPtr<Firebird::Mutex, (Firebird::InstanceControl::DtorPriority)3>
::GlobalPtr() calling Firebird::GlobalStorage::operator new(unsigned long), but
there is already provision to avoid those functions' heavy machinery for the
USE_VALGRIND case.

DEBUG_GDS_ALLOC (which would enable other inopportune overloads of global
operator new/delete) needs to be disabled in a somewhat contorted way, because
8ea07101c1613d213fd7cea17f094a947b14cd00 "external/firebird: Work around
operator new alignment violations" might explicitly set it from the outside,
which we need to undo for ASan builds.

`make check screenshot` encountered one alloc-dealloc-mismatch (already during
building firebird itself) that needs a patch to src/jrd/SimilarToMatcher.h.

Change-Id: I6bb68604095908189259be14440f72d3b23cbd4e
Reviewed-on: https://gerrit.libreoffice.org/52706
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Stephan Bergmann <sbergman@redhat.com>

4 files changed, 212 insertions, 5 deletions

diff --git a/external/firebird/ExternalProject_firebird.mk b/external/firebird/ExternalProject_firebird.mk
index e1ab197227d1..5678a6f36218 100644
--- a/external/firebird/ExternalProject_firebird.mk
+++ b/external/firebird/ExternalProject_firebird.mk
@@ -30,7 +30,7 @@ ifeq ($(COM_IS_CLANG),TRUE)
 firebird_NO_CXX11_NARROWING := -Wno-c++11-narrowing
 endif
 
-MAKE_PRE=$(call gb_Helper_extend_ld_path,$(call gb_UnpackedTarball_get_dir,icu)/source/lib) LC_ALL=C
+MAKE_PRE=LC_ALL=C
 
 MAKE_POST=$(if $(filter MACOSX,$(OS)),&& $(PERL) \
 			$(SRCDIR)/solenv/bin/macosx-change-install-names.pl shl OOO \
@@ -97,9 +97,9 @@ $(call gb_ExternalProject_get_state_target,firebird,build):
 							'<' 101200)), \
 					ac_cv_func_clock_gettime=no)) \
 		&& if [ -n "$${FB_CPU_ARG}" ]; then \
-			   $(MAKE_PRE) $(MAKE) $(if $(ENABLE_DEBUG),Debug) $(INVOKE_FPA) SHELL='$(SHELL)' $(MAKE_POST); \
+			   $(MAKE_PRE) $(MAKE) $(if $(ENABLE_DEBUG),Debug) $(INVOKE_FPA) SHELL='$(SHELL)' LIBO_TUNNEL_LIBRARY_PATH='$(subst ','\'',$(subst $$,$$$$,$(call gb_Helper_extend_ld_path,$(call gb_UnpackedTarball_get_dir,icu)/source/lib)))' $(MAKE_POST); \
 			else \
-			   $(MAKE_PRE) $(MAKE) $(if $(ENABLE_DEBUG),Debug) SHELL='$(SHELL)' $(MAKE_POST); \
+			   $(MAKE_PRE) $(MAKE) $(if $(ENABLE_DEBUG),Debug) SHELL='$(SHELL)' LIBO_TUNNEL_LIBRARY_PATH='$(subst ','\'',$(subst $$,$$$$,$(call gb_Helper_extend_ld_path,$(call gb_UnpackedTarball_get_dir,icu)/source/lib)))' $(MAKE_POST); \
 			fi \
 	)
 # vim: set noet sw=4 ts=4:
diff --git a/external/firebird/UnpackedTarball_firebird.mk b/external/firebird/UnpackedTarball_firebird.mk
index fd780b964945..aaea3a5b4231 100644
--- a/external/firebird/UnpackedTarball_firebird.mk
+++ b/external/firebird/UnpackedTarball_firebird.mk
@@ -30,6 +30,7 @@ $(eval $(call gb_UnpackedTarball_add_patches,firebird,\
 		external/firebird/0002-Backported-fix-for-CORE-5452-Segfault-when-engine-s-.patch.1 \
 		external/firebird/c++17.patch \
 		external/firebird/ubsan.patch \
+		external/firebird/asan.patch \
 ))
 
 ifeq ($(OS),WNT)
diff --git a/external/firebird/asan.patch b/external/firebird/asan.patch
new file mode 100644
index 000000000000..33d418513fba
--- /dev/null
+++ b/external/firebird/asan.patch
@@ -0,0 +1,206 @@
+--- builds/posix/Makefile.in
++++ builds/posix/Makefile.in
+@@ -323,8 +323,8 @@
+ 
+ metadata.fdb:	$(RUN_ISQL) $(SRC_ROOT)/dbs/metadata.sql
+ 	-$(RM) $@
+-	$(RUN_ISQL) -q -i $(SRC_ROOT)/dbs/metadata.sql
+-	$(RUN_GFIX) -mode read_only $@
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(RUN_ISQL) -q -i $(SRC_ROOT)/dbs/metadata.sql
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(RUN_GFIX) -mode read_only $@
+ 	$(CHMOD) 0444 $@
+ 
+ $(HELP_FDB):	help.fdb
+@@ -333,7 +333,7 @@
+ 	$(CHMOD) 0444 $@
+ 
+ help.fdb:	$(BLD_ROOT)/misc/help.gbak
+-	$(RUN_GBAK) -MODE read_only -R $< $@
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(RUN_GBAK) -MODE read_only -R $< $@
+ 	$(CHMOD) 0444 $@
+ 
+ $(SECURITY_FDB):	security.fdb
+@@ -343,18 +343,18 @@
+ security.fdb:	$(SRC_ROOT)/dbs/security.sql
+ 	-$(RM) $@
+ 	-$(RM) $(SECURITY_TMP)
+-	echo create database \'$(SECURITY_TMP)\'\; | $(RUN_ISQL)
+-	$(RUN_GFIX) -write async $(SECURITY_TMP)
+-	$(RUN_ISQL) -i $^ $(SECURITY_TMP)
++	echo create database \'$(SECURITY_TMP)\'\; | $(LIBO_TUNNEL_LIBRARY_PATH) $(RUN_ISQL)
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(RUN_GFIX) -write async $(SECURITY_TMP)
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(RUN_ISQL) -i $^ $(SECURITY_TMP)
+ 	$(CHMOD) a=rw $(SECURITY_TMP)
+ 	$(CP) $(SECURITY_TMP) $@
+-	$(RUN_GFIX) -write sync $@
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(RUN_GFIX) -write sync $@
+ 
+ msg.timestamp:	$(MSG_FILES)
+ 	-$(RM) msg.fdb
+-	echo create database \'msg.fdb\'\; | $(RUN_ISQL)
+-	$(RUN_GFIX) -write async msg.fdb
+-	for sql in $(MSG_FILES); do (echo $$sql; $(RUN_ISQL) -i $$sql msg.fdb) || exit; done
++	echo create database \'msg.fdb\'\; | $(LIBO_TUNNEL_LIBRARY_PATH) $(RUN_ISQL)
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(RUN_GFIX) -write async msg.fdb
++	for sql in $(MSG_FILES); do (echo $$sql; $(LIBO_TUNNEL_LIBRARY_PATH) $(RUN_ISQL) -i $$sql msg.fdb) || exit; done
+ 	$(TOUCH) $@
+ 
+ 
+@@ -560,7 +560,7 @@
+ message_file:	$(FIREBIRD_MSG)
+ 
+ $(FIREBIRD_MSG):	$(BUILD_FILE) msg.timestamp
+-	$(BUILD_FILE) -d msg.fdb -f $@
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(BUILD_FILE) -d msg.fdb -f $@
+ 	$(CHMOD_6) $@
+ 
+ $(BUILD_FILE):	$(BUILD_Objects) $(COMMON_LIB)
+--- builds/posix/make.rules
++++ builds/posix/make.rules
+@@ -68,17 +68,17 @@
+ .SUFFIXES: .c .e .epp .cpp
+ 
+ .e.c:
+-	$(GPRE_CURRENT) $(GPRE_FLAGS) $< $@
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(GPRE_CURRENT) $(GPRE_FLAGS) $< $@
+ 
+ 
+ $(OBJ)/jrd/%.cpp: $(SRC_ROOT)/jrd/%.epp
+-	$(GPRE_CURRENT) $(JRD_GPRE_FLAGS) $(firstword $<) $@
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(GPRE_CURRENT) $(JRD_GPRE_FLAGS) $(firstword $<) $@
+ 
+ $(OBJ)/isql/%.cpp: $(SRC_ROOT)/isql/%.epp
+-	$(GPRE_CURRENT) $(ISQL_GPRE_FLAGS) $< $@
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(GPRE_CURRENT) $(ISQL_GPRE_FLAGS) $< $@
+ 
+ $(OBJ)/%.cpp: $(SRC_ROOT)/%.epp
+-	$(GPRE_CURRENT) $(GPRE_FLAGS) $(firstword $<) $@
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(GPRE_CURRENT) $(GPRE_FLAGS) $(firstword $<) $@
+ 
+ 
+ .SUFFIXES: .lo .o .cpp .c
+--- src/common/classes/alloc.cpp
++++ src/common/classes/alloc.cpp
+@@ -2187,7 +2187,7 @@
+ 
+ void* MemPool::allocRaw(size_t size)
+ {
+-#ifndef USE_VALGRIND
++#if !(defined USE_VALGRIND || defined USE_ASAN)
+ 	if (size == DEFAULT_ALLOCATION)
+ 	{
+ 		MutexLockGuard guard(*cache_mutex, "MemPool::allocRaw");
+@@ -2267,7 +2267,7 @@
+ 
+ void MemPool::releaseRaw(bool destroying, void* block, size_t size, bool use_cache) throw ()
+ {
+-#ifndef USE_VALGRIND
++#if !(defined USE_VALGRIND || defined USE_ASAN)
+ 	if (use_cache && (size == DEFAULT_ALLOCATION))
+ 	{
+ 		MutexLockGuard guard(*cache_mutex, "MemPool::releaseRaw");
+@@ -2277,7 +2277,7 @@
+ 			return;
+ 		}
+ 	}
+-#else
++#elif defined USE_VALGRIND
+ 	// Set access protection for block to prevent memory from deleted pool being accessed
+ 	int handle = /* //VALGRIND_MAKE_NOACCESS */ VALGRIND_MAKE_MEM_DEFINED(block, size);
+ 
+--- src/common/classes/alloc.h
++++ src/common/classes/alloc.h
+@@ -295,40 +295,60 @@
+ 
+ // operators new and delete
+ 
++#if !defined USE_ASAN
+ inline void* operator new(size_t s ALLOC_PARAMS)
+ {
+ 	return MemoryPool::globalAlloc(s ALLOC_PASS_ARGS);
+ }
+ inline void* operator new[](size_t s ALLOC_PARAMS)
+ {
+ 	return MemoryPool::globalAlloc(s ALLOC_PASS_ARGS);
+ }
++#endif
+ 
+ inline void* operator new(size_t s, Firebird::MemoryPool& pool ALLOC_PARAMS)
+ {
++#if defined USE_ASAN
++	return operator new(s);
++#else
+ 	return pool.allocate(s ALLOC_PASS_ARGS);
++#endif
+ }
+ inline void* operator new[](size_t s, Firebird::MemoryPool& pool ALLOC_PARAMS)
+ {
++#if defined USE_ASAN
++	return operator new[](s);
++#else
+ 	return pool.allocate(s ALLOC_PASS_ARGS);
++#endif
+ }
+ 
++#if !defined USE_ASAN
+ inline void operator delete(void* mem ALLOC_PARAMS) throw()
+ {
+ 	MemoryPool::globalFree(mem);
+ }
+ inline void operator delete[](void* mem ALLOC_PARAMS) throw()
+ {
+ 	MemoryPool::globalFree(mem);
+ }
++#endif
+ 
+ inline void operator delete(void* mem, Firebird::MemoryPool& pool ALLOC_PARAMS) throw()
+ {
++#if defined USE_ASAN
++	return operator delete(mem);
++#else
+ 	MemoryPool::globalFree(mem);
++#endif
+ }
+ inline void operator delete[](void* mem, Firebird::MemoryPool& pool ALLOC_PARAMS) throw()
+ {
++#if defined USE_ASAN
++	return operator delete[](mem);
++#else
+ 	MemoryPool::globalFree(mem);
++#endif
+ }
+ 
+ #ifdef DEBUG_GDS_ALLOC
+--- src/include/firebird.h
++++ src/include/firebird.h
+@@ -38,10 +38,19 @@
+ #include "gen/autoconfig.h"
+ #endif
+ 
++#if defined __clang__ && defined __has_feature
++#if __has_feature(address_sanitizer)
++#define USE_ASAN
++#endif
++#endif
++
+ // Using our debugging code is pointless when we may use Valgrind features
+ #if defined(DEV_BUILD) && !defined(USE_VALGRIND)
+ #define DEBUG_GDS_ALLOC
+ #endif
++#if defined USE_ASAN
++#undef DEBUG_GDS_ALLOC
++#endif
+ 
+ #if defined(WIN_NT)
+ #define FB_DLL_EXPORT __declspec(dllexport)
+--- src/jrd/SimilarToMatcher.h
++++ src/jrd/SimilarToMatcher.h
+@@ -338,7 +338,7 @@
+ 		private:
+ 			static const unsigned INCREASE_FACTOR = 50;
+ 			unsigned size;
+-			AutoPtr<UCHAR> data;
++			AutoPtr<UCHAR, ArrayDelete<UCHAR>> data;
+ 			T* end;
+ 		};
+ #endif	// RECURSIVE_SIMILAR
diff --git a/external/firebird/firebird-cygwin-msvc.patch b/external/firebird/firebird-cygwin-msvc.patch
index 658580415ab5..12dcb3c9513a 100644
--- a/external/firebird/firebird-cygwin-msvc.patch
+++ b/external/firebird/firebird-cygwin-msvc.patch
@@ -10,8 +10,8 @@
  #include "gen/autoconfig.h"
 -#endif
  
- // Using our debugging code is pointless when we may use Valgrind features
- #if defined(DEV_BUILD) && !defined(USE_VALGRIND)
+ #if defined __clang__ && defined __has_feature
+ #if __has_feature(address_sanitizer)
 --- src/misc/writeBuildNum.sh	2016-07-07 15:57:04.538983200 +0200
 +++ src/misc/writeBuildNum.sh	2016-07-13 11:31:18.132820200 +0200
 @@ -95,9 +95,9 @@