external/cairo: Support building with ASan/UBSan

A full `make check screenshot` required lots of little "harmless" fixes in
pixman and cairo to address:

> cairo-image-compositor.c:133:34: runtime error: left shift of 255 by 24 places cannot be represented in type 'int'
during CppunitTest_emfio_emf

> pixman-fast-path.c:3089:23: runtime error: left shift of 255 by 24 places cannot be represented in type 'int'
during CppunitTest_emfio_emf

> pixman-sse2.c:5019:17: runtime error: load of misaligned address 0x7f99303dbac5 for type 'uint32_t' (aka 'unsigned int'), which requires 4 byte alignment
during CppunitTest_emfio_emf

> cairo-fixed-private.h:64:14: runtime error: left shift of negative value -8388608
during CppunitTest_emfio_wmf

> pixman-sse2.c:6443:20: runtime error: left shift of 198 by 24 places cannot be represented in type 'int'
during CppunitTest_filter_svg

> pixman-sse2.c:5976:6: runtime error: load of misaligned address 0x629000163202 for type 'uint32_t' (aka 'unsigned int'), which requires 4 byte alignment
during CppunitTest_filter_svg

> pixman-sse2.c:3259:10: runtime error: load of misaligned address 0x606000c85761 for type 'uint32_t' (aka 'unsigned int'), which requires 4 byte alignment
during CppunitTest_oox_vml

> pixman-sse2.c:521:18: runtime error: load of misaligned address 0x607000ca9d41 for type 'const uint32_t' (aka 'const unsigned int'), which requires 4 byte alignment
during CppunitTest_oox_vml

> pixman-gradient-walker.c:196:14: runtime error: left shift of 255 by 24 places cannot be represented in type 'int'
during CppunitTest_sc_tiledrendering

> pixman-combine32.c:786:1: runtime error: left shift of 255 by 24 places cannot be represented in type 'int32_t' (aka 'int')
during CppunitTest_vcl_backend_test

> pixman-fast-path.c:2761:29: runtime error: left shift of negative value -99
during CppunitTest_xmloff_draw

> pixman-bits-image.c:243:31: runtime error: left shift of negative value -99
during CppunitTest_xmloff_draw

> pixman-bits-image.c:244:31: runtime error: left shift of negative value -9
during CppunitTest_sd_tiledrendering

> pixman-fast-path.c:2762:29: runtime error: left shift of negative value -84
during CppunitTest_sw_rtfexport2

> cairo-gstate.c:2300:14: runtime error: null pointer passed as argument 1, which is declared to never be null
during CppunitTest_sw_ooxmlexport8

> pixman-access.c:389:2: runtime error: left shift of 1 by 31 places cannot be represented in type 'int'
during CppunitTest_sw_ooxmlexport15

> ERROR: AddressSanitizer: stack-use-after-scope on address 0x7ff264ae275c at pc 0x7ff238941795 bp 0x7fff6bbadb10 sp 0x7fff6bbadb08
> READ of size 4 at 0x7ff264ae275c thread T0
>  #0 in _add_clipped_edge at workdir/UnpackedTarball/cairo/src/cairo-polygon.c:351:24 (instdir/program/libcairo.so.2 +0x88c794)
during CppunitTest_sw_odfexport

> cairo-tor-scan-converter.c:1619:34: runtime error: left shift of negative value -39
during CppunitTest_sw_odfexport

> ERROR: AddressSanitizer: stack-use-after-scope on address 0x7fe6ca085750 at pc 0x000000325c3a bp 0x7fff899bedd0 sp 0x7fff899be580
> READ of size 16 at 0x7fe6ca085750 thread T0
>  #0 in __asan_memcpy at /home/sbergman/github.com/llvm/llvm-project/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cpp:22:3 (workdir/LinkTarget/Executable/cppunittester +0x325c39)
during CppunitTest_sw_odfexport

> pixman-sse2.c:3352:14: runtime error: left shift of 65535 by 16 places cannot be represented in type 'int'
during CppunitTest_sw_odfexport

> cairo-gstate.c:2355:14: runtime error: null pointer passed as argument 1, which is declared to never be null
during CppunitTest_basctl_dialogs_test

> pixman-sse2.c:3537:10: runtime error: load of misaligned address 0x615000167682 for type 'uint32_t' (aka 'unsigned int'), which requires 4 byte alignment
during CppunitTest_sc_screenshots

> cairo-image-source.c:512:10: runtime error: load of misaligned address 0x6180037aee6f for type 'uint32_t' (aka 'unsigned int'), which requires 4 byte alignment
during UITest_writer_tests7

Change-Id: Icd2a211df4751d8dbfd5903bfba424b4c4672999
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/104572
Tested-by: Jenkins
Reviewed-by: Stephan Bergmann <sbergman@redhat.com>

4 files changed, 288 insertions, 1 deletions

diff --git a/external/cairo/ExternalProject_cairo.mk b/external/cairo/ExternalProject_cairo.mk
index 43b4a7b53b2a..41e7621c789f 100644
--- a/external/cairo/ExternalProject_cairo.mk
+++ b/external/cairo/ExternalProject_cairo.mk
@@ -38,6 +38,26 @@ $(call gb_ExternalProject_get_state_target,cairo,build) :
 
 else
 
+# Including -rtlib=compiler-rt in pixman_LIBS is a BAD HACK:  At least when compiling with Clang
+# -fsanitize=undefined on Linux x86-64, the generated code references __muloti4, which is an
+# extension provided by libclang_rt.builtins-x86_64.a runtime, but not by GCC's libgcc_s.so.1 (which
+# ultimately boils down to a bug in LLVM, I would say).  I am not sure whether it should in general
+# work to mix uses of the (default on Linux, at least) GCC libgcc_s and LLVM's libclang_rt.builtins
+# runtime libraries in one process, but for this specific case of libcairo.so it appears to work
+# well:  For one, the only symbol referenced by libcairo.so from the runtime library is __muloti4;
+# for another, at least in my LLVM build, lib/clang/12.0.0/lib/linux/libclang_rt.builtins-x86_64.a
+# is only provided as a static archive; so libcairo.so will only contain a "harmless" copy of
+# __muloti4 and not have a DT_NEEDED of any libclang_rt.builtins dynamic library that it would pull
+# in at runtime.
+# But passing -rtlib=compiler-rt into cairo's configure via the more obvious LDFLAGS would fail at
+# least when building with -fsanitize=address and -fsanitize=undefined, as then the executable
+# compiled  by configure when "checking whether the C compiler works" would reference
+# _Unwind_Backtrace etc. that are provided by GCC's libgcc_s.so.1 but not by LLVM's
+# libclang_rt.builtins-x86_64.a (and whatever the reason for that inconsistency).  So
+# -rtlib=compiler-rt must be passed just into the linking of libcairo.so, but not generally into
+# cairo's configure.  And pixman_LIBS happens to offer that.  (The -Wc is necessary so that libtool
+# does not throw away the -rtlib=compiler-rt which it does not understand.)
+
 # overwrite src/cairo-version.h because that is just a dummy file and included
 # from cairo.h in non-overridable way
 
@@ -54,7 +74,10 @@ $(call gb_ExternalProject_get_state_target,cairo,build) :
 		pixman_CFLAGS="-I$(call gb_UnpackedTarball_get_dir,pixman)/pixman" \
 		pixman_LIBS="-L$(call gb_UnpackedTarball_get_dir,pixman)/pixman/.libs -lpixman-1 \
 			$(if $(filter LINUX,$(OS)),-Wl$(COMMA)-z$(COMMA)origin \
-					-Wl$(COMMA)-rpath$(COMMA)\\\$$\$$ORIGIN)" \
+					-Wl$(COMMA)-rpath$(COMMA)\\\$$\$$ORIGIN) \
+			$(if $(filter -fsanitize=%,$(CC)), \
+			    $(if $(filter LINUX-X86_64-TRUE,$(OS)-$(CPUNAME)-$(COM_IS_CLANG)), \
+			        -Wc$(COMMA)-rtlib=compiler-rt))" \
 		png_REQUIRES="trick_configure_into_using_png_CFLAGS_and_LIBS" \
 		png_CFLAGS="$(LIBPNG_CFLAGS)" png_LIBS="$(LIBPNG_LIBS)" \
 		$(if $(SYSTEM_FREETYPE),,FREETYPE_CFLAGS="-I$(call gb_UnpackedTarball_get_dir,freetype)/include") \
diff --git a/external/cairo/UnpackedTarball_cairo.mk b/external/cairo/UnpackedTarball_cairo.mk
index 75e3d58e7dee..dfd79d673be1 100644
--- a/external/cairo/UnpackedTarball_cairo.mk
+++ b/external/cairo/UnpackedTarball_cairo.mk
@@ -15,6 +15,7 @@ $(eval $(call gb_UnpackedTarball_add_patches,cairo,\
 	external/cairo/cairo/cairo-1.10.2.patch \
 	external/cairo/cairo/cairo-libtool-rpath.patch.1 \
 	external/cairo/cairo/cairo.oldfreetype.patch \
+	external/cairo/cairo/san.patch.0 \
 ))
 
 ifeq ($(OS),iOS)
diff --git a/external/cairo/cairo/san.patch.0 b/external/cairo/cairo/san.patch.0
new file mode 100644
index 000000000000..112741260ec5
--- /dev/null
+++ b/external/cairo/cairo/san.patch.0
@@ -0,0 +1,111 @@
+--- src/cairo-fixed-private.h
++++ src/cairo-fixed-private.h
+@@ -61,7 +61,7 @@
+ static inline cairo_fixed_t
+ _cairo_fixed_from_int (int i)
+ {
+-    return i << CAIRO_FIXED_FRAC_BITS;
++    return (unsigned)i << CAIRO_FIXED_FRAC_BITS;
+ }
+ 
+ /* This is the "magic number" approach to converting a double into fixed
+--- src/cairo-gstate.c
++++ src/cairo-gstate.c
+@@ -2297,7 +2297,7 @@
+ 		if (!drop || KEEP_GLYPH (transformed_glyphs[j]))
+ 		    j++;
+ 	    }
+-	    memcpy (transformed_clusters, clusters,
++	    if (num_clusters != 0) memcpy (transformed_clusters, clusters,
+ 		    num_clusters * sizeof (cairo_text_cluster_t));
+ 	} else {
+ 	    const cairo_glyph_t *cur_glyph;
+@@ -2352,7 +2352,7 @@
+ 		if (! drop || KEEP_GLYPH (transformed_glyphs[j]))
+ 		    j++;
+ 	    }
+-	    memcpy (transformed_clusters, clusters,
++	    if (num_clusters != 0) memcpy (transformed_clusters, clusters,
+ 		    num_clusters * sizeof (cairo_text_cluster_t));
+ 	} else {
+ 	    const cairo_glyph_t *cur_glyph;
+--- src/cairo-image-compositor.c
++++ src/cairo-image-compositor.c
+@@ -130,10 +130,10 @@
+ color_to_uint32 (const cairo_color_t *color)
+ {
+     return
+-        (color->alpha_short >> 8 << 24) |
+-        (color->red_short >> 8 << 16)   |
+-        (color->green_short & 0xff00)   |
+-        (color->blue_short >> 8);
++        ((uint32_t)color->alpha_short >> 8 << 24) |
++        ((uint32_t)color->red_short >> 8 << 16)   |
++        ((uint32_t)color->green_short & 0xff00)   |
++        ((uint32_t)color->blue_short >> 8);
+ }
+ 
+ static inline cairo_bool_t
+--- src/cairo-image-source.c
++++ src/cairo-image-source.c
+@@ -509,7 +509,11 @@
+ 	return pixman_image_create_solid_fill (&color);
+ 
+     case CAIRO_FORMAT_RGB24_888:
+-	pixel = *(uint32_t *) (image->data + y * image->stride + 3 * x);
++#ifdef WORDS_BIGENDIAN
++	pixel = (uint32_t)(image->data + y * image->stride + 3 * x)[3] | ((uint32_t)(image->data + y * image->stride + 3 * x)[2] << 8) | ((uint32_t)(image->data + y * image->stride + 3 * x)[1] << 16) | ((uint32_t)(image->data + y * image->stride + 3 * x)[0] << 24);
++#else
++	pixel = (uint32_t)(image->data + y * image->stride + 3 * x)[0] | ((uint32_t)(image->data + y * image->stride + 3 * x)[1] << 8) | ((uint32_t)(image->data + y * image->stride + 3 * x)[2] << 16) | ((uint32_t)(image->data + y * image->stride + 3 * x)[3] << 24);
++#endif
+ 	pixel &= 0x00ffffff; /* ignore next pixel bits */
+ 	if (pixel == 0)
+ 	    return _pixman_black_image ();
+--- src/cairo-spans-compositor.c
++++ src/cairo-spans-compositor.c
+@@ -1041,14 +1041,14 @@
+     if (status == CAIRO_INT_STATUS_UNSUPPORTED) {
+ 	cairo_polygon_t polygon;
+ 	cairo_fill_rule_t fill_rule = CAIRO_FILL_RULE_WINDING;
++	cairo_box_t limits;
+ 
+ 	if (! _cairo_rectangle_contains_rectangle (&extents->unbounded,
+ 						   &extents->mask))
+ 	{
+ 	    if (extents->clip->num_boxes == 1) {
+ 		_cairo_polygon_init (&polygon, extents->clip->boxes, 1);
+ 	    } else {
+-		cairo_box_t limits;
+ 		_cairo_box_from_rectangle (&limits, &extents->unbounded);
+ 		_cairo_polygon_init (&polygon, &limits, 1);
+ 	    }
+@@ -1128,17 +1128,17 @@
+     }
+     if (status == CAIRO_INT_STATUS_UNSUPPORTED) {
+ 	cairo_polygon_t polygon;
++	cairo_box_t limits;
+ 
+ 	TRACE((stderr, "%s - polygon\n", __FUNCTION__));
+ 
+ 	if (! _cairo_rectangle_contains_rectangle (&extents->unbounded,
+ 						   &extents->mask))
+ 	{
+ 	    TRACE((stderr, "%s - clipping to bounds\n", __FUNCTION__));
+ 	    if (extents->clip->num_boxes == 1) {
+ 		_cairo_polygon_init (&polygon, extents->clip->boxes, 1);
+ 	    } else {
+-		cairo_box_t limits;
+ 		_cairo_box_from_rectangle (&limits, &extents->unbounded);
+ 		_cairo_polygon_init (&polygon, &limits, 1);
+ 	    }
+--- src/cairo-tor-scan-converter.c
++++ src/cairo-tor-scan-converter.c
+@@ -253,7 +253,7 @@
+ #elif GRID_XY == 15
+ #  define  GRID_AREA_TO_ALPHA(c)  (((c) << 4) + (c))
+ #elif GRID_XY == 2*256*15
+-#  define  GRID_AREA_TO_ALPHA(c)  (((c) + ((c)<<4) + 256) >> 9)
++#  define  GRID_AREA_TO_ALPHA(c)  (((c) + ((uint32_t)(c)<<4) + 256) >> 9)
+ #else
+ #  define  GRID_AREA_TO_ALPHA(c)  (((c)*255 + GRID_XY/2) / GRID_XY)
+ #endif
diff --git a/external/cairo/pixman/pixman-ubsan.patch b/external/cairo/pixman/pixman-ubsan.patch
index 583706ae1764..4455d4a324e8 100644
--- a/external/cairo/pixman/pixman-ubsan.patch
+++ b/external/cairo/pixman/pixman-ubsan.patch
@@ -11,6 +11,17 @@
  #define pixman_fixed_frac(f)		((f) & pixman_fixed_1_minus_e)
 --- misc/pixman/pixman/pixman-fast-path.c
 +++ misc/build/pixman/pixman/pixman-fast-path.c
+@@ -2758,8 +2758,8 @@
+ 	 * positioned relative to a particular phase (and not relative to whatever
+ 	 * exact fraction we happen to get here).
+ 	 */
+-	x = ((vx >> x_phase_shift) << x_phase_shift) + ((1 << x_phase_shift) >> 1);
+-	y = ((vy >> y_phase_shift) << y_phase_shift) + ((1 << y_phase_shift) >> 1);
++	x = ((uint32_t)(vx >> x_phase_shift) << x_phase_shift) + ((1 << x_phase_shift) >> 1);
++	y = ((uint32_t)(vy >> y_phase_shift) << y_phase_shift) + ((1 << y_phase_shift) >> 1);
+ 
+ 	px = (x & 0xffff) >> x_phase_shift;
+ 	py = (y & 0xffff) >> y_phase_shift;
 @@ -2836,7 +2836,7 @@
  	sgtot = CLIP (sgtot, 0, 0xff);
  	sbtot = CLIP (sbtot, 0, 0xff);
@@ -20,8 +31,39 @@
  
      next:
  	vx += ux;
+@@ -3086,7 +3086,7 @@
+ static force_inline uint32_t
+ convert_a8 (const uint8_t *row, int x)
+ {
+-    return *(row + x) << 24;
++    return (uint32_t)*(row + x) << 24;
+ }
+ 
+ static force_inline uint32_t
+--- misc/pixman/pixman/pixman-access.c
++++ misc/build/pixman/pixman/pixman-access.c
+@@ -100,7 +100,7 @@
+ 	uint32_t  *__d = ((uint32_t *)(l)) + ((o) >> 5);		\
+ 	uint32_t __m, __v;						\
+ 									\
+-	__m = 1 << ((o) & 0x1f);					\
++	__m = (uint32_t)1 << ((o) & 0x1f);					\
+ 	__v = (v)? __m : 0;						\
+ 									\
+ 	WRITE((img), __d, (READ((img), __d) & ~__m) | __v);		\
 --- misc/pixman/pixman/pixman-bits-image.c
 +++ misc/build/pixman/pixman/pixman-bits-image.c
+@@ -243,8 +243,8 @@
+      * positioned relative to a particular phase (and not relative to whatever
+      * exact fraction we happen to get here).
+      */
+-    x = ((x >> x_phase_shift) << x_phase_shift) + ((1 << x_phase_shift) >> 1);
+-    y = ((y >> y_phase_shift) << y_phase_shift) + ((1 << y_phase_shift) >> 1);
++    x = ((uint32_t)(x >> x_phase_shift) << x_phase_shift) + ((1 << x_phase_shift) >> 1);
++    y = ((uint32_t)(y >> y_phase_shift) << y_phase_shift) + ((1 << y_phase_shift) >> 1);
+ 
+     px = (x & 0xffff) >> x_phase_shift;
+     py = (y & 0xffff) >> y_phase_shift;
 @@ -306,7 +306,7 @@
      sgtot = CLIP (sgtot, 0, 0xff);
      sbtot = CLIP (sbtot, 0, 0xff);
@@ -31,3 +73,113 @@
  }
  
  static force_inline uint32_t
+--- misc/pixman/pixman/pixman-combine32.c
++++ misc/build/pixman/pixman/pixman-combine32.c
+@@ -589,7 +589,7 @@
+ 	    rg = DIV_ONE_UN8 (rg);					\
+ 	    rb = DIV_ONE_UN8 (rb);					\
+ 									\
+-	    *(dest + i) = ra << 24 | rr << 16 | rg << 8 | rb;		\
++	    *(dest + i) = (uint32_t)ra << 24 | rr << 16 | rg << 8 | rb;		\
+ 	}								\
+     }									\
+     									\
+--- misc/pixman/pixman/pixman-gradient-walker.c
++++ misc/build/pixman/pixman/pixman-gradient-walker.c
+@@ -193,7 +193,7 @@
+     g8 = g + 0.5f;
+     b8 = b + 0.5f;
+ 
+-    v = ((a8 << 24) & 0xff000000) |
++    v = (((uint32_t)a8 << 24) & 0xff000000) |
+         ((r8 << 16) & 0x00ff0000) |
+         ((g8 <<  8) & 0x0000ff00) |
+         ((b8 >>  0) & 0x000000ff);
+--- misc/pixman/pixman/pixman-sse2.c
++++ misc/build/pixman/pixman/pixman-sse2.c
+@@ -516,9 +516,13 @@
+ }
+ 
+ static force_inline uint32_t
+-combine1 (const uint32_t *ps, const uint32_t *pm)
++combine1 (const void *ps, const uint32_t *pm)
+ {
+-    uint32_t s = *ps;
++#ifdef WORDS_BIGENDIAN
++    uint32_t s = (uint32_t)((const uint8_t *)ps)[3] | ((uint32_t)((const uint8_t *)ps)[2] << 8) | ((uint32_t)((const uint8_t *)ps)[1] << 16) | ((uint32_t)((const uint8_t *)ps)[0] << 24);
++#else
++    uint32_t s = (uint32_t)((const uint8_t *)ps)[0] | ((uint32_t)((const uint8_t *)ps)[1] << 8) | ((uint32_t)((const uint8_t *)ps)[2] << 16) | ((uint32_t)((const uint8_t *)ps)[3] << 24);
++#endif
+ 
+     if (pm)
+     {
+@@ -3256,7 +3260,11 @@
+ 
+ 	while (w >= 4)
+ 	{
+-	    m = *((uint32_t*)mask);
++#ifdef WORDS_BIGENDIAN
++	    m = (uint32_t)mask[3] | ((uint32_t)mask[2] << 8) | ((uint32_t)mask[1] << 16) | ((uint32_t)mask[0] << 24);
++#else
++	    m = (uint32_t)mask[0] | ((uint32_t)mask[1] << 8) | ((uint32_t)mask[2] << 16) | ((uint32_t)mask[3] << 24);
++#endif
+ 
+ 	    if (srca == 0xff && m == 0xffffffff)
+ 	    {
+@@ -3343,7 +3351,7 @@
+ 
+ 	b = filler & 0xff;
+ 	w = (b << 8) | b;
+-	filler = (w << 16) | w;
++	filler = ((uint32_t)w << 16) | w;
+     }
+     else if (bpp == 16)
+     {
+@@ -3528,7 +3536,11 @@
+ 
+ 	while (w >= 4)
+ 	{
+-	    m = *((uint32_t*)mask);
++#ifdef WORDS_BIGENDIAN
++	    m = (uint32_t)mask[3] | ((uint32_t)mask[2] << 8) | ((uint32_t)mask[1] << 16) | ((uint32_t)mask[0] << 24);
++#else
++	    m = (uint32_t)mask[0] | ((uint32_t)mask[1] << 8) | ((uint32_t)mask[2] << 16) | ((uint32_t)mask[3] << 24);
++#endif
+ 
+ 	    if (srca == 0xff && m == 0xffffffff)
+ 	    {
+@@ -5016,7 +5028,11 @@
+ 
+         while (w >= 4)
+         {
+-            m = *(uint32_t *) mask;
++#ifdef WORDS_BIGENDIAN
++            m = (uint32_t)mask[3] | ((uint32_t)mask[2] << 8) | ((uint32_t)mask[1] << 16) | ((uint32_t)mask[0] << 24);
++#else
++            m = (uint32_t)mask[0] | ((uint32_t)mask[1] << 8) | ((uint32_t)mask[2] << 16) | ((uint32_t)mask[3] << 24);
++#endif
+ 
+ 	    if (m)
+ 	    {
+@@ -5970,7 +5986,11 @@
+ 	__m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
+ 	__m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
+ 
+-	m = *(uint32_t*)mask;
++#ifdef WORDS_BIGENDIAN
++	m = (uint32_t)mask[3] | ((uint32_t)mask[2] << 8) | ((uint32_t)mask[1] << 16) | ((uint32_t)mask[0] << 24);
++#else // TODO:big endian
++	m = (uint32_t)mask[0] | ((uint32_t)mask[1] << 8) | ((uint32_t)mask[2] << 16) | ((uint32_t)mask[3] << 24);
++#endif
+ 
+ 	if (m)
+ 	{
+@@ -6437,7 +6457,7 @@
+ 
+     while (w)
+     {
+-	*dst++ = *(src++) << 24;
++	*dst++ = (uint32_t)*(src++) << 24;
+ 	w--;
+     }
+