ofz#5573 Out of memory

Change-Id: Ifb5cfdd87d7b26d18fcb66279afa7ef06beea9e3
Reviewed-on: https://gerrit.libreoffice.org/48323
Reviewed-by: Caolán McNamara <caolanm@redhat.com>
Tested-by: Caolán McNamara <caolanm@redhat.com>

1 files changed, 14 insertions, 13 deletions

diff --git a/vcl/source/filter/igif/gifread.cxx b/vcl/source/filter/igif/gifread.cxx
index ea92b3ee1a0c..5667960c5766 100644
--- a/vcl/source/filter/igif/gifread.cxx
+++ b/vcl/source/filter/igif/gifread.cxx
@@ -157,19 +157,7 @@ void GIFReader::CreateBitmaps( long nWidth, long nHeight, BitmapPalette* pPal,
 {
     const Size aSize( nWidth, nHeight );
 
-#if SAL_TYPES_SIZEOFPOINTER == 8
-    // Don't bother allocating a bitmap of a size that would fail on a
-    // 32-bit system. We have at least one unit tests that is expected
-    // to fail (loading a 65535*65535 size GIF
-    // svtools/qa/cppunit/data/gif/fail/CVE-2008-5937-1.gif), but
-    // which doesn't fail on 64-bit Mac OS X at least. Why the loading
-    // fails on 64-bit Linux, no idea.
-    if (nWidth >= 64000 && nHeight >= 64000)
-    {
-        bStatus = false;
-        return;
-    }
-#endif
+    sal_uInt64 nCombinedPixSize = nWidth * nHeight;
 
     // "Overall data compression asymptotically approaches 3839 × 8 / 12 = 2559 1/3"
     // so assume compression of 1:2560 is possible
@@ -181,6 +169,7 @@ void GIFReader::CreateBitmaps( long nWidth, long nHeight, BitmapPalette* pPal,
     {
         const Size& rSize = aAnimation.Get(i).aSizePix;
         nMinFileData += rSize.Width() * rSize.Height() / 2560;
+        nCombinedPixSize += rSize.Width() * rSize.Height();
     }
 
     if (nMaxStreamData < nMinFileData)
@@ -192,6 +181,18 @@ void GIFReader::CreateBitmaps( long nWidth, long nHeight, BitmapPalette* pPal,
         return;
     }
 
+    // Don't bother allocating a bitmap of a size that would fail on a
+    // 32-bit system. We have at least one unit tests that is expected
+    // to fail (loading a 65535*65535 size GIF
+    // svtools/qa/cppunit/data/gif/fail/CVE-2008-5937-1.gif), but
+    // which doesn't fail on 64-bit Mac OS X at least. Why the loading
+    // fails on 64-bit Linux, no idea.
+    if (nCombinedPixSize >= 64000U * 64000U)
+    {
+        bStatus = false;
+        return;
+    }
+
     if( bGCTransparent )
     {
         const Color aWhite( COL_WHITE );