From 91e6074c79ffc62e57568544c3d01f9b576b4795 Mon Sep 17 00:00:00 2001
From: Stephan Bergmann <sbergman@redhat.com>
Date: Wed, 23 Apr 2014 17:53:21 +0200
Subject: Avoid integer overflow

Change-Id: Id429ad5ebb9bd1501292756db45d9fac76f26222
---
 vcl/quartz/salbmp.cxx | 31 +++++++++++++++++++++++--------
 1 file changed, 23 insertions(+), 8 deletions(-)

(limited to 'vcl/quartz')

diff --git a/vcl/quartz/salbmp.cxx b/vcl/quartz/salbmp.cxx
index 138d0f129bf9..2704dc428da3 100644
--- a/vcl/quartz/salbmp.cxx
+++ b/vcl/quartz/salbmp.cxx
@@ -17,6 +17,11 @@
  *   the License at http://www.apache.org/licenses/LICENSE-2.0 .
  */
 
+#include <sal/config.h>
+
+#include <cstddef>
+#include <limits>
+
 #include "basebmp/scanlineformats.hxx"
 #include "basebmp/color.hxx"
 
@@ -296,21 +301,31 @@ bool QuartzSalBitmap::AllocateUserData()
         }
     }
 
-    try
+    bool alloc = false;
+    if (mnBytesPerRow != 0
+        && mnBytesPerRow <= std::numeric_limits<std::size_t>::max() / mnHeight)
     {
-        if( mnBytesPerRow )
+        try
+        {
             maUserBuffer.reset( new sal_uInt8[mnBytesPerRow * mnHeight] );
-#ifdef DBG_UTIL
-        for (size_t i = 0; i < mnBytesPerRow * mnHeight; i++)
-            maUserBuffer.get()[i] = (i & 0xFF);
-#endif
+            alloc = true;
+        }
+        catch (std::bad_alloc &) {}
     }
-    catch( const std::bad_alloc& )
+    if (!alloc)
     {
-        OSL_FAIL( "vcl::QuartzSalBitmap::AllocateUserData: bad alloc" );
+        SAL_WARN(
+            "vcl.quartz", "bad alloc " << mnBytesPerRow << "x" << mnHeight);
         maUserBuffer.reset( static_cast<sal_uInt8*>(NULL) );
         mnBytesPerRow = 0;
     }
+#ifdef DBG_UTIL
+    else
+    {
+        for (size_t i = 0; i < mnBytesPerRow * mnHeight; i++)
+            maUserBuffer.get()[i] = (i & 0xFF);
+    }
+#endif
 
     return maUserBuffer.get() != 0;
 }
-- 
cgit