ofz#1605 check multiply and shift

Change-Id: I6aad9ad23e7bf080b3b610223f92df7074530beb
Reviewed-on: https://gerrit.libreoffice.org/37642
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Miklos Vajna <vmiklos@collabora.co.uk>

1 files changed, 18 insertions, 1 deletions

diff --git a/vcl/source/gdi/salmisc.cxx b/vcl/source/gdi/salmisc.cxx
index 84c7becceea4..0218c320e92b 100644
--- a/vcl/source/gdi/salmisc.cxx
+++ b/vcl/source/gdi/salmisc.cxx
@@ -20,6 +20,7 @@
 #include <vcl/bitmapaccess.hxx>
 #include <vcl/salbtype.hxx>
 #include <bmpfast.hxx>
+#include <o3tl/safeint.hxx>
 #include <osl/diagnose.h>
 #include <memory>
 
@@ -330,7 +331,23 @@ BitmapBuffer* StretchAndConvert(
     pDstBuffer->mnFormat = nDstBitmapFormat;
     pDstBuffer->mnWidth = rTwoRect.mnDestWidth;
     pDstBuffer->mnHeight = rTwoRect.mnDestHeight;
-    pDstBuffer->mnScanlineSize = AlignedWidth4Bytes( pDstBuffer->mnBitCount * pDstBuffer->mnWidth );
+    long nScanlineBase;
+    bool bFail = o3tl::checked_multiply<long>(pDstBuffer->mnBitCount, pDstBuffer->mnWidth, nScanlineBase);
+    if (bFail)
+    {
+        SAL_WARN("vcl.gdi", "checked multiply failed");
+        pDstBuffer->mpBits = nullptr;
+        delete pDstBuffer;
+        return nullptr;
+    }
+    pDstBuffer->mnScanlineSize = AlignedWidth4Bytes(nScanlineBase);
+    if (pDstBuffer->mnScanlineSize < nScanlineBase/8)
+    {
+        SAL_WARN("vcl.gdi", "scanline calculation wraparound");
+        pDstBuffer->mpBits = nullptr;
+        delete pDstBuffer;
+        return nullptr;
+    }
     try
     {
         pDstBuffer->mpBits = new sal_uInt8[ pDstBuffer->mnScanlineSize * pDstBuffer->mnHeight ];