Related: moz#424333 pixmaps are limited to max signed short

Change-Id: I88cfd9592e11aa7dbe704df4810090b6a1ce4c66

13 files changed, 122 insertions, 28 deletions

diff --git a/svtools/qa/cppunit/data/gif/pass/CVE-2007-6715-1.gif b/svtools/qa/cppunit/data/gif/pass/CVE-2007-6715-1.gif
new file mode 100644
index 000000000000..63426f9d80c2
--- /dev/null
+++ b/svtools/qa/cppunit/data/gif/pass/CVE-2007-6715-1.gif
diff --git a/svtools/qa/cppunit/data/jpg/pass/CVE-2004-0200-5.jpg b/svtools/qa/cppunit/data/jpg/pass/CVE-2004-0200-5.jpg
new file mode 100644
index 000000000000..bc668d3e3f31
--- /dev/null
+++ b/svtools/qa/cppunit/data/jpg/pass/CVE-2004-0200-5.jpg
diff --git a/svtools/source/filter/igif/gifread.cxx b/svtools/source/filter/igif/gifread.cxx
index 607b6a877af3..210b940201d6 100644
--- a/svtools/source/filter/igif/gifread.cxx
+++ b/svtools/source/filter/igif/gifread.cxx
@@ -51,6 +51,10 @@ GIFReader::GIFReader( SvStream& rStm ) :
             nLastPos        ( rStm.Tell() ),
             nLogWidth100    ( 0UL ),
             nLogHeight100   ( 0UL ),
+            nGlobalWidth    ( 0 ),
+            nGlobalHeight   ( 0 ),
+            nImageWidth     ( 0 ),
+            nImageHeight    ( 0 ),
             nLoops          ( 1 ),
             eActAction      ( GLOBAL_HEADER_READING ),
             bGCTransparent  ( sal_False ),
diff --git a/vcl/Library_vclplug_gen.mk b/vcl/Library_vclplug_gen.mk
index 2f42059e8d43..c9c9bbe6468f 100644
--- a/vcl/Library_vclplug_gen.mk
+++ b/vcl/Library_vclplug_gen.mk
@@ -101,6 +101,7 @@ $(eval $(call gb_Library_add_exception_objects,vclplug_gen,\
     vcl/unx/generic/window/salframe \
     vcl/unx/generic/window/salobj \
     vcl/unx/x11/x11sys \
+    vcl/unx/x11/xlimits \
 ))
 
 # ultimately we want to split the x11 dependencies out
diff --git a/vcl/inc/unx/x11/xlimits.hxx b/vcl/inc/unx/x11/xlimits.hxx
new file mode 100644
index 000000000000..a90ac80ad534
--- /dev/null
+++ b/vcl/inc/unx/x11/xlimits.hxx
@@ -0,0 +1,40 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * Version: MPL 1.1 / GPLv3+ / LGPLv3+
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License or as specified alternatively below. You may obtain a copy of
+ * the License at http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * Major Contributor(s):
+ * Copyright (C) 2012 Red Hat, Inc., Caolán McNamara <caolanm@redhat.com>
+ *  (initial developer)
+ *
+ * All Rights Reserved.
+ *
+ * For minor contributions see the git repository.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 3 or later (the "GPLv3+"), or
+ * the GNU Lesser General Public License Version 3 or later (the "LGPLv3+"),
+ * in which case the provisions of the GPLv3+ or the LGPLv3+ are applicable
+ * instead of those above.
+ */
+
+#ifndef VCL_X11LIMITS
+#define VCL_X11LIMITS
+
+#include <X11/Xlib.h>
+#include <vclpluginapi.h>
+
+VCLPLUG_GEN_PUBLIC Pixmap limitXCreatePixmap(Display *display, Drawable d, unsigned int width, unsigned int height, unsigned int depth);
+
+#endif
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/vcl/unx/generic/dtrans/bmp.cxx b/vcl/unx/generic/dtrans/bmp.cxx
index 534e80a60b42..9ca88a39f50d 100644
--- a/vcl/unx/generic/dtrans/bmp.cxx
+++ b/vcl/unx/generic/dtrans/bmp.cxx
@@ -34,6 +34,7 @@
 #include <bmp.hxx>
 
 #include <X11_selection.hxx>
+#include <unx/x11/xlimits.hxx>
 #include <sal/macros.h>
 
 using namespace x11;
@@ -670,7 +671,7 @@ Pixmap PixmapHolder::setBitmapData( const sal_uInt8* pData )
     if( m_aBitmap != None )
         XFreePixmap( m_pDisplay, m_aBitmap ), m_aBitmap = None;
 
-    m_aPixmap = XCreatePixmap( m_pDisplay,
+    m_aPixmap = limitXCreatePixmap( m_pDisplay,
                                RootWindow( m_pDisplay, m_aInfo.screen ),
                                nWidth, nHeight, m_aInfo.depth );
 
@@ -723,7 +724,7 @@ Pixmap PixmapHolder::setBitmapData( const sal_uInt8* pData )
         rtl_freeMemory( aImage.data );
 
         // prepare bitmap (mask)
-        m_aBitmap = XCreatePixmap( m_pDisplay,
+        m_aBitmap = limitXCreatePixmap( m_pDisplay,
                                    RootWindow( m_pDisplay, m_aInfo.screen ),
                                    nWidth, nHeight, 1 );
         XGCValues aVal;
diff --git a/vcl/unx/generic/gdi/salbmp.cxx b/vcl/unx/generic/gdi/salbmp.cxx
index bde272b48363..013bdd61d130 100644
--- a/vcl/unx/generic/gdi/salbmp.cxx
+++ b/vcl/unx/generic/gdi/salbmp.cxx
@@ -51,6 +51,7 @@
 #include <unx/salgdi.h>
 #include <unx/salbmp.h>
 #include <unx/salinst.h>
+#include <unx/x11/xlimits.hxx>
 
 // -----------
 // - Defines -
@@ -892,7 +893,7 @@ ImplSalDDB::ImplSalDDB( XImage* pImage, Drawable aDrawable,
     SalDisplay* pSalDisp = GetGenericData()->GetSalDisplay();
     Display*    pXDisp = pSalDisp->GetDisplay();
 
-    if( (maPixmap = XCreatePixmap( pXDisp, aDrawable, ImplGetWidth(), ImplGetHeight(), ImplGetDepth() )) )
+    if( (maPixmap = limitXCreatePixmap( pXDisp, aDrawable, ImplGetWidth(), ImplGetHeight(), ImplGetDepth() )) )
     {
         XGCValues   aValues;
         GC          aGC;
@@ -928,7 +929,7 @@ ImplSalDDB::ImplSalDDB(
     SalDisplay* pSalDisp = GetGenericData()->GetSalDisplay();
     Display*    pXDisp = pSalDisp->GetDisplay();
 
-    if( (maPixmap = XCreatePixmap( pXDisp, aDrawable, nWidth, nHeight, nDrawableDepth )) )
+    if( (maPixmap = limitXCreatePixmap( pXDisp, aDrawable, nWidth, nHeight, nDrawableDepth )) )
     {
         XGCValues   aValues;
         GC          aGC;
diff --git a/vcl/unx/generic/gdi/salgdi.cxx b/vcl/unx/generic/gdi/salgdi.cxx
index b07a709555cf..c0d1be5f8bd8 100644
--- a/vcl/unx/generic/gdi/salgdi.cxx
+++ b/vcl/unx/generic/gdi/salgdi.cxx
@@ -49,6 +49,7 @@
 #include "unx/salgdi.h"
 #include "unx/salframe.h"
 #include "unx/salvd.h"
+#include <unx/x11/xlimits.hxx>
 
 #include "generic/printergfx.hxx"
 #include "xrender_peer.hxx"
@@ -469,11 +470,11 @@ BOOL X11SalGraphics::GetDitherPixmap( SalColor nSalColor )
     {
         if (hBrush_)
             XFreePixmap (GetXDisplay(), hBrush_);
-        hBrush_ = XCreatePixmap( GetXDisplay(), GetDrawable(), 8, 8, 8 );
+        hBrush_ = limitXCreatePixmap( GetXDisplay(), GetDrawable(), 8, 8, 8 );
     }
     else
     if( !hBrush_ )
-        hBrush_ = XCreatePixmap( GetXDisplay(), GetDrawable(), 8, 8, 8 );
+        hBrush_ = limitXCreatePixmap( GetXDisplay(), GetDrawable(), 8, 8, 8 );
 
     // put the ximage to the pixmap
     XPutImage( GetXDisplay(),
@@ -1141,7 +1142,7 @@ bool X11SalGraphics::drawFilledTrapezoids( const ::basegfx::B2DTrapezoid* pB2DTr
     {
         Display* pXDisplay = GetXDisplay();
 
-        rEntry.m_aPixmap = ::XCreatePixmap( pXDisplay, hDrawable_, 1, 1, 32 );
+        rEntry.m_aPixmap = limitXCreatePixmap( pXDisplay, hDrawable_, 1, 1, 32 );
         XRenderPictureAttributes aAttr;
         aAttr.repeat = true;
 
diff --git a/vcl/unx/generic/gdi/salgdi2.cxx b/vcl/unx/generic/gdi/salgdi2.cxx
index e61576ef7391..033055ab2006 100644
--- a/vcl/unx/generic/gdi/salgdi2.cxx
+++ b/vcl/unx/generic/gdi/salgdi2.cxx
@@ -39,6 +39,7 @@
 #include "unx/salgdi.h"
 #include "unx/salframe.h"
 #include "unx/salvd.h"
+#include <unx/x11/xlimits.hxx>
 #include "xrender_peer.hxx"
 
 #include "generic/printergfx.hxx"
@@ -347,7 +348,7 @@ GC X11SalGraphics::SetMask( int           &nX,
     }
 
     // - - - - create alternate clip pixmap for region clipping - - - -
-    Pixmap hPixmap  = XCreatePixmap( pDisplay, hClipMask, nDX, nDY, 1 );
+    Pixmap hPixmap = limitXCreatePixmap( pDisplay, hClipMask, nDX, nDY, 1 );
 
     if( !hPixmap )
     {
@@ -503,7 +504,7 @@ void X11SalGraphics::copyBits( const SalTwoRect *pPosAry,
             && !pSrcGraphics->bVirDev_
             && (GetDisplay()->GetProperties() & PROPERTY_BUG_XCopyArea_GXxor) )
         {
-            Pixmap hPixmap = XCreatePixmap( GetXDisplay(),
+            Pixmap hPixmap = limitXCreatePixmap( GetXDisplay(),
                                             pSrcGraphics->GetDrawable(),        // source
                                             pPosAry->mnSrcWidth, pPosAry->mnSrcHeight,
                                             pSrcGraphics->GetBitCount() );
@@ -688,9 +689,9 @@ void X11SalGraphics::drawMaskedBitmap( const SalTwoRect* pPosAry,
     const sal_uInt16    nDepth( m_pVDev ?
                             m_pVDev->GetDepth() :
                             pSalDisp->GetVisual( m_nXScreen ).GetDepth() );
-    Pixmap          aFG( XCreatePixmap( pXDisp, aDrawable, pPosAry->mnDestWidth,
+    Pixmap          aFG( limitXCreatePixmap( pXDisp, aDrawable, pPosAry->mnDestWidth,
                                         pPosAry->mnDestHeight, nDepth ) );
-    Pixmap          aBG( XCreatePixmap( pXDisp, aDrawable, pPosAry->mnDestWidth,
+    Pixmap          aBG( limitXCreatePixmap( pXDisp, aDrawable, pPosAry->mnDestWidth,
                                         pPosAry->mnDestHeight, nDepth ) );
 
     if( aFG && aBG )
@@ -862,7 +863,7 @@ bool X11SalGraphics::drawAlphaBitmap( const SalTwoRect& rTR,
         pAlphaBits, pAlphaBuffer->mnWidth, pAlphaBuffer->mnHeight,
         pAlphaFormat->depth, pAlphaBuffer->mnScanlineSize );
 
-    Pixmap aAlphaPM = XCreatePixmap( pXDisplay, hDrawable_,
+    Pixmap aAlphaPM = limitXCreatePixmap( pXDisplay, hDrawable_,
         rTR.mnDestWidth, rTR.mnDestHeight, 8 );
 
     XGCValues aAlphaGCV;
@@ -944,7 +945,7 @@ void X11SalGraphics::drawMask( const SalTwoRect* pPosAry,
     const SalDisplay*   pSalDisp = GetDisplay();
     Display*            pXDisp = pSalDisp->GetDisplay();
     Drawable            aDrawable( GetDrawable() );
-    Pixmap              aStipple( XCreatePixmap( pXDisp, aDrawable,
+    Pixmap              aStipple( limitXCreatePixmap( pXDisp, aDrawable,
                                                  pPosAry->mnDestWidth,
                                                  pPosAry->mnDestHeight, 1 ) );
 
diff --git a/vcl/unx/generic/gdi/salvd.cxx b/vcl/unx/generic/gdi/salvd.cxx
index 0bf64dfdbc97..1a044c120d9d 100644
--- a/vcl/unx/generic/gdi/salvd.cxx
+++ b/vcl/unx/generic/gdi/salvd.cxx
@@ -38,6 +38,7 @@
 #include <unx/saldisp.hxx>
 #include <unx/salgdi.h>
 #include <unx/salvd.h>
+#include <unx/x11/xlimits.hxx>
 
 #include <salinst.hxx>
 
@@ -166,7 +167,7 @@ sal_Bool X11SalVirtualDevice::Init( SalDisplay *pDisplay,
     nDepth_                 = nBitCount;
 
     if( hDrawable == None )
-        hDrawable_          = XCreatePixmap( GetXDisplay(),
+        hDrawable_          = limitXCreatePixmap( GetXDisplay(),
                                              pDisplay_->GetDrawable( m_nXScreen ),
                                              nDX_, nDY_,
                                              GetDepth() );
@@ -225,20 +226,10 @@ sal_Bool X11SalVirtualDevice::SetSize( long nDX, long nDY )
     if( bExternPixmap_ )
         return sal_False;
 
-    // #144688#
-    // the X protocol request CreatePixmap puts an upper bound
-    // of 16 bit to the size. Beyond that there may be implementation
-    // limits of the Xserver; which we should catch by a failed XCreatePixmap
-    // call. However extra large values should be caught here since we'd run into
-    // 16 bit truncation here without noticing.
-    if( nDX < 0 || nDX > 65535 ||
-        nDY < 0 || nDY > 65535 )
-        return sal_False;
-
     if( !nDX ) nDX = 1;
     if( !nDY ) nDY = 1;
 
-    Pixmap h = XCreatePixmap( GetXDisplay(),
+    Pixmap h = limitXCreatePixmap( GetXDisplay(),
                               pDisplay_->GetDrawable( m_nXScreen ),
                               nDX, nDY, nDepth_ );
 
@@ -246,7 +237,7 @@ sal_Bool X11SalVirtualDevice::SetSize( long nDX, long nDY )
     {
         if( !GetDrawable() )
         {
-            hDrawable_ = XCreatePixmap( GetXDisplay(),
+            hDrawable_ = limitXCreatePixmap( GetXDisplay(),
                                         pDisplay_->GetDrawable( m_nXScreen ),
                                         1, 1, nDepth_ );
             nDX_ = 1;
diff --git a/vcl/unx/generic/window/salframe.cxx b/vcl/unx/generic/window/salframe.cxx
index 43f6acd3b2d0..549616ca8c17 100644
--- a/vcl/unx/generic/window/salframe.cxx
+++ b/vcl/unx/generic/window/salframe.cxx
@@ -66,6 +66,7 @@
 #include "unx/i18n_ic.hxx"
 #include "unx/i18n_keysym.hxx"
 #include "unx/i18n_status.hxx"
+#include <unx/x11/xlimits.hxx>
 
 #include "generic/gensys.h"
 #include "sallayout.hxx"
@@ -873,7 +874,7 @@ void X11SalFrame::SetBackgroundBitmap( SalBitmap* pBitmap )
         if( aSize.Width() && aSize.Height() )
         {
             mhBackgroundPixmap =
-                XCreatePixmap( GetXDisplay(),
+                limitXCreatePixmap( GetXDisplay(),
                                GetWindow(),
                                aSize.Width(),
                                aSize.Height(),
diff --git a/vcl/unx/gtk/window/gtkframe.cxx b/vcl/unx/gtk/window/gtkframe.cxx
index a647bf86429d..28776f123273 100644
--- a/vcl/unx/gtk/window/gtkframe.cxx
+++ b/vcl/unx/gtk/window/gtkframe.cxx
@@ -41,6 +41,9 @@
 #include <vcl/floatwin.hxx>
 #include <vcl/svapp.hxx>
 #include <vcl/window.hxx>
+#if !GTK_CHECK_VERSION(3,0,0)
+#include <unx/x11/xlimits.hxx>
+#endif
 
 #include <tools/prex.h>
 #include <X11/Xatom.h>
@@ -2760,7 +2763,7 @@ void GtkSalFrame::SetBackgroundBitmap( SalBitmap* pBitmap )
         if( aSize.Width() && aSize.Height() )
         {
             m_hBackgroundPixmap =
-                XCreatePixmap( getDisplay()->GetDisplay(),
+                limitXCreatePixmap( getDisplay()->GetDisplay(),
                                widget_get_xid(m_pWindow),
                                aSize.Width(),
                                aSize.Height(),
diff --git a/vcl/unx/x11/xlimits.cxx b/vcl/unx/x11/xlimits.cxx
new file mode 100644
index 000000000000..1d64d8e9f0f0
--- /dev/null
+++ b/vcl/unx/x11/xlimits.cxx
@@ -0,0 +1,50 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * Version: MPL 1.1 / GPLv3+ / LGPLv3+
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License or as specified alternatively below. You may obtain a copy of
+ * the License at http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * Major Contributor(s):
+ * Copyright (C) 2012 Red Hat, Inc., Caolán McNamara <caolanm@redhat.com>
+ *  (initial developer)
+ *
+ * All Rights Reserved.
+ *
+ * For minor contributions see the git repository.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 3 or later (the "GPLv3+"), or
+ * the GNU Lesser General Public License Version 3 or later (the "LGPLv3+"),
+ * in which case the provisions of the GPLv3+ or the LGPLv3+ are applicable
+ * instead of those above.
+ */
+
+#include <sal/log.hxx>
+#include <unx/x11/xlimits.hxx>
+
+Pixmap limitXCreatePixmap(Display *display, Drawable d, unsigned int width, unsigned int height, unsigned int depth)
+{
+    // The X protocol request CreatePixmap puts an upper bound
+    // of 16 bit to the size. Beyond that there may be implementation
+    // limits of the Xserver; which we should catch by a failed XCreatePixmap
+    // call. However extra large values should be caught here since we'd run into
+    // 16 bit truncation here without noticing.
+    //
+    // see, e.g. moz#424333
+    if (width > SAL_MAX_INT16 || height > SAL_MAX_INT16)
+    {
+        SAL_WARN("vcl", "overlarge pixmap: " << width << " x " << height);
+        return None;
+    }
+    return XCreatePixmap(display, d, width, height, depth);
+}
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */