Resolves: ofz#310: Sanitize 1 bit bitmap palette indexes too

(cherry picked from commit b08e48166310ab8a53f06f4fee227a948416cf4a)
(cherry picked from commit 3bff2da31ed580817f6b5b653f48727e38078af5)

crashtesting: fix bmp->odg failure...

id:000178,src:000000,op:havoc,rep:16.bmp
id:000383,src:000047,op:flip2,pos:18,+cov.bmp
id:000403,src:000047,op:arith32,pos:22,val:-33.bmp
id:000317,src:000025,op:havoc,rep:8.bmp
id:000119,src:000000,op:havoc,rep:16.bmp
id:000167,src:000000,op:havoc,rep:8.bmp

revealed since

commit 81e3ca4f60e6ac0823c1233841c22a759cfe937f
Author: Tor Lillqvist <tml@collabora.com>
Date:   Tue Jun 21 10:34:21 2016 +0300

Use real assert() instead of DBG_ASSERT()

sanitize invalid palette entry indexes at the outer perimeter on initial load
to try and avoid having to do it in all sort of places in the interior.

we load the palette before getting here, so we should always know the palette size here

(cherry picked from commit 92cee94a262a3a2f43c87bb940c50cb90a2ebd89)

Change-Id: Ied2b71eb1c9088bbbff5fed97f7a04b43f4b4827
Reviewed-on: https://gerrit.libreoffice.org/32333
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Markus Mohrhard <markus.mohrhard@googlemail.com>

1 files changed, 48 insertions, 16 deletions

diff --git a/vcl/source/gdi/dibtools.cxx b/vcl/source/gdi/dibtools.cxx
index 75e39e6f018e..f45fa3d206c8 100644
--- a/vcl/source/gdi/dibtools.cxx
+++ b/vcl/source/gdi/dibtools.cxx
@@ -331,6 +331,22 @@ bool ImplReadDIBPalette( SvStream& rIStm, BitmapWriteAccess& rAcc, bool bQuad )
     return( rIStm.GetError() == 0UL );
 }
 
+namespace
+{
+    sal_uInt8 SanitizePaletteIndex(sal_uInt8 nIndex, bool bHasPalette, sal_uInt16 nPaletteEntryCount)
+    {
+        if (bHasPalette && nIndex >= nPaletteEntryCount)
+        {
+            auto nSanitizedIndex = nIndex % nPaletteEntryCount;
+            SAL_WARN_IF(nIndex != nSanitizedIndex, "vcl", "invalid colormap index: "
+                        << static_cast<unsigned int>(nIndex) << ", colormap len is: "
+                        << nPaletteEntryCount);
+            nIndex = nSanitizedIndex;
+        }
+        return nIndex;
+    }
+}
+
 bool ImplDecodeRLE( sal_uInt8* pBuffer, DIBV5Header& rHeader, BitmapWriteAccess& rAcc, bool bRLE4 )
 {
     Scanline pRLE = pBuffer;
@@ -342,6 +358,8 @@ bool ImplDecodeRLE( sal_uInt8* pBuffer, DIBV5Header& rHeader, BitmapWriteAccess&
     sal_uLong       nX = 0UL;
     sal_uInt8       cTmp;
     bool        bEndDecoding = false;
+    const bool bHasPalette = rAcc.HasPalette();
+    const sal_uInt16 nPaletteEntryCount = rAcc.GetPaletteEntryCount();
 
     do
     {
@@ -367,10 +385,10 @@ bool ImplDecodeRLE( sal_uInt8* pBuffer, DIBV5Header& rHeader, BitmapWriteAccess&
                         cTmp = *pRLE++;
 
                         if( nX < nWidth )
-                            rAcc.SetPixelIndex( nY, nX++, cTmp >> 4 );
+                            rAcc.SetPixelIndex(nY, nX++, SanitizePaletteIndex(cTmp >> 4, bHasPalette, nPaletteEntryCount));
 
                         if( nX < nWidth )
-                            rAcc.SetPixelIndex( nY, nX++, cTmp & 0x0f );
+                            rAcc.SetPixelIndex(nY, nX++, SanitizePaletteIndex(cTmp & 0x0f, bHasPalette, nPaletteEntryCount));
                     }
 
                     if( nRunByte & 1 )
@@ -379,7 +397,7 @@ bool ImplDecodeRLE( sal_uInt8* pBuffer, DIBV5Header& rHeader, BitmapWriteAccess&
                             return false;
 
                         if( nX < nWidth )
-                            rAcc.SetPixelIndex( nY, nX++, *pRLE >> 4 );
+                            rAcc.SetPixelIndex(nY, nX++, SanitizePaletteIndex(*pRLE >> 4, bHasPalette, nPaletteEntryCount));
 
                         pRLE++;
                     }
@@ -400,7 +418,7 @@ bool ImplDecodeRLE( sal_uInt8* pBuffer, DIBV5Header& rHeader, BitmapWriteAccess&
                             return false;
 
                         if( nX < nWidth )
-                            rAcc.SetPixelIndex( nY, nX++, *pRLE );
+                            rAcc.SetPixelIndex(nY, nX++, SanitizePaletteIndex(*pRLE, bHasPalette, nPaletteEntryCount));
 
                         pRLE++;
                     }
@@ -447,19 +465,19 @@ bool ImplDecodeRLE( sal_uInt8* pBuffer, DIBV5Header& rHeader, BitmapWriteAccess&
                 for( sal_uLong i = 0UL; i < nRunByte; i++ )
                 {
                     if( nX < nWidth )
-                        rAcc.SetPixelIndex( nY, nX++, cTmp >> 4 );
+                        rAcc.SetPixelIndex(nY, nX++, SanitizePaletteIndex(cTmp >> 4, bHasPalette, nPaletteEntryCount));
 
                     if( nX < nWidth )
-                        rAcc.SetPixelIndex( nY, nX++, cTmp & 0x0f );
+                        rAcc.SetPixelIndex(nY, nX++, SanitizePaletteIndex(cTmp & 0x0f, bHasPalette, nPaletteEntryCount));
                 }
 
                 if( ( nCountByte & 1 ) && ( nX < nWidth ) )
-                    rAcc.SetPixelIndex( nY, nX++, cTmp >> 4 );
+                    rAcc.SetPixelIndex(nY, nX++, SanitizePaletteIndex(cTmp >> 4, bHasPalette, nPaletteEntryCount));
             }
             else
             {
                 for( sal_uLong i = 0UL; ( i < nCountByte ) && ( nX < nWidth ); i++ )
-                    rAcc.SetPixelIndex( nY, nX++, cTmp );
+                    rAcc.SetPixelIndex(nY, nX++, SanitizePaletteIndex(cTmp, bHasPalette, nPaletteEntryCount));
             }
         }
     }
@@ -481,10 +499,10 @@ bool ImplReadDIBBits(SvStream& rIStm, DIBV5Header& rHeader, BitmapWriteAccess& r
     switch(rAcc.GetScanlineFormat())
     {
         case ScanlineFormat::N1BitMsbPal:
-        case ScanlineFormat::N4BitMsnPal:
-        case ScanlineFormat::N8BitPal:
         case ScanlineFormat::N24BitTcBgr:
         {
+            // we can't trust arbitrary-sourced index based formats to have correct indexes, so we exclude the pal formats
+            // from raw read and force checking their colormap indexes
             bNative = ( ( static_cast< bool >(rAcc.IsBottomUp()) != bTopDown ) && !bRLE && !bTCMask && ( rAcc.GetScanlineSize() == nAlignedWidth ) );
             break;
         }
@@ -496,7 +514,7 @@ bool ImplReadDIBBits(SvStream& rIStm, DIBV5Header& rHeader, BitmapWriteAccess& r
     }
 
     // Read data
-    if(bNative)
+    if (bNative)
     {
         if (nAlignedWidth
             > std::numeric_limits<sal_Size>::max() / rHeader.nHeight)
@@ -520,7 +538,7 @@ bool ImplReadDIBBits(SvStream& rIStm, DIBV5Header& rHeader, BitmapWriteAccess& r
             rIStm.ReadUInt32( nBMask );
         }
 
-        if(bRLE)
+        if (bRLE)
         {
             if(!rHeader.nSizeImage)
             {
@@ -550,6 +568,9 @@ bool ImplReadDIBBits(SvStream& rIStm, DIBV5Header& rHeader, BitmapWriteAccess& r
             {
                 case 1:
                 {
+                    const bool bHasPalette = rAcc.HasPalette();
+                    const sal_uInt16 nPaletteEntryCount = rAcc.GetPaletteEntryCount();
+
                     for( ; nCount--; nY += nI )
                     {
                         sal_uInt8*  pTmp;
@@ -568,7 +589,8 @@ bool ImplReadDIBBits(SvStream& rIStm, DIBV5Header& rHeader, BitmapWriteAccess& r
                                 cTmp = *pTmp++;
                             }
 
-                            rAcc.SetPixelIndex( nY, nX, (cTmp >> --nShift) & 1);
+                            auto nIndex = (cTmp >> --nShift) & 1;
+                            rAcc.SetPixelIndex(nY, nX, SanitizePaletteIndex(nIndex, bHasPalette, nPaletteEntryCount));
                         }
                     }
                 }
@@ -576,6 +598,9 @@ bool ImplReadDIBBits(SvStream& rIStm, DIBV5Header& rHeader, BitmapWriteAccess& r
 
                 case 4:
                 {
+                    const bool bHasPalette = rAcc.HasPalette();
+                    const sal_uInt16 nPaletteEntryCount = rAcc.GetPaletteEntryCount();
+
                     for( ; nCount--; nY += nI )
                     {
                         sal_uInt8*  pTmp;
@@ -594,7 +619,8 @@ bool ImplReadDIBBits(SvStream& rIStm, DIBV5Header& rHeader, BitmapWriteAccess& r
                                 cTmp = *pTmp++;
                             }
 
-                            rAcc.SetPixelIndex( nY, nX, (cTmp >> ( --nShift << 2UL ) ) & 0x0f);
+                            auto nIndex = (cTmp >> ( --nShift << 2UL ) ) & 0x0f;
+                            rAcc.SetPixelIndex(nY, nX, SanitizePaletteIndex(nIndex, bHasPalette, nPaletteEntryCount));
                         }
                     }
                 }
@@ -602,6 +628,9 @@ bool ImplReadDIBBits(SvStream& rIStm, DIBV5Header& rHeader, BitmapWriteAccess& r
 
                 case 8:
                 {
+                    const bool bHasPalette = rAcc.HasPalette();
+                    const sal_uInt16 nPaletteEntryCount = rAcc.GetPaletteEntryCount();
+
                     for( ; nCount--; nY += nI )
                     {
                         sal_uInt8*  pTmp;
@@ -612,7 +641,10 @@ bool ImplReadDIBBits(SvStream& rIStm, DIBV5Header& rHeader, BitmapWriteAccess& r
                         }
 
                         for( long nX = 0L; nX < nWidth; nX++ )
-                            rAcc.SetPixelIndex( nY, nX, *pTmp++ );
+                        {
+                            auto nIndex = *pTmp++;
+                            rAcc.SetPixelIndex(nY, nX, SanitizePaletteIndex(nIndex, bHasPalette, nPaletteEntryCount));
+                        }
                     }
                 }
                 break;
@@ -895,7 +927,7 @@ bool ImplReadDIBBody( SvStream& rIStm, Bitmap& rBmp, AlphaMask* pBmpAlpha, sal_u
         }
 
         // read palette
-        if(nColors)
+        if (nColors)
         {
             pAcc->SetPaletteEntryCount(nColors);
             ImplReadDIBPalette(*pIStm, *pAcc, aHeader.nSize != DIBCOREHEADERSIZE);