ofz#829 ensure palette can fulfil largest index used

Change-Id: I95aa4796875ee71e8b06a2bb8985845f270b8817
Reviewed-on: https://gerrit.libreoffice.org/35104
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Miklos Vajna <vmiklos@collabora.co.uk>

1 files changed, 40 insertions, 25 deletions

diff --git a/filter/source/graphicfilter/itiff/itiff.cxx b/filter/source/graphicfilter/itiff/itiff.cxx
index 48f985e12ab6..db1ddccd1e64 100644
--- a/filter/source/graphicfilter/itiff/itiff.cxx
+++ b/filter/source/graphicfilter/itiff/itiff.cxx
@@ -52,6 +52,7 @@ private:
     Bitmap                  aBitmap;
     BitmapWriteAccess*      pAcc;
     sal_uInt16              nDstBitsPerPixel;
+    int                     nLargestPixelIndex;
     AlphaMask*              pAlphaMask;
     BitmapWriteAccess*      pMaskAcc;
 
@@ -120,6 +121,9 @@ private:
         // converts a Scanline to the Windows-BMP format
 
     bool HasAlphaChannel() const;
+
+    void SetPixelIndex(BitmapWriteAccess *pAcc, long nY, long nX, sal_uInt8 cIndex);
+
 public:
 
     TIFFReader()
@@ -127,6 +131,7 @@ public:
         , pTIFF(nullptr)
         , pAcc(nullptr)
         , nDstBitsPerPixel(0)
+        , nLargestPixelIndex(-1)
         , pAlphaMask(nullptr)
         , pMaskAcc(nullptr)
         , nOrigPos(0)
@@ -783,6 +788,11 @@ sal_uLong TIFFReader::GetBits( const sal_uInt8 * pSrc, sal_uLong nBitsPos, sal_u
     return nRes;
 }
 
+void TIFFReader::SetPixelIndex(BitmapWriteAccess *pWriteAcc, long nY, long nX, sal_uInt8 cIndex)
+{
+    pWriteAcc->SetPixelIndex(nY, nX, cIndex);
+    nLargestPixelIndex = std::max<int>(nLargestPixelIndex, cIndex);
+}
 
 bool TIFFReader::ConvertScanline(sal_Int32 nY)
 {
@@ -945,7 +955,7 @@ bool TIFFReader::ConvertScanline(sal_Int32 nY)
                             for (sal_Int32 nx = 0; nx < nImageWidth; ++nx)
                             {
                                 nLast += nx == 0 ? BYTESWAP( (sal_uInt8)*pt++ ) : *pt++;
-                                pAcc->SetPixelIndex( nY, nx, nLast );
+                                SetPixelIndex(pAcc, nY, nx, nLast);
                             }
                         }
                         else
@@ -953,7 +963,7 @@ bool TIFFReader::ConvertScanline(sal_Int32 nY)
                             for (sal_Int32 nx = 0; nx < nImageWidth; ++nx)
                             {
                                 sal_uInt8 nLast = *pt++;
-                                pAcc->SetPixelIndex( nY, nx, static_cast<sal_uInt8>( (BYTESWAP((sal_uLong)nLast) - nMinSampleValue) * nMinMax ) );
+                                SetPixelIndex(pAcc, nY, nx, static_cast<sal_uInt8>( (BYTESWAP((sal_uLong)nLast) - nMinSampleValue) * nMinMax ));
                             }
                         }
                     }
@@ -965,15 +975,14 @@ bool TIFFReader::ConvertScanline(sal_Int32 nY)
                             for (sal_Int32 nx = 0; nx < nImageWidth; ++nx)
                             {
                                 nLast += *pt++;
-                                pAcc->SetPixelIndex( nY, nx, nLast );
+                                SetPixelIndex(pAcc, nY, nx, nLast);
                             }
                         }
                         else
                         {
                             for (sal_Int32 nx = 0; nx < nImageWidth; ++nx)
                             {
-                                pAcc->SetPixelIndex( nY, nx, static_cast<sal_uInt8>( ((sal_uLong)*pt++ - nMinSampleValue) * nMinMax ) );
-
+                                SetPixelIndex(pAcc, nY, nx, static_cast<sal_uInt8>( ((sal_uLong)*pt++ - nMinSampleValue) * nMinMax ));
                             }
                         }
                     }
@@ -990,7 +999,7 @@ bool TIFFReader::ConvertScanline(sal_Int32 nY)
                     for (sal_Int32 nx = 0; nx < nImageWidth; ++nx)
                     {
                         nVal = ( GetBits( pt, nx * nBitsPerSample, nBitsPerSample ) - nMinSampleValue ) * nMinMax;
-                        pAcc->SetPixelIndex( nY, nx, static_cast<sal_uInt8>(nVal));
+                        SetPixelIndex(pAcc, nY, nx, static_cast<sal_uInt8>(nVal));
                     }
                 }
                 break;
@@ -1011,28 +1020,28 @@ bool TIFFReader::ConvertScanline(sal_Int32 nY)
                         while (nByteCount--)
                         {
                             nByteVal = *pt++;
-                            pAcc->SetPixelIndex( nY, nx++, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, nx++, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, nx++, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, nx++, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, nx++, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, nx++, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, nx++, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, nx++, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, nx++, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, nx++, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, nx++, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, nx++, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, nx++, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, nx++, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, nx++, nByteVal );
+                            SetPixelIndex(pAcc, nY, nx++, nByteVal);
                         }
                         if ( nImageWidth & 7 )
                         {
                             nByteVal = *pt++;
                             while ( nx < nImageWidth )
                             {
-                                pAcc->SetPixelIndex( nY, nx++, nByteVal & 1 );
+                                SetPixelIndex(pAcc, nY, nx++, nByteVal & 1);
                                 nByteVal >>= 1;
                             }
                         }
@@ -1043,21 +1052,21 @@ bool TIFFReader::ConvertScanline(sal_Int32 nY)
                         while (nByteCount--)
                         {
                             nByteVal = *pt++;
-                            pAcc->SetPixelIndex( nY, nx, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, nx, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, --nx, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, --nx, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, --nx, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, --nx, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, --nx, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, --nx, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, --nx, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, --nx, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, --nx, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, --nx, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, --nx, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, --nx, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, --nx, nByteVal );
+                            SetPixelIndex(pAcc, nY, --nx, nByteVal);
                             nx += 15;
                         }
                         if ( nImageWidth & 7 )
@@ -1067,7 +1076,7 @@ bool TIFFReader::ConvertScanline(sal_Int32 nY)
                             nShift = 7;
                             while ( nx < nImageWidth )
                             {
-                                pAcc->SetPixelIndex( nY, nx++, ( nByteVal >> nShift ) & 1);
+                                SetPixelIndex(pAcc, nY, nx++, ( nByteVal >> nShift ) & 1);
                             }
                         }
                     }
@@ -1088,7 +1097,7 @@ bool TIFFReader::ConvertScanline(sal_Int32 nY)
             sal_uInt8*  pt = pMap[ 0 ];
             for (sal_Int32 nx = 0; nx < nImageWidth; nx++, pt += 2 )
             {
-                pAcc->SetPixelIndex( nY, nx, static_cast<sal_uInt8>( ((sal_uLong)*pt - nMinSampleValue) * nMinMax) );
+                SetPixelIndex(pAcc, nY, nx, static_cast<sal_uInt8>( ((sal_uLong)*pt - nMinSampleValue) * nMinMax));
             }
         }
     }
@@ -1110,6 +1119,11 @@ void TIFFReader::MakePalCol()
             nNumColors = (sal_uLong)1 << nBitsPerSample;
             if ( nNumColors > 256 )
                 nNumColors = 256;
+            if (nLargestPixelIndex >= static_cast<int>(nNumColors))
+            {
+                SAL_WARN("filter.tiff", "palette has less entries that largest index used. Expanding palette to match");
+                nNumColors = nLargestPixelIndex + 1;
+            }
             pAcc->SetPaletteEntryCount( (sal_uInt16)nNumColors );
             for ( i = 0; i < nNumColors; i++ )
             {
@@ -1340,6 +1354,7 @@ bool TIFFReader::ReadTIFF(SvStream & rTIFF, Graphic & rGraphic )
                 bStatus = false;
             if ( bStatus )
             {
+                nLargestPixelIndex = -1;
                 if ( nMaxSampleValue == 0 )
                 {
                     if ( nBitsPerSample == 32 )         // sj: i93300, compiler bug, 1 << 32 gives 1 one 32bit windows platforms,