valgrind + bff: Integer divide by zero

(cherry picked from commit a1fab361cb3325b2d012942e43660f2c0a2a4b96)

Conflicts:
filter/source/graphicfilter/itiff/itiff.cxx

Change-Id: I4c47d17a421f1c6d2d6cc29c7b92ec930bdf901f
Reviewed-on: https://gerrit.libreoffice.org/11022
Reviewed-by: David Tardon <dtardon@redhat.com>
Tested-by: David Tardon <dtardon@redhat.com>

1 files changed, 22 insertions, 9 deletions

diff --git a/filter/source/graphicfilter/itiff/itiff.cxx b/filter/source/graphicfilter/itiff/itiff.cxx
index 71bad87d44d9..a921b5359ebd 100644
--- a/filter/source/graphicfilter/itiff/itiff.cxx
+++ b/filter/source/graphicfilter/itiff/itiff.cxx
@@ -178,6 +178,19 @@ public:
         delete pAlphaMask;
     }
 
+    sal_uLong GetRowsPerStrip() const
+    {
+        //Rows Per Strip:
+        //
+        //(TIFF format only) The number of rows of pixels per strip to use for
+        //encoding the TIFF image. A value greater than zero specifies the
+        //number of rows per strip. A value of 0 sets the rows per strip equal
+        //to the image length, resulting in a single strip. A value of -1 (the
+        //default) sets the rows per strip equal to infinity, resulting in a
+        //single strip.
+        return nRowsPerStrip == 0 ? nImageLength : nRowsPerStrip;
+    }
+
     sal_Bool ReadTIFF( SvStream & rTIFF, Graphic & rGraphic );
 };
 
@@ -547,10 +560,10 @@ sal_Bool TIFFReader::ReadMap( sal_uLong nMinPercent, sal_uLong nMaxPercent )
         {
             for ( np = 0; np < nPlanes; np++ )
             {
-                nStrip = ny / nRowsPerStrip + np * nStripsPerPlane;
+                nStrip = ny / GetRowsPerStrip() + np * nStripsPerPlane;
                 if ( nStrip >= nNumStripOffsets )
                     return sal_False;
-                pTIFF->Seek( pStripOffsets[ nStrip ] + ( ny % nRowsPerStrip ) * nStripBytesPerRow );
+                pTIFF->Seek( pStripOffsets[ nStrip ] + ( ny % GetRowsPerStrip() ) * nStripBytesPerRow );
                 pTIFF->Read( pMap[ np ], nBytesPerRow );
                 if ( pTIFF->GetError() )
                     return sal_False;
@@ -601,9 +614,9 @@ sal_Bool TIFFReader::ReadMap( sal_uLong nMinPercent, sal_uLong nMaxPercent )
         {
             for ( np = 0; np < nPlanes; np++ )
             {
-                if ( ny / nRowsPerStrip + np * nStripsPerPlane > nStrip )
+                if ( ny / GetRowsPerStrip() + np * nStripsPerPlane > nStrip )
                 {
-                    nStrip=ny/nRowsPerStrip+np*nStripsPerPlane;
+                    nStrip=ny/GetRowsPerStrip()+np*nStripsPerPlane;
                     if ( nStrip >= nNumStripOffsets )
                         return sal_False;
                     pTIFF->Seek( pStripOffsets[ nStrip ] );
@@ -632,9 +645,9 @@ sal_Bool TIFFReader::ReadMap( sal_uLong nMinPercent, sal_uLong nMaxPercent )
         {
             for ( np = 0; np < nPlanes; np++ )
             {
-                if ( ny / nRowsPerStrip + np * nStripsPerPlane > nStrip )
+                if ( ny / GetRowsPerStrip() + np * nStripsPerPlane > nStrip )
                 {
-                    nStrip = ny / nRowsPerStrip + np * nStripsPerPlane;
+                    nStrip = ny / GetRowsPerStrip() + np * nStripsPerPlane;
                     if ( nStrip >= nNumStripOffsets )
                         return sal_False;
                     pTIFF->Seek(pStripOffsets[nStrip]);
@@ -660,9 +673,9 @@ sal_Bool TIFFReader::ReadMap( sal_uLong nMinPercent, sal_uLong nMaxPercent )
         {
             for ( np = 0; np < nPlanes; np++ )
             {
-                if ( ny / nRowsPerStrip + np * nStripsPerPlane > nStrip )
+                if ( ny / GetRowsPerStrip() + np * nStripsPerPlane > nStrip )
                 {
-                    nStrip=ny/nRowsPerStrip+np*nStripsPerPlane;
+                    nStrip=ny/GetRowsPerStrip()+np*nStripsPerPlane;
                     if ( nStrip >= nNumStripOffsets )
                         return sal_False;
                     pTIFF->Seek(pStripOffsets[nStrip]);
@@ -1318,7 +1331,7 @@ sal_Bool TIFFReader::ReadTIFF(SvStream & rTIFF, Graphic & rGraphic )
                     if ( ( nFillOrder == 2 ) && ( nCompression != 5 ) )     // in the LZW mode bits are already being inverted
                         bByteSwap = sal_True;
 
-                    nStripsPerPlane = ( nImageLength - 1 ) / nRowsPerStrip + 1;
+                    nStripsPerPlane = ( nImageLength - 1 ) / GetRowsPerStrip() + 1;
                     nBytesPerRow = ( nImageWidth * nSamplesPerPixel / nPlanes * nBitsPerSample + 7 ) >> 3;
 
                     for ( sal_uLong j = 0; j < 4; j++ )