Avoid undefined floating -> integer conversion in TIFF import

...as started to happen when reading (invalid)
filter/qa/cppunit/data/tiff/fail/RC4-crash-7.tiff in
CppunitTest_filter_tiff_test after c81765629bf0f7b3a0a8bb1dbed599a7f49ee58c
"coverity#1266496 Untrusted loop bound":

> /filter/source/graphicfilter/itiff/itiff.cxx:270:47: runtime error: value -nan is outside the range of representable values of type 'int'
>     #0 0x2b5bae7ad928 in TIFFReader::ReadIntData() /filter/source/graphicfilter/itiff/itiff.cxx:270:47
>     #1 0x2b5bae7b0017 in TIFFReader::ReadTagData(unsigned short, unsigned int) /filter/source/graphicfilter/itiff/itiff.cxx:320:27
>     #2 0x2b5bae7e80f3 in TIFFReader::ReadTIFF(SvStream&, Graphic&) /filter/source/graphicfilter/itiff/itiff.cxx:1377:21
[...]

With an error-reporting concept apparently missing here, just convert such out-
of-bounds values to zero.  (And make ReadDoubleData not go though the value-
limiting ReadIntData for floating types.)

Change-Id: I6e53e468e6b98fb7a7d5fd7f3336ee2168f76e30
Reviewed-on: https://gerrit.libreoffice.org/51700
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Stephan Bergmann <sbergman@redhat.com>

1 files changed, 46 insertions, 15 deletions

diff --git a/filter/source/graphicfilter/itiff/itiff.cxx b/filter/source/graphicfilter/itiff/itiff.cxx
index e68f87e8f9c7..f764c73e0bee 100644
--- a/filter/source/graphicfilter/itiff/itiff.cxx
+++ b/filter/source/graphicfilter/itiff/itiff.cxx
@@ -267,11 +267,27 @@ sal_uInt32 TIFFReader::ReadIntData()
         break;
         case 11 :
             pTIFF->ReadFloat( nFLOAT );
-            nUINT32a = static_cast<sal_Int32>(nFLOAT);
+            if (!rtl::math::isNan(nFLOAT) && nFLOAT > SAL_MIN_INT32 - 1.0
+                && nFLOAT < SAL_MAX_INT32 + 1.0)
+            {
+                nUINT32a = static_cast<sal_Int32>(nFLOAT);
+            }
+            else
+            {
+                SAL_INFO("filter.tiff", "float " << nFLOAT << " outsider of sal_Int32 range");
+            }
         break;
         case 12 :
             pTIFF->ReadDouble( nDOUBLE );
-            nUINT32a = static_cast<sal_Int32>(nDOUBLE);
+            if (!rtl::math::isNan(nDOUBLE) && nDOUBLE > SAL_MIN_INT32 - 1.0
+                && nDOUBLE < SAL_MAX_INT32 + 1.0)
+            {
+                nUINT32a = static_cast<sal_Int32>(nDOUBLE);
+            }
+            else
+            {
+                SAL_INFO("filter.tiff", "double " << nDOUBLE << " outsider of sal_Int32 range");
+            }
         break;
         default:
             pTIFF->ReadUInt32( nUINT32a );
@@ -282,21 +298,36 @@ sal_uInt32 TIFFReader::ReadIntData()
 
 double TIFFReader::ReadDoubleData()
 {
-    double  nd;
+    switch (nDataType) {
+    case 5:
+        {
+            sal_uInt32 nulong(0);
+            pTIFF->ReadUInt32( nulong );
+            double nd = static_cast<double>(nulong);
+            nulong = 0;
+            pTIFF->ReadUInt32( nulong );
+            if ( nulong != 0 )
+                nd /= static_cast<double>(nulong);
+            return nd;
+        }
 
-    if ( nDataType == 5 )
-    {
-        sal_uInt32 nulong(0);
-        pTIFF->ReadUInt32( nulong );
-        nd = static_cast<double>(nulong);
-        nulong = 0;
-        pTIFF->ReadUInt32( nulong );
-        if ( nulong != 0 )
-            nd /= static_cast<double>(nulong);
+    case 11:
+        {
+            float x = 0;
+            pTIFF->ReadFloat(x);
+            return x;
+        }
+
+    case 12:
+        {
+            double x = 0;
+            pTIFF->ReadDouble(x);
+            return x;
+        }
+
+    default:
+        return static_cast<double>(ReadIntData());
     }
-    else
-        nd = static_cast<double>(ReadIntData());
-    return nd;
 }
 
 void TIFFReader::ReadTagData( sal_uInt16 nTagType, sal_uInt32 nDataLen)