compute bitmap checksum for proper amount of bytes and bits

Change-Id: Ifee6fae39dd13a36c72674550cd48dc35fdd71ac
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/97761
Tested-by: Jenkins
Reviewed-by: Stephan Bergmann <sbergman@redhat.com>

2 files changed, 50 insertions, 7 deletions

diff --git a/vcl/qa/cppunit/svm/svmtest.cxx b/vcl/qa/cppunit/svm/svmtest.cxx
index 9deed5e13c10..f318aa718c4d 100644
--- a/vcl/qa/cppunit/svm/svmtest.cxx
+++ b/vcl/qa/cppunit/svm/svmtest.cxx
@@ -997,7 +997,7 @@ void SvmTest::checkBitmapExs(const GDIMetaFile& rMetaFile)
             "281fc589",
             "b8dee5da",
             "4df0e464",
-            "7d3a8da3",
+            "4322ee3a",
             "1426653b",
             "4fd547df",
             "71efc447",
@@ -1012,8 +1012,8 @@ void SvmTest::checkBitmapExs(const GDIMetaFile& rMetaFile)
             "281fc589",
             "5e01ddcc",
             "4df0e464",
-            "6c1263f9",
-            "747e8dfb",
+            "4322ee3a",
+            "794c92a9",
             "3c80d829",
             "71efc447",
         });
diff --git a/vcl/source/bitmap/salbmp.cxx b/vcl/source/bitmap/salbmp.cxx
index 6d955b6aa4d3..eae8aadc5d0e 100644
--- a/vcl/source/bitmap/salbmp.cxx
+++ b/vcl/source/bitmap/salbmp.cxx
@@ -19,6 +19,18 @@
 
 #include <salbmp.hxx>
 
+static BitmapChecksum scanlineChecksum(BitmapChecksum nCrc, const sal_uInt8* bits, int lineBitsCount, sal_uInt8 extraBitsMask)
+{
+    if( lineBitsCount / 8 > 0 )
+        nCrc = vcl_get_checksum( nCrc, bits, lineBitsCount / 8 );
+    if( extraBitsMask != 0 )
+    {
+        sal_uInt8 extraByte = bits[ lineBitsCount / 8 ] & extraBitsMask;
+        nCrc = vcl_get_checksum( nCrc, &extraByte, 1 );
+    }
+    return nCrc;
+}
+
 void SalBitmap::updateChecksum() const
 {
     if (mbChecksumValid)
@@ -30,19 +42,50 @@ void SalBitmap::updateChecksum() const
     if (pBuf)
     {
         nCrc = pBuf->maPalette.GetChecksum();
-        const int bytesPerPixel = ( pBuf->mnBitCount + 7 ) / 8;
+        const int lineBitsCount = pBuf->mnWidth * pBuf->mnBitCount;
+        // With 1bpp/4bpp format we need to check only used bits in the last byte.
+        sal_uInt8 extraBitsMask = 0;
+        if( lineBitsCount % 8 != 0 )
+        {
+            const int extraBitsCount = lineBitsCount % 8;
+            switch( RemoveScanline( pBuf->mnFormat ))
+            {
+                case ScanlineFormat::N1BitMsbPal:
+                {
+                    static const sal_uInt8 mask1Bit[] = { 0x80, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc, 0xfe, 0xff };
+                    extraBitsMask = mask1Bit[ extraBitsCount ];
+                    break;
+                }
+                case ScanlineFormat::N1BitLsbPal:
+                {
+                    static const sal_uInt8 mask1Bit[] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f, 0x7f, 0xff };
+                    extraBitsMask = mask1Bit[ extraBitsCount ];
+                    break;
+                }
+                case ScanlineFormat::N4BitMsnPal:
+                    assert(extraBitsCount == 4);
+                    extraBitsMask = 0xf0;
+                    break;
+                case ScanlineFormat::N4BitLsnPal:
+                    assert(extraBitsCount == 4);
+                    extraBitsMask = 0x0f;
+                    break;
+                default:
+                    break;
+            }
+        }
         if( pBuf->mnFormat & ScanlineFormat::TopDown )
         {
-            if( pBuf->mnScanlineSize == pBuf->mnWidth * bytesPerPixel )
+            if( pBuf->mnScanlineSize == lineBitsCount / 8 )
                 nCrc = vcl_get_checksum(nCrc, pBuf->mpBits, pBuf->mnScanlineSize * pBuf->mnHeight);
             else // Do not include padding with undefined content in the checksum.
                 for( long y = 0; y < pBuf->mnHeight; ++y )
-                    nCrc = vcl_get_checksum(nCrc, pBuf->mpBits + y * pBuf->mnScanlineSize, pBuf->mnWidth * bytesPerPixel);
+                    nCrc = scanlineChecksum(nCrc, pBuf->mpBits + y * pBuf->mnScanlineSize, lineBitsCount, extraBitsMask);
         }
         else // Compute checksum in the order of scanlines, to make it consistent between different bitmap implementations.
         {
             for( long y = pBuf->mnHeight - 1; y >= 0; --y )
-                nCrc = vcl_get_checksum(nCrc, pBuf->mpBits + y * pBuf->mnScanlineSize, pBuf->mnWidth * bytesPerPixel);
+                nCrc = scanlineChecksum(nCrc, pBuf->mpBits + y * pBuf->mnScanlineSize, lineBitsCount, extraBitsMask);
         }
         pThis->ReleaseBuffer(pBuf, BitmapAccessMode::Read);
         pThis->mnChecksum = nCrc;