tdf#156540 invert alpha mask when drawing sprites

Due to the switch from transparency to alpha in commit
81994cb2b8b32453a92bcb011830fcb884f22ff3, a sprite's
alpha mask needs to be inverted.

Additionally, fixes an oversight in vcl's alpha.cxx, where manual
blend math got mangled, also in
81994cb2b8b32453a92bcb011830fcb884f22ff3.

Change-Id: I8ebbbc7fe624d8dfc8121d8814d30875c498870d
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/155378
Reviewed-by: Patrick Luby <plubius@neooffice.org>
Tested-by: Jenkins
Reviewed-by: Noel Grandin <noel.grandin@collabora.co.uk>

2 files changed, 6 insertions, 4 deletions

diff --git a/canvas/source/vcl/spritehelper.cxx b/canvas/source/vcl/spritehelper.cxx
index 3494a5fbda8f..f188526930a0 100644
--- a/canvas/source/vcl/spritehelper.cxx
+++ b/canvas/source/vcl/spritehelper.cxx
@@ -134,12 +134,14 @@ namespace vclcanvas
                 // sprite content might contain alpha, create
                 // BmpEx, then.
                 BitmapEx aMask( mpBackBufferMask->getOutDev().GetBitmapEx( aEmptyPoint,
-                                                                       aOutputSize ) );
+                                                                           aOutputSize ) );
+                AlphaMask aAlpha( aMask.GetBitmap() );
+                aAlpha.Invert();
 
                 // Note: since we retrieved aBmp and aMask
                 // directly from an OutDev, it's already a
                 // 'display bitmap' on windows.
-                maContent = BitmapEx( aBmp.GetBitmap(), AlphaMask( aMask.GetBitmap()) );
+                maContent = BitmapEx( aBmp.GetBitmap(), aAlpha );
             }
         }
 
diff --git a/vcl/source/bitmap/alpha.cxx b/vcl/source/bitmap/alpha.cxx
index 8e082c695137..ff0d035d52d1 100644
--- a/vcl/source/bitmap/alpha.cxx
+++ b/vcl/source/bitmap/alpha.cxx
@@ -118,8 +118,8 @@ void AlphaMask::BlendWith(const AlphaMask& rOther)
             // Awkward calculation because the original used transparency, and to replicate
             // the logic we need to translate into transparency, perform the original logic,
             // then translate back to alpha.
-            auto tmp = 255 - ((255 - nGrey1) + (255 - nGrey2) - (255 - nGrey1) * (255 - nGrey2));
-            *scanline = static_cast<sal_uInt8>(tmp / 255);
+            auto tmp = 255 - ((255 - nGrey1) + (255 - nGrey2) - (255 - nGrey1) * (255 - nGrey2) / 255);
+            *scanline = static_cast<sal_uInt8>(tmp);
             ++scanline;
             ++otherScanline;
         }