Windows/--disable-skia: fix D2DWriteTextOutRenderer to properly set AA mode

This simplifies the process; in D2DWriteTextOutRenderer ctor, it
uses the final mode for the call to CreateRenderTarget, applying
the correct mode.

Also, pass the AA flag from SalGraphics, and check the setting of
GetUseFontAAFromSystem to use it, as done in e.g. CairoTextRender
after commit e6538f5bdd876911ea30f84a6512c03908e620fd (tdf#118966
vcl: add a flag to determine if AA of fonts is used from the system,
2018-07-28).

This fixes the failures on Windows with --disable-skia, as seen in
https://lists.freedesktop.org/archives/libreoffice/2024-October/092541.html

VclCjkTextTest::testVerticalText needed to be fixed by a tweak in
getCharacterRightSideHeight, to use color's IsDark, instead of
comparing to the fixed black, because the correct AA mode that is
used now, makes the vertical part of the character not completely
black.

Change-Id: Iee8fe98e29a80a242f8e761c9a23c68b34a45699
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/175188
Tested-by: Jenkins
Reviewed-by: Mike Kaganski <mike.kaganski@collabora.com>
(cherry picked from commit 85a56048a004b85d232805cc3b15db0c100d8a5c)
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/175235
Reviewed-by: Xisco Fauli <xiscofauli@libreoffice.org>

5 files changed, 71 insertions, 100 deletions

diff --git a/vcl/inc/win/DWriteTextRenderer.hxx b/vcl/inc/win/DWriteTextRenderer.hxx
index 1cdf67d04a39..3d93a463f8ec 100644
--- a/vcl/inc/win/DWriteTextRenderer.hxx
+++ b/vcl/inc/win/DWriteTextRenderer.hxx
@@ -27,33 +27,28 @@
 
 #include <win/winlayout.hxx>
 
-enum class D2DTextAntiAliasMode
-{
-    Default,
-    Aliased,
-    AntiAliased,
-    ClearType,
-};
-
 class D2DWriteTextOutRenderer : public TextOutRenderer
 {
 public:
-    explicit D2DWriteTextOutRenderer(bool bRenderingModeNatural);
+    using MODE = std::pair<D2D1_TEXT_ANTIALIAS_MODE, DWRITE_RENDERING_MODE>;
+
+    explicit D2DWriteTextOutRenderer(MODE mode);
 
     bool operator()(GenericSalLayout const &rLayout,
         SalGraphics &rGraphics,
-        HDC hDC,
-        bool bRenderingModeNatural) override;
+        HDC hDC) override;
 
     HRESULT BindDC(HDC hDC, tools::Rectangle const & rRect = tools::Rectangle(0, 0, 1, 1));
 
-    HRESULT CreateRenderTarget(bool bRenderingModeNatural);
+    HRESULT CreateRenderTarget();
 
     bool Ready() const;
 
-    void applyTextAntiAliasMode(bool bRenderingModeNatural);
+    void applyTextAntiAliasMode();
+
+    MODE GetRenderingMode() const { return maRenderingMode; }
 
-    bool GetRenderingModeNatural() const { return mbRenderingModeNatural; }
+    static MODE GetMode(bool bRenderingModeNatural, bool bAntiAlias);
 
 private:
     // This is a singleton object disable copy ctor and assignment operator
@@ -61,14 +56,13 @@ private:
     D2DWriteTextOutRenderer & operator = (const D2DWriteTextOutRenderer &) = delete;
 
     IDWriteFontFace* GetDWriteFace(const WinFontInstance& rWinFont, float * lfSize) const;
-    bool performRender(GenericSalLayout const &rLayout, SalGraphics &rGraphics, HDC hDC, bool& bRetry, bool bRenderingModeNatural);
+    bool performRender(GenericSalLayout const &rLayout, SalGraphics &rGraphics, HDC hDC, bool& bRetry);
 
     sal::systools::COMReference<ID2D1Factory> mpD2DFactory;
     sal::systools::COMReference<ID2D1DCRenderTarget> mpRT;
     const D2D1_RENDER_TARGET_PROPERTIES mRTProps;
 
-    bool mbRenderingModeNatural;
-    D2DTextAntiAliasMode meTextAntiAliasMode;
+    MODE maRenderingMode;
 };
 
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/vcl/inc/win/winlayout.hxx b/vcl/inc/win/winlayout.hxx
index cfb36e825b54..6e536d0a8dd8 100644
--- a/vcl/inc/win/winlayout.hxx
+++ b/vcl/inc/win/winlayout.hxx
@@ -74,14 +74,13 @@ protected:
     TextOutRenderer & operator = (const TextOutRenderer &) = delete;
 
 public:
-    static TextOutRenderer & get(bool bUseDWrite, bool bRenderingModeNatural);
+    static TextOutRenderer & get(bool bUseDWrite, bool bRenderingModeNatural, bool bAntiAlias);
 
     virtual ~TextOutRenderer() = default;
 
     virtual bool operator ()(GenericSalLayout const &rLayout,
         SalGraphics &rGraphics,
-        HDC hDC,
-        bool bRenderingModeNatural) = 0;
+        HDC hDC) = 0;
 };
 
 class ExTextOutRenderer : public TextOutRenderer
@@ -94,8 +93,7 @@ public:
 
     bool operator ()(GenericSalLayout const &rLayout,
         SalGraphics &rGraphics,
-        HDC hDC,
-        bool bRenderingModeNatural) override;
+        HDC hDC) override;
 };
 
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/vcl/qa/cppunit/cjktext.cxx b/vcl/qa/cppunit/cjktext.cxx
index ad00de2e8893..96ee658bb87d 100644
--- a/vcl/qa/cppunit/cjktext.cxx
+++ b/vcl/qa/cppunit/cjktext.cxx
@@ -84,7 +84,7 @@ static tools::Long getCharacterRightSideHeight(VirtualDevice* device, const Poin
     Bitmap bitmap = device->GetBitmap(Point(), device->GetOutputSizePixel());
     BitmapScopedReadAccess access(bitmap);
     tools::Long x = start.X();
-    while (x >= 0 && access->GetColor(start.Y(), x) != COL_BLACK)
+    while (x >= 0 && !access->GetColor(start.Y(), x).IsDark())
         --x;
     if (x < 0)
         return -1;
diff --git a/vcl/win/gdi/DWriteTextRenderer.cxx b/vcl/win/gdi/DWriteTextRenderer.cxx
index f25fe80cd79d..8a3ade545e36 100644
--- a/vcl/win/gdi/DWriteTextRenderer.cxx
+++ b/vcl/win/gdi/DWriteTextRenderer.cxx
@@ -38,28 +38,12 @@
 namespace
 {
 
-D2DTextAntiAliasMode lclGetSystemTextAntiAliasMode()
+D2D1_TEXT_ANTIALIAS_MODE lclGetSystemTextAntiAliasType()
 {
-    D2DTextAntiAliasMode eMode = D2DTextAntiAliasMode::Default;
-
-    BOOL bFontSmoothing;
-    if (!SystemParametersInfoW(SPI_GETFONTSMOOTHING, 0, &bFontSmoothing, 0))
-        return eMode;
-
-    if (bFontSmoothing)
-    {
-        eMode = D2DTextAntiAliasMode::AntiAliased;
-
-        UINT nType;
-        if (SystemParametersInfoW(SPI_GETFONTSMOOTHINGTYPE, 0, &nType, 0) && nType == FE_FONTSMOOTHINGCLEARTYPE)
-            eMode = D2DTextAntiAliasMode::ClearType;
-    }
-    else
-    {
-        eMode = D2DTextAntiAliasMode::Aliased;
-    }
-
-    return eMode;
+    UINT t;
+    if (SystemParametersInfoW(SPI_GETFONTSMOOTHINGTYPE, 0, &t, 0) && t == FE_FONTSMOOTHINGCLEARTYPE)
+        return D2D1_TEXT_ANTIALIAS_MODE_CLEARTYPE;
+    return D2D1_TEXT_ANTIALIAS_MODE_GRAYSCALE;
 }
 
 IDWriteRenderingParams* lclSetRenderingMode(DWRITE_RENDERING_MODE eRenderingMode)
@@ -111,59 +95,52 @@ private:
 
 } // end anonymous namespace
 
-D2DWriteTextOutRenderer::D2DWriteTextOutRenderer(bool bRenderingModeNatural)
+// static
+D2DWriteTextOutRenderer::MODE D2DWriteTextOutRenderer::GetMode(bool bRenderingModeNatural,
+                                                               bool bAntiAlias)
+{
+    D2D1_TEXT_ANTIALIAS_MODE eTextMode;
+    if (!Application::GetSettings().GetStyleSettings().GetUseFontAAFromSystem())
+        eTextMode = bAntiAlias ? lclGetSystemTextAntiAliasType() : D2D1_TEXT_ANTIALIAS_MODE_ALIASED;
+    else if (BOOL bSmoothing; SystemParametersInfoW(SPI_GETFONTSMOOTHING, 0, &bSmoothing, 0))
+        eTextMode = bSmoothing ? lclGetSystemTextAntiAliasType() : D2D1_TEXT_ANTIALIAS_MODE_ALIASED;
+    else
+        eTextMode = D2D1_TEXT_ANTIALIAS_MODE_DEFAULT;
+
+    DWRITE_RENDERING_MODE eRenderingMode;
+    if (eTextMode == D2D1_TEXT_ANTIALIAS_MODE_ALIASED)
+        eRenderingMode = DWRITE_RENDERING_MODE_ALIASED; // no way to use bRenderingModeNatural
+    else if (bRenderingModeNatural)
+        eRenderingMode = DWRITE_RENDERING_MODE_NATURAL;
+    else if (eTextMode == D2D1_TEXT_ANTIALIAS_MODE_DEFAULT)
+        eRenderingMode = DWRITE_RENDERING_MODE_DEFAULT;
+    else // D2D1_TEXT_ANTIALIAS_MODE_GRAYSCALE || D2D1_TEXT_ANTIALIAS_MODE_CLEARTYPE
+        eRenderingMode = DWRITE_RENDERING_MODE_GDI_CLASSIC;
+
+    return { eTextMode, eRenderingMode };
+}
+
+D2DWriteTextOutRenderer::D2DWriteTextOutRenderer(MODE mode)
     : mpD2DFactory(nullptr),
     mpRT(nullptr),
     mRTProps(D2D1::RenderTargetProperties(D2D1_RENDER_TARGET_TYPE_DEFAULT,
                                           D2D1::PixelFormat(DXGI_FORMAT_B8G8R8A8_UNORM, D2D1_ALPHA_MODE_PREMULTIPLIED),
                                           0, 0)),
-    mbRenderingModeNatural(bRenderingModeNatural),
-    meTextAntiAliasMode(D2DTextAntiAliasMode::Default)
+    maRenderingMode(mode)
 {
     HRESULT hr = D2D1CreateFactory(D2D1_FACTORY_TYPE_SINGLE_THREADED, __uuidof(ID2D1Factory), nullptr, IID_PPV_ARGS_Helper(&mpD2DFactory));
     if (SUCCEEDED(hr))
-        hr = CreateRenderTarget(bRenderingModeNatural);
-    meTextAntiAliasMode = lclGetSystemTextAntiAliasMode();
+        hr = CreateRenderTarget();
 }
 
-void D2DWriteTextOutRenderer::applyTextAntiAliasMode(bool bRenderingModeNatural)
-{
-    D2D1_TEXT_ANTIALIAS_MODE eTextAAMode = D2D1_TEXT_ANTIALIAS_MODE_DEFAULT;
-    DWRITE_RENDERING_MODE eRenderingMode = DWRITE_RENDERING_MODE_DEFAULT;
-    switch (meTextAntiAliasMode)
-    {
-        case D2DTextAntiAliasMode::Default:
-            eRenderingMode = DWRITE_RENDERING_MODE_DEFAULT;
-            eTextAAMode = D2D1_TEXT_ANTIALIAS_MODE_DEFAULT;
-            break;
-        case D2DTextAntiAliasMode::Aliased:
-            eRenderingMode = DWRITE_RENDERING_MODE_ALIASED;
-            eTextAAMode = D2D1_TEXT_ANTIALIAS_MODE_ALIASED;
-            break;
-        case D2DTextAntiAliasMode::AntiAliased:
-            eRenderingMode = DWRITE_RENDERING_MODE_CLEARTYPE_GDI_CLASSIC;
-            eTextAAMode = D2D1_TEXT_ANTIALIAS_MODE_GRAYSCALE;
-            break;
-        case D2DTextAntiAliasMode::ClearType:
-            eRenderingMode = DWRITE_RENDERING_MODE_CLEARTYPE_GDI_CLASSIC;
-            eTextAAMode = D2D1_TEXT_ANTIALIAS_MODE_CLEARTYPE;
-            break;
-        default:
-            break;
-    }
-
-    if (bRenderingModeNatural)
-        eRenderingMode = DWRITE_RENDERING_MODE_CLEARTYPE_NATURAL;
-
-    mpRT->SetTextRenderingParams(lclSetRenderingMode(eRenderingMode));
-    mpRT->SetTextAntialiasMode(eTextAAMode);
-}
-
-HRESULT D2DWriteTextOutRenderer::CreateRenderTarget(bool bRenderingModeNatural)
+HRESULT D2DWriteTextOutRenderer::CreateRenderTarget()
 {
     HRESULT hr = CHECKHR(mpD2DFactory->CreateDCRenderTarget(&mRTProps, &mpRT));
     if (SUCCEEDED(hr))
-        applyTextAntiAliasMode(bRenderingModeNatural);
+    {
+        mpRT->SetTextRenderingParams(lclSetRenderingMode(maRenderingMode.second));
+        mpRT->SetTextAntialiasMode(maRenderingMode.first);
+    }
     return hr;
 }
 
@@ -180,7 +157,7 @@ HRESULT D2DWriteTextOutRenderer::BindDC(HDC hDC, tools::Rectangle const & rRect)
     return CHECKHR(mpRT->BindDC(hDC, &rc));
 }
 
-bool D2DWriteTextOutRenderer::operator()(GenericSalLayout const & rLayout, SalGraphics& rGraphics, HDC hDC, bool bRenderingModeNatural)
+bool D2DWriteTextOutRenderer::operator()(GenericSalLayout const & rLayout, SalGraphics& rGraphics, HDC hDC)
 {
     bool bRetry = false;
     bool bResult = false;
@@ -188,13 +165,13 @@ bool D2DWriteTextOutRenderer::operator()(GenericSalLayout const & rLayout, SalGr
     do
     {
        bRetry = false;
-       bResult = performRender(rLayout, rGraphics, hDC, bRetry, bRenderingModeNatural);
+       bResult = performRender(rLayout, rGraphics, hDC, bRetry);
        nCount++;
     } while (bRetry && nCount < 3);
     return bResult;
 }
 
-bool D2DWriteTextOutRenderer::performRender(GenericSalLayout const & rLayout, SalGraphics& rGraphics, HDC hDC, bool& bRetry, bool bRenderingModeNatural)
+bool D2DWriteTextOutRenderer::performRender(GenericSalLayout const & rLayout, SalGraphics& rGraphics, HDC hDC, bool& bRetry)
 {
     if (!Ready())
         return false;
@@ -203,14 +180,14 @@ bool D2DWriteTextOutRenderer::performRender(GenericSalLayout const & rLayout, Sa
 
     if (hr == D2DERR_RECREATE_TARGET)
     {
-        CreateRenderTarget(bRenderingModeNatural);
+        CreateRenderTarget();
         bRetry = true;
         return false;
     }
     if (FAILED(hr))
     {
         // If for any reason we can't bind fallback to legacy APIs.
-        return ExTextOutRenderer()(rLayout, rGraphics, hDC, bRenderingModeNatural);
+        return ExTextOutRenderer()(rLayout, rGraphics, hDC);
     }
 
     const WinFontInstance& rWinFont = static_cast<const WinFontInstance&>(rLayout.GetFont());
@@ -278,7 +255,7 @@ bool D2DWriteTextOutRenderer::performRender(GenericSalLayout const & rLayout, Sa
 
     if (hr == D2DERR_RECREATE_TARGET)
     {
-        CreateRenderTarget(bRenderingModeNatural);
+        CreateRenderTarget();
         bRetry = true;
     }
 
diff --git a/vcl/win/gdi/winlayout.cxx b/vcl/win/gdi/winlayout.cxx
index 19eaae2ecee7..83c3a653b1ae 100644
--- a/vcl/win/gdi/winlayout.cxx
+++ b/vcl/win/gdi/winlayout.cxx
@@ -50,7 +50,7 @@
 #include <shlwapi.h>
 #include <winver.h>
 
-TextOutRenderer& TextOutRenderer::get(bool bUseDWrite, bool bRenderingModeNatural)
+TextOutRenderer& TextOutRenderer::get(bool bUseDWrite, bool bRenderingModeNatural, bool bAntiAlias)
 {
     SalData* const pSalData = GetSalData();
 
@@ -62,14 +62,16 @@ TextOutRenderer& TextOutRenderer::get(bool bUseDWrite, bool bRenderingModeNatura
 
     if (bUseDWrite)
     {
-        if (!pSalData->m_pD2DWriteTextOutRenderer
-            || static_cast<D2DWriteTextOutRenderer*>(pSalData->m_pD2DWriteTextOutRenderer.get())
-                       ->GetRenderingModeNatural()
-                   != bRenderingModeNatural)
+        const auto mode = D2DWriteTextOutRenderer::GetMode(bRenderingModeNatural, bAntiAlias);
+        if (pSalData->m_pD2DWriteTextOutRenderer)
         {
-            pSalData->m_pD2DWriteTextOutRenderer.reset(
-                new D2DWriteTextOutRenderer(bRenderingModeNatural));
+            auto pRenderer
+                = static_cast<D2DWriteTextOutRenderer*>(pSalData->m_pD2DWriteTextOutRenderer.get());
+            if (pRenderer->GetRenderingMode() == mode)
+                return *pSalData->m_pD2DWriteTextOutRenderer;
         }
+
+        pSalData->m_pD2DWriteTextOutRenderer.reset(new D2DWriteTextOutRenderer(mode));
         return *pSalData->m_pD2DWriteTextOutRenderer;
     }
     if (!pSalData->m_pExTextOutRenderer)
@@ -80,7 +82,7 @@ TextOutRenderer& TextOutRenderer::get(bool bUseDWrite, bool bRenderingModeNatura
 }
 
 bool ExTextOutRenderer::operator()(GenericSalLayout const& rLayout, SalGraphics& /*rGraphics*/,
-                                   HDC hDC, bool /*bRenderingModeNatural*/)
+                                   HDC hDC)
 {
     int nStart = 0;
     basegfx::B2DPoint aPos;
@@ -197,8 +199,8 @@ void WinFontInstance::SetGraphics(WinSalGraphics* pGraphics)
 void WinSalGraphics::DrawTextLayout(const GenericSalLayout& rLayout, HDC hDC, bool bUseDWrite,
                                     bool bRenderingModeNatural)
 {
-    TextOutRenderer& render = TextOutRenderer::get(bUseDWrite, bRenderingModeNatural);
-    render(rLayout, *this, hDC, bRenderingModeNatural);
+    auto& render = TextOutRenderer::get(bUseDWrite, bRenderingModeNatural, getAntiAlias());
+    render(rLayout, *this, hDC);
 }
 
 void WinSalGraphics::DrawTextLayout(const GenericSalLayout& rLayout)