Related: fdo#45355 stale gdi font handles apparently still used

When we do glyph fallback we check to see if the glyph is too large to fit the
space available, and if it is we try and rescale it smaller.

This worked wonderfully under Linux, but under Windows it seems that the
SalLayout impl keeps an old unscaled gdi font handle which would give visually
broken glyphs for me when the base font was smaller than the replacement font.

So, now instead rather than just changing the font for the fallback level, we
throw away the SalLayout using the old font, set the font for the fallback
level, and recreate a new SalLayout using the new font.

Hopefully this is the true reason behind the crashes of fdo#45355 although
that remains unreproducible for me.

Change-Id: I415b6e7a77777d4aa5e62079c52c6c18473c0ebd

2 files changed, 47 insertions, 28 deletions

diff --git a/vcl/inc/vcl/outdev.hxx b/vcl/inc/vcl/outdev.hxx
index e5e21c58ce68..aa2b8cef98ae 100644
--- a/vcl/inc/vcl/outdev.hxx
+++ b/vcl/inc/vcl/outdev.hxx
@@ -559,7 +559,7 @@ public:
     // Helper for line geometry paint with support for graphic expansion (pattern and fat_to_area)
     void impPaintLineGeometryWithEvtlExpand(const LineInfo& rInfo, basegfx::B2DPolyPolygon aLinePolyPolygon);
 
-    SAL_DLLPRIVATE void forceFallbackFontToFit(SalLayout &rFallback, ImplFontEntry &rFallbackFont,
+    SAL_DLLPRIVATE SalLayout* getFallbackFontThatFits(ImplFontEntry &rFallbackFont,
         FontSelectPattern &rFontSelData, int nFallbackLevel,
         ImplLayoutArgs& rLayoutArgs, const ImplFontMetricData& rOrigMetric) const;
 protected:
diff --git a/vcl/source/gdi/outdev3.cxx b/vcl/source/gdi/outdev3.cxx
index fc166ae5f34c..e1bebd8385a5 100644
--- a/vcl/source/gdi/outdev3.cxx
+++ b/vcl/source/gdi/outdev3.cxx
@@ -6210,15 +6210,30 @@ SalLayout* OutputDevice::ImplLayout( const String& rOrigStr,
     return pSalLayout;
 }
 
-void OutputDevice::forceFallbackFontToFit(SalLayout &rFallback, ImplFontEntry &rFallbackFont,
+SalLayout* OutputDevice::getFallbackFontThatFits(ImplFontEntry &rFallbackFont,
     FontSelectPattern &rFontSelData, int nFallbackLevel,
     ImplLayoutArgs& rLayoutArgs, const ImplFontMetricData& rOrigMetric) const
 {
+    rFallbackFont.mnSetFontFlags = mpGraphics->SetFont( &rFontSelData, nFallbackLevel );
+
+    rLayoutArgs.ResetPos();
+    SalLayout* pFallback = mpGraphics->GetTextLayout( rLayoutArgs, nFallbackLevel );
+
+    if (!pFallback)
+        return NULL;
+
+    if (!pFallback->LayoutText(rLayoutArgs))
+    {
+        // there is no need for a font that couldn't resolve anything
+        pFallback->Release();
+        return NULL;
+    }
+
     Rectangle aBoundRect;
     bool bHaveBounding = false;
     Rectangle aRectangle;
 
-    rFallback.AdjustLayout( rLayoutArgs );
+    pFallback->AdjustLayout( rLayoutArgs );
 
     //All we care about here is getting the vertical bounds of this text and
     //make sure it will fit inside the available space
@@ -6226,7 +6241,7 @@ void OutputDevice::forceFallbackFontToFit(SalLayout &rFallback, ImplFontEntry &r
     for( int nStart = 0;;)
     {
         sal_GlyphId nLGlyph;
-        if( !rFallback.GetNextGlyphs( 1, &nLGlyph, aPos, nStart ) )
+        if( !pFallback->GetNextGlyphs( 1, &nLGlyph, aPos, nStart ) )
             break;
 
         sal_GlyphId nFontTag = nFallbackLevel << GF_FONTSHIFT;
@@ -6255,11 +6270,28 @@ void OutputDevice::forceFallbackFontToFit(SalLayout &rFallback, ImplFontEntry &r
         if (fScale < 1)
         {
             long nOrigHeight = rFontSelData.mnHeight;
-            rFontSelData.mnHeight = static_cast<int>(static_cast<float>(rFontSelData.mnHeight) * fScale);
+            long nNewHeight = static_cast<int>(static_cast<float>(rFontSelData.mnHeight) * fScale);
+
+            if (nNewHeight == nOrigHeight)
+                --nNewHeight;
+
+            pFallback->Release();
+
+            rFontSelData.mnHeight = nNewHeight;
             rFallbackFont.mnSetFontFlags = mpGraphics->SetFont( &rFontSelData, nFallbackLevel );
             rFontSelData.mnHeight = nOrigHeight;
+
+            rLayoutArgs.ResetPos();
+            pFallback = mpGraphics->GetTextLayout( rLayoutArgs, nFallbackLevel );
+            if (pFallback && !pFallback->LayoutText(rLayoutArgs))
+            {
+                pFallback->Release();
+                pFallback = NULL;
+            }
+            SAL_WARN_IF(pFallback, "vcl.gdi", "we couldn't layout text with a smaller point size that worked with a bigger one");
         }
     }
+    return pFallback;
 }
 
 // -----------------------------------------------------------------------
@@ -6320,30 +6352,17 @@ SalLayout* OutputDevice::ImplGlyphFallbackLayout( SalLayout* pSalLayout, ImplLay
             }
         }
 
-        pFallbackFont->mnSetFontFlags = mpGraphics->SetFont( &aFontSelData, nFallbackLevel );
-
         // create and add glyph fallback layout to multilayout
-        rLayoutArgs.ResetPos();
-        SalLayout* pFallback = mpGraphics->GetTextLayout( rLayoutArgs, nFallbackLevel );
-        if( pFallback )
-        {
-            if( pFallback->LayoutText( rLayoutArgs ) )
-            {
-                forceFallbackFontToFit(*pFallback, *pFallbackFont, aFontSelData,
-                    nFallbackLevel, rLayoutArgs, aOrigMetric);
-
-                if( !pMultiSalLayout )
-                    pMultiSalLayout = new MultiSalLayout( *pSalLayout );
-                pMultiSalLayout->AddFallback( *pFallback,
-                    rLayoutArgs.maRuns, aFontSelData.mpFontData );
-                if (nFallbackLevel == MAX_FALLBACK-1)
-                    pMultiSalLayout->SetInComplete();
-            }
-            else
-            {
-                // there is no need for a font that couldn't resolve anything
-                pFallback->Release();
-            }
+        SalLayout* pFallback = getFallbackFontThatFits(*pFallbackFont, aFontSelData,
+            nFallbackLevel, rLayoutArgs, aOrigMetric);
+        if (pFallback)
+        {
+            if( !pMultiSalLayout )
+                pMultiSalLayout = new MultiSalLayout( *pSalLayout );
+            pMultiSalLayout->AddFallback( *pFallback,
+                rLayoutArgs.maRuns, aFontSelData.mpFontData );
+            if (nFallbackLevel == MAX_FALLBACK-1)
+                pMultiSalLayout->SetInComplete();
         }
 
         mpFontCache->Release( pFallbackFont );