tdf#138039 tdf#134298 sw: layout: fix overlap of fly and table

The layout is horribly borked, the fly anchored in the body-level
paragraph messed with the preceding table:

page id="1" top="284" width="11905" height="16837" bottom="17120"
tab id="3" top="794"
row id="4" top="17121"
fly id="8" top="16725"
txt id="7" top="1394"
fly ptr="0x6ce5510" id="10" top="1302"

SwTabFrame::CalcFlyOffsets() detects an overlap with the large fly, and
since it has wrap NONE it resizes to below the large image.

Then the SwTabFrame doesn't fit on the page, so it is split, but the split
fails because nDistanceToUpperPrtBottom is -720 (negative); hence it is
joined again.

Meanwhile the fly was invalidated, so now CalcFlyOffsets() ignores it and
the table shrinks again.

Once the fly is positioned again, the process repeats from the start.

Fix this in SwTabFrame::CalcFlyOffsets() by ignoring flys with wrap NONE that
extend below the body of the document and are anchored in a frame in the
next-chain of the table frame: these must move to the next page with their
anchor frame.

For the bugdoc this gives the same layout as LO 5.2.

Reportedly this problem started to happen since commit
6f5024de2e1a5cc533527e45b33d9a415467c48d, but it's not obvious why.

Reviewed-on: https://gerrit.libreoffice.org/c/core/+/105809
Tested-by: Jenkins
Reviewed-by: Michael Stahl <michael.stahl@cib.de>
(cherry picked from commit 6b92d2e8522ecc98d2c5532f5076c20ae295168e)
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/105940
Tested-by: Michael Stahl <michael.stahl@cib.de>

Change-Id: Iafb8a6afcba634f11c5db73869313ded0fe13bbd

3 files changed, 57 insertions, 3 deletions

diff --git a/sw/qa/extras/layout/data/tdf138039.odt b/sw/qa/extras/layout/data/tdf138039.odt
new file mode 100644
index 000000000000..f355fd1349a6
--- /dev/null
+++ b/sw/qa/extras/layout/data/tdf138039.odt
diff --git a/sw/qa/extras/layout/layout.cxx b/sw/qa/extras/layout/layout.cxx
index 682b8b27a8d7..8d9f2a476f2a 100644
--- a/sw/qa/extras/layout/layout.cxx
+++ b/sw/qa/extras/layout/layout.cxx
@@ -1173,6 +1173,30 @@ CPPUNIT_TEST_FIXTURE(SwLayoutWriter, testRedlineFlysInFootnote)
     }
 }
 
+CPPUNIT_TEST_FIXTURE(SwLayoutWriter, testTdf138039)
+{
+    createDoc("tdf138039.odt");
+
+    xmlDocPtr pXmlDoc = parseLayoutDump();
+
+    // there are 3 pages
+    assertXPath(pXmlDoc, "/root/page", 3);
+    // table on first page
+    assertXPath(pXmlDoc, "/root/page[1]/body/tab", 1);
+    assertXPath(pXmlDoc, "/root/page[1]/body/txt", 0);
+    // paragraph with large fly on second page
+    assertXPath(pXmlDoc, "/root/page[2]/body/tab", 0);
+    assertXPath(pXmlDoc, "/root/page[2]/body/txt", 1);
+    assertXPath(pXmlDoc, "/root/page[2]/body/txt[1]/anchored/fly", 1);
+    assertXPath(pXmlDoc, "/root/page[2]/body/txt[1]/anchored/fly[1]/infos/bounds", "top", "17915");
+    assertXPath(pXmlDoc, "/root/page[2]/body/txt[1]/anchored/fly[1]/infos/bounds", "height",
+                "15819");
+    // paragraph on third page
+    assertXPath(pXmlDoc, "/root/page[3]/body/tab", 0);
+    assertXPath(pXmlDoc, "/root/page[3]/body/txt", 1);
+    assertXPath(pXmlDoc, "/root/page[3]/body/txt[1]/anchored", 0);
+}
+
 CPPUNIT_TEST_FIXTURE(SwLayoutWriter, testTableOverlapFooterFly)
 {
     // Load a document that has a fly anchored in the footer.
diff --git a/sw/source/core/layout/tabfrm.cxx b/sw/source/core/layout/tabfrm.cxx
index a27b7d3c1c22..781355964448 100644
--- a/sw/source/core/layout/tabfrm.cxx
+++ b/sw/source/core/layout/tabfrm.cxx
@@ -2724,6 +2724,21 @@ void SwTabFrame::MakeAll(vcl::RenderContext* pRenderContext)
         aNotify.SetInvaKeep();
 }
 
+static bool IsNextOnSamePage(SwPageFrame const& rPage,
+        SwTabFrame const& rTabFrame, SwTextFrame const& rAnchorFrame)
+{
+    for (SwContentFrame const* pContentFrame = rTabFrame.FindNextCnt();
+        pContentFrame && pContentFrame->FindPageFrame() == &rPage;
+        pContentFrame = pContentFrame->FindNextCnt())
+    {
+        if (pContentFrame == &rAnchorFrame)
+        {
+            return true;
+        }
+    }
+    return false;
+}
+
 /// Calculate the offsets arising because of FlyFrames
 bool SwTabFrame::CalcFlyOffsets( SwTwips& rUpper,
                                long& rLeftOffset,
@@ -2823,10 +2838,25 @@ bool SwTabFrame::CalcFlyOffsets( SwTwips& rUpper,
                     const SwFormatHoriOrient &rHori= pFly->GetFormat()->GetHoriOrient();
                     if ( css::text::WrapTextMode_NONE == rSur.GetSurround() )
                     {
+                        // possible cases:
+                        //        both in body
+                        //        both in same fly
+                        //        any comb. of body, footnote, header/footer
+                        // to keep it safe, check only in doc body vs page margin for now
                         long nBottom = aRectFnSet.GetBottom(aFlyRect);
-                        if( aRectFnSet.YDiff( nPrtPos, nBottom ) < 0 )
-                            nPrtPos = nBottom;
-                        bInvalidatePrtArea = true;
+                        // tdf#138039 don't grow beyond the page body
+                        // if the fly is anchored below the table; the fly
+                        // must move with its anchor frame to the next page
+                        SwRectFnSet fnPage(pPage);
+                        if (!IsInDocBody() // TODO
+                            || fnPage.YDiff(fnPage.GetBottom(aFlyRect), fnPage.GetPrtBottom(*pPage)) <= 0
+                            || !IsNextOnSamePage(*pPage, *this,
+                                *static_cast<SwTextFrame*>(pFly->GetAnchorFrameContainingAnchPos())))
+                        {
+                            if (aRectFnSet.YDiff( nPrtPos, nBottom ) < 0)
+                                nPrtPos = nBottom;
+                            bInvalidatePrtArea = true;
+                        }
                     }
                     if ( (css::text::WrapTextMode_RIGHT    == rSur.GetSurround() ||
                           css::text::WrapTextMode_PARALLEL == rSur.GetSurround())&&