tdf#138039 tdf#134298 sw: layout: fix overlap of fly and table

The layout is horribly borked, the fly anchored in the body-level
paragraph messed with the preceding table:

page id="1" top="284" width="11905" height="16837" bottom="17120"
tab id="3" top="794"
row id="4" top="17121"
fly id="8" top="16725"
txt id="7" top="1394"
fly ptr="0x6ce5510" id="10" top="1302"

SwTabFrame::CalcFlyOffsets() detects an overlap with the large fly, and
since it has wrap NONE it resizes to below the large image.

Then the SwTabFrame doesn't fit on the page, so it is split, but the split
fails because nDistanceToUpperPrtBottom is -720 (negative); hence it is
joined again.

Meanwhile the fly was invalidated, so now CalcFlyOffsets() ignores it and
the table shrinks again.

Once the fly is positioned again, the process repeats from the start.

Fix this in SwTabFrame::CalcFlyOffsets() by ignoring flys with wrap NONE that
extend below the body of the document and are anchored in a frame in the
next-chain of the table frame: these must move to the next page with their
anchor frame.

For the bugdoc this gives the same layout as LO 5.2.

Reportedly this problem started to happen since commit
6f5024de2e1a5cc533527e45b33d9a415467c48d, but it's not obvious why.

Change-Id: Iafb8a6afcba634f11c5db73869313ded0fe13bbd
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/105809
Tested-by: Jenkins
Reviewed-by: Michael Stahl <michael.stahl@cib.de>
(cherry picked from commit 6b92d2e8522ecc98d2c5532f5076c20ae295168e)
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/105940
Tested-by: Michael Stahl <michael.stahl@cib.de>

3 files changed, 57 insertions, 3 deletions

diff --git a/sw/qa/extras/layout/data/tdf138039.odt b/sw/qa/extras/layout/data/tdf138039.odt
new file mode 100644
index 000000000000..f355fd1349a6
--- /dev/null
+++ b/sw/qa/extras/layout/data/tdf138039.odt
diff --git a/sw/qa/extras/layout/layout.cxx b/sw/qa/extras/layout/layout.cxx
index c94811b4d5e7..b872cf9087b3 100644
--- a/sw/qa/extras/layout/layout.cxx
+++ b/sw/qa/extras/layout/layout.cxx
@@ -1139,6 +1139,30 @@ CPPUNIT_TEST_FIXTURE(SwLayoutWriter, testRedlineFlysInFootnote)
     }
 }
 
+CPPUNIT_TEST_FIXTURE(SwLayoutWriter, testTdf138039)
+{
+    createDoc("tdf138039.odt");
+
+    xmlDocPtr pXmlDoc = parseLayoutDump();
+
+    // there are 3 pages
+    assertXPath(pXmlDoc, "/root/page", 3);
+    // table on first page
+    assertXPath(pXmlDoc, "/root/page[1]/body/tab", 1);
+    assertXPath(pXmlDoc, "/root/page[1]/body/txt", 0);
+    // paragraph with large fly on second page
+    assertXPath(pXmlDoc, "/root/page[2]/body/tab", 0);
+    assertXPath(pXmlDoc, "/root/page[2]/body/txt", 1);
+    assertXPath(pXmlDoc, "/root/page[2]/body/txt[1]/anchored/fly", 1);
+    assertXPath(pXmlDoc, "/root/page[2]/body/txt[1]/anchored/fly[1]/infos/bounds", "top", "17915");
+    assertXPath(pXmlDoc, "/root/page[2]/body/txt[1]/anchored/fly[1]/infos/bounds", "height",
+                "15819");
+    // paragraph on third page
+    assertXPath(pXmlDoc, "/root/page[3]/body/tab", 0);
+    assertXPath(pXmlDoc, "/root/page[3]/body/txt", 1);
+    assertXPath(pXmlDoc, "/root/page[3]/body/txt[1]/anchored", 0);
+}
+
 CPPUNIT_TEST_FIXTURE(SwLayoutWriter, testTdf116486)
 {
     SwDoc* pDoc = createDoc("tdf116486.docx");
diff --git a/sw/source/core/layout/tabfrm.cxx b/sw/source/core/layout/tabfrm.cxx
index 060d5151e3bc..719f08e5a363 100644
--- a/sw/source/core/layout/tabfrm.cxx
+++ b/sw/source/core/layout/tabfrm.cxx
@@ -2670,6 +2670,21 @@ void SwTabFrame::MakeAll(vcl::RenderContext* pRenderContext)
         aNotify.SetInvaKeep();
 }
 
+static bool IsNextOnSamePage(SwPageFrame const& rPage,
+        SwTabFrame const& rTabFrame, SwTextFrame const& rAnchorFrame)
+{
+    for (SwContentFrame const* pContentFrame = rTabFrame.FindNextCnt();
+        pContentFrame && pContentFrame->FindPageFrame() == &rPage;
+        pContentFrame = pContentFrame->FindNextCnt())
+    {
+        if (pContentFrame == &rAnchorFrame)
+        {
+            return true;
+        }
+    }
+    return false;
+}
+
 /// Calculate the offsets arising because of FlyFrames
 bool SwTabFrame::CalcFlyOffsets( SwTwips& rUpper,
                                long& rLeftOffset,
@@ -2769,10 +2784,25 @@ bool SwTabFrame::CalcFlyOffsets( SwTwips& rUpper,
                     const SwFormatHoriOrient &rHori= pFly->GetFormat()->GetHoriOrient();
                     if ( css::text::WrapTextMode_NONE == rSur.GetSurround() )
                     {
+                        // possible cases:
+                        //        both in body
+                        //        both in same fly
+                        //        any comb. of body, footnote, header/footer
+                        // to keep it safe, check only in doc body vs page margin for now
                         long nBottom = aRectFnSet.GetBottom(aFlyRect);
-                        if( aRectFnSet.YDiff( nPrtPos, nBottom ) < 0 )
-                            nPrtPos = nBottom;
-                        bInvalidatePrtArea = true;
+                        // tdf#138039 don't grow beyond the page body
+                        // if the fly is anchored below the table; the fly
+                        // must move with its anchor frame to the next page
+                        SwRectFnSet fnPage(pPage);
+                        if (!IsInDocBody() // TODO
+                            || fnPage.YDiff(fnPage.GetBottom(aFlyRect), fnPage.GetPrtBottom(*pPage)) <= 0
+                            || !IsNextOnSamePage(*pPage, *this,
+                                *static_cast<SwTextFrame*>(pFly->GetAnchorFrameContainingAnchPos())))
+                        {
+                            if (aRectFnSet.YDiff( nPrtPos, nBottom ) < 0)
+                                nPrtPos = nBottom;
+                            bInvalidatePrtArea = true;
+                        }
                     }
                     if ( (css::text::WrapTextMode_RIGHT    == rSur.GetSurround() ||
                           css::text::WrapTextMode_PARALLEL == rSur.GetSurround())&&