sw floattable: fix legacy max height of split flys with negative vert offset

A reduced bugdoc from tdf#155002 had 2 floating tables, the one on the
2nd page was split to two fly frames, so the first and the second row
overlapped.

This is similar to what was fixed in commit
4cb6e54a3dcdd771ef76bd98b58f0bf1c4be4c45 (sw floattable: fix missing
table join when moving master fly to next page, 2023-05-18), but here we
are in the legacy layout mode, which allows floating tables outside the
body frame, somewhat. The exact detail here is that the vertical offset
of the floating table is -179 twips, and Word also splits the table to 2
floating frames in case there is no such vertical offset.

Fix the problem by ignoring the part of the fly frame that is
overlapping with the top margin area, this way the 2nd floating table
fits the 2nd page and our layout matches Word.

The crash with the original tdf#155002 bugdoc needs more work, still.

(cherry picked from commit 632f36cc972116cd8da8245590f74014c22532db)

Change-Id: I61ac54116480904320c7fa7cc557e0fcaf792739
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/152078
Tested-by: Jenkins CollaboraOffice <jenkinscollaboraoffice@gmail.com>
Reviewed-by: Miklos Vajna <vmiklos@collabora.com>

3 files changed, 39 insertions, 1 deletions

diff --git a/sw/qa/core/layout/data/floattable-tab-join-legacy.docx b/sw/qa/core/layout/data/floattable-tab-join-legacy.docx
new file mode 100644
index 000000000000..f4056bf21f5c
--- /dev/null
+++ b/sw/qa/core/layout/data/floattable-tab-join-legacy.docx
diff --git a/sw/qa/core/layout/flycnt.cxx b/sw/qa/core/layout/flycnt.cxx
index 8aec18461bee..3c6a061cb962 100644
--- a/sw/qa/core/layout/flycnt.cxx
+++ b/sw/qa/core/layout/flycnt.cxx
@@ -855,6 +855,36 @@ CPPUNIT_TEST_FIXTURE(Test, testSplitFlyTabJoin)
     CPPUNIT_ASSERT(pPage3);
     CPPUNIT_ASSERT(!pPage3->GetSortedObjs());
 }
+
+CPPUNIT_TEST_FIXTURE(Test, testSplitFlyTabJoinLegacy)
+{
+    // Given a document with 3 pages and 2 tables: table on first and second page, 3rd page has no
+    // table (Word 2010 mode):
+    createSwDoc("floattable-tab-join-legacy.docx");
+
+    // When laying out that document:
+    calcLayout();
+
+    // Then make sure that all pages have the expected amount of fly frames:
+    SwDoc* pDoc = getSwDoc();
+    SwRootFrame* pLayout = pDoc->getIDocumentLayoutAccess().GetCurrentLayout();
+    auto pPage1 = dynamic_cast<SwPageFrame*>(pLayout->Lower());
+    CPPUNIT_ASSERT(pPage1);
+    const SwSortedObjs& rPage1Objs = *pPage1->GetSortedObjs();
+    CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(1), rPage1Objs.size());
+    auto pPage2 = dynamic_cast<SwPageFrame*>(pPage1->GetNext());
+    CPPUNIT_ASSERT(pPage2);
+    const SwSortedObjs& rPage2Objs = *pPage2->GetSortedObjs();
+    // Without the accompanying fix in place, this test would have failed with:
+    // - Expected: 1
+    // - Actual  : 2
+    // i.e. the 2nd page had 2 fly frames, hosting a split table, instead of joining that table and
+    // having 1 fly frame (even after the non-legacy case was fixed already).
+    CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(1), rPage2Objs.size());
+    auto pPage3 = dynamic_cast<SwPageFrame*>(pPage2->GetNext());
+    CPPUNIT_ASSERT(pPage3);
+    CPPUNIT_ASSERT(!pPage3->GetSortedObjs());
+}
 }
 
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/sw/source/core/layout/fly.cxx b/sw/source/core/layout/fly.cxx
index c5ee8815a71b..b598346216ee 100644
--- a/sw/source/core/layout/fly.cxx
+++ b/sw/source/core/layout/fly.cxx
@@ -107,11 +107,19 @@ SwTwips GetFlyAnchorBottom(SwFlyFrame* pFly, const SwFrame& rAnchor)
         // See if the fly height would fit at least the page height, ignoring the vertical offset.
         SwTwips nFlyHeight = aRectFnSet.GetHeight(pFly->getFrameArea());
         SwTwips nPageHeight = aRectFnSet.GetHeight(pPage->getFramePrintArea());
+        SwTwips nFlyTop = aRectFnSet.GetTop(pFly->getFrameArea());
+        SwTwips nBodyTop = aRectFnSet.GetTop(pBody->getFrameArea());
+        if (nFlyTop < nBodyTop)
+        {
+            // Fly frame overlaps with the top margin area, ignore that part of the fly frame for
+            // top/height purposes.
+            nFlyHeight -= nBodyTop - nFlyTop;
+            nFlyTop = nBodyTop;
+        }
         if (nFlyHeight <= nPageHeight)
         {
             // Yes, it would fit: allow overlap if there is no problematic vertical offset.
             SwTwips nDeadline = aRectFnSet.GetBottom(pPage->getFrameArea());
-            SwTwips nFlyTop = aRectFnSet.GetTop(pFly->getFrameArea());
             SwTwips nBodyHeight = aRectFnSet.GetHeight(pBody->getFramePrintArea());
             if (nDeadline - nFlyTop > nBodyHeight)
             {