Related: tdf#154005 sc ods fileopen: fix dropdown form control size

Fixing the crashtesting assert/crash after the original change.
Also a little clean-up.

Change-Id: I35453fbc55b3d5d4064179e84755334c2d3a01ca
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/149583
Tested-by: Jenkins
Reviewed-by: Caolán McNamara <caolanm@redhat.com>
Reviewed-by: Balazs Varga <balazs.varga.extern@allotropia.de>
(cherry picked from commit 69cc8bdd5f9109804d912b52d5ee1040d6e1868f)
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/149597
Reviewed-by: Xisco Fauli <xiscofauli@libreoffice.org>

5 files changed, 58 insertions, 8 deletions

diff --git a/sc/inc/document.hxx b/sc/inc/document.hxx
index f8558cc21070..97b47ee94d00 100644
--- a/sc/inc/document.hxx
+++ b/sc/inc/document.hxx
@@ -2035,7 +2035,8 @@ public:
     SC_DLLPUBLIC SCROW          FirstVisibleRow(SCROW nStartRow, SCROW nEndRow, SCTAB nTab) const;
     SC_DLLPUBLIC SCROW          LastVisibleRow(SCROW nStartRow, SCROW nEndRow, SCTAB nTab) const;
     SCROW                       CountVisibleRows(SCROW nStartRow, SCROW nEndRow, SCTAB nTab) const;
-    SCCOL                       CountVisibleCols(SCROW nStartCol, SCROW nEndCol, SCTAB nTab) const;
+    SCROW                       CountHiddenRows(SCROW nStartRow, SCROW nEndRow, SCTAB nTab) const;
+    SCCOL                       CountHiddenCols(SCROW nStartCol, SCROW nEndCol, SCTAB nTab) const;
 
     SC_DLLPUBLIC bool           RowFiltered(SCROW nRow, SCTAB nTab, SCROW* pFirstRow = nullptr, SCROW* pLastRow = nullptr) const;
     bool                        HasFilteredRows(SCROW nStartRow, SCROW nEndRow, SCTAB nTab) const;
diff --git a/sc/inc/table.hxx b/sc/inc/table.hxx
index f81e3925964e..851248a99c19 100644
--- a/sc/inc/table.hxx
+++ b/sc/inc/table.hxx
@@ -964,9 +964,11 @@ public:
     SCROW       FirstVisibleRow(SCROW nStartRow, SCROW nEndRow) const;
     SCROW       LastVisibleRow(SCROW nStartRow, SCROW nEndRow) const;
     SCROW       CountVisibleRows(SCROW nStartRow, SCROW nEndRow) const;
+    SCROW       CountHiddenRows(SCROW nStartRow, SCROW nEndRow) const;
     tools::Long GetTotalRowHeight(SCROW nStartRow, SCROW nEndRow, bool bHiddenAsZero = true) const;
 
     SCCOL       CountVisibleCols(SCCOL nStartCol, SCCOL nEndCol) const;
+    SCCOL       CountHiddenCols(SCCOL nStartCol, SCCOL nEndCol) const;
 
     SCCOLROW    LastHiddenColRow(SCCOLROW nPos, bool bCol) const;
 
diff --git a/sc/source/core/data/document.cxx b/sc/source/core/data/document.cxx
index e31e8f385592..79ce684d87be 100644
--- a/sc/source/core/data/document.cxx
+++ b/sc/source/core/data/document.cxx
@@ -4571,12 +4571,20 @@ SCROW ScDocument::CountVisibleRows(SCROW nStartRow, SCROW nEndRow, SCTAB nTab) c
     return maTabs[nTab]->CountVisibleRows(nStartRow, nEndRow);
 }
 
-SCCOL ScDocument::CountVisibleCols(SCROW nStartCol, SCROW nEndCol, SCTAB nTab) const
+SCROW ScDocument::CountHiddenRows(SCROW nStartRow, SCROW nEndRow, SCTAB nTab) const
 {
     if (!ValidTab(nTab) || nTab >= static_cast<SCTAB>(maTabs.size()) || !maTabs[nTab])
         return 0;
 
-    return maTabs[nTab]->CountVisibleCols(nStartCol, nEndCol);
+    return maTabs[nTab]->CountHiddenRows(nStartRow, nEndRow);
+}
+
+SCCOL ScDocument::CountHiddenCols(SCROW nStartCol, SCROW nEndCol, SCTAB nTab) const
+{
+    if (!ValidTab(nTab) || nTab >= static_cast<SCTAB>(maTabs.size()) || !maTabs[nTab])
+        return 0;
+
+    return maTabs[nTab]->CountHiddenCols(nStartCol, nEndCol);
 }
 
 bool ScDocument::RowFiltered(SCROW nRow, SCTAB nTab, SCROW* pFirstRow, SCROW* pLastRow) const
diff --git a/sc/source/core/data/drwlayer.cxx b/sc/source/core/data/drwlayer.cxx
index e6203697a4e7..5627c8b889ec 100644
--- a/sc/source/core/data/drwlayer.cxx
+++ b/sc/source/core/data/drwlayer.cxx
@@ -708,11 +708,8 @@ void lcl_SetLogicRectFromAnchor(SdrObject* pObj, const ScDrawObjData& rAnchor, c
     // tdf#154005: Handle hidden row/col: remove hidden row/cols size from the ScDrawObjData shape size in case of forms
     if (pObj->GetObjIdentifier() == SdrObjKind::UNO && pObj->GetObjInventor() == SdrInventor::FmForm)
     {
-        nHiddenRows = ((rAnchor.maEnd.Row() - rAnchor.maStart.Row()) + 1) -
-            (pDoc->CountVisibleRows(rAnchor.maStart.Row(), rAnchor.maEnd.Row(), rAnchor.maStart.Tab()));
-
-        nHiddenCols = ((rAnchor.maEnd.Col() - rAnchor.maStart.Col()) + 1) -
-            (pDoc->CountVisibleCols(rAnchor.maStart.Col(), rAnchor.maEnd.Col(), rAnchor.maStart.Tab()));
+        nHiddenRows = pDoc->CountHiddenRows(rAnchor.maStart.Row(), rAnchor.maEnd.Row(), rAnchor.maStart.Tab());
+        nHiddenCols = pDoc->CountHiddenCols(rAnchor.maStart.Col(), rAnchor.maEnd.Col(), rAnchor.maStart.Tab());
     }
 
     // In case of a vertical mirrored custom shape, LibreOffice uses internally an additional 180deg
diff --git a/sc/source/core/data/table5.cxx b/sc/source/core/data/table5.cxx
index ad136f53bb4b..40f59f9b0151 100644
--- a/sc/source/core/data/table5.cxx
+++ b/sc/source/core/data/table5.cxx
@@ -769,6 +769,27 @@ SCROW ScTable::CountVisibleRows(SCROW nStartRow, SCROW nEndRow) const
     return nCount;
 }
 
+SCROW ScTable::CountHiddenRows(SCROW nStartRow, SCROW nEndRow) const
+{
+    SCROW nCount = 0;
+    SCROW nRow = nStartRow;
+    ScFlatBoolRowSegments::RangeData aData;
+    while (nRow <= nEndRow)
+    {
+        if (!mpHiddenRows->getRangeData(nRow, aData))
+            break;
+
+        if (aData.mnRow2 > nEndRow)
+            aData.mnRow2 = nEndRow;
+
+        if (aData.mbValue)
+            nCount += aData.mnRow2 - nRow + 1;
+
+        nRow = aData.mnRow2 + 1;
+    }
+    return nCount;
+}
+
 tools::Long ScTable::GetTotalRowHeight(SCROW nStartRow, SCROW nEndRow, bool bHiddenAsZero) const
 {
     tools::Long nHeight = 0;
@@ -814,6 +835,27 @@ SCCOL ScTable::CountVisibleCols(SCCOL nStartCol, SCCOL nEndCol) const
     return nCount;
 }
 
+SCCOL ScTable::CountHiddenCols(SCCOL nStartCol, SCCOL nEndCol) const
+{
+    SCCOL nCount = 0;
+    SCCOL nCol = nStartCol;
+    ScFlatBoolColSegments::RangeData aData;
+    while (nCol <= nEndCol)
+    {
+        if (!mpHiddenCols->getRangeData(nCol, aData))
+            break;
+
+        if (aData.mnCol2 > nEndCol)
+            aData.mnCol2 = nEndCol;
+
+        if (aData.mbValue)
+            nCount += aData.mnCol2 - nCol + 1;
+
+        nCol = aData.mnCol2 + 1;
+    }
+    return nCount;
+}
+
 SCCOLROW ScTable::LastHiddenColRow(SCCOLROW nPos, bool bCol) const
 {
     if (bCol)