tdf#107945: properly iterate over mtv during pivot cache loading.

This reduces the total time required for populating the pivot cache
by ~60%.

Reviewed-on: https://gerrit.libreoffice.org/37971
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Kohei Yoshida <libreoffice@kohei.us>
(cherry picked from commit d4cd8677889ec3807c194ef5b462f8e031807e5b)

Conflicts:
sc/inc/document.hxx
sc/source/core/data/formulacell.cxx

Change-Id: I6a8511959c20231a8a5dbd0b0a9a3d0930a1fa0c

8 files changed, 223 insertions, 18 deletions

diff --git a/sc/source/core/data/cellvalue.cxx b/sc/source/core/data/cellvalue.cxx
index efe66f789596..75b7c5bbb12f 100644
--- a/sc/source/core/data/cellvalue.cxx
+++ b/sc/source/core/data/cellvalue.cxx
@@ -17,6 +17,7 @@
 #include "editutil.hxx"
 #include "tokenarray.hxx"
 #include <formula/token.hxx>
+#include <formula/errorcodes.hxx>
 #include <svl/sharedstring.hxx>
 
 namespace {
@@ -178,6 +179,27 @@ OUString getStringImpl( const CellT& rCell, const ScDocument* pDoc )
     return EMPTY_OUSTRING;
 }
 
+template<typename CellT>
+OUString getRawStringImpl( const CellT& rCell, const ScDocument* pDoc )
+{
+    switch (rCell.meType)
+    {
+        case CELLTYPE_VALUE:
+            return OUString::number(rCell.mfValue);
+        case CELLTYPE_STRING:
+            return rCell.mpString->getString();
+        case CELLTYPE_EDIT:
+            if (rCell.mpEditText)
+                return ScEditUtil::GetString(*rCell.mpEditText, pDoc);
+        break;
+        case CELLTYPE_FORMULA:
+            return rCell.mpFormula->GetRawString().getString();
+        default:
+            ;
+    }
+    return EMPTY_OUSTRING;
+}
+
 }
 
 ScCellValue::ScCellValue() : meType(CELLTYPE_NONE), mfValue(0.0) {}
@@ -544,6 +566,11 @@ bool ScRefCellValue::hasNumeric() const
     return hasNumericImpl(meType, mpFormula);
 }
 
+bool ScRefCellValue::hasError() const
+{
+    return meType == CELLTYPE_FORMULA && mpFormula->GetErrCode() != FormulaError::NONE;
+}
+
 double ScRefCellValue::getValue()
 {
     switch (meType)
@@ -558,11 +585,30 @@ double ScRefCellValue::getValue()
     return 0.0;
 }
 
+double ScRefCellValue::getRawValue() const
+{
+    switch (meType)
+    {
+        case CELLTYPE_VALUE:
+            return mfValue;
+        case CELLTYPE_FORMULA:
+            return mpFormula->GetRawValue();
+        default:
+            ;
+    }
+    return 0.0;
+}
+
 OUString ScRefCellValue::getString( const ScDocument* pDoc )
 {
     return getStringImpl(*this, pDoc);
 }
 
+OUString ScRefCellValue::getRawString( const ScDocument* pDoc ) const
+{
+    return getRawStringImpl(*this, pDoc);
+}
+
 bool ScRefCellValue::isEmpty() const
 {
     return meType == CELLTYPE_NONE;
diff --git a/sc/source/core/data/column4.cxx b/sc/source/core/data/column4.cxx
index 69d95f96fa64..a50092c0ea9b 100644
--- a/sc/source/core/data/column4.cxx
+++ b/sc/source/core/data/column4.cxx
@@ -14,6 +14,7 @@
 #include <document.hxx>
 #include <cellvalues.hxx>
 #include <columnspanset.hxx>
+#include <columniterator.hxx>
 #include <listenercontext.hxx>
 #include <tokenstringcontext.hxx>
 #include <mtvcellfunc.hxx>
@@ -30,6 +31,7 @@
 #include <sharedformula.hxx>
 
 #include <svl/sharedstringpool.hxx>
+#include <o3tl/make_unique.hxx>
 
 #include <vector>
 #include <cassert>
@@ -1574,4 +1576,28 @@ void ScColumn::SetNeedsListeningGroup( SCROW nRow )
         (*pp)->SetNeedsListening(true);
 }
 
+std::unique_ptr<sc::ColumnIterator> ScColumn::GetColumnIterator( SCROW nRow1, SCROW nRow2 ) const
+{
+    if (!ValidRow(nRow1) || !ValidRow(nRow2) || nRow1 > nRow2)
+        return std::unique_ptr<sc::ColumnIterator>();
+
+    return o3tl::make_unique<sc::ColumnIterator>(maCells, nRow1, nRow2);
+}
+
+void ScColumn::EnsureFormulaCellResults( SCROW nRow1, SCROW nRow2 )
+{
+    if (!ValidRow(nRow1) || !ValidRow(nRow2) || nRow1 > nRow2)
+        return;
+
+    if (!HasFormulaCell(nRow1, nRow2))
+        return;
+
+    sc::ProcessFormula(maCells.begin(), maCells, nRow1, nRow2,
+        []( size_t /*nRow*/, ScFormulaCell* pCell )
+        {
+            pCell->MaybeInterpret();
+        }
+    );
+}
+
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/sc/source/core/data/columniterator.cxx b/sc/source/core/data/columniterator.cxx
index 4e363f8f319e..10f87459f91e 100644
--- a/sc/source/core/data/columniterator.cxx
+++ b/sc/source/core/data/columniterator.cxx
@@ -168,4 +168,41 @@ void ScColumnTextWidthIterator::checkEndRow()
     miBlockCur = miBlockEnd;
 }
 
+namespace sc {
+
+ColumnIterator::ColumnIterator( const CellStoreType& rCells, SCROW nRow1, SCROW nRow2 ) :
+    maPos(rCells.position(nRow1)),
+    maPosEnd(rCells.position(maPos.first, nRow2+1))
+{
+}
+
+ColumnIterator::~ColumnIterator() {}
+
+void ColumnIterator::next()
+{
+    maPos = CellStoreType::next_position(maPos);
+}
+
+SCROW ColumnIterator::getRow() const
+{
+    return CellStoreType::logical_position(maPos);
+}
+
+bool ColumnIterator::hasCell() const
+{
+    return maPos != maPosEnd;
+}
+
+mdds::mtv::element_t ColumnIterator::getType() const
+{
+    return maPos.first->type;
+}
+
+ScRefCellValue ColumnIterator::getCell() const
+{
+    return toRefCell(maPos.first, maPos.second);
+}
+
+}
+
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/sc/source/core/data/document10.cxx b/sc/source/core/data/document10.cxx
index 35632d44536f..a2dc93205f91 100644
--- a/sc/source/core/data/document10.cxx
+++ b/sc/source/core/data/document10.cxx
@@ -20,6 +20,7 @@
 #include <bcaslot.hxx>
 #include <cellvalues.hxx>
 #include <docpool.hxx>
+#include <columniterator.hxx>
 
 #include "dociter.hxx"
 #include "patattr.hxx"
@@ -903,4 +904,26 @@ bool ScDocument::IsEditActionAllowed(
     return true;
 }
 
+std::unique_ptr<sc::ColumnIterator> ScDocument::GetColumnIterator( SCTAB nTab, SCCOL nCol, SCROW nRow1, SCROW nRow2 ) const
+{
+    const ScTable* pTab = FetchTable(nTab);
+    if (!pTab)
+        return std::unique_ptr<sc::ColumnIterator>();
+
+    return pTab->GetColumnIterator(nCol, nRow1, nRow2);
+}
+
+void ScDocument::EnsureFormulaCellResults( const ScRange& rRange )
+{
+    for (SCTAB nTab = rRange.aStart.Tab(); nTab <= rRange.aEnd.Tab(); ++nTab)
+    {
+        ScTable* pTab = FetchTable(nTab);
+        if (!pTab)
+            continue;
+
+        pTab->EnsureFormulaCellResults(
+            rRange.aStart.Col(), rRange.aStart.Row(), rRange.aEnd.Col(), rRange.aEnd.Row());
+    }
+}
+
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/sc/source/core/data/dpcache.cxx b/sc/source/core/data/dpcache.cxx
index fd41b12a3782..2db6ff88a8b8 100644
--- a/sc/source/core/data/dpcache.cxx
+++ b/sc/source/core/data/dpcache.cxx
@@ -29,6 +29,8 @@
 #include "dpitemdata.hxx"
 #include "dputil.hxx"
 #include "dpnumgroupinfo.hxx"
+#include <columniterator.hxx>
+#include <cellvalue.hxx>
 
 #include <rtl/math.hxx>
 #include <unotools/textsearch.hxx>
@@ -104,9 +106,10 @@ private:
     ScDocument* mpDoc;
 };
 
-OUString createLabelString(ScDocument* pDoc, SCCOL nCol, SCROW nRow, SCTAB nTab)
+OUString createLabelString( const ScDocument* pDoc, SCCOL nCol, const ScRefCellValue& rCell )
 {
-    OUString aDocStr = pDoc->GetString(nCol, nRow, nTab);
+    OUString aDocStr = rCell.getRawString(pDoc);
+
     if (aDocStr.isEmpty())
     {
         // Replace an empty label string with column name.
@@ -122,25 +125,24 @@ OUString createLabelString(ScDocument* pDoc, SCCOL nCol, SCROW nRow, SCTAB nTab)
 }
 
 void initFromCell(
-    ScDPCache& rCache, ScDocument* pDoc, SCCOL nCol, SCROW nRow, SCTAB nTab,
+    ScDPCache& rCache, ScDocument* pDoc, const ScAddress& rPos,
+    const ScRefCellValue& rCell,
     ScDPItemData& rData, sal_uInt32& rNumFormat)
 {
-    OUString aDocStr = pDoc->GetString(nCol, nRow, nTab);
+    OUString aDocStr = rCell.getRawString(pDoc);
     rNumFormat = 0;
 
-    ScAddress aPos(nCol, nRow, nTab);
-
-    if (pDoc->GetErrCode(aPos) != FormulaError::NONE)
+    if (rCell.hasError())
     {
         rData.SetErrorString(rCache.InternString(aDocStr));
     }
-    else if (pDoc->HasValueData(nCol, nRow, nTab))
+    else if (rCell.hasNumeric())
     {
-        double fVal = pDoc->GetValue(aPos);
-        rNumFormat = pDoc->GetNumberFormat(aPos);
+        double fVal = rCell.getRawValue();
+        rNumFormat = pDoc->GetNumberFormat(rPos);
         rData.SetValue(fVal);
     }
-    else if (pDoc->HasData(nCol, nRow, nTab))
+    else if (!rCell.isEmpty())
     {
         rData.SetString(rCache.InternString(aDocStr));
     }
@@ -321,21 +323,33 @@ void ScDPCache::InitFromDoc(ScDocument* pDoc, const ScRange& rRange)
 
     maLabelNames.reserve(mnColumnCount+1);
 
+    // Ensure that none of the formula cells in the data range are dirty.
+    pDoc->EnsureFormulaCellResults(rRange);
+
     ScDPItemData aData;
     for (sal_uInt16 nCol = nStartCol; nCol <= nEndCol; ++nCol)
     {
-        AddLabel(createLabelString(pDoc, nCol, nStartRow, nDocTab));
+        std::unique_ptr<sc::ColumnIterator> pIter =
+            pDoc->GetColumnIterator(nDocTab, nCol, nStartRow, nEndRow);
+        assert(pIter);
+        assert(pIter->hasCell());
+
+        AddLabel(createLabelString(pDoc, nCol, pIter->getCell()));
+        pIter->next();
+
         Field& rField = *maFields[nCol-nStartCol].get();
         std::vector<Bucket> aBuckets;
         aBuckets.reserve(nEndRow-nStartRow); // skip the topmost label cell.
 
         // Push back all original values.
-        SCROW nOffset = nStartRow + 1;
-        for (SCROW i = 0, n = nEndRow-nStartRow; i < n; ++i)
+        for (SCROW i = 0, n = nEndRow-nStartRow; i < n; ++i, pIter->next())
         {
-            SCROW nRow = i + nOffset;
+            assert(pIter->hasCell());
+
             sal_uInt32 nNumFormat = 0;
-            initFromCell(*this, pDoc, nCol, nRow, nDocTab, aData, nNumFormat);
+            ScAddress aPos(nCol, pIter->getRow(), nDocTab);
+            initFromCell(*this, pDoc, aPos, pIter->getCell(), aData, nNumFormat);
+
             aBuckets.push_back(Bucket(aData, i));
 
             if (!aData.IsEmpty())
diff --git a/sc/source/core/data/formulacell.cxx b/sc/source/core/data/formulacell.cxx
index b19bfba89bc9..4c6471f781a0 100644
--- a/sc/source/core/data/formulacell.cxx
+++ b/sc/source/core/data/formulacell.cxx
@@ -2608,15 +2608,25 @@ bool ScFormulaCell::IsValueNoError() const
 double ScFormulaCell::GetValue()
 {
     MaybeInterpret();
+    return GetRawValue();
+}
+
+svl::SharedString ScFormulaCell::GetString()
+{
+    MaybeInterpret();
+    return GetRawString();
+}
+
+double ScFormulaCell::GetRawValue() const
+{
     if ((pCode->GetCodeError() == FormulaError::NONE || pCode->GetCodeError() == FormulaError::DoubleRef) &&
             aResult.GetResultError() == FormulaError::NONE)
         return aResult.GetDouble();
     return 0.0;
 }
 
-svl::SharedString ScFormulaCell::GetString()
+svl::SharedString ScFormulaCell::GetRawString() const
 {
-    MaybeInterpret();
     if ((pCode->GetCodeError() == FormulaError::NONE || pCode->GetCodeError() == FormulaError::DoubleRef) &&
             aResult.GetResultError() == FormulaError::NONE)
         return aResult.GetString();
diff --git a/sc/source/core/data/mtvcellfunc.cxx b/sc/source/core/data/mtvcellfunc.cxx
new file mode 100644
index 000000000000..98f6998cc120
--- /dev/null
+++ b/sc/source/core/data/mtvcellfunc.cxx
@@ -0,0 +1,31 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+#include <mtvcellfunc.hxx>
+
+namespace sc {
+
+CellStoreType::iterator ProcessFormula(
+    const CellStoreType::iterator& it, CellStoreType& rStore, SCROW nRow1, SCROW nRow2,
+    std::function<void(size_t,ScFormulaCell*)> aFuncElem )
+{
+    using FuncType = std::function<void(size_t,ScFormulaCell*)>;
+    using ElseFuncType = std::function<void(mdds::mtv::element_t, size_t, size_t)>;
+
+    // empty function for handling the 'else' part.
+    static ElseFuncType aFuncElse =
+        [](mdds::mtv::element_t,size_t,size_t) {};
+
+    return ProcessElements1<
+        CellStoreType, formula_block,
+        FuncType, ElseFuncType>(
+            it, rStore, nRow1, nRow2, aFuncElem, aFuncElse);
+}
+
+}
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/sc/source/core/data/table7.cxx b/sc/source/core/data/table7.cxx
index 6179737aec93..b8bd98e5d1a9 100644
--- a/sc/source/core/data/table7.cxx
+++ b/sc/source/core/data/table7.cxx
@@ -17,6 +17,7 @@
 #include <cellvalues.hxx>
 #include "olinetab.hxx"
 #include <tabprotection.hxx>
+#include <columniterator.hxx>
 
 bool ScTable::IsMerged( SCCOL nCol, SCROW nRow ) const
 {
@@ -392,6 +393,23 @@ bool ScTable::IsEditActionAllowed(
     return false;
 }
 
+std::unique_ptr<sc::ColumnIterator> ScTable::GetColumnIterator( SCCOL nCol, SCROW nRow1, SCROW nRow2 ) const
+{
+    if (!ValidCol(nCol) || nCol >= aCol.size())
+        return std::unique_ptr<sc::ColumnIterator>();
+
+    return aCol[nCol].GetColumnIterator(nRow1, nRow2);
+}
+
+void ScTable::EnsureFormulaCellResults( SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2 )
+{
+    if (nCol2 < nCol1 || !ValidCol(nCol1) || !ValidCol(nCol2))
+        return;
+
+    for (SCCOL nCol = nCol1; nCol <= nCol2; ++nCol)
+        aCol[nCol].EnsureFormulaCellResults(nRow1, nRow2);
+}
+
 void ScTable::finalizeOutlineImport()
 {
     if (pOutlineTable && pRowFlags)