From d468958331f36310d11265ba55d7c27366ab58ab Mon Sep 17 00:00:00 2001 From: Noel Grandin Date: Wed, 20 Nov 2019 13:46:49 +0200 Subject: tdf#128812 speed up loading calc doc with lots of countif by creating a copy of ScQueryCellIterator that is specialised for this use-case. Takes the opening time from 50s to 8s on my machine. Change-Id: I193a7c181a5dfed6fecf75e871729d73625d0df6 Reviewed-on: https://gerrit.libreoffice.org/83299 Tested-by: Jenkins Reviewed-by: Noel Grandin --- sc/inc/column.hxx | 1 + sc/inc/dociter.hxx | 25 +++++++ sc/inc/document.hxx | 1 + sc/inc/table.hxx | 1 + sc/source/core/data/dociter.cxx | 140 +++++++++++++++++++++++++++++++++++++++ sc/source/core/tool/interpr1.cxx | 12 +--- 6 files changed, 170 insertions(+), 10 deletions(-) (limited to 'sc') diff --git a/sc/inc/column.hxx b/sc/inc/column.hxx index 18cf3de6231f..74fa4e8b37f9 100644 --- a/sc/inc/column.hxx +++ b/sc/inc/column.hxx @@ -141,6 +141,7 @@ friend class ScValueIterator; friend class ScHorizontalValueIterator; friend class ScDBQueryDataIterator; friend class ScQueryCellIterator; +friend class ScCountIfCellIterator; friend class ScFormulaGroupIterator; friend class ScCellIterator; friend class ScHorizontalCellIterator; diff --git a/sc/inc/dociter.hxx b/sc/inc/dociter.hxx index a6a8d370270b..3479ee67ca08 100644 --- a/sc/inc/dociter.hxx +++ b/sc/inc/dociter.hxx @@ -367,6 +367,31 @@ public: bool FindEqualOrSortedLastInRange( SCCOL& nFoundCol, SCROW& nFoundRow ); }; +// Used by ScInterpreter::ScCountIf. +// Walk through all non-empty cells in an area. +class ScCountIfCellIterator +{ + typedef sc::CellStoreType::const_position_type PositionType; + PositionType maCurPos; + ScQueryParam maParam; + ScDocument* pDoc; + const ScInterpreterContext& mrContext; + SCTAB nTab; + SCCOL nCol; + SCROW nRow; + + /** Initialize position for new column. */ + void InitPos(); + void IncPos(); + void IncBlock(); + void AdvanceQueryParamEntryField(); + +public: + ScCountIfCellIterator(ScDocument* pDocument, const ScInterpreterContext& rContext, SCTAB nTable, + const ScQueryParam& aParam); + int GetCount(); +}; + class ScDocAttrIterator // all attribute areas { private: diff --git a/sc/inc/document.hxx b/sc/inc/document.hxx index 66ed8fefc000..d774cdbdd3ce 100644 --- a/sc/inc/document.hxx +++ b/sc/inc/document.hxx @@ -316,6 +316,7 @@ friend class ScDBQueryDataIterator; friend class ScFormulaGroupIterator; friend class ScCellIterator; friend class ScQueryCellIterator; +friend class ScCountIfCellIterator; friend class ScHorizontalCellIterator; friend class ScHorizontalAttrIterator; friend class ScDocAttrIterator; diff --git a/sc/inc/table.hxx b/sc/inc/table.hxx index a31e6191eb6e..49ff6b2a96ea 100644 --- a/sc/inc/table.hxx +++ b/sc/inc/table.hxx @@ -250,6 +250,7 @@ friend class ScDBQueryDataIterator; friend class ScFormulaGroupIterator; friend class ScCellIterator; friend class ScQueryCellIterator; +friend class ScCountIfCellIterator; friend class ScHorizontalCellIterator; friend class ScHorizontalAttrIterator; friend class ScDocAttrIterator; diff --git a/sc/source/core/data/dociter.cxx b/sc/source/core/data/dociter.cxx index 3c8e369a575a..919c41c783e5 100644 --- a/sc/source/core/data/dociter.cxx +++ b/sc/source/core/data/dociter.cxx @@ -1449,6 +1449,146 @@ bool ScQueryCellIterator::FindEqualOrSortedLastInRange( SCCOL& nFoundCol, return (nFoundCol <= pDoc->MaxCol()) && (nFoundRow <= pDoc->MaxRow()); } +ScCountIfCellIterator::ScCountIfCellIterator(ScDocument* pDocument, const ScInterpreterContext& rContext, SCTAB nTable, + const ScQueryParam& rParam ) : + maParam(rParam), + pDoc( pDocument ), + mrContext( rContext ), + nTab( nTable) +{ + nCol = maParam.nCol1; + nRow = maParam.nRow1; +} + +void ScCountIfCellIterator::InitPos() +{ + nRow = maParam.nRow1; + if (maParam.bHasHeader && maParam.bByRow) + ++nRow; + ScColumn* pCol = &(pDoc->maTabs[nTab])->aCol[nCol]; + maCurPos = pCol->maCells.position(nRow); +} + +void ScCountIfCellIterator::IncPos() +{ + if (maCurPos.second + 1 < maCurPos.first->size) + { + // Move within the same block. + ++maCurPos.second; + ++nRow; + } + else + // Move to the next block. + IncBlock(); +} + +void ScCountIfCellIterator::IncBlock() +{ + ++maCurPos.first; + maCurPos.second = 0; + + nRow = maCurPos.first->position; +} + +int ScCountIfCellIterator::GetCount() +{ + assert(nTab < pDoc->GetTableCount() && "try to access index out of bounds, FIX IT"); + nCol = maParam.nCol1; + InitPos(); + + const ScQueryEntry& rEntry = maParam.GetEntry(0); + const ScQueryEntry::Item& rItem = rEntry.GetQueryItem(); + const bool bSingleQueryItem = rEntry.GetQueryItems().size() == 1; + int count = 0; + + ScColumn* pCol = &(pDoc->maTabs[nTab])->aCol[nCol]; + while (true) + { + bool bNextColumn = maCurPos.first == pCol->maCells.end(); + if (!bNextColumn) + { + if (nRow > maParam.nRow2) + bNextColumn = true; + } + + if (bNextColumn) + { + do + { + ++nCol; + if (nCol > maParam.nCol2 || nCol >= pDoc->maTabs[nTab]->GetAllocatedColumnsCount()) + return count; // Over and out + AdvanceQueryParamEntryField(); + pCol = &(pDoc->maTabs[nTab])->aCol[nCol]; + } + while (!rItem.mbMatchEmpty && pCol->IsEmptyData()); + + InitPos(); + } + + if (maCurPos.first->type == sc::element_type_empty) + { + if (rItem.mbMatchEmpty && bSingleQueryItem) + { + // This shortcut, instead of determining if any SC_OR query + // exists or this query is SC_AND'ed (which wouldn't make + // sense, but..) and evaluating them in ValidQuery(), is + // possible only because the interpreter is the only caller + // that sets mbMatchEmpty and there is only one item in those + // cases. + // XXX this would have to be reworked if other filters used it + // in different manners and evaluation would have to be done in + // ValidQuery(). + count++; + IncPos(); + continue; + } + else + { + IncBlock(); + continue; + } + } + + ScRefCellValue aCell = sc::toRefCell(maCurPos.first, maCurPos.second); + + if ( pDoc->maTabs[nTab]->ValidQuery( nRow, maParam, + (nCol == static_cast(rEntry.nField) ? &aCell : nullptr), + nullptr, + &mrContext) ) + { + if (aCell.isEmpty()) + return count; + count++; + IncPos(); + continue; + } + else + IncPos(); + } + return count; +} + +void ScCountIfCellIterator::AdvanceQueryParamEntryField() +{ + SCSIZE nEntries = maParam.GetEntryCount(); + for ( SCSIZE j = 0; j < nEntries; j++ ) + { + ScQueryEntry& rEntry = maParam.GetEntry( j ); + if ( rEntry.bDoQuery ) + { + if ( rEntry.nField < pDoc->MaxCol() ) + rEntry.nField++; + else + { + OSL_FAIL( "AdvanceQueryParamEntryField: ++rEntry.nField > MAXCOL" ); + } + } + else + break; // for + } +} + namespace { /** diff --git a/sc/source/core/tool/interpr1.cxx b/sc/source/core/tool/interpr1.cxx index 151247180655..9069138c3013 100644 --- a/sc/source/core/tool/interpr1.cxx +++ b/sc/source/core/tool/interpr1.cxx @@ -5810,16 +5810,8 @@ void ScInterpreter::ScCountIf() } else { - ScQueryCellIterator aCellIter(pDok, mrContext, nTab1, rParam, false); - // Keep Entry.nField in iterator on column change - aCellIter.SetAdvanceQueryParamEntryField( true ); - if ( aCellIter.GetFirst() ) - { - do - { - fCount++; - } while ( aCellIter.GetNext() ); - } + ScCountIfCellIterator aCellIter(pDok, mrContext, nTab1, rParam); + fCount += aCellIter.GetCount(); } } else -- cgit