Resolves: tdf#144135 Rework Sort with area extras

Since

commit 774a61afa9fc281290e7bfad4e28c05978b82d73
CommitDate: Wed Apr 14 08:46:03 2021 +0200

tdf#126678 - Consider "Include formats" option during sort

a sheet formatted with visible attributes like cell background
colour up to the end if for Sort all columns and/or rows are
selected lead to an excessive memory allocation and slow execution
time if it didn't get killed by the operating system before due to
memory exhaustion.

The same could had happened already before if graphics or comments
were to be included that could had resulted in a similar large
range. However, cell formats across sheets are more likely.

This changes the strategy how the to be sorted data range is
determined (range only with data) and additional area extras
ranges without data that are only to be rearranged. Those are then
processed in chunks (limited to ~512MB per chunk).

Cell formats that are identical within one column's rows range do
not even need to be covered as they are not rearranged, in the
best case leading to all trailing formats' ranges being excluded
from the sort.

Additionally optimize the cell gathering of formats, graphics and
comments such that for the area extras they are only collected if
actually requested.

The overall performance gain is in an order of magnitudes even if
some extras are to be collected.

Change-Id: If3abbaeaa615aaff7d88a82a5b3fc7ac633d770d
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/122013
Reviewed-by: Eike Rathke <erack@redhat.com>
Tested-by: Jenkins
(cherry picked from commit 0a9b68c9f9880655576e3220d8b70064b367dbee)
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/121981
Reviewed-by: Mike Kaganski <mike.kaganski@collabora.com>

1 files changed, 23 insertions, 18 deletions

diff --git a/sc/source/core/data/sortparam.cxx b/sc/source/core/data/sortparam.cxx
index cd98dd07a982..cb369baae99c 100644
--- a/sc/source/core/data/sortparam.cxx
+++ b/sc/source/core/data/sortparam.cxx
@@ -33,11 +33,13 @@ ScSortParam::ScSortParam()
 }
 
 ScSortParam::ScSortParam( const ScSortParam& r ) :
-        nCol1(r.nCol1),nRow1(r.nRow1),nCol2(r.nCol2),nRow2(r.nRow2),nUserIndex(r.nUserIndex),
+        nCol1(r.nCol1),nRow1(r.nRow1),nCol2(r.nCol2),nRow2(r.nRow2),
+        aDataAreaExtras(r.aDataAreaExtras),
+        nUserIndex(r.nUserIndex),
         bHasHeader(r.bHasHeader),bByRow(r.bByRow),bCaseSens(r.bCaseSens),
-        bNaturalSort(r.bNaturalSort),bIncludeComments(r.bIncludeComments),
-        bIncludeGraphicObjects(r.bIncludeGraphicObjects),bUserDef(r.bUserDef),
-        bIncludePattern(r.bIncludePattern),bInplace(r.bInplace),
+        bNaturalSort(r.bNaturalSort),
+        bUserDef(r.bUserDef),
+        bInplace(r.bInplace),
         nDestTab(r.nDestTab),nDestCol(r.nDestCol),nDestRow(r.nDestRow),
         maKeyState( r.maKeyState ),
         aCollatorLocale( r.aCollatorLocale ), aCollatorAlgorithm( r.aCollatorAlgorithm ),
@@ -53,13 +55,14 @@ void ScSortParam::Clear()
 
     nCol1=nCol2=nDestCol = 0;
     nRow1=nRow2=nDestRow = 0;
+    aDataAreaExtras = ScDataAreaExtras();
+    aDataAreaExtras.mbCellDrawObjects = true;
+    aDataAreaExtras.mbCellFormats = true;
     nCompatHeader = 2;
     nDestTab = 0;
     nUserIndex = 0;
     bHasHeader=bCaseSens=bUserDef=bNaturalSort = false;
-    bIncludeComments = false;
-    bIncludeGraphicObjects = true;
-    bByRow=bIncludePattern=bInplace = true;
+    bByRow = bInplace = true;
     aCollatorLocale = css::lang::Locale();
     aCollatorAlgorithm.clear();
 
@@ -77,15 +80,13 @@ ScSortParam& ScSortParam::operator=( const ScSortParam& r )
     nRow1           = r.nRow1;
     nCol2           = r.nCol2;
     nRow2           = r.nRow2;
+    aDataAreaExtras = r.aDataAreaExtras;
     nUserIndex      = r.nUserIndex;
     bHasHeader      = r.bHasHeader;
     bByRow          = r.bByRow;
     bCaseSens       = r.bCaseSens;
     bNaturalSort    = r.bNaturalSort;
-    bIncludeComments= r.bIncludeComments;
-    bIncludeGraphicObjects = r.bIncludeGraphicObjects;
     bUserDef        = r.bUserDef;
-    bIncludePattern = r.bIncludePattern;
     bInplace        = r.bInplace;
     nDestTab        = r.nDestTab;
     nDestCol        = r.nDestCol;
@@ -123,15 +124,13 @@ bool ScSortParam::operator==( const ScSortParam& rOther ) const
         && (nRow1           == rOther.nRow1)
         && (nCol2           == rOther.nCol2)
         && (nRow2           == rOther.nRow2)
+        && (aDataAreaExtras == rOther.aDataAreaExtras)
         && (bHasHeader      == rOther.bHasHeader)
         && (bByRow          == rOther.bByRow)
         && (bCaseSens       == rOther.bCaseSens)
         && (bNaturalSort    == rOther.bNaturalSort)
-        && (bIncludeComments== rOther.bIncludeComments)
-        && (bIncludeGraphicObjects == rOther.bIncludeGraphicObjects)
         && (bUserDef        == rOther.bUserDef)
         && (nUserIndex      == rOther.nUserIndex)
-        && (bIncludePattern == rOther.bIncludePattern)
         && (bInplace        == rOther.bInplace)
         && (nDestTab        == rOther.nDestTab)
         && (nDestCol        == rOther.nDestCol)
@@ -155,15 +154,19 @@ bool ScSortParam::operator==( const ScSortParam& rOther ) const
 }
 
 ScSortParam::ScSortParam( const ScSubTotalParam& rSub, const ScSortParam& rOld ) :
-        nCol1(rSub.nCol1),nRow1(rSub.nRow1),nCol2(rSub.nCol2),nRow2(rSub.nRow2),nUserIndex(rSub.nUserIndex),
+        nCol1(rSub.nCol1),nRow1(rSub.nRow1),nCol2(rSub.nCol2),nRow2(rSub.nRow2),
+        aDataAreaExtras(rOld.aDataAreaExtras),
+        nUserIndex(rSub.nUserIndex),
         bHasHeader(true),bByRow(true),bCaseSens(rSub.bCaseSens),bNaturalSort(rOld.bNaturalSort),
-        bIncludeComments(rOld.bIncludeComments),bIncludeGraphicObjects(rOld.bIncludeGraphicObjects),
-        bUserDef(rSub.bUserDef),bIncludePattern(rSub.bIncludePattern),
+        bUserDef(rSub.bUserDef),
         bInplace(true),
         nDestTab(0),nDestCol(0),nDestRow(0),
         aCollatorLocale( rOld.aCollatorLocale ), aCollatorAlgorithm( rOld.aCollatorAlgorithm ),
         nCompatHeader( rOld.nCompatHeader )
 {
+    aDataAreaExtras.mbCellFormats = rSub.bIncludePattern;
+    aDataAreaExtras.resetArea();
+
     sal_uInt16 i;
 
     //  first the groups from the partial results
@@ -201,12 +204,14 @@ ScSortParam::ScSortParam( const ScSubTotalParam& rSub, const ScSortParam& rOld )
 ScSortParam::ScSortParam( const ScQueryParam& rParam, SCCOL nCol ) :
         nCol1(nCol),nRow1(rParam.nRow1),nCol2(nCol),nRow2(rParam.nRow2),nUserIndex(0),
         bHasHeader(rParam.bHasHeader),bByRow(true),bCaseSens(rParam.bCaseSens),
-        bNaturalSort(false),bIncludeComments(false),bIncludeGraphicObjects(true),
+        bNaturalSort(false),
 //TODO: what about Locale and Algorithm?
-        bUserDef(false),bIncludePattern(false),
+        bUserDef(false),
         bInplace(true),
         nDestTab(0),nDestCol(0),nDestRow(0), nCompatHeader(2)
 {
+    aDataAreaExtras.mbCellDrawObjects = true;
+
     ScSortKeyState aKeyState;
     aKeyState.bDoSort = true;
     aKeyState.nField = nCol;