introduce a way to write a simple data representation to a stream

The format used is column orientated and allows quick import and
export of the table content. This will be used for the external data
to cache the results of each transformation step in the UI.

Change-Id: I6e1bfd3b3384cbfadeb98fb995dfd0b03d5e6eb6
Reviewed-on: https://gerrit.libreoffice.org/41198
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Markus Mohrhard <markus.mohrhard@googlemail.com>

10 files changed, 323 insertions, 0 deletions

diff --git a/sc/CppunitTest_sc_cache_test.mk b/sc/CppunitTest_sc_cache_test.mk
new file mode 100644
index 000000000000..af495db849d9
--- /dev/null
+++ b/sc/CppunitTest_sc_cache_test.mk
@@ -0,0 +1,119 @@
+# -*- Mode: makefile-gmake; tab-width: 4; indent-tabs-mode: t -*-
+#
+# This file is part of the LibreOffice project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+
+$(eval $(call gb_CppunitTest_CppunitTest,sc_cache_test))
+
+$(eval $(call gb_CppunitTest_use_externals,sc_cache_test, \
+	boost_headers \
+    icu_headers \
+    icui18n \
+    icuuc \
+	libxml2 \
+	mdds_headers \
+))
+
+$(eval $(call gb_CppunitTest_add_exception_objects,sc_cache_test, \
+    sc/qa/unit/datacache \
+))
+
+$(eval $(call gb_CppunitTest_use_libraries,sc_cache_test, \
+    basegfx \
+    comphelper \
+    cppu \
+    cppuhelper \
+    drawinglayer \
+    editeng \
+    for \
+    forui \
+    i18nlangtag \
+    msfilter \
+    oox \
+    sal \
+    salhelper \
+    sax \
+    sb \
+    sc \
+    scqahelper \
+    sfx \
+    sot \
+    svl \
+    svt \
+    svx \
+    svxcore \
+	test \
+    tk \
+    tl \
+    ucbhelper \
+	unotest \
+    utl \
+    vbahelper \
+    vcl \
+    xo \
+))
+
+$(eval $(call gb_CppunitTest_set_include,sc_cache_test,\
+    -I$(SRCDIR)/sc/source/ui/inc \
+    -I$(SRCDIR)/sc/inc \
+    $$(INCLUDE) \
+))
+
+$(eval $(call gb_CppunitTest_use_api,sc_cache_test,\
+    offapi \
+    oovbaapi \
+    udkapi \
+))
+
+$(eval $(call gb_CppunitTest_use_ure,sc_cache_test))
+$(eval $(call gb_CppunitTest_use_vcl,sc_cache_test))
+
+$(eval $(call gb_CppunitTest_use_components,sc_cache_test,\
+    basic/util/sb \
+    chart2/source/chartcore \
+    chart2/source/controller/chartcontroller \
+    comphelper/util/comphelp \
+    configmgr/source/configmgr \
+    dbaccess/util/dba \
+    embeddedobj/util/embobj \
+    eventattacher/source/evtatt \
+    filter/source/config/cache/filterconfig1 \
+    forms/util/frm \
+    framework/util/fwk \
+    i18npool/util/i18npool \
+    oox/util/oox \
+    package/source/xstor/xstor \
+    package/util/package2 \
+    sax/source/expatwrap/expwrap \
+    scaddins/source/analysis/analysis \
+    scaddins/source/datefunc/date \
+    scripting/source/basprov/basprov \
+    scripting/util/scriptframe \
+    sc/util/sc \
+    sc/util/scd \
+    sc/util/scfilt \
+    $(call gb_Helper_optional,SCRIPTING, \
+	    sc/util/vbaobj) \
+    sfx2/util/sfx \
+    sot/util/sot \
+    svl/source/fsstor/fsstorage \
+    svl/util/svl \
+    svx/util/svx \
+    svx/util/svxcore \
+    toolkit/util/tk \
+    ucb/source/core/ucb1 \
+    ucb/source/ucp/file/ucpfile1 \
+    ucb/source/ucp/tdoc/ucptdoc1 \
+    unotools/util/utl \
+    unoxml/source/rdf/unordf \
+    unoxml/source/service/unoxml \
+    xmloff/util/xo \
+))
+
+$(eval $(call gb_CppunitTest_use_configuration,sc_cache_test))
+
+# vim: set noet sw=4 ts=4:
diff --git a/sc/Module_sc.mk b/sc/Module_sc.mk
index 6db1bd21224b..766069c2676d 100644
--- a/sc/Module_sc.mk
+++ b/sc/Module_sc.mk
@@ -45,6 +45,7 @@ $(eval $(call gb_Module_add_check_targets,sc,\
 	CppunitTest_sc_core \
 	CppunitTest_sc_dataprovider \
 	CppunitTest_sc_datatransformation \
+	CppunitTest_sc_cache_test \
 ))
 
 ifneq ($(ENABLE_HEADLESS),TRUE)
diff --git a/sc/README b/sc/README
index 8dfb5ec613d7..89b74f744045 100644
--- a/sc/README
+++ b/sc/README
@@ -14,3 +14,28 @@ Dumps the graphic objects and their position and size in pixel.
 
 Dumps the SfxItemSet representing the cell properties' of the
 current selection as a xml file. The file will be named dump.xml
+
+=== The Cache Format ===
+
+ScDocument::StoreTabToCache allows storing the content (not the formatting)
+of a table to a binary cache format.
+
+The format is column orientated which allows quick serialization of the table.
+
+Header:
+    * Number of Columns: 64 bit unsigned integer
+
+Column:
+    * Column Index: 64 bit unsigned integer
+    * Column Size: 64 bit unsigned integer
+    * For each cell type block a new ColumnBlock
+
+ColumnBlock:
+    * Start Row: 64 bit unsigned integer
+    * Block Size: 64 bit unsigned integer
+    * Type: 8 bit unsigned integer
+        - 0 : empty
+        - 1 : numeric
+            * for each cell: 64 bit IEEE 754 double precision value
+        - 2 : string
+            * for each cell: 32 bit signed string length followed by string length bytes of the string (UTF-8)
diff --git a/sc/inc/column.hxx b/sc/inc/column.hxx
index 35693f87aa8b..d76f3f0bd992 100644
--- a/sc/inc/column.hxx
+++ b/sc/inc/column.hxx
@@ -672,6 +672,8 @@ public:
 
     void EnsureFormulaCellResults( SCROW nRow1, SCROW nRow2 );
 
+    void StoreToCache(SvStream& rStrm) const;
+
 #if DUMP_COLUMN_STORAGE
     void DumpColumnStorage() const;
 #endif
diff --git a/sc/inc/document.hxx b/sc/inc/document.hxx
index f0f417baf9bd..a32eac54bf1e 100644
--- a/sc/inc/document.hxx
+++ b/sc/inc/document.hxx
@@ -2304,6 +2304,8 @@ public:
 
     std::unique_ptr<sc::ColumnIterator> GetColumnIterator( SCTAB nTab, SCCOL nCol, SCROW nRow1, SCROW nRow2 ) const;
 
+    SC_DLLPUBLIC void StoreTabToCache(SCTAB nTab, SvStream& rStrm) const;
+
 #if DUMP_COLUMN_STORAGE
     SC_DLLPUBLIC void DumpColumnStorage( SCTAB nTab, SCCOL nCol ) const;
 #endif
diff --git a/sc/inc/table.hxx b/sc/inc/table.hxx
index 80f4b7dfc7de..a4d828c1f96a 100644
--- a/sc/inc/table.hxx
+++ b/sc/inc/table.hxx
@@ -1008,6 +1008,8 @@ public:
 
     void finalizeOutlineImport();
 
+    void StoreToCache(SvStream& rStrm) const;
+
 #if DUMP_COLUMN_STORAGE
     void DumpColumnStorage( SCCOL nCol ) const;
 #endif
diff --git a/sc/qa/unit/datacache.cxx b/sc/qa/unit/datacache.cxx
new file mode 100644
index 000000000000..492e31c3fa3e
--- /dev/null
+++ b/sc/qa/unit/datacache.cxx
@@ -0,0 +1,71 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+#include <sal/config.h>
+#include <test/bootstrapfixture.hxx>
+#include <unotools/configmgr.hxx>
+#include "helper/qahelper.hxx"
+
+#include "global.hxx"
+#include "document.hxx"
+
+#include <tools/stream.hxx>
+
+class ScCacheTest : public CppUnit::TestFixture
+{
+public:
+
+    void testCacheSimple();
+    void testCacheString();
+
+    CPPUNIT_TEST_SUITE(ScCacheTest);
+    CPPUNIT_TEST(testCacheSimple);
+    CPPUNIT_TEST(testCacheString);
+    CPPUNIT_TEST_SUITE_END();
+
+public:
+    virtual void setUp() override
+    {
+        utl::ConfigManager::EnableAvoidConfig();
+        ScDLL::Init();
+        ScGlobal::Init();
+    }
+};
+
+void ScCacheTest::testCacheSimple()
+{
+    ScDocument aDoc;
+    aDoc.InsertTab(0, "test");
+    for (SCROW nRow = 0; nRow < 10; ++nRow)
+        aDoc.SetValue(0, nRow, 0, nRow);
+
+    aDoc.SetValue(0, 100000, 0, -10);
+
+    SvMemoryStream aStrm;
+    aDoc.StoreTabToCache(0, aStrm);
+}
+
+void ScCacheTest::testCacheString()
+{
+    ScDocument aDoc;
+    aDoc.InsertTab(0, "test");
+
+    aDoc.SetString(0, 0, 0, "TestString");
+    aDoc.SetString(0, 1, 0, "asjdaonfdssda");
+    aDoc.SetString(0, 2, 0, "da");
+
+    SvMemoryStream aStrm;
+    aDoc.StoreTabToCache(0, aStrm);
+}
+
+CPPUNIT_TEST_SUITE_REGISTRATION(ScCacheTest);
+
+CPPUNIT_PLUGIN_IMPLEMENT();
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/sc/source/core/data/column4.cxx b/sc/source/core/data/column4.cxx
index 7132ed02f732..e7104a856d99 100644
--- a/sc/source/core/data/column4.cxx
+++ b/sc/source/core/data/column4.cxx
@@ -1632,4 +1632,80 @@ void ScColumn::EnsureFormulaCellResults( SCROW nRow1, SCROW nRow2 )
     );
 }
 
+namespace {
+
+class StoreToCacheFunc
+{
+    SvStream& mrStrm;
+public:
+
+    StoreToCacheFunc(SvStream& rStrm):
+        mrStrm(rStrm)
+    {
+    }
+
+    void operator() ( const sc::CellStoreType::value_type& node, size_t nOffset, size_t nDataSize )
+    {
+        SCROW nStartRow = node.position + nOffset;
+        mrStrm.WriteUInt64(nStartRow);
+        mrStrm.WriteUInt64(nDataSize);
+        switch (node.type)
+        {
+            case sc::element_type_empty:
+            {
+                mrStrm.WriteUChar(0);
+            }
+            break;
+            case sc::element_type_numeric:
+            {
+                mrStrm.WriteUChar(1);
+                sc::numeric_block::const_iterator it = sc::numeric_block::begin(*node.data);
+                std::advance(it, nOffset);
+                sc::numeric_block::const_iterator itEnd = it;
+                std::advance(itEnd, nDataSize);
+
+                for (; it != itEnd; ++it)
+                {
+                    mrStrm.WriteDouble(*it);
+                }
+            }
+            break;
+            case sc::element_type_string:
+            {
+                mrStrm.WriteUChar(2);
+                sc::string_block::const_iterator it = sc::string_block::begin(*node.data);
+                std::advance(it, nOffset);
+                sc::string_block::const_iterator itEnd = it;
+                std::advance(itEnd, nDataSize);
+
+                for (; it != itEnd; ++it)
+                {
+                    OString aStr = OUStringToOString(it->getString(), RTL_TEXTENCODING_UTF8);
+                    sal_Int32 nStrLength = aStr.getLength();
+                    mrStrm.WriteInt32(nStrLength);
+                    mrStrm.WriteCharPtr(aStr.getStr());
+                }
+            }
+            break;
+            case sc::element_type_formula:
+            {
+                mrStrm.WriteUChar(2);
+            }
+            break;
+        }
+    }
+};
+
+}
+
+void ScColumn::StoreToCache(SvStream& rStrm) const
+{
+    rStrm.WriteUInt64(nCol);
+    SCROW nLastRow = GetLastDataPos();
+    rStrm.WriteUInt64(nLastRow + 1); // the rows are zero based
+
+    StoreToCacheFunc aFunc(rStrm);
+    sc::ParseBlock(maCells.begin(), maCells, aFunc, (SCROW)0, nLastRow);
+}
+
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/sc/source/core/data/document10.cxx b/sc/source/core/data/document10.cxx
index 89c1975ab0cb..bd6f6e862cf5 100644
--- a/sc/source/core/data/document10.cxx
+++ b/sc/source/core/data/document10.cxx
@@ -939,4 +939,13 @@ sc::ExternalDataMapper& ScDocument::GetExternalDataMapper()
     return *mpDataMapper;
 }
 
+void ScDocument::StoreTabToCache(SCTAB nTab, SvStream& rStrm) const
+{
+    const ScTable* pTab = FetchTable(nTab);
+    if (!pTab)
+        return;
+
+    pTab->StoreToCache(rStrm);
+}
+
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/sc/source/core/data/table7.cxx b/sc/source/core/data/table7.cxx
index e20f5b44d777..340dc7476fae 100644
--- a/sc/source/core/data/table7.cxx
+++ b/sc/source/core/data/table7.cxx
@@ -417,4 +417,20 @@ void ScTable::finalizeOutlineImport()
     }
 }
 
+void ScTable::StoreToCache(SvStream& rStrm) const
+{
+    SCCOL nStartCol = 0;
+    SCCOL nEndCol = MAXCOL;
+    SCROW nStartRow = 0;
+    SCROW nEndRow = MAXROW;
+
+    GetDataArea(nStartCol, nStartRow, nEndCol, nEndRow, false, false);
+
+    rStrm.WriteUInt64(nEndCol + 1);
+    for (SCCOL nCol = 0; nCol <= nEndCol; ++nCol)
+    {
+        aCol[nCol].StoreToCache(rStrm);
+    }
+}
+
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */