Use orcus csv parser to streamd data which has much less overhead.

Also, use DocumentStreamAccess to modify ScDocument's content.

Change-Id: I516260cff1f2315267afcff05e36e620798a1aed

8 files changed, 218 insertions, 74 deletions

diff --git a/sc/Library_sc.mk b/sc/Library_sc.mk
index b60af822f237..01a4ab964e5b 100644
--- a/sc/Library_sc.mk
+++ b/sc/Library_sc.mk
@@ -45,6 +45,12 @@ $(eval $(call gb_Library_use_externals,sc,\
     mdds_headers \
 ))
 
+ifeq ($(SYSTEM_LIBORCUS),YES)
+$(eval $(call gb_Library_use_externals,sc,orcus))
+else
+$(eval $(call gb_Library_use_externals,sc,orcus-parser))
+endif
+
 ifeq ($(ENABLE_TELEPATHY),TRUE)
 $(eval $(call gb_Library_use_libraries,sc,tubes))
 
diff --git a/sc/Library_scqahelper.mk b/sc/Library_scqahelper.mk
index 37b1d54f3c02..cbdc99df6441 100644
--- a/sc/Library_scqahelper.mk
+++ b/sc/Library_scqahelper.mk
@@ -18,11 +18,15 @@ $(eval $(call gb_Library_set_include,scqahelper,\
 $(eval $(call gb_Library_use_externals,scqahelper, \
 	boost_headers \
 	mdds_headers \
-	orcus \
-	orcus-parser \
 	cppunit \
 ))
 
+ifeq ($(SYSTEM_LIBORCUS),YES)
+$(eval $(call gb_Library_use_externals,scqahelper,orcus))
+else
+$(eval $(call gb_Library_use_externals,scqahelper,orcus-parser))
+endif
+
 $(eval $(call gb_Library_add_defs,scqahelper,\
 	-DSCQAHELPER_DLLIMPLEMENTATION \
 ))
diff --git a/sc/inc/documentstreamaccess.hxx b/sc/inc/documentstreamaccess.hxx
index 1450f211d8d9..74f8914f4dce 100644
--- a/sc/inc/documentstreamaccess.hxx
+++ b/sc/inc/documentstreamaccess.hxx
@@ -10,7 +10,11 @@
 #ifndef SC_DOCUMENTSTREAMACCESS_HXX
 #define SC_DOCUMENTSTREAMACCESS_HXX
 
+#include <rtl/ustring.hxx>
+
 class ScDocument;
+class ScAddress;
+class ScRange;
 
 namespace sc {
 
@@ -28,6 +32,26 @@ class DocumentStreamAccess
 
 public:
     DocumentStreamAccess( ScDocument& rDoc );
+
+    void setStringCell( const ScAddress& rPos, const OUString& rStr );
+
+    /**
+     * Clear its internal state, and more importantly all the block position
+     * hints currently held.
+     */
+    void reset();
+
+    /**
+     * Pop the top row inside specified range, shift all the other rows up by
+     * one, then set the bottom row empty.
+     */
+    void shiftRangeUp( const ScRange& rRange );
+
+    /**
+     * Top the bottom row inside specified range, shift all the other rows
+     * above downward by one by inserting an empty row at the top.
+     */
+    void shiftRangeDown( const ScRange& rRange );
 };
 
 }
diff --git a/sc/inc/mtvelements.hxx b/sc/inc/mtvelements.hxx
index e5efbf13f804..0fcafdecce98 100644
--- a/sc/inc/mtvelements.hxx
+++ b/sc/inc/mtvelements.hxx
@@ -145,6 +145,8 @@ public:
     ColumnBlockPositionSet(ScDocument& rDoc);
 
     ColumnBlockPosition* getBlockPosition(SCTAB nTab, SCCOL nCol);
+
+    void clear();
 };
 
 ScRefCellValue toRefCell( const sc::CellStoreType::const_iterator& itPos, size_t nOffset );
diff --git a/sc/source/core/data/documentstreamaccess.cxx b/sc/source/core/data/documentstreamaccess.cxx
index 2d427af022f0..81ca1609f0fb 100644
--- a/sc/source/core/data/documentstreamaccess.cxx
+++ b/sc/source/core/data/documentstreamaccess.cxx
@@ -9,19 +9,114 @@
 
 #include "documentstreamaccess.hxx"
 #include "document.hxx"
+#include "table.hxx"
+#include "column.hxx"
+#include "mtvelements.hxx"
+
+#include "svl/sharedstringpool.hxx"
 
 namespace sc {
 
 struct DocumentStreamAccessImpl
 {
     ScDocument& mrDoc;
+    ColumnBlockPositionSet maBlockPosSet;
 
-    DocumentStreamAccessImpl( ScDocument& rDoc ) : mrDoc(rDoc) {}
+    DocumentStreamAccessImpl( ScDocument& rDoc ) :
+        mrDoc(rDoc),
+        maBlockPosSet(rDoc)
+    {}
 };
 
 DocumentStreamAccess::DocumentStreamAccess( ScDocument& rDoc ) :
     mpImpl(new DocumentStreamAccessImpl(rDoc)) {}
 
+void DocumentStreamAccess::setStringCell( const ScAddress& rPos, const OUString& rStr )
+{
+    ScTable* pTab = mpImpl->mrDoc.FetchTable(rPos.Tab());
+    if (!pTab)
+        return;
+
+    ColumnBlockPosition* pBlockPos =
+        mpImpl->maBlockPosSet.getBlockPosition(rPos.Tab(), rPos.Col());
+
+    if (!pBlockPos)
+        return;
+
+    svl::SharedString aSS = mpImpl->mrDoc.GetSharedStringPool().intern(rStr);
+    if (!aSS.getData())
+        return;
+
+    // Set the string.
+    CellStoreType& rCells = pTab->aCol[rPos.Col()].maCells;
+    pBlockPos->miCellPos = rCells.set(pBlockPos->miCellPos, rPos.Row(), aSS);
+
+    // Be sure to set the corresponding text attribute to the default value.
+    CellTextAttrStoreType& rAttrs = pTab->aCol[rPos.Col()].maCellTextAttrs;
+    pBlockPos->miCellTextAttrPos = rAttrs.set(pBlockPos->miCellTextAttrPos, rPos.Row(), CellTextAttr());
+}
+
+void DocumentStreamAccess::reset()
+{
+    mpImpl->maBlockPosSet.clear();
+}
+
+void DocumentStreamAccess::shiftRangeUp( const ScRange& rRange )
+{
+    ScTable* pTab = mpImpl->mrDoc.FetchTable(rRange.aStart.Tab());
+    if (!pTab)
+        return;
+
+    SCROW nTopRow = rRange.aStart.Row();
+    SCROW nLastRow = rRange.aEnd.Row();
+
+    for (SCCOL nCol = rRange.aStart.Col(); nCol <= rRange.aEnd.Col(); ++nCol)
+    {
+        ColumnBlockPosition* pBlockPos =
+            mpImpl->maBlockPosSet.getBlockPosition(rRange.aStart.Tab(), nCol);
+
+        if (!pBlockPos)
+            return;
+
+        CellStoreType& rCells = pTab->aCol[nCol].maCells;
+        rCells.erase(nTopRow, nTopRow); // Erase the top, and shift the rest up.
+        pBlockPos->miCellPos = rCells.insert_empty(nLastRow, 1);
+
+        // Do the same for the text attribute storage.
+        CellTextAttrStoreType& rAttrs = pTab->aCol[nCol].maCellTextAttrs;
+        rAttrs.erase(nTopRow, nTopRow);
+        pBlockPos->miCellTextAttrPos = rAttrs.insert_empty(nLastRow, 1);
+    }
+}
+
+void DocumentStreamAccess::shiftRangeDown( const ScRange& rRange )
+{
+    ScTable* pTab = mpImpl->mrDoc.FetchTable(rRange.aStart.Tab());
+    if (!pTab)
+        return;
+
+    SCROW nTopRow = rRange.aStart.Row();
+    SCROW nLastRow = rRange.aEnd.Row();
+
+    for (SCCOL nCol = rRange.aStart.Col(); nCol <= rRange.aEnd.Col(); ++nCol)
+    {
+        ColumnBlockPosition* pBlockPos =
+            mpImpl->maBlockPosSet.getBlockPosition(rRange.aStart.Tab(), nCol);
+
+        if (!pBlockPos)
+            return;
+
+        CellStoreType& rCells = pTab->aCol[nCol].maCells;
+        rCells.erase(nLastRow, nLastRow); // Erase the bottom.
+        pBlockPos->miCellPos = rCells.insert_empty(nTopRow, 1); // insert at the top and shift everything down.
+
+        // Do the same for the text attribute storage.
+        CellTextAttrStoreType& rAttrs = pTab->aCol[nCol].maCellTextAttrs;
+        rAttrs.erase(nLastRow, nLastRow);
+        pBlockPos->miCellTextAttrPos = rAttrs.insert_empty(nTopRow, 1);
+    }
+}
+
 }
 
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/sc/source/core/data/mtvelements.cxx b/sc/source/core/data/mtvelements.cxx
index 5a946067e032..1110ab6d2a04 100644
--- a/sc/source/core/data/mtvelements.cxx
+++ b/sc/source/core/data/mtvelements.cxx
@@ -67,6 +67,12 @@ ColumnBlockPosition* ColumnBlockPositionSet::getBlockPosition(SCTAB nTab, SCCOL
     return &it->second;
 }
 
+void ColumnBlockPositionSet::clear()
+{
+    osl::MutexGuard aGuard(&maMtxTables);
+    maTables.clear();
+}
+
 ScRefCellValue toRefCell( const sc::CellStoreType::const_iterator& itPos, size_t nOffset )
 {
     switch (itPos->type)
diff --git a/sc/source/ui/docshell/datastream.cxx b/sc/source/ui/docshell/datastream.cxx
index f053371a2c1c..a798b84b9362 100644
--- a/sc/source/ui/docshell/datastream.cxx
+++ b/sc/source/ui/docshell/datastream.cxx
@@ -28,6 +28,15 @@
 #include <tabvwsh.hxx>
 #include <viewdata.hxx>
 
+#include <config_orcus.h>
+
+#if ENABLE_ORCUS
+#if defined WNT
+#define __ORCUS_STATIC_LIB
+#endif
+#include <orcus/csv_parser.hpp>
+#endif
+
 #include <queue>
 
 namespace sc {
@@ -316,6 +325,7 @@ void DataStream::StartImport()
 {
     if (mbRunning)
         return;
+
     if (!mxReaderThread.is())
     {
         SvStream *pStream = 0;
@@ -327,6 +337,7 @@ void DataStream::StartImport()
         mxReaderThread->launch();
     }
     mbRunning = true;
+    maDocAccess.reset();
     mxThread->maStart.set();
 }
 
@@ -334,6 +345,7 @@ void DataStream::StopImport()
 {
     if (!mbRunning)
         return;
+
     mbRunning = false;
     Repaint();
 }
@@ -362,23 +374,32 @@ void DataStream::MoveData()
     switch (meMove)
     {
         case RANGE_DOWN:
+        {
             if (mnCurRow == mpEndRange->aStart.Row())
                 meMove = MOVE_UP;
-            break;
+        }
+        break;
         case MOVE_UP:
+        {
             // Remove the top row and shift the remaining rows upward. Then
             // insert a new row at the end row position.
-            mpDoc->DeleteRow(maStartRange);
-            mpDoc->InsertRow(*mpEndRange);
-            break;
+            ScRange aRange = maStartRange;
+            aRange.aEnd = mpEndRange->aEnd;
+            maDocAccess.shiftRangeUp(aRange);
+        }
+        break;
         case MOVE_DOWN:
+        {
             // Remove the end row and shift the remaining rows downward by
             // inserting a new row at the top row.
-            mpDoc->DeleteRow(*mpEndRange);
-            mpDoc->InsertRow(maStartRange);
-            break;
+            ScRange aRange = maStartRange;
+            aRange.aEnd = mpEndRange->aEnd;
+            maDocAccess.shiftRangeDown(aRange);
+        }
+        break;
         case NO_MOVE:
-            break;
+        default:
+            ;
     }
 }
 
@@ -388,77 +409,61 @@ IMPL_LINK_NOARG(DataStream, RefreshHdl)
     return 0;
 }
 
-//  lcl_ScanString and Text2Doc is simplified version
-//  of code from sc/source/ui/docshell/impex.cxx
-const sal_Unicode* lcl_ScanString( const sal_Unicode* p, OUString& rString, sal_Unicode cStr)
+#if ENABLE_ORCUS
+
+namespace {
+
+/**
+ * This handler handles a single line CSV input.
+ */
+class CSVHandler
 {
-    const sal_Unicode* p0 = p;
-    for( ;; )
+    DocumentStreamAccess& mrDoc;
+    ScAddress maPos;
+    SCROW mnRow;
+    SCCOL mnCol;
+    SCCOL mnEndCol;
+    SCTAB mnTab;
+
+public:
+    CSVHandler( DocumentStreamAccess& rDoc, const ScAddress& rPos, SCCOL nEndCol ) :
+        mrDoc(rDoc), maPos(rPos), mnEndCol(nEndCol) {}
+
+    void begin_parse() {}
+    void end_parse() {}
+    void begin_row() {}
+    void end_row() {}
+
+    void cell(const char* p, size_t n)
     {
-        if (!*p)
-            break;
-        if (*p == cStr)
+        if (maPos.Col() <= mnEndCol)
         {
-            if (*++p != cStr)
-                break;
-            p++;
+            mrDoc.setStringCell(maPos, OUString(p, n, RTL_TEXTENCODING_UTF8));
         }
-        else
-            p++;
+        maPos.IncCol();
     }
-    if (p0 < p)
-        if (rString.getLength() + (p - p0) <= STRING_MAXLEN)
-            rString += OUString( p0, sal::static_int_cast<sal_Int32>( p - p0 ) );
-    return p;
+};
+
 }
 
 void DataStream::Text2Doc()
 {
-    sal_Unicode cSep(',');
-    sal_Unicode cStr('"');
-    SCCOL nStartCol = maStartRange.aStart.Col();
-    SCCOL nEndCol = maStartRange.aEnd.Col();
-    OUString aCell;
-    ScDocumentImport aDocImport(*mpDoc);
-
-    SCCOL nCol = nStartCol;
-    OUString sLine( OStringToOUString(ConsumeLine(), RTL_TEXTENCODING_UTF8) );
-    const sal_Unicode* p = sLine.getStr();
-    while (*p)
-    {
-        aCell = "";
-        const sal_Unicode* q = p;
-        while (*p && *p != cSep)
-        {
-            // Always look for a pairing quote and ignore separator in between.
-            while (*p && *p == cStr)
-                q = p = lcl_ScanString(p, aCell, cStr);
-            // All until next separator or quote.
-            while (*p && *p != cSep && *p != cStr)
-                ++p;
-            if (aCell.getLength() + (p - q) <= STRING_MAXLEN)
-                aCell += OUString( q, sal::static_int_cast<sal_Int32>( p - q ) );
-            q = p;
-        }
-        if (*p)
-            ++p;
-        if (nCol <= nEndCol)
-        {
-            ScAddress aAddress(nCol, mnCurRow, maStartRange.aStart.Tab());
-            if (aCell == "0" || ( aCell.indexOf(':') == -1 && aCell.toDouble() ))
-                aDocImport.setNumericCell(aAddress, aCell.toDouble());
-            else
-                aDocImport.setStringCell(aAddress, aCell);
-        }
-        ++nCol;
-    }
+    OString aLine = ConsumeLine();
+    orcus::csv_parser_config aConfig;
+    aConfig.delimiters.push_back(',');
+    aConfig.text_qualifier = '"';
+    CSVHandler aHdl(maDocAccess, ScAddress(maStartRange.aStart.Col(), mnCurRow, maStartRange.aStart.Tab()), maStartRange.aEnd.Col());
+    orcus::csv_parser<CSVHandler> parser(aLine.getStr(), aLine.getLength(), aHdl, aConfig);
+    parser.parse();
+
     ++mnRepaintCounter;
+}
 
-    aDocImport.finalize();
+#else
 
-    ScRange aBCRange(nStartCol, mnCurRow, maStartRange.aStart.Tab(), nEndCol, mnCurRow, maStartRange.aStart.Tab());
-    maBroadcastRanges.Join(aBCRange);
-}
+void DataStream::Text2Doc() {}
+
+#endif
 
 bool DataStream::ImportData()
 {
@@ -474,6 +479,7 @@ bool DataStream::ImportData()
     }
     else
     {
+#if 0 // TODO : temporarily disable this code.
         ScDocumentImport aDocImport(*mpDoc);
         // read more lines at once but not too much
         for (int i = 0; i < 10; ++i)
@@ -496,8 +502,9 @@ bool DataStream::ImportData()
             maBroadcastRanges.Join(aAddress);
         }
         aDocImport.finalize();
+#endif
     }
-    mpDocShell->SetDocumentModified();
+
     if (meMove == NO_MOVE)
         return mbRunning;
 
diff --git a/sc/source/ui/inc/datastream.hxx b/sc/source/ui/inc/datastream.hxx
index 70857dcd00eb..5f20bf4e1904 100644
--- a/sc/source/ui/inc/datastream.hxx
+++ b/sc/source/ui/inc/datastream.hxx
@@ -78,8 +78,8 @@ private:
     void Broadcast();
 
 private:
-    ScDocShell *mpDocShell;
-    ScDocument *mpDoc;
+    ScDocShell* mpDocShell;
+    ScDocument* mpDoc;
     DocumentStreamAccess maDocAccess;
     OUString msURL;
     OUString msMove;
@@ -88,7 +88,7 @@ private:
     MoveEnum meMove;
     bool mbRunning;
     bool mbValuesInLine;
-    LinesList *mpLines;
+    LinesList* mpLines;
     size_t mnLinesCount;
     size_t mnRepaintCounter;
     SCROW mnCurRow;