parallel deflate compression (one stream, multiple threads)

ZipPackageStream::saveChild() already had one threaded compression,
but that still uses only one thread for one stream. Many documents
contain many streams (where this is useful), but large documents
often contain one huge content.xml, which then would be compressed
using just one thread.
But it is in fact possible to do deflate in parallel on the same data,
at the cost of somewhat increased CPU usage (spread over threads).
This is handled separately from the background thread path, as
integrating these two approaches would probably be needlessly complex
(since they both internally use ThreadPool, the tasks should often
intermix and parallelize anyway).
On my 4-core (8 HT threads) machine this reduces the compression time
of tdf#113042 from 3s to 1s.

Change-Id: Ifbc889a27966f97eb1ce2ce01c5fb0b151a1bdf8
Reviewed-on: https://gerrit.libreoffice.org/73032
Tested-by: Jenkins
Reviewed-by: Luboš Luňák <l.lunak@collabora.com>

2 files changed, 121 insertions, 9 deletions

diff --git a/package/inc/ThreadedDeflater.hxx b/package/inc/ThreadedDeflater.hxx
new file mode 100644
index 000000000000..90801700a37e
--- /dev/null
+++ b/package/inc/ThreadedDeflater.hxx
@@ -0,0 +1,62 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * This file incorporates work covered by the following license notice:
+ *
+ *   Licensed to the Apache Software Foundation (ASF) under one or more
+ *   contributor license agreements. See the NOTICE file distributed
+ *   with this work for additional information regarding copyright
+ *   ownership. The ASF licenses this file to you under the Apache
+ *   License, Version 2.0 (the "License"); you may not use this file
+ *   except in compliance with the License. You may obtain a copy of
+ *   the License at http://www.apache.org/licenses/LICENSE-2.0 .
+ */
+
+#ifndef INCLUDED_PACKAGE_THREADEDDEFLATER_HXX
+#define INCLUDED_PACKAGE_THREADEDDEFLATER_HXX
+
+#include <com/sun/star/uno/Sequence.hxx>
+#include <package/packagedllapi.hxx>
+#include <comphelper/threadpool.hxx>
+#include <atomic>
+#include <memory>
+
+namespace ZipUtils
+{
+/// Parallel compression a stream using the libz deflate algorithm.
+///
+/// Almost a replacement for the Deflater class. Call startDeflate() with the data,
+/// check with finished() or waitForTasks() and retrieve result with getOutput().
+/// The class will internally split into multiple threads.
+class ThreadedDeflater final
+{
+    class Task;
+    // Note: All this should be lock-less. Each task writes only to its part
+    // of the data, flags are atomic.
+    css::uno::Sequence<sal_Int8> inBuffer;
+    int zlibLevel;
+    std::shared_ptr<comphelper::ThreadTaskTag> threadTaskTag;
+    std::atomic<int> pendingTasksCount;
+    std::vector<std::vector<sal_Int8>> outBuffers;
+
+public:
+    // Unlike with Deflater class, bNoWrap is always true.
+    ThreadedDeflater(sal_Int32 nSetLevel);
+    ~ThreadedDeflater();
+    void startDeflate(const css::uno::Sequence<sal_Int8>& rBuffer);
+    void waitForTasks();
+    bool finished() const;
+    css::uno::Sequence<sal_Int8> getOutput() const;
+    void clear();
+};
+
+} // namespace
+
+#endif
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/package/inc/ZipOutputEntry.hxx b/package/inc/ZipOutputEntry.hxx
index af6528f04ea1..7234d890f4c2 100644
--- a/package/inc/ZipOutputEntry.hxx
+++ b/package/inc/ZipOutputEntry.hxx
@@ -35,11 +35,9 @@ struct ZipEntry;
 class ZipPackageBuffer;
 class ZipPackageStream;
 
-class ZipOutputEntry
+class ZipOutputEntryBase
 {
 protected:
-    css::uno::Sequence< sal_Int8 > m_aDeflateBuffer;
-    ZipUtils::Deflater m_aDeflater;
     css::uno::Reference< css::uno::XComponentContext > m_xContext;
     css::uno::Reference< css::io::XOutputStream > m_xOutStream;
 
@@ -53,10 +51,9 @@ protected:
     bool const          m_bEncryptCurrentEntry;
 
 public:
-    ZipOutputEntry(
-        const css::uno::Reference< css::io::XOutputStream >& rxOutStream,
-        const css::uno::Reference< css::uno::XComponentContext >& rxContext,
-        ZipEntry& rEntry, ZipPackageStream* pStream, bool bEncrypt);
+    virtual ~ZipOutputEntryBase() = default;
+
+    virtual void writeStream(const css::uno::Reference< css::io::XInputStream >& xInStream) = 0;
 
     ZipEntry* getZipEntry() { return m_pCurrentEntry; }
     ZipPackageStream* getZipPackageStream() { return m_pCurrentStream; }
@@ -64,7 +61,36 @@ public:
 
     void closeEntry();
 
-    void writeStream(const css::uno::Reference< css::io::XInputStream >& xInStream);
+protected:
+    ZipOutputEntryBase(
+        const css::uno::Reference< css::io::XOutputStream >& rxOutStream,
+        const css::uno::Reference< css::uno::XComponentContext >& rxContext,
+        ZipEntry& rEntry, ZipPackageStream* pStream, bool bEncrypt, bool checkStream);
+
+    // Inherited classes call this with deflated data buffer.
+    void processDeflated( const css::uno::Sequence< sal_Int8 >& deflateBuffer, sal_Int32 nLength );
+    // Inherited classes call this with the input buffer.
+    void processInput( const css::uno::Sequence< sal_Int8 >& rBuffer );
+
+    virtual void finishDeflater() = 0;
+    virtual sal_Int64 getDeflaterTotalIn() const = 0;
+    virtual sal_Int64 getDeflaterTotalOut() const = 0;
+    virtual void deflaterReset() = 0;
+    virtual bool isDeflaterFinished() const = 0;
+};
+
+// Normal non-threaded case.
+class ZipOutputEntry : public ZipOutputEntryBase
+{
+    css::uno::Sequence< sal_Int8 > m_aDeflateBuffer;
+    ZipUtils::Deflater m_aDeflater;
+
+public:
+    ZipOutputEntry(
+        const css::uno::Reference< css::io::XOutputStream >& rxOutStream,
+        const css::uno::Reference< css::uno::XComponentContext >& rxContext,
+        ZipEntry& rEntry, ZipPackageStream* pStream, bool bEncrypt);
+    void writeStream(const css::uno::Reference< css::io::XInputStream >& xInStream) override;
     void write(const css::uno::Sequence< sal_Int8 >& rBuffer);
 
 protected:
@@ -72,10 +98,15 @@ protected:
         const css::uno::Reference< css::io::XOutputStream >& rxOutStream,
         const css::uno::Reference< css::uno::XComponentContext >& rxContext,
         ZipEntry& rEntry, ZipPackageStream* pStream, bool bEncrypt, bool checkStream);
+    virtual void finishDeflater() override;
+    virtual sal_Int64 getDeflaterTotalIn() const override;
+    virtual sal_Int64 getDeflaterTotalOut() const override;
+    virtual void deflaterReset() override;
+    virtual bool isDeflaterFinished() const override;
     void doDeflate();
 };
 
-// Class that runs the compression in a thread.
+// Class that runs the compression in a background thread.
 class ZipOutputEntryInThread : public ZipOutputEntry
 {
     class Task;
@@ -103,6 +134,25 @@ private:
     void setFinished() { m_bFinished = true; }
 };
 
+// Class that synchronously runs the compression in multiple threads (using ThreadDeflater).
+class ZipOutputEntryParallel : public ZipOutputEntryBase
+{
+    sal_Int64 totalIn;
+    sal_Int64 totalOut;
+public:
+    ZipOutputEntryParallel(
+        const css::uno::Reference< css::io::XOutputStream >& rxOutStream,
+        const css::uno::Reference< css::uno::XComponentContext >& rxContext,
+        ZipEntry& rEntry, ZipPackageStream* pStream, bool bEncrypt);
+    void writeStream(const css::uno::Reference< css::io::XInputStream >& xInStream) override;
+protected:
+    virtual void finishDeflater() override;
+    virtual sal_Int64 getDeflaterTotalIn() const override;
+    virtual sal_Int64 getDeflaterTotalOut() const override;
+    virtual void deflaterReset() override;
+    virtual bool isDeflaterFinished() const override;
+};
+
 #endif
 
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */