BinaryDataContainer swap out implementation. private/mmeeks/binarydatacache

We can easily accumulate a large number of in-memory graphic
objects, and swapping these as well as the un-compressed
images can become important.

Despite the shared pImpl, we retained the reference counting
on the underling vector to keep this immutable while we hand
out a MemoryStream reference to it.

Change-Id: Ib7ca45afb8499460b1852461f7c11afca3f3cdfa
Signed-off-by: Michael Meeks <michael.meeks@collabora.com>

3 files changed, 113 insertions, 18 deletions

diff --git a/include/vcl/BinaryDataContainer.hxx b/include/vcl/BinaryDataContainer.hxx
index e178ad6c4d62..f6f07f0c5ef6 100644
--- a/include/vcl/BinaryDataContainer.hxx
+++ b/include/vcl/BinaryDataContainer.hxx
@@ -13,6 +13,7 @@
 #include <sal/config.h>
 
 #include <com/sun/star/uno/Sequence.hxx>
+#include <unotools/tempfile.hxx>
 #include <tools/stream.hxx>
 #include <vcl/dllapi.h>
 
@@ -26,9 +27,11 @@
  */
 class VCL_DLLPUBLIC BinaryDataContainer final
 {
-private:
-    // the binary data
-    std::shared_ptr<std::vector<sal_uInt8>> mpData;
+    struct Impl;
+
+    std::shared_ptr<Impl> mpImpl;
+
+    void ensureSwappedIn() const;
 
 public:
     BinaryDataContainer() = default;
@@ -56,6 +59,9 @@ public:
     /// return the in-memory size in bytes as of now.
     std::size_t getSizeBytes() const;
 
+    /// swap out to disk for now
+    void swapOut() const;
+
     size_t calculateHash() const;
 };
 
diff --git a/vcl/source/gdi/impgraph.cxx b/vcl/source/gdi/impgraph.cxx
index 76598c9945e1..ce49a8e42706 100644
--- a/vcl/source/gdi/impgraph.cxx
+++ b/vcl/source/gdi/impgraph.cxx
@@ -1253,6 +1253,9 @@ bool ImpGraphic::swapOutGraphic(SvStream& rStream)
             break;
     }
 
+    if (mpGfxLink)
+        mpGfxLink->getDataContainer().swapOut();
+
     return true;
 }
 
diff --git a/vcl/source/graphic/BinaryDataContainer.cxx b/vcl/source/graphic/BinaryDataContainer.cxx
index 31561d9e16e3..a99cae99e023 100644
--- a/vcl/source/graphic/BinaryDataContainer.cxx
+++ b/vcl/source/graphic/BinaryDataContainer.cxx
@@ -10,21 +10,75 @@
 
 #include <vcl/BinaryDataContainer.hxx>
 #include <o3tl/hash_combine.hxx>
+#include <unotools/tempfile.hxx>
+#include <comphelper/lok.hxx>
+#include <sal/log.hxx>
+
+struct BinaryDataContainer::Impl
+{
+    // temp file to store the data out of RAM if necessary
+    std::unique_ptr<utl::TempFileNamed> mpFile;
+    // the binary data
+    std::shared_ptr<std::vector<sal_uInt8>> mpData;
+
+    /// Populate mpData from the stream
+    void readData(SvStream& stream, size_t size)
+    {
+        auto pData = std::make_shared<std::vector<sal_uInt8>>(size);
+        if (stream.ReadBytes(pData->data(), pData->size()) == size)
+            mpData = std::move(pData);
+    }
+
+    /// ensure the data is in-RAM
+    void ensureSwappedIn()
+    {
+        if (mpData || !mpFile)
+            return;
+
+        auto pStream = mpFile->GetStream(StreamMode::READ);
+        pStream->Seek(0);
+        readData(*pStream, pStream->remainingSize());
+
+        // Horrifying data loss ...
+        SAL_WARN_IF(pStream->GetError(), "vcl",
+                    "Inconsistent system - failed to swap image back in");
+        SAL_DEBUG("Swap in: " << pStream->GetError());
+    }
+
+    void swapOut()
+    {
+        if (mpFile)
+        {
+            // we already have it swapped out.
+            mpData.reset();
+            return;
+        }
+
+        if (!mpData || mpData->empty())
+            return;
+
+        mpFile.reset(new utl::TempFileNamed());
+        auto pStream = mpFile->GetStream(StreamMode::READWRITE);
+
+        pStream->WriteBytes(mpData->data(), mpData->size());
+
+        mpData.reset();
+    }
+};
 
 BinaryDataContainer::BinaryDataContainer(SvStream& stream, size_t size)
 {
-    auto pBuffer = std::make_shared<std::vector<sal_uInt8>>(size);
-    if (stream.ReadBytes(pBuffer->data(), pBuffer->size()) == size)
-        mpData = std::move(pBuffer);
+    mpImpl.reset(new Impl());
+    mpImpl->readData(stream, size);
 }
 
 size_t BinaryDataContainer::calculateHash() const
 {
     size_t nSeed = 0;
-    if (mpData)
+    if (mpImpl && mpImpl->mpData && !mpImpl->mpData->empty())
     {
         o3tl::hash_combine(nSeed, getSize());
-        for (sal_uInt8 const& rByte : *mpData)
+        for (sal_uInt8 const& rByte : *mpImpl->mpData)
             o3tl::hash_combine(nSeed, rByte);
     }
     return nSeed;
@@ -34,10 +88,11 @@ css::uno::Sequence<sal_Int8> BinaryDataContainer::getCopyAsByteSequence() const
 {
     if (isEmpty())
         return css::uno::Sequence<sal_Int8>();
+    assert(mpImpl);
 
     css::uno::Sequence<sal_Int8> aData(getSize());
 
-    std::copy(mpData->cbegin(), mpData->cend(), aData.getArray());
+    std::copy(mpImpl->mpData->cbegin(), mpImpl->mpData->cend(), aData.getArray());
 
     return aData;
 }
@@ -53,10 +108,9 @@ class ReferencedMemoryStream : public SvMemoryStream
     std::shared_ptr<std::vector<sal_uInt8>> mpData;
 
 public:
-    ReferencedMemoryStream(const std::shared_ptr<std::vector<sal_uInt8>>& rData)
-        : SvMemoryStream(rData ? rData->data() : nullptr, rData ? rData->size() : 0,
-                         StreamMode::READ)
-        , mpData(rData)
+    ReferencedMemoryStream(const std::shared_ptr<std::vector<sal_uInt8>>& pData)
+        : SvMemoryStream(mpData->data(), mpData->size(), StreamMode::READ)
+        , mpData(pData)
     {
     }
 };
@@ -64,20 +118,52 @@ public:
 
 std::shared_ptr<SvStream> BinaryDataContainer::getAsStream()
 {
-    return std::make_shared<ReferencedMemoryStream>(mpData);
+    ensureSwappedIn(); // TODO: transfer in streamed chunks
+    return std::make_shared<ReferencedMemoryStream>(mpImpl->mpData);
 }
 
 std::size_t BinaryDataContainer::writeToStream(SvStream& rStream) const
 {
+    ensureSwappedIn(); // TODO: transfer in streamed chunks
     return rStream.WriteBytes(getData(), getSize());
 }
 
-size_t BinaryDataContainer::getSize() const { return mpData ? mpData->size() : 0; }
+size_t BinaryDataContainer::getSize() const
+{
+    ensureSwappedIn();
+    return mpImpl && mpImpl->mpData ? mpImpl->mpData->size() : 0;
+}
 
-size_t BinaryDataContainer::getSizeBytes() const { return getSize(); }
+size_t BinaryDataContainer::getSizeBytes() const
+{
+    return mpImpl && mpImpl->mpData ? mpImpl->mpData->size() : 0;
+}
 
-bool BinaryDataContainer::isEmpty() const { return !mpData || mpData->empty(); }
+bool BinaryDataContainer::isEmpty() const
+{
+    ensureSwappedIn();
+    return !mpImpl || !mpImpl->mpData || mpImpl->mpData->empty();
+}
 
-const sal_uInt8* BinaryDataContainer::getData() const { return mpData ? mpData->data() : nullptr; }
+const sal_uInt8* BinaryDataContainer::getData() const
+{
+    ensureSwappedIn();
+    return mpImpl && mpImpl->mpData ? mpImpl->mpData->data() : nullptr;
+}
+
+void BinaryDataContainer::ensureSwappedIn() const
+{
+    if (mpImpl)
+        mpImpl->ensureSwappedIn();
+}
+
+void BinaryDataContainer::swapOut() const
+{
+    // Only bother reducing memory footprint in kit mode - for mobile/online etc.
+    if (!mpImpl || !comphelper::LibreOfficeKit::isActive())
+        return;
+
+    mpImpl->swapOut();
+}
 
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */