3 files changed, 49 insertions, 33 deletions
diff --git a/vcl/source/gdi/pdfwriter_impl.cxx b/vcl/source/gdi/pdfwriter_impl.cxx
index f86962425fb6..2641256b30c4 100644
--- a/vcl/source/gdi/pdfwriter_impl.cxx
+++ b/vcl/source/gdi/pdfwriter_impl.cxx
@@ -1737,14 +1737,13 @@ bool PDFWriterImpl::writeBufferBytes( const void* pBuffer, sal_uInt64 nBytes )
     }
     else
     {
-        if (m_aPDFEncryptor.m_bEncryptThisStream)
+        if (m_aPDFEncryptor.isStreamEncryptionEnabled())
         {
-            /* implement the encryption part of the PDF spec encryption algorithm 3.1 */
             m_vEncryptionBuffer.resize(nBytes);
-            rtl_cipher_encodeARCFOUR(m_aPDFEncryptor.m_aCipher, pBuffer, static_cast<sal_Size>(nBytes), m_vEncryptionBuffer.data(), static_cast<sal_Size>(nBytes));
+            m_aPDFEncryptor.encrypt(pBuffer, nBytes, m_vEncryptionBuffer.data(), nBytes);
         }
 
-        const void* pWriteBuffer = m_aPDFEncryptor.m_bEncryptThisStream ? m_vEncryptionBuffer.data()  : pBuffer;
+        const void* pWriteBuffer = m_aPDFEncryptor.isStreamEncryptionEnabled() ? m_vEncryptionBuffer.data() : pBuffer;
         m_DocDigest.update(static_cast<unsigned char const*>(pWriteBuffer), static_cast<sal_uInt32>(nBytes));
 
         if (m_aFile.write(pWriteBuffer, nBytes, nWritten) != osl::File::E_None)
diff --git a/vcl/source/gdi/pdfwriter_impl2.cxx b/vcl/source/gdi/pdfwriter_impl2.cxx
index 75ccbe9f03a0..354397af60df 100644
--- a/vcl/source/gdi/pdfwriter_impl2.cxx
+++ b/vcl/source/gdi/pdfwriter_impl2.cxx
@@ -1077,43 +1077,27 @@ void PDFWriterImpl::playMetafile( const GDIMetaFile& i_rMtf, vcl::PDFExtOutDevDa
 
 // Encryption methods
 
-void PDFWriterImpl::checkAndEnableStreamEncryption( sal_Int32 nObject )
+void PDFWriterImpl::checkAndEnableStreamEncryption(sal_Int32 nObject)
 {
-    if( !m_aContext.Encryption.Encrypt() )
+    if (!m_aContext.Encryption.Encrypt())
         return;
 
-    m_aPDFEncryptor.m_bEncryptThisStream = true;
-    sal_Int32 i = m_aPDFEncryptor.getKeyLength();
-    m_aContext.Encryption.EncryptionKey[i++] = static_cast<sal_uInt8>(nObject);
-    m_aContext.Encryption.EncryptionKey[i++] = static_cast<sal_uInt8>( nObject >> 8 );
-    m_aContext.Encryption.EncryptionKey[i++] = static_cast<sal_uInt8>( nObject >> 16 );
-    // the other location of m_nEncryptionKey is already set to 0, our fixed generation number
-    // do the MD5 hash
-    ::std::vector<unsigned char> const nMD5Sum(::comphelper::Hash::calculateHash(
-        m_aContext.Encryption.EncryptionKey.data(), i+2, ::comphelper::HashType::MD5));
-    // the i+2 to take into account the generation number, always zero
-    // initialize the RC4 with the key
-    // key length: see algorithm 3.1, step 4: (N+5) max 16
-    rtl_cipher_initARCFOUR(m_aPDFEncryptor.m_aCipher, rtl_Cipher_DirectionEncode, nMD5Sum.data(), m_aPDFEncryptor.getRC4KeyLength(), nullptr, 0);
+    m_aPDFEncryptor.enableStreamEncryption();
+    m_aPDFEncryptor.setupEncryption(m_aContext.Encryption.EncryptionKey, nObject);
 }
 
-void PDFWriterImpl::enableStringEncryption( sal_Int32 nObject )
+void PDFWriterImpl::disableStreamEncryption()
 {
-    if( !m_aContext.Encryption.Encrypt() )
+    m_aPDFEncryptor.disableStreamEncryption();
+
+}
+
+void PDFWriterImpl::enableStringEncryption(sal_Int32 nObject)
+{
+    if (!m_aContext.Encryption.Encrypt())
         return;
 
-    sal_Int32 i = m_aPDFEncryptor.getKeyLength();
-    m_aContext.Encryption.EncryptionKey[i++] = static_cast<sal_uInt8>(nObject);
-    m_aContext.Encryption.EncryptionKey[i++] = static_cast<sal_uInt8>( nObject >> 8 );
-    m_aContext.Encryption.EncryptionKey[i++] = static_cast<sal_uInt8>( nObject >> 16 );
-    // the other location of m_nEncryptionKey is already set to 0, our fixed generation number
-    // do the MD5 hash
-    // the i+2 to take into account the generation number, always zero
-    ::std::vector<unsigned char> const nMD5Sum(::comphelper::Hash::calculateHash(
-        m_aContext.Encryption.EncryptionKey.data(), i+2, ::comphelper::HashType::MD5));
-    // initialize the RC4 with the key
-    // key length: see algorithm 3.1, step 4: (N+5) max 16
-    rtl_cipher_initARCFOUR(m_aPDFEncryptor.m_aCipher, rtl_Cipher_DirectionEncode, nMD5Sum.data(), m_aPDFEncryptor.getRC4KeyLength(), nullptr, 0);
+    m_aPDFEncryptor.setupEncryption(m_aContext.Encryption.EncryptionKey, nObject);
 }
 
 const tools::Long unsetRun[256] =
diff --git a/vcl/source/pdf/PDFEncryptor.cxx b/vcl/source/pdf/PDFEncryptor.cxx
index 2c6d37296d42..16b88ec160b0 100644
--- a/vcl/source/pdf/PDFEncryptor.cxx
+++ b/vcl/source/pdf/PDFEncryptor.cxx
@@ -380,6 +380,7 @@ bool PDFEncryptor::prepareEncryption(
     return bSuccess;
 }
 
+/* this function implements part of the PDF spec algorithm 3.1 in encryption */
 void PDFEncryptor::setupKeysAndCheck(vcl::PDFEncryptionProperties& rProperties)
 {
     // sanity check
@@ -399,6 +400,38 @@ void PDFEncryptor::setupKeysAndCheck(vcl::PDFEncryptionProperties& rProperties)
     }
 }
 
+void PDFEncryptor::enableStreamEncryption() { m_bEncryptThisStream = true; }
+
+void PDFEncryptor::disableStreamEncryption() { m_bEncryptThisStream = false; }
+
+void PDFEncryptor::setupEncryption(std::vector<sal_uInt8> const& rEncryptionKey, sal_Int32 nObject)
+{
+    std::vector<sal_uInt8> aKey(rEncryptionKey.begin(), rEncryptionKey.begin() + m_nKeyLength);
+    std::vector<sal_uInt8> aObjectArray{
+        sal_uInt8(nObject), sal_uInt8(nObject >> 8), sal_uInt8(nObject >> 16),
+        0, // generation number, always zero
+        0 // generation number, always zero
+    };
+    aKey.insert(aKey.end(), aObjectArray.begin(), aObjectArray.end());
+
+    // do the MD5 hash
+    auto const nMD5Sum
+        = comphelper::Hash::calculateHash(aKey.data(), aKey.size(), ::comphelper::HashType::MD5);
+
+    // initialize the RC4 with the key
+    // key length: see algorithm 3.1, step 4: (N+5) max 16
+    rtl_cipher_initARCFOUR(m_aCipher, rtl_Cipher_DirectionEncode, nMD5Sum.data(), getRC4KeyLength(),
+                           nullptr, 0);
+}
+
+/* implement the encryption part of the PDF spec encryption algorithm 3.1 */
+void PDFEncryptor::encrypt(const void* pInput, sal_uInt64 nInputSize, sal_uInt8* pOutput,
+                           sal_uInt64 nOutputsSize)
+{
+    rtl_cipher_encodeARCFOUR(m_aCipher, pInput, sal_Size(nInputSize), pOutput,
+                             sal_Size(nOutputsSize));
+}
+
 } // end vcl::pdf
 
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */