Prepare to handle OOXML Agile Encryption password hash as well

... that prepends the iteration count to the hash instead of
appending it

Change-Id: I090393e6337c110029e35baaa259b40ef4e5d416

2 files changed, 17 insertions, 15 deletions

diff --git a/comphelper/source/misc/docpasswordhelper.cxx b/comphelper/source/misc/docpasswordhelper.cxx
index e6055b29cebc..97b06af2e689 100644
--- a/comphelper/source/misc/docpasswordhelper.cxx
+++ b/comphelper/source/misc/docpasswordhelper.cxx
@@ -263,12 +263,13 @@ css::uno::Sequence<sal_Int8> DocPasswordHelper::GetOoxHashAsSequence(
         const rtl::OUString& rPassword,
         const rtl::OUString& rSaltValue,
         sal_uInt32 nSpinCount,
+        bool bPrependNotAppend,
         const rtl::OUString& rAlgorithmName)
 {
     comphelper::HashType eType;
-    if (rAlgorithmName == "SHA-512")
+    if (rAlgorithmName == "SHA-512" || rAlgorithmName == "SHA512")
         eType = comphelper::HashType::SHA512;
-    else if (rAlgorithmName == "SHA-256")
+    else if (rAlgorithmName == "SHA-256" || rAlgorithmName == "SHA256")
         eType = comphelper::HashType::SHA256;
     else if (rAlgorithmName == "SHA-1")
         eType = comphelper::HashType::SHA1;
@@ -285,7 +286,8 @@ css::uno::Sequence<sal_Int8> DocPasswordHelper::GetOoxHashAsSequence(
         aSaltVec = comphelper::sequenceToContainer<std::vector<unsigned char>>( aSaltSeq);
     }
 
-    std::vector<unsigned char> hash( comphelper::Hash::calculateHash( rPassword, aSaltVec, nSpinCount, eType));
+    std::vector<unsigned char> hash( comphelper::Hash::calculateHash( rPassword, aSaltVec, nSpinCount,
+                bPrependNotAppend, eType));
 
     return comphelper::containerToSequence<sal_Int8>( hash);
 }
@@ -294,9 +296,11 @@ OUString DocPasswordHelper::GetOoxHashAsBase64(
         const rtl::OUString& rPassword,
         const rtl::OUString& rSaltValue,
         sal_uInt32 nSpinCount,
+        bool bPrependNotAppend,
         const rtl::OUString& rAlgorithmName)
 {
-    css::uno::Sequence<sal_Int8> aSeq( GetOoxHashAsSequence( rPassword, rSaltValue, nSpinCount, rAlgorithmName));
+    css::uno::Sequence<sal_Int8> aSeq( GetOoxHashAsSequence( rPassword, rSaltValue, nSpinCount,
+                bPrependNotAppend, rAlgorithmName));
 
     OUStringBuffer aBuf;
     comphelper::Base64::encode( aBuf, aSeq);
diff --git a/comphelper/source/misc/hash.cxx b/comphelper/source/misc/hash.cxx
index b629d8d17530..ad0a247ae7f6 100644
--- a/comphelper/source/misc/hash.cxx
+++ b/comphelper/source/misc/hash.cxx
@@ -157,6 +157,7 @@ std::vector<unsigned char> Hash::calculateHash(
         const unsigned char* pInput, size_t nLength,
         const unsigned char* pSalt, size_t nSaltLen,
         sal_uInt32 nSpinCount,
+        bool bPrependNotAppend,
         HashType eType)
 {
     if (!pSalt)
@@ -184,16 +185,11 @@ std::vector<unsigned char> Hash::calculateHash(
     {
         // https://msdn.microsoft.com/en-us/library/dd920692
         // says the iteration is concatenated after the hash.
-        // XXX NOTE: oox/source/crypto/AgileEngine.cxx
-        // AgileEngine::calculateHashFinal() prepends the iteration value, they
-        // do things differently for write protection and encryption passwords.
-        // https://msdn.microsoft.com/en-us/library/dd924776
-        /* TODO: maybe pass a flag whether to prepend or append, and then let
-         * AgileEngine::calculateHashFinal() call this function. */
-        const size_t nIterPos = hash.size();
-        const size_t nHashPos = 0;
-        //const size_t nIterPos = 0;
-        //const size_t nHashPos = 4;
+        // https://msdn.microsoft.com/en-us/library/dd924776 and
+        // https://msdn.microsoft.com/en-us/library/dd925430
+        // say the iteration is prepended to the hash.
+        const size_t nIterPos = (bPrependNotAppend ? 0 : hash.size());
+        const size_t nHashPos = (bPrependNotAppend ? 4 : 0);
         std::vector<unsigned char> data( hash.size() + 4, 0);
         for (sal_uInt32 i = 0; i < nSpinCount; ++i)
         {
@@ -222,11 +218,13 @@ std::vector<unsigned char> Hash::calculateHash(
         const OUString& rPassword,
         const std::vector<unsigned char>& rSaltValue,
         sal_uInt32 nSpinCount,
+        bool bPrependNotAppend,
         HashType eType)
 {
     const unsigned char* pPassBytes = reinterpret_cast<const unsigned char*>(rPassword.getStr());
     const size_t nPassBytesLen = rPassword.getLength() * 2;
-    return calculateHash( pPassBytes, nPassBytesLen, rSaltValue.data(), rSaltValue.size(), nSpinCount, eType);
+    return calculateHash( pPassBytes, nPassBytesLen, rSaltValue.data(), rSaltValue.size(), nSpinCount,
+            bPrependNotAppend, eType);
 }
 
 }