Prepare to handle OOXML Agile Encryption password hash as well

... that prepends the iteration count to the hash instead of
appending it

Change-Id: I090393e6337c110029e35baaa259b40ef4e5d416

3 files changed, 19 insertions, 17 deletions

diff --git a/comphelper/qa/unit/test_hash.cxx b/comphelper/qa/unit/test_hash.cxx
index 11dcbd0bd10f..5d2ece34dc83 100644
--- a/comphelper/qa/unit/test_hash.cxx
+++ b/comphelper/qa/unit/test_hash.cxx
@@ -99,7 +99,7 @@ void TestHash::testSHA512_NoSaltNoSpin()
     const char* const pInput = "";
     std::vector<unsigned char> calculate_hash =
         comphelper::Hash::calculateHash( reinterpret_cast<const unsigned char*>(pInput), 0,
-                nullptr, 0, 0, comphelper::HashType::SHA512);
+                nullptr, 0, 0, false, comphelper::HashType::SHA512);
     CPPUNIT_ASSERT_EQUAL(size_t(64), calculate_hash.size());
     std::string aStr("cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e");
     CPPUNIT_ASSERT_EQUAL(aStr, tostring(calculate_hash));
@@ -112,7 +112,7 @@ void TestHash::testSHA512_saltspin()
     const OUString aPass("pwd");
     const OUString aAlgo("SHA-512");
     const OUString aSalt("876MLoKTq42+/DLp415iZQ==");
-    const OUString aHash = comphelper::DocPasswordHelper::GetOoxHashAsBase64( aPass, aSalt, 100000, aAlgo);
+    const OUString aHash = comphelper::DocPasswordHelper::GetOoxHashAsBase64( aPass, aSalt, 100000, false, aAlgo);
     const OUString aStr("5l3mgNHXpWiFaBPv5Yso1Xd/UifWvQWmlDnl/hsCYbFT2sJCzorjRmBCQ/3qeDu6Q/4+GIE8a1DsdaTwYh1q2g==");
     CPPUNIT_ASSERT_EQUAL(aStr, aHash);
 }
diff --git a/comphelper/source/misc/docpasswordhelper.cxx b/comphelper/source/misc/docpasswordhelper.cxx
index e6055b29cebc..97b06af2e689 100644
--- a/comphelper/source/misc/docpasswordhelper.cxx
+++ b/comphelper/source/misc/docpasswordhelper.cxx
@@ -263,12 +263,13 @@ css::uno::Sequence<sal_Int8> DocPasswordHelper::GetOoxHashAsSequence(
         const rtl::OUString& rPassword,
         const rtl::OUString& rSaltValue,
         sal_uInt32 nSpinCount,
+        bool bPrependNotAppend,
         const rtl::OUString& rAlgorithmName)
 {
     comphelper::HashType eType;
-    if (rAlgorithmName == "SHA-512")
+    if (rAlgorithmName == "SHA-512" || rAlgorithmName == "SHA512")
         eType = comphelper::HashType::SHA512;
-    else if (rAlgorithmName == "SHA-256")
+    else if (rAlgorithmName == "SHA-256" || rAlgorithmName == "SHA256")
         eType = comphelper::HashType::SHA256;
     else if (rAlgorithmName == "SHA-1")
         eType = comphelper::HashType::SHA1;
@@ -285,7 +286,8 @@ css::uno::Sequence<sal_Int8> DocPasswordHelper::GetOoxHashAsSequence(
         aSaltVec = comphelper::sequenceToContainer<std::vector<unsigned char>>( aSaltSeq);
     }
 
-    std::vector<unsigned char> hash( comphelper::Hash::calculateHash( rPassword, aSaltVec, nSpinCount, eType));
+    std::vector<unsigned char> hash( comphelper::Hash::calculateHash( rPassword, aSaltVec, nSpinCount,
+                bPrependNotAppend, eType));
 
     return comphelper::containerToSequence<sal_Int8>( hash);
 }
@@ -294,9 +296,11 @@ OUString DocPasswordHelper::GetOoxHashAsBase64(
         const rtl::OUString& rPassword,
         const rtl::OUString& rSaltValue,
         sal_uInt32 nSpinCount,
+        bool bPrependNotAppend,
         const rtl::OUString& rAlgorithmName)
 {
-    css::uno::Sequence<sal_Int8> aSeq( GetOoxHashAsSequence( rPassword, rSaltValue, nSpinCount, rAlgorithmName));
+    css::uno::Sequence<sal_Int8> aSeq( GetOoxHashAsSequence( rPassword, rSaltValue, nSpinCount,
+                bPrependNotAppend, rAlgorithmName));
 
     OUStringBuffer aBuf;
     comphelper::Base64::encode( aBuf, aSeq);
diff --git a/comphelper/source/misc/hash.cxx b/comphelper/source/misc/hash.cxx
index b629d8d17530..ad0a247ae7f6 100644
--- a/comphelper/source/misc/hash.cxx
+++ b/comphelper/source/misc/hash.cxx
@@ -157,6 +157,7 @@ std::vector<unsigned char> Hash::calculateHash(
         const unsigned char* pInput, size_t nLength,
         const unsigned char* pSalt, size_t nSaltLen,
         sal_uInt32 nSpinCount,
+        bool bPrependNotAppend,
         HashType eType)
 {
     if (!pSalt)
@@ -184,16 +185,11 @@ std::vector<unsigned char> Hash::calculateHash(
     {
         // https://msdn.microsoft.com/en-us/library/dd920692
         // says the iteration is concatenated after the hash.
-        // XXX NOTE: oox/source/crypto/AgileEngine.cxx
-        // AgileEngine::calculateHashFinal() prepends the iteration value, they
-        // do things differently for write protection and encryption passwords.
-        // https://msdn.microsoft.com/en-us/library/dd924776
-        /* TODO: maybe pass a flag whether to prepend or append, and then let
-         * AgileEngine::calculateHashFinal() call this function. */
-        const size_t nIterPos = hash.size();
-        const size_t nHashPos = 0;
-        //const size_t nIterPos = 0;
-        //const size_t nHashPos = 4;
+        // https://msdn.microsoft.com/en-us/library/dd924776 and
+        // https://msdn.microsoft.com/en-us/library/dd925430
+        // say the iteration is prepended to the hash.
+        const size_t nIterPos = (bPrependNotAppend ? 0 : hash.size());
+        const size_t nHashPos = (bPrependNotAppend ? 4 : 0);
         std::vector<unsigned char> data( hash.size() + 4, 0);
         for (sal_uInt32 i = 0; i < nSpinCount; ++i)
         {
@@ -222,11 +218,13 @@ std::vector<unsigned char> Hash::calculateHash(
         const OUString& rPassword,
         const std::vector<unsigned char>& rSaltValue,
         sal_uInt32 nSpinCount,
+        bool bPrependNotAppend,
         HashType eType)
 {
     const unsigned char* pPassBytes = reinterpret_cast<const unsigned char*>(rPassword.getStr());
     const size_t nPassBytesLen = rPassword.getLength() * 2;
-    return calculateHash( pPassBytes, nPassBytesLen, rSaltValue.data(), rSaltValue.size(), nSpinCount, eType);
+    return calculateHash( pPassBytes, nPassBytesLen, rSaltValue.data(), rSaltValue.size(), nSpinCount,
+            bPrependNotAppend, eType);
 }
 
 }