Implement OOXML password hashing algorithm, tdf#104250 prep

As per https://msdn.microsoft.com/en-us/library/dd920692

Change-Id: Iebacaf3549dab28fd3033f9c241130fd66782b25
Reviewed-on: https://gerrit.libreoffice.org/50259
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Eike Rathke <erack@redhat.com>

2 files changed, 141 insertions, 0 deletions

diff --git a/comphelper/qa/unit/test_hash.cxx b/comphelper/qa/unit/test_hash.cxx
index f83f91d3286a..beec2c537cf4 100644
--- a/comphelper/qa/unit/test_hash.cxx
+++ b/comphelper/qa/unit/test_hash.cxx
@@ -9,6 +9,7 @@
 
 #include <comphelper/hash.hxx>
 
+#include <rtl/ustring.hxx>
 #include <sal/log.hxx>
 #include <iomanip>
 
@@ -22,12 +23,16 @@ public:
     void testSHA1();
     void testSHA256();
     void testSHA512();
+    void testSHA512_NoSaltNoSpin();
+    void testSHA512_saltspin();
 
     CPPUNIT_TEST_SUITE(TestHash);
     CPPUNIT_TEST(testMD5);
     CPPUNIT_TEST(testSHA1);
     CPPUNIT_TEST(testSHA256);
     CPPUNIT_TEST(testSHA512);
+    CPPUNIT_TEST(testSHA512_NoSaltNoSpin);
+    CPPUNIT_TEST(testSHA512_saltspin);
     CPPUNIT_TEST_SUITE_END();
 };
 
@@ -87,6 +92,30 @@ void TestHash::testSHA512()
     CPPUNIT_ASSERT_EQUAL(aStr, tostring(calculate_hash));
 }
 
+// Must be identical to testSHA512()
+void TestHash::testSHA512_NoSaltNoSpin()
+{
+    const char* const pInput = "";
+    std::vector<unsigned char> calculate_hash =
+        comphelper::Hash::calculateHash( reinterpret_cast<const unsigned char*>(pInput), 0,
+                nullptr, 0, 0, comphelper::HashType::SHA512);
+    CPPUNIT_ASSERT_EQUAL(size_t(64), calculate_hash.size());
+    std::string aStr("cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e");
+    CPPUNIT_ASSERT_EQUAL(aStr, tostring(calculate_hash));
+}
+
+// Password, salt, hash and spin count taken from OOXML sheetProtection of
+// tdf#104250 https://bugs.documentfoundation.org/attachment.cgi?id=129104
+void TestHash::testSHA512_saltspin()
+{
+    const OUString aPass("pwd");
+    const OUString aAlgo("SHA-512");
+    const OUString aSalt("876MLoKTq42+/DLp415iZQ==");
+    const OUString aHash = comphelper::Hash::calculateHash( aPass, aSalt, 100000, aAlgo);
+    OUString aStr("5l3mgNHXpWiFaBPv5Yso1Xd/UifWvQWmlDnl/hsCYbFT2sJCzorjRmBCQ/3qeDu6Q/4+GIE8a1DsdaTwYh1q2g==");
+    CPPUNIT_ASSERT_EQUAL(aStr, aHash);
+}
+
 CPPUNIT_TEST_SUITE_REGISTRATION(TestHash);
 
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/comphelper/source/misc/hash.cxx b/comphelper/source/misc/hash.cxx
index 8da77a792e6a..7a8c3fecd7e9 100644
--- a/comphelper/source/misc/hash.cxx
+++ b/comphelper/source/misc/hash.cxx
@@ -8,6 +8,11 @@
  */
 
 #include <comphelper/hash.hxx>
+#include <comphelper/base64.hxx>
+#include <comphelper/sequence.hxx>
+#include <rtl/ustring.hxx>
+#include <rtl/alloc.h>
+#include <osl/endian.h>
 #include <config_oox.h>
 
 #if USE_TLS_NSS
@@ -150,6 +155,113 @@ std::vector<unsigned char> Hash::calculateHash(const unsigned char* pInput, size
     return aHash.finalize();
 }
 
+std::vector<unsigned char> Hash::calculateHash(
+        const unsigned char* pInput, size_t nLength,
+        const unsigned char* pSalt, size_t nSaltLen,
+        sal_uInt32 nSpinCount,
+        HashType eType)
+{
+    if (!pSalt)
+        nSaltLen = 0;
+
+    if (!nSaltLen && !nSpinCount)
+        return calculateHash( pInput, nLength, eType);
+
+    Hash aHash(eType);
+    std::vector<unsigned char> hash;
+    if (nSaltLen)
+    {
+        std::vector<unsigned char> initialData( nSaltLen + nLength);
+        std::copy( pSalt, pSalt + nSaltLen, initialData.begin());
+        std::copy( pInput, pInput + nLength, initialData.begin() + nSaltLen);
+        aHash.update( initialData.data(), initialData.size());
+        rtl_secureZeroMemory( initialData.data(), initialData.size());
+    }
+    else
+    {
+        aHash.update( pInput, nLength);
+    }
+    hash = aHash.finalize();
+
+    if (nSpinCount)
+    {
+        // https://msdn.microsoft.com/en-us/library/dd920692
+        // says the iteration is concatenated after the hash.
+        // XXX NOTE: oox/source/crypto/AgileEngine.cxx
+        // AgileEngine::calculateHashFinal() prepends the iteration value, they
+        // do things differently for write protection and encryption passwords.
+        // https://msdn.microsoft.com/en-us/library/dd924776
+        /* TODO: maybe pass a flag whether to prepend or append, and then let
+         * AgileEngine::calculateHashFinal() call this function. */
+        const size_t nIterPos = hash.size();
+        const size_t nHashPos = 0;
+        //const size_t nIterPos = 0;
+        //const size_t nHashPos = 4;
+        std::vector<unsigned char> data( hash.size() + 4, 0);
+        for (sal_uInt32 i = 0; i < nSpinCount; ++i)
+        {
+            std::copy( hash.begin(), hash.end(), data.begin() + nHashPos);
+#ifdef OSL_BIGENDIAN
+            sal_uInt32 be = i;
+            sal_uInt8* p = reinterpret_cast<sal_uInt8*>(&be);
+            std::swap( p[0], p[3] );
+            std::swap( p[1], p[2] );
+            memcpy( data.data() + nIterPos, &be, 4);
+#else
+            memcpy( data.data() + nIterPos, &i, 4);
+#endif
+            /* TODO: isn't there something better than
+             * creating/finalizing/destroying on each iteration? */
+            Hash aReHash(eType);
+            aReHash.update( data.data(), data.size());
+            hash = aReHash.finalize();
+        }
+    }
+
+    return hash;
+}
+
+std::vector<unsigned char> Hash::calculateHash(
+        const OUString& rPassword,
+        const std::vector<unsigned char>& rSaltValue,
+        sal_uInt32 nSpinCount,
+        HashType eType)
+{
+    const unsigned char* pPassBytes = reinterpret_cast<const unsigned char*>(rPassword.getStr());
+    const size_t nPassBytesLen = rPassword.getLength() * 2;
+    return calculateHash( pPassBytes, nPassBytesLen, rSaltValue.data(), rSaltValue.size(), nSpinCount, eType);
+}
+
+OUString Hash::calculateHash(
+        const rtl::OUString& rPassword,
+        const rtl::OUString& rSaltValue,
+        sal_uInt32 nSpinCount,
+        const rtl::OUString& rAlgorithmName)
+{
+    HashType eType;
+    if (rAlgorithmName == "SHA-512")
+        eType = HashType::SHA512;
+    else if (rAlgorithmName == "SHA-256")
+        eType = HashType::SHA256;
+    else if (rAlgorithmName == "SHA-1")
+        eType = HashType::SHA1;
+    else if (rAlgorithmName == "MD5")
+        eType = HashType::MD5;
+    else
+        return OUString();
+
+    css::uno::Sequence<sal_Int8> aSaltSeq;
+    comphelper::Base64::decode( aSaltSeq, rSaltValue);
+
+    std::vector<unsigned char> hash = calculateHash( rPassword,
+            comphelper::sequenceToContainer<std::vector<unsigned char>>( aSaltSeq),
+            nSpinCount, eType);
+
+    OUStringBuffer aBuf;
+    comphelper::Base64::encode( aBuf, comphelper::containerToSequence<sal_Int8>( hash));
+    return aBuf.makeStringAndClear();
+}
+
 }
 
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */