pdf: generate U, UE and O, OE keys for R6 encryption

Also test the algorithm against the known values from an example,
to be sure we are calculating the values correctly. For this we
need a couple of decryption algorithms, but those do mostly just
the reverse of the encryption.

Change-Id: I5499ed0b57671f44e48fe68961e07cde22be6b39
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/176881
Tested-by: Jenkins CollaboraOffice <jenkinscollaboraoffice@gmail.com>
Reviewed-by: Miklos Vajna <vmiklos@collabora.com>
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/178755
Tested-by: Jenkins
Reviewed-by: Tomaž Vajngerl <quikee@gmail.com>

3 files changed, 212 insertions, 5 deletions

diff --git a/vcl/inc/pdf/PDFEncryptorR6.hxx b/vcl/inc/pdf/PDFEncryptorR6.hxx
index 7234e96f6af1..e6efdeb5768d 100644
--- a/vcl/inc/pdf/PDFEncryptorR6.hxx
+++ b/vcl/inc/pdf/PDFEncryptorR6.hxx
@@ -9,17 +9,73 @@
 
 #pragma once
 
+#include <rtl/ustring.hxx>
 #include <string_view>
 #include <vector>
 #include <vcl/dllapi.h>
 
 namespace vcl::pdf
 {
+/** Algorithm 2.B: Computing a hash (revision 6 and later)
+ *
+ * Described in ISO 32000-2:2020(E) - 7.6.4.3.4
+ */
 VCL_DLLPUBLIC std::vector<sal_uInt8>
 computeHashR6(const sal_uInt8* pPassword, size_t nPasswordLength,
               std::vector<sal_uInt8> const& rValidationSalt,
               std::vector<sal_uInt8> const& rUserKey = std::vector<sal_uInt8>());
 
+/** Algorithm 11: Authenticating the user password (Security handlers of revision 6)
+ *
+ * Described in ISO 32000-2:2020(E) - 7.6.4.4.10
+ */
+VCL_DLLPUBLIC bool validateUserPassword(const sal_uInt8* pUserPass, size_t nPasswordLength,
+                                        std::vector<sal_uInt8>& U);
+
+/** Algorithm 12: Authenticating the owner password (Security handlers of revision 6)
+ *
+ * Described in ISO 32000-2:2020(E) - 7.6.4.4.11
+ */
+VCL_DLLPUBLIC bool validateOwnerPassword(const sal_uInt8* pUserPass, size_t nPasswordLength,
+                                         std::vector<sal_uInt8>& U, std::vector<sal_uInt8>& O);
+
+/** Generates the encryption key - random data 32-byte */
+VCL_DLLPUBLIC std::vector<sal_uInt8> generateKey();
+
+/** Algorithm 8: U and UE
+ *
+ * Computing the encryption dictionary’s U (user password) and UE (user encryption) values
+ * (Security handlers of revision 6)
+ *
+ * Described in ISO 32000-2:2020(E) - 7.6.4.4.7
+ */
+VCL_DLLPUBLIC void generateUandUE(const sal_uInt8* pUserPass, size_t nPasswordLength,
+                                  std::vector<sal_uInt8>& rFileEncryptionKey,
+                                  std::vector<sal_uInt8>& U, std::vector<sal_uInt8>& UE);
+
+/** Algorithm 9: O and OE
+ *
+ * Computing the encryption dictionary’s O (owner password) and OE (owner encryption) values
+ * (Security handlers of revision 6)
+ *
+ * Described in ISO 32000-2:2020(E) - 7.6.4.4.8
+ */
+VCL_DLLPUBLIC void generateOandOE(const sal_uInt8* pUserPass, size_t nPasswordLength,
+                                  std::vector<sal_uInt8>& rFileEncryptionKey,
+                                  std::vector<sal_uInt8>& U, std::vector<sal_uInt8>& O,
+                                  std::vector<sal_uInt8>& OE);
+
+/** Algorithm 8 step b) in reverse
+ *
+ * Described in ISO 32000-2:2020(E) - 7.6.4.4.7
+ *
+ * - compute the hash with password and user key salt
+ * - decrypt with hash as key and zero IV
+ */
+VCL_DLLPUBLIC std::vector<sal_uInt8> decryptKey(const sal_uInt8* pUserPass, size_t nPasswordLength,
+                                                std::vector<sal_uInt8>& U,
+                                                std::vector<sal_uInt8>& UE);
+
 } // end vcl::pdf
 
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/vcl/qa/cppunit/pdfexport/PDFEncryptionTest.cxx b/vcl/qa/cppunit/pdfexport/PDFEncryptionTest.cxx
index 4f2239bc01b8..6f1e868564e6 100644
--- a/vcl/qa/cppunit/pdfexport/PDFEncryptionTest.cxx
+++ b/vcl/qa/cppunit/pdfexport/PDFEncryptionTest.cxx
@@ -127,6 +127,59 @@ CPPUNIT_TEST_FIXTURE(PDFEncryptionTest, testComputeHashForR6)
     }
 }
 
+CPPUNIT_TEST_FIXTURE(PDFEncryptionTest, testGenerateUandUE)
+{
+    // Checks we calculate U and UE correctly
+    const sal_uInt8 pUserPass[] = { 'T', 'e', 's', 't' };
+
+    std::vector<sal_uInt8> aInputKey = vcl::pdf::generateKey();
+
+    std::vector<sal_uInt8> U;
+    std::vector<sal_uInt8> UE;
+
+    // Generate the U and UE from the user password and encrypt the
+    // encryption key into UE
+    vcl::pdf::generateUandUE(pUserPass, 4, aInputKey, U, UE);
+
+    // Checks that the U validates the password (would fail if the U
+    // would be calculated wrongly).
+    CPPUNIT_ASSERT_EQUAL(true, vcl::pdf::validateUserPassword(pUserPass, 4, U));
+
+    // Decrypt the key - this would fail if U and UE would be calculated
+    // wrongly
+    auto aDecryptedKey = vcl::pdf::decryptKey(pUserPass, 4, U, UE);
+
+    // Decrypted key and input key need to match
+    CPPUNIT_ASSERT_EQUAL(comphelper::hashToString(aInputKey),
+                         comphelper::hashToString(aDecryptedKey));
+}
+
+CPPUNIT_TEST_FIXTURE(PDFEncryptionTest, testGenerateOandOE)
+{
+    // Checks we calculate O and OE correctly
+
+    const auto aUserPass = std::to_array<sal_uInt8>({ 'T', 'e', 's', 't' });
+    const auto aOwnerPass = std::to_array<sal_uInt8>({ 'T', 'e', 's', 't', '2' });
+
+    std::vector<sal_uInt8> aInputKey = vcl::pdf::generateKey();
+
+    std::vector<sal_uInt8> U;
+    std::vector<sal_uInt8> UE;
+    std::vector<sal_uInt8> O;
+    std::vector<sal_uInt8> OE;
+
+    // Generates U and UE - we need U in generateOandOE
+    vcl::pdf::generateUandUE(aUserPass.data(), aUserPass.size(), aInputKey, U, UE);
+    vcl::pdf::generateOandOE(aOwnerPass.data(), aOwnerPass.size(), aInputKey, U, O, OE);
+
+    // Checks the user password is valid
+    CPPUNIT_ASSERT_EQUAL(true,
+                         vcl::pdf::validateUserPassword(aUserPass.data(), aUserPass.size(), U));
+
+    // Checks the owner password is valid
+    CPPUNIT_ASSERT_EQUAL(
+        true, vcl::pdf::validateOwnerPassword(aOwnerPass.data(), aOwnerPass.size(), U, O));
+}
 } // end anonymous namespace
 
 CPPUNIT_PLUGIN_IMPLEMENT();
diff --git a/vcl/source/pdf/PDFEncryptorR6.cxx b/vcl/source/pdf/PDFEncryptorR6.cxx
index 64e668dca598..9f0670f1fe1f 100644
--- a/vcl/source/pdf/PDFEncryptorR6.cxx
+++ b/vcl/source/pdf/PDFEncryptorR6.cxx
@@ -18,6 +18,10 @@ namespace vcl::pdf
 {
 namespace
 {
+constexpr size_t IV_SIZE = 16;
+constexpr size_t KEY_SIZE = 32;
+constexpr size_t SALT_SIZE = 8;
+
 /** Calculates modulo 3 of the 128-bit integer, using the first 16 bytes of the vector */
 sal_Int32 calculateModulo3(std::vector<sal_uInt8> const& rInput)
 {
@@ -26,12 +30,106 @@ sal_Int32 calculateModulo3(std::vector<sal_uInt8> const& rInput)
         nSum += rInput[i];
     return nSum % 3;
 }
+
+void generateBytes(std::vector<sal_uInt8>& rBytes, size_t nSize)
+{
+    rBytes.resize(nSize);
+
+    for (size_t i = 0; i < rBytes.size(); ++i)
+        rBytes[i] = sal_uInt8(comphelper::rng::uniform_uint_distribution(0, 0xFF));
 }
 
-/** Algorithm 2.B: Computing a hash (revision 6 and later)
- *
- * Described in ISO 32000-2:2020(E) - 7.6.4.3.4
- */
+} // end anonymous
+
+std::vector<sal_uInt8> generateKey()
+{
+    std::vector<sal_uInt8> aKey;
+    generateBytes(aKey, KEY_SIZE);
+    return aKey;
+}
+
+bool validateUserPassword(const sal_uInt8* pPass, size_t nLength, std::vector<sal_uInt8>& U)
+{
+    std::vector<sal_uInt8> aHash(U.begin(), U.begin() + KEY_SIZE);
+    std::vector<sal_uInt8> aValidationSalt(U.begin() + KEY_SIZE, U.begin() + KEY_SIZE + SALT_SIZE);
+    std::vector<sal_uInt8> aCalculatedHash
+        = vcl::pdf::computeHashR6(pPass, nLength, aValidationSalt);
+    return aHash == aCalculatedHash;
+}
+
+bool validateOwnerPassword(const sal_uInt8* pPass, size_t nLength, std::vector<sal_uInt8>& U,
+                           std::vector<sal_uInt8>& O)
+{
+    std::vector<sal_uInt8> aHash(O.begin(), O.begin() + KEY_SIZE);
+    std::vector<sal_uInt8> aValidationSalt(O.begin() + KEY_SIZE, O.begin() + KEY_SIZE + SALT_SIZE);
+    std::vector<sal_uInt8> aCalculatedHash
+        = vcl::pdf::computeHashR6(pPass, nLength, aValidationSalt, U);
+    return aHash == aCalculatedHash;
+}
+
+/** Algorithm 8 */
+void generateUandUE(const sal_uInt8* pPass, size_t nLength,
+                    std::vector<sal_uInt8>& rFileEncryptionKey, std::vector<sal_uInt8>& U,
+                    std::vector<sal_uInt8>& UE)
+{
+    std::vector<sal_uInt8> aValidationSalt;
+    generateBytes(aValidationSalt, SALT_SIZE);
+    std::vector<sal_uInt8> aKeySalt;
+    generateBytes(aKeySalt, SALT_SIZE);
+
+    U = vcl::pdf::computeHashR6(pPass, nLength, aValidationSalt);
+    U.insert(U.end(), aValidationSalt.begin(), aValidationSalt.end());
+    U.insert(U.end(), aKeySalt.begin(), aKeySalt.end());
+
+    std::vector<sal_uInt8> aKeyHash = vcl::pdf::computeHashR6(pPass, nLength, aKeySalt);
+    std::vector<sal_uInt8> iv(IV_SIZE, 0); // zero IV
+    UE = std::vector<sal_uInt8>(rFileEncryptionKey.size(), 0);
+    comphelper::Encrypt aEncrypt(aKeyHash, iv, comphelper::CryptoType::AES_256_CBC);
+    aEncrypt.update(UE, rFileEncryptionKey);
+}
+
+/** Algorithm 9 */
+void generateOandOE(const sal_uInt8* pPass, size_t nLength,
+                    std::vector<sal_uInt8>& rFileEncryptionKey, std::vector<sal_uInt8>& U,
+                    std::vector<sal_uInt8>& O, std::vector<sal_uInt8>& OE)
+{
+    std::vector<sal_uInt8> aValidationSalt;
+    generateBytes(aValidationSalt, SALT_SIZE);
+    std::vector<sal_uInt8> aKeySalt;
+    generateBytes(aKeySalt, SALT_SIZE);
+
+    O = vcl::pdf::computeHashR6(pPass, nLength, aValidationSalt, U);
+    O.insert(O.end(), aValidationSalt.begin(), aValidationSalt.end());
+    O.insert(O.end(), aKeySalt.begin(), aKeySalt.end());
+
+    std::vector<sal_uInt8> aKeyHash = vcl::pdf::computeHashR6(pPass, nLength, aKeySalt, U);
+    std::vector<sal_uInt8> iv(IV_SIZE, 0); // zero IV
+    OE = std::vector<sal_uInt8>(rFileEncryptionKey.size(), 0);
+    comphelper::Encrypt aEncrypt(aKeyHash, iv, comphelper::CryptoType::AES_256_CBC);
+    aEncrypt.update(OE, rFileEncryptionKey);
+}
+
+/** Algorithm 8 step b) */
+std::vector<sal_uInt8> decryptKey(const sal_uInt8* pPass, size_t nLength, std::vector<sal_uInt8>& U,
+                                  std::vector<sal_uInt8>& UE)
+{
+    std::vector<sal_uInt8> aKeySalt(U.begin() + KEY_SIZE + SALT_SIZE,
+                                    U.begin() + KEY_SIZE + SALT_SIZE + SALT_SIZE);
+
+    auto aKeyHash = vcl::pdf::computeHashR6(pPass, nLength, aKeySalt);
+
+    std::vector<sal_uInt8> aEncryptedKey(UE.begin(), UE.begin() + KEY_SIZE);
+    std::vector<sal_uInt8> iv(IV_SIZE, 0);
+
+    comphelper::Decrypt aDecryptCBC(aKeyHash, iv, comphelper::CryptoType::AES_256_CBC);
+    std::vector<sal_uInt8> aFileEncryptionKey(aEncryptedKey.size());
+    sal_uInt32 nDecrypted = aDecryptCBC.update(aFileEncryptionKey, aEncryptedKey);
+    if (nDecrypted == 0)
+        return std::vector<sal_uInt8>();
+    return aFileEncryptionKey;
+}
+
+/** Algorithm 2.B: Computing a hash (revision 6 and later) */
 std::vector<sal_uInt8> computeHashR6(const sal_uInt8* pPassword, size_t nPasswordLength,
                                      std::vector<sal_uInt8> const& rValidationSalt,
                                      std::vector<sal_uInt8> const& rUserKey)
@@ -47,7 +145,7 @@ std::vector<sal_uInt8> computeHashR6(const sal_uInt8* pPassword, size_t nPasswor
 
     std::vector<sal_uInt8> E;
 
-    sal_Int32 nRound = 1;
+    sal_Int32 nRound = 1; // round 0 is done already
     do
     {
         // Step a)