pdf: R6 hash algorithm and test, introduce PDFEncryptorR6

This adds PDFEncryptorR6 and adds R6 hash implementation and makes
sure it is correct with a test.

Change-Id: I11ca746a6b676bb294723b4ef76069f1d4f3a182
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/176451
Reviewed-by: Miklos Vajna <vmiklos@collabora.com>
Tested-by: Jenkins CollaboraOffice <jenkinscollaboraoffice@gmail.com>

6 files changed, 318 insertions, 0 deletions

diff --git a/vcl/CppunitTest_vcl_pdf_encryption.mk b/vcl/CppunitTest_vcl_pdf_encryption.mk
new file mode 100644
index 000000000000..d275b087a767
--- /dev/null
+++ b/vcl/CppunitTest_vcl_pdf_encryption.mk
@@ -0,0 +1,54 @@
+# -*- Mode: makefile-gmake; tab-width: 4; indent-tabs-mode: t -*-
+#
+# This file is part of the LibreOffice project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+
+$(eval $(call gb_CppunitTest_CppunitTest,vcl_pdf_encryption))
+
+$(eval $(call gb_CppunitTest_add_exception_objects,vcl_pdf_encryption, \
+    vcl/qa/cppunit/pdfexport/PDFEncryptionTest \
+))
+
+$(eval $(call gb_CppunitTest_set_include,vcl_pdf_encryption,\
+    $$(INCLUDE) \
+    -I$(SRCDIR)/vcl/inc \
+))
+
+$(eval $(call gb_CppunitTest_use_libraries,vcl_pdf_encryption, \
+	basegfx \
+	comphelper \
+	cppu \
+	cppuhelper \
+	sal \
+	subsequenttest \
+	test \
+	unotest \
+	utl \
+	tl \
+	vcl \
+	xmlsecurity \
+))
+
+$(eval $(call gb_CppunitTest_use_externals,vcl_pdf_encryption, \
+	boost_headers \
+	$(if $(filter PDFIUM,$(BUILD_TYPE)),pdfium) \
+))
+
+ifeq ($(TLS),NSS)
+$(eval $(call gb_CppunitTest_use_externals,vcl_pdf_encryption,\
+       plc4 \
+       nss3 \
+))
+endif
+
+$(eval $(call gb_CppunitTest_use_sdk_api,vcl_pdf_encryption))
+$(eval $(call gb_CppunitTest_use_ure,vcl_pdf_encryption))
+$(eval $(call gb_CppunitTest_use_vcl,vcl_pdf_encryption))
+$(eval $(call gb_CppunitTest_use_rdb,vcl_pdf_encryption,services))
+$(eval $(call gb_CppunitTest_use_configuration,vcl_pdf_encryption))
+
+# vim: set noet sw=4 ts=4:
diff --git a/vcl/Library_vcl.mk b/vcl/Library_vcl.mk
index 421f04e6af73..fcc608cfa2e1 100644
--- a/vcl/Library_vcl.mk
+++ b/vcl/Library_vcl.mk
@@ -498,6 +498,7 @@ $(eval $(call gb_Library_add_exception_objects,vcl,\
     vcl/source/pdf/ExternalPDFStreams \
     vcl/source/pdf/PDFiumTools \
     vcl/source/pdf/PDFEncryptor \
+    vcl/source/pdf/PDFEncryptorR6 \
     vcl/source/pdf/PdfConfig \
     vcl/source/pdf/ResourceDict \
     vcl/source/pdf/Matrix3 \
diff --git a/vcl/Module_vcl.mk b/vcl/Module_vcl.mk
index 9791991650ac..d02764a139a0 100644
--- a/vcl/Module_vcl.mk
+++ b/vcl/Module_vcl.mk
@@ -287,6 +287,7 @@ ifneq (,$(filter PDFIUM,$(BUILD_TYPE)))
 $(eval $(call gb_Module_add_slowcheck_targets,vcl,\
     CppunitTest_vcl_pdfexport \
     CppunitTest_vcl_pdfexport2 \
+    CppunitTest_vcl_pdf_encryption \
     CppunitTest_vcl_filter_ipdf \
 ))
 endif
diff --git a/vcl/inc/pdf/PDFEncryptorR6.hxx b/vcl/inc/pdf/PDFEncryptorR6.hxx
new file mode 100644
index 000000000000..7234e96f6af1
--- /dev/null
+++ b/vcl/inc/pdf/PDFEncryptorR6.hxx
@@ -0,0 +1,25 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+#pragma once
+
+#include <string_view>
+#include <vector>
+#include <vcl/dllapi.h>
+
+namespace vcl::pdf
+{
+VCL_DLLPUBLIC std::vector<sal_uInt8>
+computeHashR6(const sal_uInt8* pPassword, size_t nPasswordLength,
+              std::vector<sal_uInt8> const& rValidationSalt,
+              std::vector<sal_uInt8> const& rUserKey = std::vector<sal_uInt8>());
+
+} // end vcl::pdf
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/vcl/qa/cppunit/pdfexport/PDFEncryptionTest.cxx b/vcl/qa/cppunit/pdfexport/PDFEncryptionTest.cxx
new file mode 100644
index 000000000000..4f2239bc01b8
--- /dev/null
+++ b/vcl/qa/cppunit/pdfexport/PDFEncryptionTest.cxx
@@ -0,0 +1,134 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+#include <sal/config.h>
+#include <config_oox.h>
+
+#include <algorithm>
+#include <memory>
+#include <string_view>
+
+#include <test/unoapi_test.hxx>
+#include <o3tl/string_view.hxx>
+
+#include <vcl/filter/PDFiumLibrary.hxx>
+#include <vcl/pdfread.hxx>
+#include <comphelper/propertyvalue.hxx>
+#include <cmath>
+
+#include <comphelper/crypto/Crypto.hxx>
+#include <comphelper/hash.hxx>
+#include <comphelper/random.hxx>
+
+#include <pdf/PDFEncryptorR6.hxx>
+
+#if USE_TLS_NSS
+#include <nss.h>
+#endif
+
+using namespace ::com::sun::star;
+
+namespace
+{
+class PDFEncryptionTest : public UnoApiTest
+{
+public:
+    PDFEncryptionTest()
+        : UnoApiTest("/vcl/qa/cppunit/pdfexport/data/")
+    {
+    }
+
+    ~PDFEncryptionTest()
+    {
+#if USE_TLS_NSS
+        NSS_Shutdown();
+#endif
+    }
+};
+
+// TODO: taken from GUID
+sal_uInt8 gethex(char nChar)
+{
+    if (nChar >= '0' && nChar <= '9')
+        return nChar - '0';
+    else if (nChar >= 'a' && nChar <= 'f')
+        return nChar - 'a' + 10;
+    else if (nChar >= 'A' && nChar <= 'F')
+        return nChar - 'A' + 10;
+    else
+        return 0;
+}
+
+// TODO: taken from GUID
+sal_uInt8 convertHexChar(char high, char low) { return (gethex(high) << 4) + gethex(low); }
+
+std::vector<sal_uInt8> parseHex(std::string_view rString)
+{
+    std::vector<sal_uInt8> aResult;
+    aResult.reserve(rString.size() / 2);
+    for (size_t i = 0; i < rString.size(); i += 2)
+    {
+        aResult.push_back(convertHexChar(rString[i], rString[i + 1]));
+    }
+    return aResult;
+}
+
+CPPUNIT_TEST_FIXTURE(PDFEncryptionTest, testComputeHashForR6)
+{
+    const sal_uInt8 pOwnerPass[] = { 'T', 'e', 's', 't' };
+    const sal_uInt8 pUserPass[] = { 'T', 'e', 's', 't' };
+
+    std::vector<sal_uInt8> U = parseHex("7BD210807A0277FECC52C261C442F02E1AD62C1A23553348B8F8AF7320"
+                                        "DC9978FAB7E65E1BF4CA76F4BE5E6D2AA8C7D5");
+    CPPUNIT_ASSERT_EQUAL(size_t(48), U.size());
+
+    std::vector<sal_uInt8> O = parseHex("E4507A474CEFBBA1AF76BA0EB40EC322C91C1900D3FD65FEC98B873BA1"
+                                        "9B27F89FBC9331D5E14DBCEE2A0ADDA52267C9");
+    CPPUNIT_ASSERT_EQUAL(size_t(48), O.size());
+
+    // User Password
+    {
+        std::vector<sal_uInt8> aUserHash(U.begin(), U.begin() + 32);
+        CPPUNIT_ASSERT_EQUAL(size_t(32), aUserHash.size());
+
+        CPPUNIT_ASSERT_EQUAL(
+            std::string("7bd210807a0277fecc52c261c442f02e1ad62c1a23553348b8f8af7320dc9978"),
+            comphelper::hashToString(aUserHash));
+
+        std::vector<sal_uInt8> aUserValidationSalt(U.begin() + 32, U.begin() + 32 + 8);
+        auto aComputedHash = vcl::pdf::computeHashR6(pUserPass, 4, aUserValidationSalt);
+        CPPUNIT_ASSERT_EQUAL(
+            std::string("7bd210807a0277fecc52c261c442f02e1ad62c1a23553348b8f8af7320dc9978"),
+            comphelper::hashToString(aComputedHash));
+    }
+
+    // Owner Password
+    {
+        std::vector<sal_uInt8> aOwnerHash(O.begin(), O.begin() + 32);
+        CPPUNIT_ASSERT_EQUAL(size_t(32), aOwnerHash.size());
+
+        std::vector<sal_uInt8> aOwnerValidationSalt(O.begin() + 32, O.begin() + 32 + 8);
+        CPPUNIT_ASSERT_EQUAL(size_t(8), aOwnerValidationSalt.size());
+
+        CPPUNIT_ASSERT_EQUAL(
+            std::string("e4507a474cefbba1af76ba0eb40ec322c91c1900d3fd65fec98b873ba19b27f8"),
+            comphelper::hashToString(aOwnerHash));
+
+        auto RO = vcl::pdf::computeHashR6(pOwnerPass, 4, aOwnerValidationSalt, U);
+        CPPUNIT_ASSERT_EQUAL(
+            std::string("e4507a474cefbba1af76ba0eb40ec322c91c1900d3fd65fec98b873ba19b27f8"),
+            comphelper::hashToString(RO));
+    }
+}
+
+} // end anonymous namespace
+
+CPPUNIT_PLUGIN_IMPLEMENT();
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/vcl/source/pdf/PDFEncryptorR6.cxx b/vcl/source/pdf/PDFEncryptorR6.cxx
new file mode 100644
index 000000000000..64e668dca598
--- /dev/null
+++ b/vcl/source/pdf/PDFEncryptorR6.cxx
@@ -0,0 +1,103 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+#include <pdf/PDFEncryptorR6.hxx>
+#include <pdf/EncryptionHashTransporter.hxx>
+#include <pdf/pdfwriter_impl.hxx>
+#include <comphelper/crypto/Crypto.hxx>
+#include <comphelper/hash.hxx>
+#include <comphelper/random.hxx>
+
+namespace vcl::pdf
+{
+namespace
+{
+/** Calculates modulo 3 of the 128-bit integer, using the first 16 bytes of the vector */
+sal_Int32 calculateModulo3(std::vector<sal_uInt8> const& rInput)
+{
+    sal_Int32 nSum = 0;
+    for (size_t i = 0; i < 16; ++i)
+        nSum += rInput[i];
+    return nSum % 3;
+}
+}
+
+/** Algorithm 2.B: Computing a hash (revision 6 and later)
+ *
+ * Described in ISO 32000-2:2020(E) - 7.6.4.3.4
+ */
+std::vector<sal_uInt8> computeHashR6(const sal_uInt8* pPassword, size_t nPasswordLength,
+                                     std::vector<sal_uInt8> const& rValidationSalt,
+                                     std::vector<sal_uInt8> const& rUserKey)
+{
+    // Round 0
+    comphelper::Hash aHash(comphelper::HashType::SHA256);
+    aHash.update(pPassword, nPasswordLength);
+    aHash.update(rValidationSalt);
+    if (!rUserKey.empty()) // if calculating owner key
+        aHash.update(rUserKey);
+
+    std::vector<sal_uInt8> K = aHash.finalize();
+
+    std::vector<sal_uInt8> E;
+
+    sal_Int32 nRound = 1;
+    do
+    {
+        // Step a)
+        std::vector<sal_uInt8> K1;
+        for (sal_Int32 nRepetition = 0; nRepetition < 64; ++nRepetition)
+        {
+            K1.insert(K1.end(), pPassword, pPassword + nPasswordLength);
+            K1.insert(K1.end(), K.begin(), K.end());
+            if (!rUserKey.empty()) // if calculating owner key
+                K1.insert(K1.end(), rUserKey.begin(), rUserKey.end());
+        }
+
+        // Step b)
+        std::vector<sal_uInt8> aKey(K.begin(), K.begin() + 16);
+        std::vector<sal_uInt8> aInitVector(K.begin() + 16, K.end());
+
+        E = std::vector<sal_uInt8>(K1.size(), 0);
+
+        comphelper::Encrypt aEncrypt(aKey, aInitVector, comphelper::CryptoType::AES_128_CBC);
+        aEncrypt.update(E, K1);
+
+        // Step c)
+        sal_Int32 nModulo3Result = calculateModulo3(E);
+
+        // Step d)
+        comphelper::HashType eType;
+        switch (nModulo3Result)
+        {
+            case 0:
+                eType = comphelper::HashType::SHA256;
+                break;
+            case 1:
+                eType = comphelper::HashType::SHA384;
+                break;
+            default:
+                eType = comphelper::HashType::SHA512;
+                break;
+        }
+        K = comphelper::Hash::calculateHash(E.data(), E.size(), eType);
+
+        nRound++;
+    }
+    // Step e) and f)
+    // We stop iteration if we do at least 64 rounds and (the last element of E <= round number - 32)
+    while (nRound < 64 || E.back() > (nRound - 32));
+
+    // Output - first 32 bytes
+    return std::vector<sal_uInt8>(K.begin(), K.begin() + 32);
+}
+
+} // end vcl::pdf
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */