xmlsecurity: handle MDP permission during PDF verify

(cherry picked from commit 586f6abee92af3cdabdce034b607b9a046ed3946)

Conflicts:
include/vcl/filter/PDFiumLibrary.hxx
vcl/source/pdf/PDFiumLibrary.cxx
xmlsecurity/source/helper/pdfsignaturehelper.cxx

(cherry picked from commit 00479937dc071246cc27f33fd6397668448a7ed9)

Change-Id: I626fca7c03079fb0374c577dcfe024e7db6ed5b3
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/106067
Tested-by: Jenkins CollaboraOffice <jenkinscollaboraoffice@gmail.com>
Reviewed-by: Miklos Vajna <vmiklos@collabora.com>

2 files changed, 84 insertions, 11 deletions

diff --git a/vcl/source/filter/ipdf/pdfdocument.cxx b/vcl/source/filter/ipdf/pdfdocument.cxx
index 701b96e38606..dd1355d924a8 100644
--- a/vcl/source/filter/ipdf/pdfdocument.cxx
+++ b/vcl/source/filter/ipdf/pdfdocument.cxx
@@ -38,7 +38,6 @@ namespace filter
 {
 const int MAX_SIGNATURE_CONTENT_LENGTH = 50000;
 
-
 XRefEntry::XRefEntry() = default;
 
 PDFDocument::PDFDocument() = default;
@@ -1943,10 +1942,8 @@ static void visitPages(PDFObjectElement* pPages, std::vector<PDFObjectElement*>&
     pPages->setVisiting(false);
 }
 
-std::vector<PDFObjectElement*> PDFDocument::GetPages()
+PDFObjectElement* PDFDocument::GetCatalog()
 {
-    std::vector<PDFObjectElement*> aRet;
-
     PDFReferenceElement* pRoot = nullptr;
 
     PDFTrailerElement* pTrailer = nullptr;
@@ -1966,11 +1963,18 @@ std::vector<PDFObjectElement*> PDFDocument::GetPages()
 
     if (!pRoot)
     {
-        SAL_WARN("vcl.filter", "PDFDocument::GetPages: trailer has no Root key");
-        return aRet;
+        SAL_WARN("vcl.filter", "PDFDocument::GetCatalog: trailer has no Root key");
+        return nullptr;
     }
 
-    PDFObjectElement* pCatalog = pRoot->LookupObject();
+    return pRoot->LookupObject();
+}
+
+std::vector<PDFObjectElement*> PDFDocument::GetPages()
+{
+    std::vector<PDFObjectElement*> aRet;
+
+    PDFObjectElement* pCatalog = GetCatalog();
     if (!pCatalog)
     {
         SAL_WARN("vcl.filter", "PDFDocument::GetPages: trailer has no catalog");
@@ -2043,6 +2047,71 @@ std::vector<PDFObjectElement*> PDFDocument::GetSignatureWidgets()
     return aRet;
 }
 
+int PDFDocument::GetMDPPerm()
+{
+    int nRet = 3;
+
+    std::vector<PDFObjectElement*> aSignatures = GetSignatureWidgets();
+    if (aSignatures.empty())
+    {
+        return nRet;
+    }
+
+    for (const auto& pSignature : aSignatures)
+    {
+        vcl::filter::PDFObjectElement* pSig = pSignature->LookupObject("V");
+        if (!pSig)
+        {
+            SAL_WARN("vcl.filter", "PDFDocument::GetMDPPerm: can't find signature object");
+            continue;
+        }
+
+        auto pReference = dynamic_cast<PDFArrayElement*>(pSig->Lookup("Reference"));
+        if (!pReference || pReference->GetElements().empty())
+        {
+            continue;
+        }
+
+        auto pFirstReference = dynamic_cast<PDFDictionaryElement*>(pReference->GetElements()[0]);
+        if (!pFirstReference)
+        {
+            SAL_WARN("vcl.filter",
+                     "PDFDocument::GetMDPPerm: reference array doesn't contain a dictionary");
+            continue;
+        }
+
+        PDFElement* pTransformParams = pFirstReference->LookupElement("TransformParams");
+        auto pTransformParamsDict = dynamic_cast<PDFDictionaryElement*>(pTransformParams);
+        if (!pTransformParamsDict)
+        {
+            auto pTransformParamsRef = dynamic_cast<PDFReferenceElement*>(pTransformParams);
+            if (pTransformParamsRef)
+            {
+                PDFObjectElement* pTransformParamsObj = pTransformParamsRef->LookupObject();
+                if (pTransformParamsObj)
+                {
+                    pTransformParamsDict = pTransformParamsObj->GetDictionary();
+                }
+            }
+        }
+
+        if (!pTransformParamsDict)
+        {
+            continue;
+        }
+
+        auto pP = dynamic_cast<PDFNumberElement*>(pTransformParamsDict->LookupElement("P"));
+        if (!pP)
+        {
+            return 2;
+        }
+
+        return pP->GetValue();
+    }
+
+    return nRet;
+}
+
 std::vector<unsigned char> PDFDocument::DecodeHexString(PDFHexStringElement const* pElement)
 {
     return svl::crypto::DecodeHexString(pElement->GetValue());
diff --git a/vcl/source/pdf/PDFiumLibrary.cxx b/vcl/source/pdf/PDFiumLibrary.cxx
index 952a1e2803ca..3ccb1bf4135e 100644
--- a/vcl/source/pdf/PDFiumLibrary.cxx
+++ b/vcl/source/pdf/PDFiumLibrary.cxx
@@ -333,7 +333,7 @@ std::unique_ptr<PDFiumPathSegment> PDFiumPageObject::getPathSegment(int index)
     return pPDFiumPathSegment;
 }
 
-BitmapChecksum PDFiumPage::getChecksum()
+BitmapChecksum PDFiumPage::getChecksum(int nMDPPerm)
 {
     size_t nPageWidth = FPDF_GetPageWidth(mpPage);
     size_t nPageHeight = FPDF_GetPageHeight(mpPage);
@@ -343,10 +343,14 @@ BitmapChecksum PDFiumPage::getChecksum()
         return 0;
     }
 
-    // Intentionally not using FPDF_ANNOT here, annotations/commenting is OK to not affect the
-    // checksum, signature verification wants this.
+    int nFlags = 0;
+    if (nMDPPerm != 3)
+    {
+        // Annotations/commenting should affect the checksum, signature verification wants this.
+        nFlags = FPDF_ANNOT;
+    }
     FPDF_RenderPageBitmap(pPdfBitmap, mpPage, /*start_x=*/0, /*start_y=*/0, nPageWidth, nPageHeight,
-                          /*rotate=*/0, /*flags=*/0);
+                          /*rotate=*/0, nFlags);
     Bitmap aBitmap(Size(nPageWidth, nPageHeight), 24);
     {
         BitmapScopedWriteAccess pWriteAccess(aBitmap);