xmlsecurity: handle MDP permission during PDF verify

(cherry picked from commit 586f6abee92af3cdabdce034b607b9a046ed3946)

Conflicts:
include/vcl/filter/PDFiumLibrary.hxx
vcl/source/filter/ipdf/pdfdocument.cxx
vcl/source/pdf/PDFiumLibrary.cxx
xmlsecurity/qa/unit/pdfsigning/pdfsigning.cxx
xmlsecurity/source/helper/pdfsignaturehelper.cxx

(cherry picked from commit 00479937dc071246cc27f33fd6397668448a7ed9)

Change-Id: I626fca7c03079fb0374c577dcfe024e7db6ed5b3
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/106766
Tested-by: Jenkins CollaboraOffice <jenkinscollaboraoffice@gmail.com>
Reviewed-by: Miklos Vajna <vmiklos@collabora.com>

2 files changed, 84 insertions, 10 deletions

diff --git a/vcl/source/filter/ipdf/pdfdocument.cxx b/vcl/source/filter/ipdf/pdfdocument.cxx
index 655450e28aa0..b668cb2d654a 100644
--- a/vcl/source/filter/ipdf/pdfdocument.cxx
+++ b/vcl/source/filter/ipdf/pdfdocument.cxx
@@ -1857,10 +1857,8 @@ static void visitPages(PDFObjectElement* pPages, std::vector<PDFObjectElement*>&
     pPages->setVisiting(false);
 }
 
-std::vector<PDFObjectElement*> PDFDocument::GetPages()
+PDFObjectElement* PDFDocument::GetCatalog()
 {
-    std::vector<PDFObjectElement*> aRet;
-
     PDFReferenceElement* pRoot = nullptr;
 
     PDFTrailerElement* pTrailer = nullptr;
@@ -1880,11 +1878,18 @@ std::vector<PDFObjectElement*> PDFDocument::GetPages()
 
     if (!pRoot)
     {
-        SAL_WARN("vcl.filter", "PDFDocument::GetPages: trailer has no Root key");
-        return aRet;
+        SAL_WARN("vcl.filter", "PDFDocument::GetCatalog: trailer has no Root key");
+        return nullptr;
     }
 
-    PDFObjectElement* pCatalog = pRoot->LookupObject();
+    return pRoot->LookupObject();
+}
+
+std::vector<PDFObjectElement*> PDFDocument::GetPages()
+{
+    std::vector<PDFObjectElement*> aRet;
+
+    PDFObjectElement* pCatalog = GetCatalog();
     if (!pCatalog)
     {
         SAL_WARN("vcl.filter", "PDFDocument::GetPages: trailer has no catalog");
@@ -1957,6 +1962,71 @@ std::vector<PDFObjectElement*> PDFDocument::GetSignatureWidgets()
     return aRet;
 }
 
+int PDFDocument::GetMDPPerm()
+{
+    int nRet = 3;
+
+    std::vector<PDFObjectElement*> aSignatures = GetSignatureWidgets();
+    if (aSignatures.empty())
+    {
+        return nRet;
+    }
+
+    for (const auto& pSignature : aSignatures)
+    {
+        vcl::filter::PDFObjectElement* pSig = pSignature->LookupObject("V");
+        if (!pSig)
+        {
+            SAL_WARN("vcl.filter", "PDFDocument::GetMDPPerm: can't find signature object");
+            continue;
+        }
+
+        auto pReference = dynamic_cast<PDFArrayElement*>(pSig->Lookup("Reference"));
+        if (!pReference || pReference->GetElements().empty())
+        {
+            continue;
+        }
+
+        auto pFirstReference = dynamic_cast<PDFDictionaryElement*>(pReference->GetElements()[0]);
+        if (!pFirstReference)
+        {
+            SAL_WARN("vcl.filter",
+                     "PDFDocument::GetMDPPerm: reference array doesn't contain a dictionary");
+            continue;
+        }
+
+        PDFElement* pTransformParams = pFirstReference->LookupElement("TransformParams");
+        auto pTransformParamsDict = dynamic_cast<PDFDictionaryElement*>(pTransformParams);
+        if (!pTransformParamsDict)
+        {
+            auto pTransformParamsRef = dynamic_cast<PDFReferenceElement*>(pTransformParams);
+            if (pTransformParamsRef)
+            {
+                PDFObjectElement* pTransformParamsObj = pTransformParamsRef->LookupObject();
+                if (pTransformParamsObj)
+                {
+                    pTransformParamsDict = pTransformParamsObj->GetDictionary();
+                }
+            }
+        }
+
+        if (!pTransformParamsDict)
+        {
+            continue;
+        }
+
+        auto pP = dynamic_cast<PDFNumberElement*>(pTransformParamsDict->LookupElement("P"));
+        if (!pP)
+        {
+            return 2;
+        }
+
+        return pP->GetValue();
+    }
+
+    return nRet;
+}
+
 std::vector<unsigned char> PDFDocument::DecodeHexString(PDFHexStringElement const* pElement)
 {
     return svl::crypto::DecodeHexString(pElement->GetValue());
diff --git a/vcl/source/pdf/PDFiumLibrary.cxx b/vcl/source/pdf/PDFiumLibrary.cxx
index 5b4c56a3cef3..114afb5097c4 100644
--- a/vcl/source/pdf/PDFiumLibrary.cxx
+++ b/vcl/source/pdf/PDFiumLibrary.cxx
@@ -33,7 +33,7 @@ PDFium::PDFium()
 
 PDFium::~PDFium() { FPDF_DestroyLibrary(); }
 
-BitmapChecksum PDFiumPage::getChecksum()
+BitmapChecksum PDFiumPage::getChecksum(int nMDPPerm)
 {
     size_t nPageWidth = FPDF_GetPageWidth(mpPage);
     size_t nPageHeight = FPDF_GetPageHeight(mpPage);
@@ -43,10 +43,14 @@ BitmapChecksum PDFiumPage::getChecksum()
         return 0;
     }
 
-    // Intentionally not using FPDF_ANNOT here, annotations/commenting is OK to not affect the
-    // checksum, signature verification wants this.
+    int nFlags = 0;
+    if (nMDPPerm != 3)
+    {
+        // Annotations/commenting should affect the checksum, signature verification wants this.
+        nFlags = FPDF_ANNOT;
+    }
     FPDF_RenderPageBitmap(pPdfBitmap, mpPage, /*start_x=*/0, /*start_y=*/0, nPageWidth, nPageHeight,
-                          /*rotate=*/0, /*flags=*/0);
+                          /*rotate=*/0, nFlags);
     Bitmap aBitmap(Size(nPageWidth, nPageHeight), 24);
     {
         BitmapScopedWriteAccess pWriteAccess(aBitmap);