xmlsecurity PDF NSS verify: handle SHA1_WITH_RSA

SHA1_WITH_RSA is a signing algorithm, not a digest one, but let's
accept it, so LO on Linux can verify a signature generated by LO on
Windows.

It's annoying that equivalent mapping in NSS is not part of their public
API.

Change-Id: I97186fcc1d118f922e5ee3cb472aa5b52bc4b5ca

4 files changed, 34 insertions, 2 deletions

diff --git a/xmlsecurity/inc/pdfio/pdfdocument.hxx b/xmlsecurity/inc/pdfio/pdfdocument.hxx
index 37457c024d42..ca70c94ffbe3 100644
--- a/xmlsecurity/inc/pdfio/pdfdocument.hxx
+++ b/xmlsecurity/inc/pdfio/pdfdocument.hxx
@@ -116,6 +116,8 @@ public:
     PDFDocument();
     PDFDocument& operator=(const PDFDocument&) = delete;
     PDFDocument(const PDFDocument&) = delete;
+    /// @name Low-level functions, to be used by PDFElement subclasses.
+    //@{
     static OString ReadKeyword(SvStream& rStream);
     static size_t FindStartXRef(SvStream& rStream);
     void ReadXRef(SvStream& rStream);
@@ -136,13 +138,17 @@ public:
     bool Tokenize(SvStream& rStream, TokenizeMode eMode, std::vector< std::unique_ptr<PDFElement> >& rElements, PDFObjectElement* pObject);
     /// Register an object (owned directly or indirectly by m_aElements) as a provder for a given ID.
     void SetIDObject(size_t nID, PDFObjectElement* pObject);
+    //@}
 
+    /// @name High-level functions, to be used by others.
+    //@{
     /// Read elements from the start of the stream till its end.
     bool Read(SvStream& rStream);
     /// Sign the read document with xCertificate in the edit buffer.
     bool Sign(const css::uno::Reference<css::security::XCertificate>& xCertificate, const OUString& rDescription);
     /// Serializes the contents of the edit buffer.
     bool Write(SvStream& rStream);
+    /// Get a list of signatures embedded into this document.
     std::vector<PDFObjectElement*> GetSignatureWidgets();
     /**
      * @param rInformation The actual result.
@@ -152,6 +158,7 @@ public:
     static bool ValidateSignature(SvStream& rStream, PDFObjectElement* pSignature, SignatureInformation& rInformation, bool bLast);
     /// Remove the nth signature from read document in the edit buffer.
     bool RemoveSignature(size_t nPosition);
+    //@}
 };
 
 } // namespace pdfio
diff --git a/xmlsecurity/qa/unit/pdfsigning/data/pdf14lowin.pdf b/xmlsecurity/qa/unit/pdfsigning/data/pdf14lowin.pdf
new file mode 100644
index 000000000000..5270151ebc31
--- /dev/null
+++ b/xmlsecurity/qa/unit/pdfsigning/data/pdf14lowin.pdf
diff --git a/xmlsecurity/qa/unit/pdfsigning/pdfsigning.cxx b/xmlsecurity/qa/unit/pdfsigning/pdfsigning.cxx
index 5a95586e072b..a6c764d6e3dd 100644
--- a/xmlsecurity/qa/unit/pdfsigning/pdfsigning.cxx
+++ b/xmlsecurity/qa/unit/pdfsigning/pdfsigning.cxx
@@ -58,6 +58,8 @@ public:
     void testPDF14Adobe();
     /// Test a PDF 1.6 document, signed by Adobe.
     void testPDF16Adobe();
+    /// Test a PDF 1.4 document, signed by LO on Windows.
+    void testPDF14LOWin();
 
     CPPUNIT_TEST_SUITE(PDFSigningTest);
     CPPUNIT_TEST(testPDFAdd);
@@ -66,6 +68,7 @@ public:
     CPPUNIT_TEST(testPDFRemoveAll);
     CPPUNIT_TEST(testPDF14Adobe);
     CPPUNIT_TEST(testPDF16Adobe);
+    CPPUNIT_TEST(testPDF14LOWin);
     CPPUNIT_TEST_SUITE_END();
 };
 
@@ -267,6 +270,15 @@ void PDFSigningTest::testPDF16Adobe()
     verify(m_directories.getURLFromSrc(DATA_DIRECTORY) + "pdf16adobe.pdf", 1);
 }
 
+void PDFSigningTest::testPDF14LOWin()
+{
+    // mscrypto used SEC_OID_PKCS1_SHA1_WITH_RSA_ENCRYPTION as a digest
+    // algorithm when it meant SEC_OID_SHA1, make sure we tolerate that on all
+    // platforms.
+    // This failed, as NSS HASH_Create() didn't handle the sign algorithm.
+    verify(m_directories.getURLFromSrc(DATA_DIRECTORY) + "pdf14lowin.pdf", 1);
+}
+
 CPPUNIT_TEST_SUITE_REGISTRATION(PDFSigningTest);
 
 CPPUNIT_PLUGIN_IMPLEMENT();
diff --git a/xmlsecurity/source/pdfio/pdfdocument.cxx b/xmlsecurity/source/pdfio/pdfdocument.cxx
index 6822e149c47b..bfd66d240f29 100644
--- a/xmlsecurity/source/pdfio/pdfdocument.cxx
+++ b/xmlsecurity/source/pdfio/pdfdocument.cxx
@@ -1760,7 +1760,20 @@ bool PDFDocument::ValidateSignature(SvStream& rStream, PDFObjectElement* pSignat
     }
 
     SECItem aAlgorithm = NSS_CMSSignedData_GetDigestAlgs(pCMSSignedData)[0]->algorithm;
-    HASH_HashType eHashType = HASH_GetHashTypeByOidTag(SECOID_FindOIDTag(&aAlgorithm));
+    SECOidTag eOidTag = SECOID_FindOIDTag(&aAlgorithm);
+
+    // Map a sign algorithm to a digest algorithm.
+    // See NSS_CMSUtil_MapSignAlgs(), which is private to us.
+    switch (eOidTag)
+    {
+    case SEC_OID_PKCS1_SHA1_WITH_RSA_ENCRYPTION:
+        eOidTag = SEC_OID_SHA1;
+        break;
+    default:
+        break;
+    }
+
+    HASH_HashType eHashType = HASH_GetHashTypeByOidTag(eOidTag);
     HASHContext* pHASHContext = HASH_Create(eHashType);
     if (!pHASHContext)
     {
@@ -1796,7 +1809,7 @@ bool PDFDocument::ValidateSignature(SvStream& rStream, PDFObjectElement* pSignat
 
     // Find out what is the expected length of the hash.
     unsigned int nMaxResultLen = 0;
-    switch (SECOID_FindOIDTag(&aAlgorithm))
+    switch (eOidTag)
     {
     case SEC_OID_SHA1:
         nMaxResultLen = msfilter::SHA1_HASH_LENGTH;