xmlsecurity PDF sign: use a predictor when compressing the xref stream

With this our xref stream output is close enough to Acrobat so that the
existing signature verifier runs without any problems.

Change-Id: I6eca7966890365759c269b465e4bf4d86d335219

2 files changed, 46 insertions, 5 deletions

diff --git a/xmlsecurity/qa/unit/pdfsigning/pdfsigning.cxx b/xmlsecurity/qa/unit/pdfsigning/pdfsigning.cxx
index a6c764d6e3dd..7a8df3f4236e 100644
--- a/xmlsecurity/qa/unit/pdfsigning/pdfsigning.cxx
+++ b/xmlsecurity/qa/unit/pdfsigning/pdfsigning.cxx
@@ -58,6 +58,8 @@ public:
     void testPDF14Adobe();
     /// Test a PDF 1.6 document, signed by Adobe.
     void testPDF16Adobe();
+    /// Test adding a signature to a PDF 1.6 document.
+    void testPDF16Add();
     /// Test a PDF 1.4 document, signed by LO on Windows.
     void testPDF14LOWin();
 
@@ -68,6 +70,7 @@ public:
     CPPUNIT_TEST(testPDFRemoveAll);
     CPPUNIT_TEST(testPDF14Adobe);
     CPPUNIT_TEST(testPDF16Adobe);
+    CPPUNIT_TEST(testPDF16Add);
     CPPUNIT_TEST(testPDF14LOWin);
     CPPUNIT_TEST_SUITE_END();
 };
@@ -270,6 +273,19 @@ void PDFSigningTest::testPDF16Adobe()
     verify(m_directories.getURLFromSrc(DATA_DIRECTORY) + "pdf16adobe.pdf", 1);
 }
 
+void PDFSigningTest::testPDF16Add()
+{
+    // Contains PDF 1.6 features, make sure we can add a signature using that
+    // markup correctly.
+    OUString aSourceDir = m_directories.getURLFromSrc(DATA_DIRECTORY);
+    OUString aInURL = aSourceDir + "pdf16adobe.pdf";
+    OUString aTargetDir = m_directories.getURLFromWorkdir("/CppunitTest/xmlsecurity_pdfsigning.test.user/");
+    OUString aOutURL = aTargetDir + "add.pdf";
+    // This failed: verification broke as incorrect xref stream was written as
+    // part of the new signature.
+    sign(aInURL, aOutURL, 1);
+}
+
 void PDFSigningTest::testPDF14LOWin()
 {
     // mscrypto used SEC_OID_PKCS1_SHA1_WITH_RSA_ENCRYPTION as a digest
diff --git a/xmlsecurity/source/pdfio/pdfdocument.cxx b/xmlsecurity/source/pdfio/pdfdocument.cxx
index c73ee5d55ac9..f3b55a8b67ee 100644
--- a/xmlsecurity/source/pdfio/pdfdocument.cxx
+++ b/xmlsecurity/source/pdfio/pdfdocument.cxx
@@ -699,6 +699,15 @@ bool PDFDocument::Sign(const uno::Reference<security::XCertificate>& xCertificat
 
         // Write stream data.
         SvMemoryStream aXRefStream;
+        const size_t nOffsetLen = 3;
+        // 3 additional bytes: predictor, the first and the third field.
+        const size_t nLineLength = nOffsetLen + 3;
+        // This is the line as it appears before tweaking according to the predictor.
+        std::vector<unsigned char> aOrigLine(nLineLength);
+        // This is the previous line.
+        std::vector<unsigned char> aPrevLine(nLineLength);
+        // This is the line as written to the stream.
+        std::vector<unsigned char> aFilteredLine(nLineLength);
         for (const auto& rXRef : m_aXRef)
         {
             const XRefEntry& rEntry = rXRef.second;
@@ -706,6 +715,11 @@ bool PDFDocument::Sign(const uno::Reference<security::XCertificate>& xCertificat
             if (!rEntry.m_bDirty)
                 continue;
 
+            // Predictor.
+            size_t nPos = 0;
+            // PNG prediction: up (on all rows).
+            aOrigLine[nPos++] = 2;
+
             // First field.
             unsigned char nType = 0;
             switch (rEntry.m_eType)
@@ -720,25 +734,36 @@ bool PDFDocument::Sign(const uno::Reference<security::XCertificate>& xCertificat
                 nType = 2;
                 break;
             }
-            aXRefStream.WriteUChar(nType);
+            aOrigLine[nPos++] = nType;
 
             // Second field.
-            const size_t nOffsetLen = 3;
             for (size_t i = 0; i < nOffsetLen; ++i)
             {
                 size_t nByte = nOffsetLen - i - 1;
                 // Fields requiring more than one byte are stored with the
                 // high-order byte first.
                 unsigned char nCh = (rEntry.m_nOffset & (0xff << (nByte * 8))) >> (nByte * 8);
-                aXRefStream.WriteUChar(nCh);
+                aOrigLine[nPos++] = nCh;
             }
 
             // Third field.
-            aXRefStream.WriteUChar(0);
+            aOrigLine[nPos++] = 0;
+
+            // Now apply the predictor.
+            aFilteredLine[0] = aOrigLine[0];
+            for (size_t i = 1; i < nLineLength; ++i)
+            {
+                // Count the delta vs the previous line.
+                aFilteredLine[i] = aOrigLine[i] - aPrevLine[i];
+                // Remember the new reference.
+                aPrevLine[i] = aOrigLine[i];
+            }
+
+            aXRefStream.WriteBytes(aFilteredLine.data(), aFilteredLine.size());
         }
 
         m_aEditBuffer.WriteUInt32AsString(nXRefStreamId);
-        m_aEditBuffer.WriteCharPtr(" 0 obj\n<</Filter/FlateDecode");
+        m_aEditBuffer.WriteCharPtr(" 0 obj\n<</DecodeParms<</Columns 5/Predictor 12>>/Filter/FlateDecode");
 
         // ID.
         auto pID = dynamic_cast<PDFArrayElement*>(m_pXRefStream->Lookup("ID"));