iRelated rhbz#1618703: Properly handle failure en-/decoding PDF file

...when e.g. FIPS mode makes the various calls to rtl_cipher_initARCFOUR fail.

Change-Id: Id1b2222249c151470e233ab814b21228f3a8b561
Reviewed-on: https://gerrit.libreoffice.org/59543
Tested-by: Jenkins
Reviewed-by: Stephan Bergmann <sbergman@redhat.com>

1 files changed, 31 insertions, 11 deletions

diff --git a/sdext/source/pdfimport/pdfparse/pdfentries.cxx b/sdext/source/pdfimport/pdfparse/pdfentries.cxx
index 724dc766afe1..5288f57bb954 100644
--- a/sdext/source/pdfimport/pdfparse/pdfentries.cxx
+++ b/sdext/source/pdfimport/pdfparse/pdfentries.cxx
@@ -1157,9 +1157,13 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData )
         memset( nEncryptedEntry, 0, sizeof(nEncryptedEntry) );
         // see PDF reference 1.4 Algorithm 3.4
         // encrypt pad string
-        rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
-                                aKey, nKeyLen,
-                                nullptr, 0 );
+        if (rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
+                                    aKey, nKeyLen,
+                                    nullptr, 0 )
+            != rtl_Cipher_E_None)
+        {
+            return false; //TODO: differentiate "failed to decrypt" from "wrong password"
+        }
         rtl_cipher_encodeARCFOUR( pData->m_aCipher, nPadString, sizeof( nPadString ),
                                   nEncryptedEntry, sizeof( nEncryptedEntry ) );
         bValid = (memcmp( nEncryptedEntry, pData->m_aUEntry, 32 ) == 0);
@@ -1171,8 +1175,12 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData )
         aDigest.update(nPadString, sizeof(nPadString));
         aDigest.update(reinterpret_cast<unsigned char const*>(pData->m_aDocID.getStr()), pData->m_aDocID.getLength());
         ::std::vector<unsigned char> nEncryptedEntry(aDigest.finalize());
-        rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
-                                aKey, sizeof(aKey), nullptr, 0 );
+        if (rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
+                                    aKey, sizeof(aKey), nullptr, 0 )
+            != rtl_Cipher_E_None)
+        {
+            return false; //TODO: differentiate "failed to decrypt" from "wrong password"
+        }
         rtl_cipher_encodeARCFOUR( pData->m_aCipher,
                                   nEncryptedEntry.data(), 16,
                                   nEncryptedEntry.data(), 16 ); // encrypt in place
@@ -1182,8 +1190,12 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData )
             for( sal_uInt32 j = 0; j < sizeof(aTempKey); j++ )
                 aTempKey[j] = static_cast<sal_uInt8>( aKey[j] ^ i );
 
-            rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
-                                    aTempKey, sizeof(aTempKey), nullptr, 0 );
+            if (rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
+                                        aTempKey, sizeof(aTempKey), nullptr, 0 )
+                != rtl_Cipher_E_None)
+            {
+                return false; //TODO: differentiate "failed to decrypt" from "wrong password"
+            }
             rtl_cipher_encodeARCFOUR( pData->m_aCipher,
                                       nEncryptedEntry.data(), 16,
                                       nEncryptedEntry.data(), 16 ); // encrypt in place
@@ -1227,8 +1239,12 @@ bool PDFFile::setupDecryptionData( const OString& rPwd ) const
         sal_uInt32 nKeyLen = password_to_key( rPwd, aKey, m_pData.get(), true );
         if( m_pData->m_nStandardRevision == 2 )
         {
-            rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode,
-                                    aKey, nKeyLen, nullptr, 0 );
+            if (rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode,
+                                        aKey, nKeyLen, nullptr, 0 )
+                != rtl_Cipher_E_None)
+            {
+                return false; //TODO: differentiate "failed to decrypt" from "wrong password"
+            }
             rtl_cipher_decodeARCFOUR( m_pData->m_aCipher,
                                       m_pData->m_aOEntry, 32,
                                       nPwd, 32 );
@@ -1241,8 +1257,12 @@ bool PDFFile::setupDecryptionData( const OString& rPwd ) const
                 sal_uInt8 nTempKey[ENCRYPTION_KEY_LEN];
                 for( unsigned int j = 0; j < sizeof(nTempKey); j++ )
                     nTempKey[j] = sal_uInt8(aKey[j] ^ i);
-                rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode,
-                                        nTempKey, nKeyLen, nullptr, 0 );
+                if (rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode,
+                                            nTempKey, nKeyLen, nullptr, 0 )
+                    != rtl_Cipher_E_None)
+                {
+                    return false; //TODO: differentiate "failed to decrypt" from "wrong password"
+                }
                 rtl_cipher_decodeARCFOUR( m_pData->m_aCipher,
                                           nPwd, 32,
                                           nPwd, 32 ); // decrypt inplace