iRelated rhbz#1618703: Properly handle failure en-/decoding PDF file

...when e.g. FIPS mode makes the various calls to rtl_cipher_initARCFOUR fail.

Change-Id: Id1b2222249c151470e233ab814b21228f3a8b561
Reviewed-on: https://gerrit.libreoffice.org/59543
Tested-by: Jenkins
Reviewed-by: Stephan Bergmann <sbergman@redhat.com>

3 files changed, 69 insertions, 29 deletions

diff --git a/filter/source/pdf/impdialog.cxx b/filter/source/pdf/impdialog.cxx
index 861611cc51c9..255aaa7d0257 100644
--- a/filter/source/pdf/impdialog.cxx
+++ b/filter/source/pdf/impdialog.cxx
@@ -22,6 +22,7 @@
 #include "impdialog.hxx"
 #include <strings.hrc>
 #include <officecfg/Office/Common.hxx>
+#include <vcl/errinf.hxx>
 #include <vcl/settings.hxx>
 #include <vcl/svapp.hxx>
 #include <vcl/weld.hxx>
@@ -1173,6 +1174,15 @@ IMPL_LINK_NOARG(ImpPDFTabSecurityPage, ClickmaPbSetPwdHdl, weld::Button&, void)
         mbHaveOwnerPassword = !aOwnerPW.isEmpty();
 
         mxPreparedPasswords = vcl::PDFWriter::InitEncryption( aOwnerPW, aUserPW );
+        if (!mxPreparedPasswords.is()) {
+            OUString msg;
+            ErrorHandler::GetErrorString(ERRCODE_IO_NOTSUPPORTED, msg); //TOOD: handle failure
+            std::unique_ptr<weld::MessageDialog>(
+                Application::CreateMessageDialog(
+                    GetFrameWeld(), VclMessageType::Error, VclButtonsType::Ok, msg))
+                ->run();
+            return;
+        }
 
         if( mbHaveOwnerPassword )
         {
diff --git a/sdext/source/pdfimport/pdfparse/pdfentries.cxx b/sdext/source/pdfimport/pdfparse/pdfentries.cxx
index 724dc766afe1..5288f57bb954 100644
--- a/sdext/source/pdfimport/pdfparse/pdfentries.cxx
+++ b/sdext/source/pdfimport/pdfparse/pdfentries.cxx
@@ -1157,9 +1157,13 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData )
         memset( nEncryptedEntry, 0, sizeof(nEncryptedEntry) );
         // see PDF reference 1.4 Algorithm 3.4
         // encrypt pad string
-        rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
-                                aKey, nKeyLen,
-                                nullptr, 0 );
+        if (rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
+                                    aKey, nKeyLen,
+                                    nullptr, 0 )
+            != rtl_Cipher_E_None)
+        {
+            return false; //TODO: differentiate "failed to decrypt" from "wrong password"
+        }
         rtl_cipher_encodeARCFOUR( pData->m_aCipher, nPadString, sizeof( nPadString ),
                                   nEncryptedEntry, sizeof( nEncryptedEntry ) );
         bValid = (memcmp( nEncryptedEntry, pData->m_aUEntry, 32 ) == 0);
@@ -1171,8 +1175,12 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData )
         aDigest.update(nPadString, sizeof(nPadString));
         aDigest.update(reinterpret_cast<unsigned char const*>(pData->m_aDocID.getStr()), pData->m_aDocID.getLength());
         ::std::vector<unsigned char> nEncryptedEntry(aDigest.finalize());
-        rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
-                                aKey, sizeof(aKey), nullptr, 0 );
+        if (rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
+                                    aKey, sizeof(aKey), nullptr, 0 )
+            != rtl_Cipher_E_None)
+        {
+            return false; //TODO: differentiate "failed to decrypt" from "wrong password"
+        }
         rtl_cipher_encodeARCFOUR( pData->m_aCipher,
                                   nEncryptedEntry.data(), 16,
                                   nEncryptedEntry.data(), 16 ); // encrypt in place
@@ -1182,8 +1190,12 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData )
             for( sal_uInt32 j = 0; j < sizeof(aTempKey); j++ )
                 aTempKey[j] = static_cast<sal_uInt8>( aKey[j] ^ i );
 
-            rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
-                                    aTempKey, sizeof(aTempKey), nullptr, 0 );
+            if (rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
+                                        aTempKey, sizeof(aTempKey), nullptr, 0 )
+                != rtl_Cipher_E_None)
+            {
+                return false; //TODO: differentiate "failed to decrypt" from "wrong password"
+            }
             rtl_cipher_encodeARCFOUR( pData->m_aCipher,
                                       nEncryptedEntry.data(), 16,
                                       nEncryptedEntry.data(), 16 ); // encrypt in place
@@ -1227,8 +1239,12 @@ bool PDFFile::setupDecryptionData( const OString& rPwd ) const
         sal_uInt32 nKeyLen = password_to_key( rPwd, aKey, m_pData.get(), true );
         if( m_pData->m_nStandardRevision == 2 )
         {
-            rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode,
-                                    aKey, nKeyLen, nullptr, 0 );
+            if (rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode,
+                                        aKey, nKeyLen, nullptr, 0 )
+                != rtl_Cipher_E_None)
+            {
+                return false; //TODO: differentiate "failed to decrypt" from "wrong password"
+            }
             rtl_cipher_decodeARCFOUR( m_pData->m_aCipher,
                                       m_pData->m_aOEntry, 32,
                                       nPwd, 32 );
@@ -1241,8 +1257,12 @@ bool PDFFile::setupDecryptionData( const OString& rPwd ) const
                 sal_uInt8 nTempKey[ENCRYPTION_KEY_LEN];
                 for( unsigned int j = 0; j < sizeof(nTempKey); j++ )
                     nTempKey[j] = sal_uInt8(aKey[j] ^ i);
-                rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode,
-                                        nTempKey, nKeyLen, nullptr, 0 );
+                if (rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode,
+                                            nTempKey, nKeyLen, nullptr, 0 )
+                    != rtl_Cipher_E_None)
+                {
+                    return false; //TODO: differentiate "failed to decrypt" from "wrong password"
+                }
                 rtl_cipher_decodeARCFOUR( m_pData->m_aCipher,
                                           nPwd, 32,
                                           nPwd, 32 ); // decrypt inplace
diff --git a/vcl/source/gdi/pdfwriter_impl2.cxx b/vcl/source/gdi/pdfwriter_impl2.cxx
index 1551a6e36d67..11bffa24682d 100644
--- a/vcl/source/gdi/pdfwriter_impl2.cxx
+++ b/vcl/source/gdi/pdfwriter_impl2.cxx
@@ -1407,29 +1407,39 @@ bool PDFWriterImpl::computeODictionaryValue( const sal_uInt8* i_pPaddedOwnerPass
         //Step 4, the key is in nMD5Sum
         //step 5 already done, data is in i_pPaddedUserPassword
         //step 6
-        rtl_cipher_initARCFOUR( aCipher, rtl_Cipher_DirectionEncode,
-                                 nMD5Sum.data(), i_nKeyLength , nullptr, 0 );
-        // encrypt the user password using the key set above
-        rtl_cipher_encodeARCFOUR( aCipher, i_pPaddedUserPassword, ENCRYPTED_PWD_SIZE, // the data to be encrypted
-                                  &io_rOValue[0], sal_Int32(io_rOValue.size()) ); //encrypted data
-        //Step 7, only if 128 bit
-        if( i_nKeyLength == SECUR_128BIT_KEY )
+        if (rtl_cipher_initARCFOUR( aCipher, rtl_Cipher_DirectionEncode,
+                                    nMD5Sum.data(), i_nKeyLength , nullptr, 0 )
+            == rtl_Cipher_E_None)
         {
-            sal_uInt32 i, y;
-            sal_uInt8 nLocalKey[ SECUR_128BIT_KEY ]; // 16 = 128 bit key
-
-            for( i = 1; i <= 19; i++ ) // do it 19 times, start with 1
+            // encrypt the user password using the key set above
+            rtl_cipher_encodeARCFOUR( aCipher, i_pPaddedUserPassword, ENCRYPTED_PWD_SIZE, // the data to be encrypted
+                                      &io_rOValue[0], sal_Int32(io_rOValue.size()) ); //encrypted data
+            //Step 7, only if 128 bit
+            if( i_nKeyLength == SECUR_128BIT_KEY )
             {
-                for( y = 0; y < sizeof( nLocalKey ); y++ )
-                    nLocalKey[y] = static_cast<sal_uInt8>( nMD5Sum[y] ^ i );
+                sal_uInt32 i, y;
+                sal_uInt8 nLocalKey[ SECUR_128BIT_KEY ]; // 16 = 128 bit key
 
-                rtl_cipher_initARCFOUR( aCipher, rtl_Cipher_DirectionEncode,
-                                        nLocalKey, SECUR_128BIT_KEY, nullptr, 0 ); //destination data area, on init can be NULL
-                rtl_cipher_encodeARCFOUR( aCipher, &io_rOValue[0], sal_Int32(io_rOValue.size()), // the data to be encrypted
-                                          &io_rOValue[0], sal_Int32(io_rOValue.size()) ); // encrypted data, can be the same as the input, encrypt "in place"
-                //step 8, store in class data member
+                for( i = 1; i <= 19; i++ ) // do it 19 times, start with 1
+                {
+                    for( y = 0; y < sizeof( nLocalKey ); y++ )
+                        nLocalKey[y] = static_cast<sal_uInt8>( nMD5Sum[y] ^ i );
+
+                    if (rtl_cipher_initARCFOUR( aCipher, rtl_Cipher_DirectionEncode,
+                                                nLocalKey, SECUR_128BIT_KEY, nullptr, 0 ) //destination data area, on init can be NULL
+                        != rtl_Cipher_E_None)
+                    {
+                        bSuccess = false;
+                        break;
+                    }
+                    rtl_cipher_encodeARCFOUR( aCipher, &io_rOValue[0], sal_Int32(io_rOValue.size()), // the data to be encrypted
+                                              &io_rOValue[0], sal_Int32(io_rOValue.size()) ); // encrypted data, can be the same as the input, encrypt "in place"
+                    //step 8, store in class data member
+                }
             }
         }
+        else
+            bSuccess = false;
     }
     else
         bSuccess = false;