summaryrefslogtreecommitdiff
path: root/sdext
diff options
context:
space:
mode:
authorStephan Bergmann <sbergman@redhat.com>2018-08-24 10:27:01 +0200
committerStephan Bergmann <sbergman@redhat.com>2018-08-24 13:14:44 +0200
commit185a14525f114e58b48236284ed8e8644bc40e48 (patch)
tree1e2e481ddc6d3480f474078a9cf01effba8e0285 /sdext
parent91a70317a34826549ed2b1089b3a8e5b0200bdd6 (diff)
iRelated rhbz#1618703: Properly handle failure en-/decoding PDF file
...when e.g. FIPS mode makes the various calls to rtl_cipher_initARCFOUR fail. Change-Id: Id1b2222249c151470e233ab814b21228f3a8b561 Reviewed-on: https://gerrit.libreoffice.org/59543 Tested-by: Jenkins Reviewed-by: Stephan Bergmann <sbergman@redhat.com>
Diffstat (limited to 'sdext')
-rw-r--r--sdext/source/pdfimport/pdfparse/pdfentries.cxx42
1 files changed, 31 insertions, 11 deletions
diff --git a/sdext/source/pdfimport/pdfparse/pdfentries.cxx b/sdext/source/pdfimport/pdfparse/pdfentries.cxx
index 724dc766afe1..5288f57bb954 100644
--- a/sdext/source/pdfimport/pdfparse/pdfentries.cxx
+++ b/sdext/source/pdfimport/pdfparse/pdfentries.cxx
@@ -1157,9 +1157,13 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData )
memset( nEncryptedEntry, 0, sizeof(nEncryptedEntry) );
// see PDF reference 1.4 Algorithm 3.4
// encrypt pad string
- rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
- aKey, nKeyLen,
- nullptr, 0 );
+ if (rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
+ aKey, nKeyLen,
+ nullptr, 0 )
+ != rtl_Cipher_E_None)
+ {
+ return false; //TODO: differentiate "failed to decrypt" from "wrong password"
+ }
rtl_cipher_encodeARCFOUR( pData->m_aCipher, nPadString, sizeof( nPadString ),
nEncryptedEntry, sizeof( nEncryptedEntry ) );
bValid = (memcmp( nEncryptedEntry, pData->m_aUEntry, 32 ) == 0);
@@ -1171,8 +1175,12 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData )
aDigest.update(nPadString, sizeof(nPadString));
aDigest.update(reinterpret_cast<unsigned char const*>(pData->m_aDocID.getStr()), pData->m_aDocID.getLength());
::std::vector<unsigned char> nEncryptedEntry(aDigest.finalize());
- rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
- aKey, sizeof(aKey), nullptr, 0 );
+ if (rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
+ aKey, sizeof(aKey), nullptr, 0 )
+ != rtl_Cipher_E_None)
+ {
+ return false; //TODO: differentiate "failed to decrypt" from "wrong password"
+ }
rtl_cipher_encodeARCFOUR( pData->m_aCipher,
nEncryptedEntry.data(), 16,
nEncryptedEntry.data(), 16 ); // encrypt in place
@@ -1182,8 +1190,12 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData )
for( sal_uInt32 j = 0; j < sizeof(aTempKey); j++ )
aTempKey[j] = static_cast<sal_uInt8>( aKey[j] ^ i );
- rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
- aTempKey, sizeof(aTempKey), nullptr, 0 );
+ if (rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
+ aTempKey, sizeof(aTempKey), nullptr, 0 )
+ != rtl_Cipher_E_None)
+ {
+ return false; //TODO: differentiate "failed to decrypt" from "wrong password"
+ }
rtl_cipher_encodeARCFOUR( pData->m_aCipher,
nEncryptedEntry.data(), 16,
nEncryptedEntry.data(), 16 ); // encrypt in place
@@ -1227,8 +1239,12 @@ bool PDFFile::setupDecryptionData( const OString& rPwd ) const
sal_uInt32 nKeyLen = password_to_key( rPwd, aKey, m_pData.get(), true );
if( m_pData->m_nStandardRevision == 2 )
{
- rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode,
- aKey, nKeyLen, nullptr, 0 );
+ if (rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode,
+ aKey, nKeyLen, nullptr, 0 )
+ != rtl_Cipher_E_None)
+ {
+ return false; //TODO: differentiate "failed to decrypt" from "wrong password"
+ }
rtl_cipher_decodeARCFOUR( m_pData->m_aCipher,
m_pData->m_aOEntry, 32,
nPwd, 32 );
@@ -1241,8 +1257,12 @@ bool PDFFile::setupDecryptionData( const OString& rPwd ) const
sal_uInt8 nTempKey[ENCRYPTION_KEY_LEN];
for( unsigned int j = 0; j < sizeof(nTempKey); j++ )
nTempKey[j] = sal_uInt8(aKey[j] ^ i);
- rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode,
- nTempKey, nKeyLen, nullptr, 0 );
+ if (rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode,
+ nTempKey, nKeyLen, nullptr, 0 )
+ != rtl_Cipher_E_None)
+ {
+ return false; //TODO: differentiate "failed to decrypt" from "wrong password"
+ }
rtl_cipher_decodeARCFOUR( m_pData->m_aCipher,
nPwd, 32,
nPwd, 32 ); // decrypt inplace