implement CryptoAPI RC4+SHA1 encryption scheme for xls import

there might be other variants out there in practice, but this
works for default encrypted xls of excel 2013

(cherry picked from commit 1473ce030314027c01c98f513407ed0897328585)

contains...

be more c++y

Change-Id: I673b00e111a734bc626ba3d3f6ecf9692f1ce7db
(cherry picked from commit fcf7f503b64b2cf7dbef019fb43dde033e3538e8)

hash len is 20 for xls cryptoapi configuration

Change-Id: I6e06b8578959b8147043179db57e875b1d98d57d
(cherry picked from commit 491884350ce74f36044b3348bd66356ad1c234af)

clarify hash lengths a bit more

Change-Id: I66cea4b9efb82d29e6c052ac11a6f2eaca823ce0
(cherry picked from commit 8db1b13e721683d0263925c8e0300dd86a022814)

hash len isn't going to change depending on who implements it

Change-Id: Iee585cba4acad74c11d083085153e2af96c8894f
(cherry picked from commit fcc846e8f29839eaace7e1d28746abea8f4b598a)

move some useful header information to mscodec for reuse

Change-Id: Ic7adf3ed3c8279cc93a06975d6fb337210d1af87
(cherry picked from commit fc514cbf30be1613fdf4d4b7c12cbd55ca08b9b0)

rework things in light of now available documentation

Change-Id: If5f75e27191017a8c6a3929e2a70d21840f157c6
(cherry picked from commit 3fabbd0a22219464545f933fc28c869a6fa89546)

split MSCodec_Std97 into a baseclass MSCodec97

(cherry picked from commit 06916c839b16866b47235306d2db50850df0ad7c)

Change-Id: Ia3c41a048169c78684800def94e53fc9f3201e30

Change-Id: I91c0e1d1d95fbd1c68966650e7ac7d23276bcbe3
Reviewed-on: https://gerrit.libreoffice.org/30128
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Michael Stahl <mstahl@redhat.com>

3 files changed, 179 insertions, 105 deletions

diff --git a/include/filter/msfilter/mscodec.hxx b/include/filter/msfilter/mscodec.hxx
index a87ac3dd3de7..97442dc4b1e1 100644
--- a/include/filter/msfilter/mscodec.hxx
+++ b/include/filter/msfilter/mscodec.hxx
@@ -26,6 +26,12 @@
 #include <rtl/cipher.h>
 #include <rtl/digest.h>
 #include <filter/msfilter/msfilterdllapi.h>
+#include <sal/types.h>
+#include <vector>
+
+namespace com { namespace sun { namespace star {
+    namespace beans { struct NamedValue; }
+} } }
 
 namespace msfilter {
 
@@ -171,18 +177,11 @@ public:
     virtual void                Decode( sal_uInt8* pnData, sal_Size nBytes ) override;
 };
 
-
-/** Encodes and decodes data from protected MSO 97+ documents.
-
-    This is a wrapper class around low level cryptographic functions from RTL.
-    Implementation is based on the wvDecrypt package by Caolan McNamara:
-    http://www.csn.ul.ie/~caolan/docs/wvDecrypt.html
- */
-class MSFILTER_DLLPUBLIC MSCodec_Std97
+class MSFILTER_DLLPUBLIC MSCodec97
 {
 public:
-    explicit            MSCodec_Std97();
-                        ~MSCodec_Std97();
+    MSCodec97(size_t nHashLen);
+    virtual ~MSCodec97();
 
     /** Initializes the algorithm with the encryption data.
 
@@ -190,7 +189,7 @@ public:
             The sequence contains the necessary data to initialize
             the codec.
      */
-    bool                InitCodec( const css::uno::Sequence< css::beans::NamedValue >& aData );
+    bool InitCodec(const css::uno::Sequence< css::beans::NamedValue >& aData);
 
     /** Retrieves the encryption data
 
@@ -200,7 +199,6 @@ public:
      */
     css::uno::Sequence< css::beans::NamedValue > GetEncryptionData();
 
-
     /** Initializes the algorithm with the specified password and document ID.
 
         @param pPassData
@@ -209,9 +207,9 @@ public:
         @param pDocId
             Unique document identifier read from or written to the file.
      */
-    void                InitKey(
-                            const sal_uInt16 pPassData[ 16 ],
-                            const sal_uInt8 pDocId[ 16 ] );
+    virtual void InitKey(const sal_uInt16 pPassData[16],
+                         const sal_uInt8 pDocId[16]) = 0;
+
 
     /** Verifies the validity of the password using the passed salt data.
 
@@ -227,9 +225,9 @@ public:
         @return
             true = Test was successful.
      */
-    bool                VerifyKey(
-                            const sal_uInt8 pSaltData[ 16 ],
-                            const sal_uInt8 pSaltDigest[ 16 ] );
+    bool VerifyKey(const sal_uInt8* pSaltData, const sal_uInt8* pSaltDigest);
+
+    virtual void GetDigestFromSalt(const sal_uInt8* pSaltData, sal_uInt8* pDigest) = 0;
 
     /** Rekeys the codec using the specified counter.
 
@@ -246,11 +244,7 @@ public:
         @param nCounter
             Block counter used to rekey the cipher.
      */
-    bool                InitCipher( sal_uInt32 nCounter );
-
-    /** Creates an MD5 digest of salt digest. */
-    void               CreateSaltDigest(
-                            const sal_uInt8 nSaltData[16], sal_uInt8 nSaltDigest[16] );
+    virtual bool                InitCipher(sal_uInt32 nCounter) = 0;
 
     /** Encodes a block of memory.
 
@@ -274,9 +268,8 @@ public:
         @return
             true = Encoding was successful (no error occurred).
     */
-    bool                Encode(
-                            const void* pData, sal_Size nDatLen,
-                            sal_uInt8* pBuffer, sal_Size nBufLen );
+    bool                Encode(const void* pData, std::size_t nDatLen,
+                               sal_uInt8* pBuffer, std::size_t nBufLen);
 
     /** Decodes a block of memory.
 
@@ -300,9 +293,8 @@ public:
         @return
             true = Decoding was successful (no error occurred).
     */
-    bool                Decode(
-                            const void* pData, sal_Size nDatLen,
-                            sal_uInt8* pBuffer, sal_Size nBufLen );
+    bool                Decode(const void* pData, std::size_t nDatLen,
+                               sal_uInt8* pBuffer, std::size_t nBufLen);
 
     /** Lets the cipher skip a specific amount of bytes.
 
@@ -316,7 +308,66 @@ public:
         @param nDatLen
             Number of bytes to be skipped (cipher "seeks" forward).
      */
-    bool                Skip( sal_Size nDatLen );
+    bool                Skip(std::size_t nDatLen);
+
+    /* allows to get the unique document id from the codec
+     */
+    void                GetDocId( sal_uInt8 pDocId[16] );
+
+private:
+                        MSCodec97(const MSCodec97&) = delete;
+    MSCodec97&          operator=(const MSCodec97&) = delete;
+
+protected:
+    size_t              m_nHashLen;
+    rtlCipher           m_hCipher;
+    sal_uInt8           m_pDocId[16];
+    std::vector<sal_uInt8> m_aDigestValue;
+};
+
+/** Encodes and decodes data from protected MSO 97+ documents.
+
+    This is a wrapper class around low level cryptographic functions from RTL.
+    Implementation is based on the wvDecrypt package by Caolan McNamara:
+    http://www.csn.ul.ie/~caolan/docs/wvDecrypt.html
+ */
+class MSFILTER_DLLPUBLIC MSCodec_Std97 :  public MSCodec97
+{
+public:
+    MSCodec_Std97();
+    virtual ~MSCodec_Std97() override;
+
+    /** Initializes the algorithm with the specified password and document ID.
+
+        @param pPassData
+            Wide character array containing the password. Must be zero
+            terminated, which results in a maximum length of 15 characters.
+        @param pDocId
+            Unique document identifier read from or written to the file.
+     */
+    virtual void InitKey(const sal_uInt16 pPassData[16],
+                         const sal_uInt8 pDocId[16]) override;
+
+    /** Rekeys the codec using the specified counter.
+
+        After reading a specific amount of data the cipher algorithm needs to
+        be rekeyed using a counter that counts the data blocks.
+
+        The block size is for example 512 Bytes for Word files and 1024 Bytes
+        for Excel files.
+
+        @precond
+            The codec must be initialized with InitKey() before this function
+            can be used.
+
+        @param nCounter
+            Block counter used to rekey the cipher.
+     */
+    virtual bool InitCipher(sal_uInt32 nCounter) override;
+
+    /** Creates an MD5 digest of salt digest. */
+    void               CreateSaltDigest(
+                            const sal_uInt8 nSaltData[16], sal_uInt8 nSaltDigest[16] );
 
     /** Gets salt data and salt digest.
 
@@ -336,22 +387,108 @@ public:
                             sal_uInt8 pSaltData[16],
                             sal_uInt8 pSaltDigest[16]);
 
-    /* allows to get the unique document id from the codec
-     */
-    void                GetDocId( sal_uInt8 pDocId[16] );
-
-    void                GetDigestFromSalt( const sal_uInt8 pSaltData[16], sal_uInt8 pDigest[16] );
+    virtual void        GetDigestFromSalt(const sal_uInt8* pSaltData, sal_uInt8* pDigest) override;
 
 private:
                         MSCodec_Std97( const MSCodec_Std97& ) = delete;
     MSCodec_Std97&      operator=( const MSCodec_Std97& ) = delete;
 
-    rtlCipher           m_hCipher;
     rtlDigest           m_hDigest;
-    sal_uInt8           m_pDigestValue[ RTL_DIGEST_LENGTH_MD5 ];
-    sal_uInt8           m_pDocId[16];
 };
 
+class MSFILTER_DLLPUBLIC MSCodec_CryptoAPI :  public MSCodec97
+{
+public:
+    MSCodec_CryptoAPI();
+
+    virtual void InitKey(const sal_uInt16 pPassData[16],
+                         const sal_uInt8 pDocId[16]) override;
+    virtual bool InitCipher(sal_uInt32 nCounter) override;
+    virtual void GetDigestFromSalt(const sal_uInt8* pSaltData, sal_uInt8* pDigest) override;
+};
+
+const sal_uInt32 ENCRYPTINFO_CRYPTOAPI      = 0x00000004;
+const sal_uInt32 ENCRYPTINFO_DOCPROPS       = 0x00000008;
+const sal_uInt32 ENCRYPTINFO_EXTERNAL       = 0x00000010;
+const sal_uInt32 ENCRYPTINFO_AES            = 0x00000020;
+
+const sal_uInt32 ENCRYPT_ALGO_AES128        = 0x0000660E;
+const sal_uInt32 ENCRYPT_ALGO_AES192        = 0x0000660F;
+const sal_uInt32 ENCRYPT_ALGO_AES256        = 0x00006610;
+const sal_uInt32 ENCRYPT_ALGO_RC4           = 0x00006801;
+
+const sal_uInt32 ENCRYPT_HASH_SHA1          = 0x00008004;
+
+const sal_uInt32 ENCRYPT_KEY_SIZE_AES_128   = 0x00000080;
+const sal_uInt32 ENCRYPT_KEY_SIZE_AES_192   = 0x000000C0;
+const sal_uInt32 ENCRYPT_KEY_SIZE_AES_256   = 0x00000100;
+
+const sal_uInt32 ENCRYPT_PROVIDER_TYPE_AES  = 0x00000018;
+const sal_uInt32 ENCRYPT_PROVIDER_TYPE_RC4  = 0x00000001;
+
+// version of encryption info used in MS Office 1997 (major = 1, minor = 1)
+const sal_uInt32 VERSION_INFO_1997_FORMAT       = 0x00010001;
+// version of encryption info used in MS Office 2007 (major = 3, minor = 2)
+const sal_uInt32 VERSION_INFO_2007_FORMAT       = 0x00020003;
+// version of encryption info used in MS Office 2007 SP2 and older (major = 4, minor = 2)
+const sal_uInt32 VERSION_INFO_2007_FORMAT_SP2   = 0x00020004;
+
+// version of encryption info - agile (major = 4, minor = 4)
+const sal_uInt32 VERSION_INFO_AGILE         = 0x00040004;
+
+const sal_uInt32 SALT_LENGTH                    = 16;
+const sal_uInt32 ENCRYPTED_VERIFIER_LENGTH      = 16;
+const sal_uInt32 SHA1_HASH_LENGTH = RTL_DIGEST_LENGTH_SHA1; // 20
+const sal_uInt32 SHA256_HASH_LENGTH = 32;
+const sal_uInt32 SHA512_HASH_LENGTH = 64;
+
+struct MSFILTER_DLLPUBLIC EncryptionStandardHeader
+{
+    sal_uInt32 flags;
+    sal_uInt32 sizeExtra;       // 0
+    sal_uInt32 algId;           // if flag AES && CRYPTOAPI this defaults to 128-bit AES
+    sal_uInt32 algIdHash;       // 0: determine by flags - defaults to SHA-1 if not external
+    sal_uInt32 keyBits;         // key size in bits: 0 (determine by flags), 128, 192, 256
+    sal_uInt32 providedType;    // AES or RC4
+    sal_uInt32 reserved1;       // 0
+    sal_uInt32 reserved2;       // 0
+
+    EncryptionStandardHeader();
+};
+
+struct MSFILTER_DLLPUBLIC EncryptionVerifierAES
+{
+    sal_uInt32 saltSize;                                                // must be 0x00000010
+    sal_uInt8  salt[SALT_LENGTH];                                       // random generated salt value
+    sal_uInt8  encryptedVerifier[ENCRYPTED_VERIFIER_LENGTH];            // randomly generated verifier value
+    sal_uInt32 encryptedVerifierHashSize;                               // actually written hash size - depends on algorithm
+    sal_uInt8  encryptedVerifierHash[SHA256_HASH_LENGTH];               // verifier value hash - itself also encrypted
+
+    EncryptionVerifierAES();
+};
+
+struct MSFILTER_DLLPUBLIC EncryptionVerifierRC4
+{
+    sal_uInt32 saltSize;                                                // must be 0x00000010
+    sal_uInt8  salt[SALT_LENGTH];                                       // random generated salt value
+    sal_uInt8  encryptedVerifier[ENCRYPTED_VERIFIER_LENGTH];            // randomly generated verifier value
+    sal_uInt32 encryptedVerifierHashSize;                               // actually written hash size - depends on algorithm
+    sal_uInt8  encryptedVerifierHash[SHA1_HASH_LENGTH];                 // verifier value hash - itself also encrypted
+
+    EncryptionVerifierRC4();
+};
+
+struct MSFILTER_DLLPUBLIC StandardEncryptionInfo
+{
+    EncryptionStandardHeader header;
+    EncryptionVerifierAES    verifier;
+};
+
+struct MSFILTER_DLLPUBLIC RC4EncryptionInfo
+{
+    EncryptionStandardHeader header;
+    EncryptionVerifierRC4 verifier;
+};
 
 } // namespace msfilter
 
diff --git a/include/oox/crypto/CryptTools.hxx b/include/oox/crypto/CryptTools.hxx
index d963be2719c4..d4fdda2c1cfb 100644
--- a/include/oox/crypto/CryptTools.hxx
+++ b/include/oox/crypto/CryptTools.hxx
@@ -123,9 +123,6 @@ public:
         SHA512
     };
 
-    static const sal_uInt32 DIGEST_LENGTH_SHA1;
-    static const sal_uInt32 DIGEST_LENGTH_SHA512;
-
 private:
     DigestType meType;
 
diff --git a/include/oox/crypto/Standard2007Engine.hxx b/include/oox/crypto/Standard2007Engine.hxx
index 05bdc995e1a8..d6a1ca0ce272 100644
--- a/include/oox/crypto/Standard2007Engine.hxx
+++ b/include/oox/crypto/Standard2007Engine.hxx
@@ -11,7 +11,9 @@
 #ifndef INCLUDED_OOX_CRYPTO_STANDARD2007ENGINE_HXX
 #define INCLUDED_OOX_CRYPTO_STANDARD2007ENGINE_HXX
 
+#include <filter/msfilter/mscodec.hxx>
 #include <oox/crypto/CryptoEngine.hxx>
+#include <rtl/digest.h>
 #include <rtl/ustring.hxx>
 #include <sal/types.h>
 
@@ -23,71 +25,9 @@ namespace oox {
 namespace oox {
 namespace core {
 
-const sal_uInt32 ENCRYPTINFO_CRYPTOAPI      = 0x00000004;
-const sal_uInt32 ENCRYPTINFO_DOCPROPS       = 0x00000008;
-const sal_uInt32 ENCRYPTINFO_EXTERNAL       = 0x00000010;
-const sal_uInt32 ENCRYPTINFO_AES            = 0x00000020;
-
-const sal_uInt32 ENCRYPT_ALGO_AES128        = 0x0000660E;
-const sal_uInt32 ENCRYPT_ALGO_AES192        = 0x0000660F;
-const sal_uInt32 ENCRYPT_ALGO_AES256        = 0x00006610;
-const sal_uInt32 ENCRYPT_ALGO_RC4           = 0x00006801;
-
-const sal_uInt32 ENCRYPT_HASH_SHA1          = 0x00008004;
-
-const sal_uInt32 ENCRYPT_KEY_SIZE_AES_128   = 0x00000080;
-const sal_uInt32 ENCRYPT_KEY_SIZE_AES_192   = 0x000000C0;
-const sal_uInt32 ENCRYPT_KEY_SIZE_AES_256   = 0x00000100;
-
-const sal_uInt32 ENCRYPT_PROVIDER_TYPE_AES  = 0x00000018;
-const sal_uInt32 ENCRYPT_PROVIDER_TYPE_RC4  = 0x00000001;
-
-// version of encryption info used in MS Office 2007 (major = 3, minor = 2)
-const sal_uInt32 VERSION_INFO_2007_FORMAT       = 0x00020003;
-// version of encryption info used in MS Office 2007 SP2 and older (major = 4, minor = 2)
-const sal_uInt32 VERSION_INFO_2007_FORMAT_SP2   = 0x00020004;
-
-// version of encryption info - agile (major = 4, minor = 4)
-const sal_uInt32 VERSION_INFO_AGILE         = 0x00040004;
-
-const sal_uInt32 SALT_LENGTH                    = 16;
-const sal_uInt32 ENCRYPTED_VERIFIER_LENGTH      = 16;
-const sal_uInt32 ENCRYPTED_VERIFIER_HASH_LENGTH = 32;
-
-struct EncryptionStandardHeader
-{
-    sal_uInt32 flags;
-    sal_uInt32 sizeExtra;       // 0
-    sal_uInt32 algId;           // if flag AES && CRYPTOAPI this defaults to 128-bit AES
-    sal_uInt32 algIdHash;       // 0: determine by flags - defaults to SHA-1 if not external
-    sal_uInt32 keyBits;         // key size in bits: 0 (determine by flags), 128, 192, 256
-    sal_uInt32 providedType;    // AES or RC4
-    sal_uInt32 reserved1;       // 0
-    sal_uInt32 reserved2;       // 0
-
-    EncryptionStandardHeader();
-};
-
-struct EncryptionVerifierAES
-{
-    sal_uInt32 saltSize;                                                // must be 0x00000010
-    sal_uInt8  salt[SALT_LENGTH];                                       // random generated salt value
-    sal_uInt8  encryptedVerifier[ENCRYPTED_VERIFIER_LENGTH];            // randomly generated verifier value
-    sal_uInt32 encryptedVerifierHashSize;                               // actually written hash size - depends on algorithm
-    sal_uInt8  encryptedVerifierHash[ENCRYPTED_VERIFIER_HASH_LENGTH];   // verifier value hash - itself also encrypted
-
-    EncryptionVerifierAES();
-};
-
-struct StandardEncryptionInfo
-{
-    EncryptionStandardHeader header;
-    EncryptionVerifierAES    verifier;
-};
-
 class Standard2007Engine : public CryptoEngine
 {
-    StandardEncryptionInfo mInfo;
+    msfilter::StandardEncryptionInfo mInfo;
 
     bool generateVerifier();
     bool calculateEncryptionKey(const OUString& rPassword);
@@ -96,7 +36,7 @@ public:
     Standard2007Engine();
     virtual ~Standard2007Engine();
 
-    StandardEncryptionInfo& getInfo() { return mInfo;}
+    msfilter::StandardEncryptionInfo& getInfo() { return mInfo;}
 
     virtual bool generateEncryptionKey(const OUString& rPassword) override;