xmlsecurity windows: let cert picker and PDF sign find ECDSA keys

Need to incrementally migrate the remaining places (ODF, OOXML signing)
to CNG, then flip the default. SVL_CRYPTO_CNG=1 is needed till then.

(The testcase passes with and without the fix when SVL_CRYPTO_CNG is not
specified; it fails without the fix when SVL_CRYPTO_CNG is specified.)

Change-Id: Ide9d3b109bbd955a9cb83b18bba6aa72269f4d34
Reviewed-on: https://gerrit.libreoffice.org/55030
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Miklos Vajna <vmiklos@collabora.co.uk>

2 files changed, 34 insertions, 5 deletions

diff --git a/xmlsecurity/source/xmlsec/mscrypt/securityenvironment_mscryptimpl.cxx b/xmlsecurity/source/xmlsec/mscrypt/securityenvironment_mscryptimpl.cxx
index ecfdd15d1895..4f1b7e81221f 100644
--- a/xmlsecurity/source/xmlsec/mscrypt/securityenvironment_mscryptimpl.cxx
+++ b/xmlsecurity/source/xmlsec/mscrypt/securityenvironment_mscryptimpl.cxx
@@ -45,6 +45,7 @@
 #include <osl/nlsupport.h>
 #include <osl/process.h>
 #include <o3tl/char16_t2wchar_t.hxx>
+#include <svl/cryptosign.hxx>
 
 using namespace ::com::sun::star;
 using namespace ::com::sun::star::lang ;
@@ -344,6 +345,7 @@ uno::Sequence< uno::Reference < XCertificate > > SecurityEnvironment_MSCryptImpl
         HCERTSTORE hSystemKeyStore ;
         DWORD      dwKeySpec;
         HCRYPTPROV hCryptProv;
+        NCRYPT_KEY_HANDLE hCryptKey;
 
 #ifdef SAL_LOG_INFO
         CertEnumSystemStore(CERT_SYSTEM_STORE_CURRENT_USER, nullptr, nullptr, cert_enum_system_store_callback);
@@ -355,10 +357,17 @@ uno::Sequence< uno::Reference < XCertificate > > SecurityEnvironment_MSCryptImpl
             while (pCertContext)
             {
                 // for checking whether the certificate is a personal certificate or not.
+                DWORD dwFlags = CRYPT_ACQUIRE_COMPARE_KEY_FLAG;
+                HCRYPTPROV_OR_NCRYPT_KEY_HANDLE* phCryptProvOrNCryptKey = &hCryptProv;
+                if (svl::crypto::isMSCng())
+                {
+                    dwFlags |= CRYPT_ACQUIRE_ONLY_NCRYPT_KEY_FLAG;
+                    phCryptProvOrNCryptKey = &hCryptKey;
+                }
                 if(!(CryptAcquireCertificatePrivateKey(pCertContext,
-                        CRYPT_ACQUIRE_COMPARE_KEY_FLAG,
+                        dwFlags,
                         nullptr,
-                        &hCryptProv,
+                        phCryptProvOrNCryptKey,
                         &dwKeySpec,
                         nullptr)))
                 {
@@ -969,10 +978,18 @@ sal_Int32 SecurityEnvironment_MSCryptImpl::getCertificateCharacters( const css::
         BOOL    fCallerFreeProv ;
         DWORD   dwKeySpec ;
         HCRYPTPROV  hProv ;
+        NCRYPT_KEY_HANDLE hKey = 0;
+        DWORD dwFlags = 0;
+        HCRYPTPROV_OR_NCRYPT_KEY_HANDLE* phCryptProvOrNCryptKey = &hProv;
+        if (svl::crypto::isMSCng())
+        {
+            dwFlags |= CRYPT_ACQUIRE_ONLY_NCRYPT_KEY_FLAG;
+            phCryptProvOrNCryptKey = &hKey;
+        }
         if( CryptAcquireCertificatePrivateKey( pCertContext ,
-                   0 ,
+                   dwFlags,
                    nullptr ,
-                   &hProv,
+                   phCryptProvOrNCryptKey,
                    &dwKeySpec,
                    &fCallerFreeProv )
         ) {
@@ -980,6 +997,8 @@ sal_Int32 SecurityEnvironment_MSCryptImpl::getCertificateCharacters( const css::
 
             if( hProv != NULL && fCallerFreeProv )
                 CryptReleaseContext( hProv, 0 ) ;
+            else if (hKey && fCallerFreeProv)
+                NCryptFreeObject(hKey);
         } else {
             characters &= ~ css::security::CertificateCharacters::HAS_PRIVATE_KEY ;
         }
diff --git a/xmlsecurity/source/xmlsec/mscrypt/x509certificate_mscryptimpl.cxx b/xmlsecurity/source/xmlsec/mscrypt/x509certificate_mscryptimpl.cxx
index d213f21631f5..1e3bd93880f9 100644
--- a/xmlsecurity/source/xmlsec/mscrypt/x509certificate_mscryptimpl.cxx
+++ b/xmlsecurity/source/xmlsec/mscrypt/x509certificate_mscryptimpl.cxx
@@ -573,7 +573,17 @@ uno::Sequence<sal_Int8> X509Certificate_MSCryptImpl::getSHA256Thumbprint()
 
 svl::crypto::SignatureMethodAlgorithm X509Certificate_MSCryptImpl::getSignatureMethodAlgorithm()
 {
-    return svl::crypto::SignatureMethodAlgorithm::RSA;
+    svl::crypto::SignatureMethodAlgorithm nRet = svl::crypto::SignatureMethodAlgorithm::RSA;
+
+    if (!m_pCertContext || !m_pCertContext->pCertInfo)
+        return nRet;
+
+    CRYPT_ALGORITHM_IDENTIFIER algorithm = m_pCertContext->pCertInfo->SubjectPublicKeyInfo.Algorithm;
+    OString aObjId(algorithm.pszObjId);
+    if (aObjId == szOID_ECC_PUBLIC_KEY)
+        nRet = svl::crypto::SignatureMethodAlgorithm::ECDSA;
+
+    return nRet;
 }
 
 css::uno::Sequence< sal_Int8 > SAL_CALL X509Certificate_MSCryptImpl::getSHA1Thumbprint()