Assert flags passed to rtl_convertTextToUnicode/UnicodeToText are valid

...so that at least some typos of using OUSTRING_TO_OSTRING_CVTFLAGS (0x566)
instead of OSTRING_TO_OUSTRING_CVTFLAGS (0x333) can be found.  (Unfortunately,
in the other direction, 0x333 is a valid combination of
RTL_UNICODETOTEXT_FLAGS_*.)

Change-Id: I7cfb3677b103ae90de88833cc93b0a5384607e15
Reviewed-on: https://gerrit.libreoffice.org/42288
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Stephan Bergmann <sbergman@redhat.com>

3 files changed, 46 insertions, 5 deletions

diff --git a/sal/osl/unx/security.cxx b/sal/osl/unx/security.cxx
index d1d39c389b41..dafa69fae663 100644
--- a/sal/osl/unx/security.cxx
+++ b/sal/osl/unx/security.cxx
@@ -163,7 +163,7 @@ sal_Bool SAL_CALL osl_getUserIdent(oslSecurity Security, rtl_uString **ustrIdent
 
     bRet = osl_psz_getUserIdent(Security,pszIdent,sizeof(pszIdent));
 
-    rtl_string2UString( ustrIdent, pszIdent, rtl_str_getLength( pszIdent ), osl_getThreadTextEncoding(), OUSTRING_TO_OSTRING_CVTFLAGS );
+    rtl_string2UString( ustrIdent, pszIdent, rtl_str_getLength( pszIdent ), osl_getThreadTextEncoding(), OSTRING_TO_OUSTRING_CVTFLAGS );
     SAL_WARN_IF(*ustrIdent == nullptr, "sal.osl", "*ustrIdent == NULL");
 
     return bRet;
@@ -197,7 +197,7 @@ sal_Bool SAL_CALL osl_getUserName(oslSecurity Security, rtl_uString **ustrName)
 
     bRet = osl_psz_getUserName(Security,pszName,sizeof(pszName));
 
-    rtl_string2UString( ustrName, pszName, rtl_str_getLength( pszName ), osl_getThreadTextEncoding(), OUSTRING_TO_OSTRING_CVTFLAGS );
+    rtl_string2UString( ustrName, pszName, rtl_str_getLength( pszName ), osl_getThreadTextEncoding(), OSTRING_TO_OUSTRING_CVTFLAGS );
     SAL_WARN_IF(*ustrName == nullptr, "sal.osl", "ustrName == NULL");
 
     return bRet;
@@ -231,7 +231,7 @@ sal_Bool SAL_CALL osl_getHomeDir(oslSecurity Security, rtl_uString **pustrDirect
 
     if ( bRet )
     {
-        rtl_string2UString( pustrDirectory, pszDirectory, rtl_str_getLength( pszDirectory ), osl_getThreadTextEncoding(), OUSTRING_TO_OSTRING_CVTFLAGS );
+        rtl_string2UString( pustrDirectory, pszDirectory, rtl_str_getLength( pszDirectory ), osl_getThreadTextEncoding(), OSTRING_TO_OUSTRING_CVTFLAGS );
         SAL_WARN_IF(*pustrDirectory == nullptr, "sal.osl", "*pustrDirectory == NULL");
         osl_getFileURLFromSystemPath( *pustrDirectory, pustrDirectory );
     }
@@ -339,7 +339,7 @@ sal_Bool SAL_CALL osl_getConfigDir(oslSecurity Security, rtl_uString **pustrDire
 
     if ( bRet )
     {
-        rtl_string2UString( pustrDirectory, pszDirectory, rtl_str_getLength( pszDirectory ), osl_getThreadTextEncoding(), OUSTRING_TO_OSTRING_CVTFLAGS );
+        rtl_string2UString( pustrDirectory, pszDirectory, rtl_str_getLength( pszDirectory ), osl_getThreadTextEncoding(), OSTRING_TO_OUSTRING_CVTFLAGS );
         SAL_WARN_IF(*pustrDirectory == nullptr, "sal.osl", "*pustrDirectory == NULL");
         osl_getFileURLFromSystemPath( *pustrDirectory, pustrDirectory );
     }
diff --git a/sal/osl/w32/file_url.cxx b/sal/osl/w32/file_url.cxx
index 5b0b545bc212..bb6cd00215b6 100644
--- a/sal/osl/w32/file_url.cxx
+++ b/sal/osl/w32/file_url.cxx
@@ -539,7 +539,7 @@ static bool osl_decodeURL_( rtl_String* strUTF8, rtl_uString** pstrDecodedURL )
 
     if ( bValidEncoded )
     {
-        rtl_string2UString( pstrDecodedURL, pBuffer, rtl_str_getLength(pBuffer), RTL_TEXTENCODING_UTF8, OUSTRING_TO_OSTRING_CVTFLAGS );
+        rtl_string2UString( pstrDecodedURL, pBuffer, rtl_str_getLength(pBuffer), RTL_TEXTENCODING_UTF8, OSTRING_TO_OUSTRING_CVTFLAGS );
         OSL_ASSERT(*pstrDecodedURL != nullptr);
     }
 
diff --git a/sal/textenc/textcvt.cxx b/sal/textenc/textcvt.cxx
index b1024976eecd..03914833544b 100644
--- a/sal/textenc/textcvt.cxx
+++ b/sal/textenc/textcvt.cxx
@@ -19,6 +19,8 @@
 
 #include "sal/config.h"
 
+#include <cassert>
+
 #include "rtl/textcvt.h"
 
 #include "gettextencodingdata.hxx"
@@ -157,6 +159,24 @@ sal_Size SAL_CALL rtl_convertTextToUnicode( rtl_TextToUnicodeConverter hConverte
                                             sal_uInt32 nFlags, sal_uInt32* pInfo,
                                             sal_Size* pSrcCvtBytes )
 {
+    assert(
+        (nFlags & RTL_TEXTTOUNICODE_FLAGS_UNDEFINED_MASK)
+        <= RTL_TEXTTOUNICODE_FLAGS_UNDEFINED_DEFAULT);
+    assert(
+        (nFlags & RTL_TEXTTOUNICODE_FLAGS_MBUNDEFINED_MASK)
+        <= RTL_TEXTTOUNICODE_FLAGS_MBUNDEFINED_DEFAULT);
+    assert(
+        (nFlags & RTL_TEXTTOUNICODE_FLAGS_INVALID_MASK)
+        <= RTL_TEXTTOUNICODE_FLAGS_INVALID_DEFAULT);
+    assert(
+        (nFlags
+         & ~(RTL_TEXTTOUNICODE_FLAGS_UNDEFINED_MASK
+             | RTL_TEXTTOUNICODE_FLAGS_MBUNDEFINED_MASK
+             | RTL_TEXTTOUNICODE_FLAGS_INVALID_MASK
+             | RTL_TEXTTOUNICODE_FLAGS_FLUSH
+             | RTL_TEXTTOUNICODE_FLAGS_GLOBAL_SIGNATURE))
+        == 0);
+
     const ImplTextConverter* pConverter = static_cast<const ImplTextConverter*>(hConverter);
 
     /* Only temporary, because we don't want die, if we don't have a
@@ -233,6 +253,27 @@ sal_Size SAL_CALL rtl_convertUnicodeToText( rtl_UnicodeToTextConverter hConverte
                                             sal_uInt32 nFlags, sal_uInt32* pInfo,
                                             sal_Size* pSrcCvtChars )
 {
+    assert(
+        (nFlags & RTL_UNICODETOTEXT_FLAGS_UNDEFINED_MASK)
+        <= RTL_UNICODETOTEXT_FLAGS_UNDEFINED_DEFAULT);
+    assert(
+        (nFlags & RTL_UNICODETOTEXT_FLAGS_INVALID_MASK)
+        <= RTL_UNICODETOTEXT_FLAGS_INVALID_DEFAULT);
+    assert(
+        (nFlags
+         & ~(RTL_UNICODETOTEXT_FLAGS_UNDEFINED_MASK
+             | RTL_UNICODETOTEXT_FLAGS_INVALID_MASK
+             | RTL_UNICODETOTEXT_FLAGS_UNDEFINED_REPLACE
+             | RTL_UNICODETOTEXT_FLAGS_UNDEFINED_REPLACESTR
+             | RTL_UNICODETOTEXT_FLAGS_PRIVATE_MAPTO0
+             | RTL_UNICODETOTEXT_FLAGS_NONSPACING_IGNORE
+             | RTL_UNICODETOTEXT_FLAGS_CONTROL_IGNORE
+             | RTL_UNICODETOTEXT_FLAGS_PRIVATE_IGNORE
+             | RTL_UNICODETOTEXT_FLAGS_NOCOMPOSITE
+             | RTL_UNICODETOTEXT_FLAGS_FLUSH
+             | RTL_UNICODETOTEXT_FLAGS_GLOBAL_SIGNATURE))
+        == 0);
+
     const ImplTextConverter* pConverter = static_cast<const ImplTextConverter*>(hConverter);
 
     /* Only temporary, because we don't want die, if we don't have a