tdf#149447: use proper UNO types for marshalling unsigned Basic types

cppu::UnoType<sal_uInt8> corresponds to UNO boolean, so its use made the
unsigned values be converted to a true/false in Any; and additionally,
using such a boolean Any, even having a 'true' value, as a parameter to
a double argument would make operator >>=(const Any &, double &) return
false, giving the resulting double equal to 0.

The wrong conversion of UShorts and ULongs to cppu::UnoType<sal_uInt8>
(aka TypeClass_BOOLEAN) was introduced in commit
11f9aa4fcb2ad0a5fada8561c7041e7f25a059fc
Author Andreas Bregas <ab@openoffice.org>
Date   Thu May 10 14:22:42 2001 +0000
#79615# sbxToUnoValue(): Choose smallest possible type for numeric values

Treating of unsigned Basic Byte as a signed cppu::UnoType<sal_Int8> (aka
TypeClass_BYTE) was already introduced in
commit c25ec0608a167bcf1d891043f02273761c351701
Author Jens-Heiner Rechtien <hr@openoffice.org>
Date   Mon Sep 18 15:18:56 2000 +0000
initial import

Then, in commit 553cf2a834fcca17be6f7712c53e190e90e260eb
Author Andreas Bregas <ab@openoffice.org>
Date   Fri Jun 08 14:59:57 2001 +0000
#87927# Map TypeClass_BYTE to SbxINTEGER instead of SbxBYTE because of signed/unsigned problem

an attempt was made to handle obviously this same problem, changing the
corresponding UNO type to TypeClass_SHORT. But it seems that it created
problems when passing arrays of Byte through UNO to COM, where it needed
to convert to a safearray, so this decision was reverted in commit
dd6ba6b64aec20d37a402bee233e742e29285e88
Author Mikhail Voytenko <mav@openoffice.org>
Date   Thu Jul 08 21:33:48 2010 +0200
mib17: #162917# let basic byte use one byte, let olebridge convert sequence to safearray correctly

This change tries to avoid the problem that caused the latter revert,
by only treating Bytes as UNO shorts in getUnoTypeForSbxValue, where
scalar Byte values are considered, keeping old handling for arrays.

Change-Id: I805108743376e2fc27dd21a27c31759b76dc0d09
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/136526
Tested-by: Jenkins
Reviewed-by: Mike Kaganski <mike.kaganski@collabora.com>

2 files changed, 53 insertions, 10 deletions

diff --git a/basic/qa/basic_coverage/test_unsigned_integers.bas b/basic/qa/basic_coverage/test_unsigned_integers.bas
new file mode 100644
index 000000000000..d7f2385ecff5
--- /dev/null
+++ b/basic/qa/basic_coverage/test_unsigned_integers.bas
@@ -0,0 +1,46 @@
+'
+' This file is part of the LibreOffice project.
+'
+' This Source Code Form is subject to the terms of the Mozilla Public
+' License, v. 2.0. If a copy of the MPL was not distributed with this
+' file, You can obtain one at http://mozilla.org/MPL/2.0/.
+'
+
+Option Explicit
+
+Function doUnitTest() As String
+    TestUtil.TestInit
+    verify_testUnsignedIntegers
+    doUnitTest = TestUtil.GetResult()
+End Function
+
+Function convertToDouble(n)
+    Dim conv As Object
+    conv = CreateUnoService("com.sun.star.script.Converter")
+    convertToDouble = conv.convertToSimpleType(n, com.sun.star.uno.TypeClass.DOUBLE)
+End Function
+
+Sub verify_testUnsignedIntegers()
+    On Error GoTo errorHandler
+
+    Dim t As New com.sun.star.util.Time ' has both unsigned long and unsigned short
+    t.Seconds = 201
+    t.NanoSeconds = 202
+    Dim u8 As Byte, u16, u32
+    u8 = 200
+    u16 = t.Seconds ' UShort
+    u32 = t.NanoSeconds ' ULong
+
+    TestUtil.AssertEqual(TypeName(u8), "Byte", "TypeName(u8)")
+    TestUtil.AssertEqual(convertToDouble(u8), 200, "convertToDouble(u8)")
+
+    TestUtil.AssertEqual(TypeName(u16), "UShort", "TypeName(u16)")
+    TestUtil.AssertEqual(convertToDouble(u16), 201, "convertToDouble(u16)")
+
+    TestUtil.AssertEqual(TypeName(u32), "ULong", "TypeName(u32)")
+    TestUtil.AssertEqual(convertToDouble(u32), 202, "convertToDouble(u32)")
+
+    Exit Sub
+errorHandler:
+    TestUtil.ReportErrorHandler("verify_testUnsignedIntegers", Err, Error$, Erl)
+End Sub
diff --git a/basic/source/classes/sbunoobj.cxx b/basic/source/classes/sbunoobj.cxx
index 51b1b956b170..8ad9e3fe9e66 100644
--- a/basic/source/classes/sbunoobj.cxx
+++ b/basic/source/classes/sbunoobj.cxx
@@ -969,6 +969,12 @@ static Type getUnoTypeForSbxValue( const SbxValue* pVal )
     // No object, convert basic type
     else
     {
+        if (eBaseType == SbxBYTE && pVal->GetByte() > 127)
+        {
+            // Basic Byte type is unsigned; cppu::UnoType<sal_uInt8> corresponds to UNO boolean,
+            // so values 128-255 are only representable starting with UNO short types
+            eBaseType = SbxUSHORT;
+        }
         aRetType = getUnoTypeForSbxBaseType( eBaseType );
     }
     return aRetType;
@@ -1049,19 +1055,10 @@ static Any sbxToUnoValueImpl( const SbxValue* pVar, bool bBlockConversionToSmall
                     aType = ::cppu::UnoType<sal_Int16>::get();
                 break;
             }
-            case TypeClass_UNSIGNED_SHORT:
-            {
-                sal_uInt16 n = pVar->GetUShort();
-                if( n <= 255 )
-                    aType = cppu::UnoType<sal_uInt8>::get();
-                break;
-            }
             case TypeClass_UNSIGNED_LONG:
             {
                 sal_uInt32 n = pVar->GetLong();
-                if( n <= 255 )
-                    aType = cppu::UnoType<sal_uInt8>::get();
-                else if( n <= SbxMAXUINT )
+                if( n <= SbxMAXUINT )
                     aType = cppu::UnoType<cppu::UnoUnsignedShortType>::get();
                 break;
             }