fix for fdo#62323 bad conversion of Hex strings for certain values

Basic hex literals are basic Integer ( e.g. 4 byte ) types with
-2,147,483,648 through 2,147,483,647 range. Interally the scanner
was using a long to form/scan the literal, this led to bad behaviour
on 64bit linux ( where normally long -> 8 bytes )

Change-Id: I1d0fcc55ed0eda636da1445329737d1684e69f33

5 files changed, 75 insertions, 1 deletions

diff --git a/basic/qa/basic_coverage/test_hexliteral_negIntLimit-2.vb b/basic/qa/basic_coverage/test_hexliteral_negIntLimit-2.vb
new file mode 100644
index 000000000000..f33d74cf8dd9
--- /dev/null
+++ b/basic/qa/basic_coverage/test_hexliteral_negIntLimit-2.vb
@@ -0,0 +1,18 @@
+'
+' This file is part of the LibreOffice project.
+'
+' This Source Code Form is subject to the terms of the Mozilla Public
+' License, v. 2.0. If a copy of the MPL was not distributed with this
+' file, You can obtain one at http://mozilla.org/MPL/2.0/.
+'
+
+
+Function doUnitTest as Integer
+  Dim lngDecimal as Long
+  lngDecimal = Clng(&HFFFFFFFF)
+  If lngDecimal = -1 Then
+     doUnitTest = 1
+  Else
+     doUnitTest = 0
+  End If
+End Function
diff --git a/basic/qa/basic_coverage/test_hexliteral_negIntLimit.vb b/basic/qa/basic_coverage/test_hexliteral_negIntLimit.vb
new file mode 100644
index 000000000000..f33d74cf8dd9
--- /dev/null
+++ b/basic/qa/basic_coverage/test_hexliteral_negIntLimit.vb
@@ -0,0 +1,18 @@
+'
+' This file is part of the LibreOffice project.
+'
+' This Source Code Form is subject to the terms of the Mozilla Public
+' License, v. 2.0. If a copy of the MPL was not distributed with this
+' file, You can obtain one at http://mozilla.org/MPL/2.0/.
+'
+
+
+Function doUnitTest as Integer
+  Dim lngDecimal as Long
+  lngDecimal = Clng(&HFFFFFFFF)
+  If lngDecimal = -1 Then
+     doUnitTest = 1
+  Else
+     doUnitTest = 0
+  End If
+End Function
diff --git a/basic/qa/basic_coverage/test_hexliteral_posIntLimit.vb b/basic/qa/basic_coverage/test_hexliteral_posIntLimit.vb
new file mode 100644
index 000000000000..c660486a5618
--- /dev/null
+++ b/basic/qa/basic_coverage/test_hexliteral_posIntLimit.vb
@@ -0,0 +1,18 @@
+'
+' This file is part of the LibreOffice project.
+'
+' This Source Code Form is subject to the terms of the Mozilla Public
+' License, v. 2.0. If a copy of the MPL was not distributed with this
+' file, You can obtain one at http://mozilla.org/MPL/2.0/.
+'
+
+
+Function doUnitTest as Integer
+  Dim lngDecimal as Long
+  lngDecimal = Clng(&H80000000)
+  If lngDecimal = -2147483648 Then
+     doUnitTest = 1
+  Else
+     doUnitTest = 0
+  End If
+End Function
diff --git a/basic/qa/basic_coverage/test_hexliteral_zeroIntLimit.vb b/basic/qa/basic_coverage/test_hexliteral_zeroIntLimit.vb
new file mode 100644
index 000000000000..1a02bdebfb6f
--- /dev/null
+++ b/basic/qa/basic_coverage/test_hexliteral_zeroIntLimit.vb
@@ -0,0 +1,18 @@
+'
+' This file is part of the LibreOffice project.
+'
+' This Source Code Form is subject to the terms of the Mozilla Public
+' License, v. 2.0. If a copy of the MPL was not distributed with this
+' file, You can obtain one at http://mozilla.org/MPL/2.0/.
+'
+
+
+Function doUnitTest as Integer
+  Dim lngDecimal as Long
+  lngDecimal = Clng(&H0)
+  If lngDecimal = 0 Then
+     doUnitTest = 1
+  Else
+     doUnitTest = 0
+  End If
+End Function
diff --git a/basic/source/comp/scanner.cxx b/basic/source/comp/scanner.cxx
index 728187ce0fda..b59e22ac263c 100644
--- a/basic/source/comp/scanner.cxx
+++ b/basic/source/comp/scanner.cxx
@@ -413,7 +413,9 @@ bool SbiScanner::NextSym()
                 return true;
         }
         bNumber = true;
-        long l = 0;
+        // Hex literals are signed Integers ( as defined by basic
+        // e.g. -2,147,483,648 through 2,147,483,647 (signed)
+        sal_Int32 l = 0;
         int i;
         bool bBufOverflow = false;
         while(nCol < aLine.getLength() &&  theBasicCharClass::get().isAlphaNumeric(aLine[nCol] & 0xFF, bCompatible))