Avoid signed integer overflow during BrowseBox::AutoSizeLastColumn

Observed with UBSan with `soffice --base`, "Finish", "Save", "Tables", "Create
Table in Design View...":

> include/tools/gen.hxx:485:37: runtime error: signed integer overflow: 288 + 9223372036854775803 cannot be represented in type 'long'
>     #0 0x7fe2d7f827dd in tools::Rectangle::Rectangle(Point const&, Size const&) include/tools/gen.hxx:485:37
>     #1 0x7fe2d7f73833 in BrowseBox::ImplFieldRectPixel(long, unsigned short) const svtools/source/brwbox/brwbox1.cxx:2039:12
>     #2 0x7fe2d7f57d46 in BrowseBox::GetFieldRectPixel(long, unsigned short, bool) const svtools/source/brwbox/brwbox1.cxx:1977:29
>     #3 0x7fe2d7f42bdb in BrowseBox::GetFieldRect(unsigned short) const svtools/source/brwbox/brwbox1.cxx:2068:12
>     #4 0x7fe2d7f4502b in BrowseBox::SetColumnWidth(unsigned short, unsigned long) svtools/source/brwbox/brwbox1.cxx:542:19
>     #5 0x7fe2d7f48d18 in BrowseBox::AutoSizeLastColumn() svtools/source/brwbox/brwbox1.cxx:633:9
>     #6 0x7fe2d7fd299c in BrowseBox::Resize() svtools/source/brwbox/brwbox2.cxx:537:5
>     #7 0x7fe2d807ee7a in svt::EditBrowseBox::Resize() svtools/source/brwbox/editbrowsebox.cxx:1095:20
>     #8 0x7fe2cdbe5711 in vcl::Window::ImplCallResize() vcl/source/window/event.cxx:522:5
>     #9 0x7fe2ce1c8f71 in vcl::Window::Show(bool, ShowFlags) vcl/source/window/window.cxx:2261:13
>     #10 0x7fe25d27b58f in dbaui::OTableDesignView::initialize() dbaccess/source/ui/tabledesign/TableDesignView.cxx:199:22
>     #11 0x7fe25d23403b in dbaui::OTableController::impl_initialize() dbaccess/source/ui/tabledesign/TableController.cxx:519:20
>     #12 0x7fe25c3dd649 in dbaui::OGenericUnoController::initialize(com::sun::star::uno::Sequence<com::sun::star::uno::Any> const&) dbaccess/source/ui/browser/genericcontroller.cxx:270:9
>     #13 0x7fe25c3361cd in DBContentLoader::load(com::sun::star::uno::Reference<com::sun::star::frame::XFrame> const&, rtl::OUString const&, com::sun::star::uno::Sequence<com::sun::star::beans::PropertyValue> const&, com::sun::star::uno::Reference<com::sun::star::frame::XLoadEventListener> const&) dbaccess/source/ui/browser/dbloader.cxx:270:19
>     #14 0x7fe28521412a in framework::LoadEnv::impl_loadContent() framework/source/loadenv/loadenv.cxx:1101:23
>     #15 0x7fe285209e0a in framework::LoadEnv::startLoading() framework/source/loadenv/loadenv.cxx:375:20
>     #16 0x7fe28520738b in framework::LoadEnv::loadComponentFromURL(com::sun::star::uno::Reference<com::sun::star::frame::XComponentLoader> const&, com::sun::star::uno::Reference<com::sun::star::uno::XComponentContext> const&, rtl::OUString const&, rtl::OUString const&, int, com::sun::star::uno::Sequence<com::sun::star::beans::PropertyValue> const&) framework/source/loadenv/loadenv.cxx:161:14
>     #17 0x7fe285332ba8 in (anonymous namespace)::Frame::loadComponentFromURL(rtl::OUString const&, rtl::OUString const&, int, com::sun::star::uno::Sequence<com::sun::star::beans::PropertyValue> const&) framework/source/services/frame.cxx:589:12
>     #18 0x7fe285349f7d in non-virtual thunk to (anonymous namespace)::Frame::loadComponentFromURL(rtl::OUString const&, rtl::OUString const&, int, com::sun::star::uno::Sequence<com::sun::star::beans::PropertyValue> const&) framework/source/services/frame.cxx
>     #19 0x7fe25cba4dae in dbaui::DatabaseObjectView::doDispatch(comphelper::NamedValueCollection const&) dbaccess/source/ui/misc/databaseobjectview.cxx:140:41
>     #20 0x7fe25cba3514 in dbaui::DatabaseObjectView::doCreateView(com::sun::star::uno::Any const&, rtl::OUString const&, comphelper::NamedValueCollection const&) dbaccess/source/ui/misc/databaseobjectview.cxx:97:16
>     #21 0x7fe25cba667c in dbaui::TableDesigner::doCreateView(com::sun::star::uno::Any const&, rtl::OUString const&, comphelper::NamedValueCollection const&) dbaccess/source/ui/misc/databaseobjectview.cxx:233:40
>     #22 0x7fe25cba2fec in dbaui::DatabaseObjectView::createNew(com::sun::star::uno::Reference<com::sun::star::sdbc::XDataSource> const&, comphelper::NamedValueCollection const&) dbaccess/source/ui/misc/databaseobjectview.cxx:79:16
>     #23 0x7fe25c07bdfe in dbaui::OApplicationController::newElement(dbaui::ElementType, comphelper::NamedValueCollection const&, com::sun::star::uno::Reference<com::sun::star::lang::XComponent>&) dbaccess/source/ui/app/AppController.cxx:1968:40
>     #24 0x7fe25c076546 in dbaui::OApplicationController::Execute(unsigned short, com::sun::star::uno::Sequence<com::sun::star::beans::PropertyValue> const&) dbaccess/source/ui/app/AppController.cxx:1223:25
>     #25 0x7fe25c3ef474 in dbaui::OGenericUnoController::executeChecked(com::sun::star::util::URL const&, com::sun::star::uno::Sequence<com::sun::star::beans::PropertyValue> const&) dbaccess/source/ui/browser/genericcontroller.cxx:1061:13
>     #26 0x7fe25c1dd91a in dbaui::OCreationList::onSelected(SvTreeListEntry const*) const dbaccess/source/ui/app/AppDetailView.cxx:309:81
>     #27 0x7fe25c1e0de9 in dbaui::OCreationList::MouseButtonUp(MouseEvent const&) dbaccess/source/ui/app/AppDetailView.cxx:275:9
>     #28 0x7fe2ce263026 in ImplHandleMouseEvent(VclPtr<vcl::Window> const&, MouseNotifyEvent, bool, long, long, unsigned long, unsigned short, MouseEventModifiers) vcl/source/window/winproc.cxx:711:25
>     #29 0x7fe2ce298c14 in ImplHandleSalMouseButtonUp(vcl::Window*, SalMouseEvent const*) vcl/source/window/winproc.cxx:1994:12
>     #30 0x7fe2ce27c59c in ImplWindowFrameProc(vcl::Window*, SalEvent, void const*) vcl/source/window/winproc.cxx:2329:20
>     #31 0x7fe29bebd05a in SalFrame::CallCallback(SalEvent, void const*) const vcl/inc/salframe.hxx:275:29
>     #32 0x7fe29be5cfad in GtkSalFrame::CallCallbackExc(SalEvent, void const*) const vcl/unx/gtk3/gtk3gtkframe.cxx:4318:16
>     #33 0x7fe29be755ac in GtkSalFrame::signalButton(_GtkWidget*, _GdkEventButton*, void*) vcl/unx/gtk3/gtk3gtkframe.cxx:2620:16
>     #34 0x7fe29b2a4a7a  (/lib64/libgtk-3.so.0+0x233a7a)
>     #35 0x7fe2f60c373c in g_closure_invoke (/lib64/libgobject-2.0.so.0+0xf73c)
>     #36 0x7fe2f60d64dd  (/lib64/libgobject-2.0.so.0+0x224dd)
>     #37 0x7fe2f60de69e in g_signal_emit_valist (/lib64/libgobject-2.0.so.0+0x2a69e)
>     #38 0x7fe2f60df66e in g_signal_emit (/lib64/libgobject-2.0.so.0+0x2b66e)
>     #39 0x7fe29b3efcd3  (/lib64/libgtk-3.so.0+0x37ecd3)
>     #40 0x7fe29b2a1a4d  (/lib64/libgtk-3.so.0+0x230a4d)
>     #41 0x7fe29b2a3b6f in gtk_main_do_event (/lib64/libgtk-3.so.0+0x232b6f)
>     #42 0x7fe29adb1304  (/lib64/libgdk-3.so.0+0x37304)
>     #43 0x7fe29ae0ddf1  (/lib64/libgdk-3.so.0+0x93df1)
>     #44 0x7fe2f5deab76 in g_main_context_dispatch (/lib64/libglib-2.0.so.0+0x4ab76)
>     #45 0x7fe2f5deaf1f  (/lib64/libglib-2.0.so.0+0x4af1f)
>     #46 0x7fe2f5deafab in g_main_context_iteration (/lib64/libglib-2.0.so.0+0x4afab)
>     #47 0x7fe29bcbdbca in GtkSalData::Yield(bool, bool) vcl/unx/gtk3/gtk3gtkdata.cxx:459:31
>     #48 0x7fe29bccd0d2 in GtkInstance::DoYield(bool, bool) vcl/unx/gtk3/../gtk/gtkinst.cxx:410:29
>     #49 0x7fe2cf6a65e3 in ImplYield(bool, bool) vcl/source/app/svapp.cxx:469:48
>     #50 0x7fe2cf68ef1b in Application::Yield() vcl/source/app/svapp.cxx:534:5
>     #51 0x7fe2cf68eda3 in Application::Execute() vcl/source/app/svapp.cxx:449:9
>     #52 0x7fe2f7e1eef8 in desktop::Desktop::Main() desktop/source/app/app.cxx:1622:17
>     #53 0x7fe2cf6d8687 in ImplSVMain() vcl/source/app/svmain.cxx:194:35
>     #54 0x7fe2cf6e1f7f in SVMain() vcl/source/app/svmain.cxx:232:16
>     #55 0x7fe2f7f466b4 in soffice_main desktop/source/app/sofficemain.cxx:166:12
>     #56 0x42a83c in sal_main desktop/source/app/main.c:48:15
>     #57 0x42a816 in main desktop/source/app/main.c:47:1
>     #58 0x7fe2f67a6009 in __libc_start_main (/lib64/libc.so.6+0x21009)
>     #59 0x402e69 in _start (instdir/program/soffice.bin+0x402e69)

Change-Id: I6a92a87b481396955cfc1f21bf88e2f0ad9cea9f
Reviewed-on: https://gerrit.libreoffice.org/49279
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Stephan Bergmann <sbergman@redhat.com>

2 files changed, 4 insertions, 3 deletions

diff --git a/include/tools/gen.hxx b/include/tools/gen.hxx
index ca0c43d272a5..ab74f8c6e386 100644
--- a/include/tools/gen.hxx
+++ b/include/tools/gen.hxx
@@ -482,8 +482,8 @@ inline tools::Rectangle::Rectangle( const Point& rLT, const Size& rSize )
 {
     nLeft   = rLT.X();
     nTop    = rLT.Y();
-    nRight  = rSize.Width()  ? nLeft+rSize.Width()-1 : RECT_EMPTY;
-    nBottom = rSize.Height() ? nTop+rSize.Height()-1 : RECT_EMPTY;
+    nRight  = rSize.Width()  ? nLeft+(rSize.Width()-1) : RECT_EMPTY;
+    nBottom = rSize.Height() ? nTop+(rSize.Height()-1) : RECT_EMPTY;
 }
 
 inline bool tools::Rectangle::IsEmpty() const
diff --git a/svtools/source/brwbox/brwbox1.cxx b/svtools/source/brwbox/brwbox1.cxx
index fd773ff62638..8f53a16e506b 100644
--- a/svtools/source/brwbox/brwbox1.cxx
+++ b/svtools/source/brwbox/brwbox1.cxx
@@ -2038,7 +2038,8 @@ tools::Rectangle BrowseBox::ImplFieldRectPixel( long nRow, sal_uInt16 nColumnId
     // assemble the Rectangle relative to DataWin
     return tools::Rectangle(
         Point( nColX + MIN_COLUMNWIDTH, nRowY ),
-        Size( pCols[ nCol ]->Width() - 2*MIN_COLUMNWIDTH,
+        Size( (pCols[nCol]->Width() == LONG_MAX
+               ? LONG_MAX - (nColX + MIN_COLUMNWIDTH) : pCols[ nCol ]->Width() - 2*MIN_COLUMNWIDTH),
               GetDataRowHeight() - 1 ) );
 }